From 3a423a1e0ce38051a46522b34441114b37e26556 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ram=C3=B3n=20Mart=C3=ADnez?= Date: Mon, 21 Sep 2026 22:31:26 +0200 Subject: [PATCH 0001/1112] fix: exempt chat:edited/deleted/reaction-remove markers from conversational treatment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit isConversationalInjection treated only chat:reaction as a marker, so the sibling RFC-001 tags — chat:edited, chat:deleted, chat:reaction-remove — counted as real conversational input: an "[edited] ..." synthetic message arriving right after an explicit send cleared same-round suppression and re-opened auto-routing (surfaced by the zulip_mcp#23 rollout, which starts delivering edits/moves/deletions as tagged incoming messages). The four tags now live in NON_CONVERSATIONAL_TAGS (src/mcpl/tags.ts, next to the core-tag closure they belong to) with a hasNonConversationalTag helper used by both consumers: - framework.ts isConversationalInjection exempts all four; - ChannelRegistry skips them when tracking defaultPublish{Channel, MessageId,ThreadId}, so a marker cannot retarget outbound speech or become context.incoming's reply anchor (review sibling from zulip_mcp#23 round 2). The open=true transport repair stays unconditional — a marker is still authoritative liveness evidence. Tags remain non-authority (§16.6): the set only withholds default conversational treatment. Fixes #157 Co-Authored-By: Claude Fable 5 --- .../nonconversational-markers.fixed.md | 1 + src/framework.ts | 9 +++-- src/mcpl/channel-registry.ts | 13 ++++--- src/mcpl/tags.ts | 21 ++++++++++++ test/channel-registry-routing.test.ts | 34 +++++++++++++++++-- test/present-while-acting.test.ts | 30 ++++++++++++++++ 6 files changed, 99 insertions(+), 9 deletions(-) create mode 100644 changelog.d/nonconversational-markers.fixed.md diff --git a/changelog.d/nonconversational-markers.fixed.md b/changelog.d/nonconversational-markers.fixed.md new file mode 100644 index 000000000..076bd684f --- /dev/null +++ b/changelog.d/nonconversational-markers.fixed.md @@ -0,0 +1 @@ +- **Edit/delete/reaction-remove markers no longer count as conversational input** (#157). `isConversationalInjection` exempted only `chat:reaction`, so the RFC-001 marker tags `chat:edited`, `chat:deleted` and `chat:reaction-remove` (now delivered as synthetic incoming messages by e.g. zulip_mcp) cleared explicit-send suppression — an edit notice arriving right after a `send_message` re-opened auto-routing for the same round's prose. The four tags live in a shared `NON_CONVERSATIONAL_TAGS` set (`src/mcpl/tags.ts`), and `ChannelRegistry` now also skips them when tracking the default publish locus, so a marker can no longer retarget outbound speech or become the reply anchor. diff --git a/src/framework.ts b/src/framework.ts index dec36d20a..821219eb8 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -62,6 +62,7 @@ import { ProcessQueueImpl } from './queue.js'; import { REFUSAL_REACTIONS, REFUSAL_REACTION_FALLBACK } from './refusal-reactions.js'; import { Agent } from './agent.js'; import { ModuleRegistry, isStateExistsError } from './module-registry.js'; +import { hasNonConversationalTag } from './mcpl/tags.js'; import { McplServerRegistry } from './mcpl/server-registry.js'; import { FeatureSetManager } from './mcpl/feature-set-manager.js'; import { computeGrant, CapabilityGrant, expandAdvertisementShorthand } from './mcpl/capability-grant.js'; @@ -166,14 +167,16 @@ const isSilencingTool = (name: string): boolean => { * True when an injected message is real conversational input — something the * agent might actually be replying to — rather than ambient machinery. * System markers (`system: true` — send-failed notices, routing notices) and - * reactions (`chat:reaction` tag, MCPL RFC-001) don't count: they must not - * clear explicit-send suppression, and they never influence routing. + * non-conversational marker tags (reactions added/removed, edits, deletions + * — MCPL RFC-001) don't count: they are about a message the agent has + * already seen, so they must not clear explicit-send suppression, and they + * never influence routing. */ const isConversationalInjection = (metadata?: MessageMetadata): boolean => { if (!metadata) return true; const m = metadata as Record; if (m.system === true) return false; - if (Array.isArray(m.tags) && m.tags.includes('chat:reaction')) return false; + if (hasNonConversationalTag(m.tags)) return false; return true; }; diff --git a/src/mcpl/channel-registry.ts b/src/mcpl/channel-registry.ts index 636e032dd..d9d920d5c 100644 --- a/src/mcpl/channel-registry.ts +++ b/src/mcpl/channel-registry.ts @@ -35,7 +35,7 @@ import type { import type { McplServerRegistry } from './server-registry.js'; import type { FeatureSetManager } from './feature-set-manager.js'; import type { ToolDefinition, ToolResult, ProcessEvent } from '../types/index.js'; -import { expandCoreTags } from './tags.js'; +import { expandCoreTags, hasNonConversationalTag } from './tags.js'; import { CapabilityGrant } from './capability-grant.js'; // ============================================================================ @@ -870,9 +870,14 @@ export class ChannelRegistry { // deliberately after §14.5 validation: a rejected message from an // unregistered channel must not retarget outbound speech (the locus is // exactly the authority a self-attested channel would be stealing). - this.defaultPublishChannel = message.channelId; - this.defaultPublishMessageId = message.messageId; - this.defaultPublishThreadId = message.threadId; + // Non-conversational markers (reactions, edits, deletions) are ABOUT a + // message, not one the agent replies to: they must not retarget the + // default publish locus or become `context.incoming`'s reply anchor. + if (!hasNonConversationalTag(message.tags)) { + this.defaultPublishChannel = message.channelId; + this.defaultPublishMessageId = message.messageId; + this.defaultPublishThreadId = message.threadId; + } { // A server sending channels/incoming is authoritative evidence that // the transport is actually open. This repairs transient status only; diff --git a/src/mcpl/tags.ts b/src/mcpl/tags.ts index a09285c12..3bccef840 100644 --- a/src/mcpl/tags.ts +++ b/src/mcpl/tags.ts @@ -18,6 +18,27 @@ const CORE_IMPLIES: Readonly> = { 'chat:dm': ['chat:addressed', 'chat:private'], }; +/** + * RFC-001 core tags that mark a message as being ABOUT the conversation + * rather than part of it: reactions (added or removed), edits and deletions + * of messages the agent has already seen. These are markers, not turns — + * they must not clear explicit-send suppression, count as a new + * conversational round, or retarget the default publish channel. Tags are + * still never authority (§16.6): this set only ever WITHHOLDS default + * conversational treatment, it never grants anything. + */ +export const NON_CONVERSATIONAL_TAGS: ReadonlySet = new Set([ + 'chat:reaction', + 'chat:reaction-remove', + 'chat:edited', + 'chat:deleted', +]); + +/** True when a (possibly untyped) tag list carries a non-conversational marker tag. */ +export function hasNonConversationalTag(tags: unknown): boolean { + return Array.isArray(tags) && tags.some((t) => typeof t === 'string' && NON_CONVERSATIONAL_TAGS.has(t)); +} + /** * Expand a raw tag list over the normative core closure, then resolve the * §16.2 mutual exclusion: `chat:addressed` and `chat:ambient` cannot both diff --git a/test/channel-registry-routing.test.ts b/test/channel-registry-routing.test.ts index 8468c2d37..46ca1dccf 100644 --- a/test/channel-registry-routing.test.ts +++ b/test/channel-registry-routing.test.ts @@ -77,14 +77,15 @@ function makeRegistry( }; } -function incoming(channelId: string, text: string, channelName?: string) { +function incoming(channelId: string, text: string, channelName?: string, tags?: string[]) { return { messages: [{ channelId, - messageId: 'm1', + messageId: tags ? `marker-${channelId}` : 'm1', author: { id: 'u1', name: 'Antra' }, timestamp: '2026-05-30T00:00:00.000Z', content: [{ type: 'text' as const, text }], + tags, metadata: channelName ? { channelName } : undefined, }], }; @@ -526,3 +527,32 @@ test('DM prose targets resolve people-first: @name, prefix-lenient names, and <@ assert.ok('error' in missing && /send_dm/.test(missing.error), 'no-match points at send_dm'); assert.ok('error' in missing && (missing.candidates?.length ?? 0) === 2, 'lists known DMs by name'); }); + +test('non-conversational markers do not retarget the default publish locus (#157)', async () => { + // A reaction/edit/delete marker (RFC-001 tags) is ABOUT a message, not one + // the agent replies to: arriving on chanB it must not flip the + // process-global defaultPublishChannel away from the real conversation on + // chanA, and must not become context.incoming's reply anchor. + const { registry } = makeRegistry(undefined); + seedRegistered(registry, 'discord', 'chanA'); + seedRegistered(registry, 'discord', 'chanB'); + registry.handleIncoming('discord', incoming('chanA', 'hi from A')); + + for (const tag of ['chat:reaction', 'chat:reaction-remove', 'chat:edited', 'chat:deleted']) { + registry.handleIncoming('discord', incoming('chanB', `[${tag}] marker`, undefined, [tag])); + assert.equal(registry.getDefaultPublishChannel(), 'chanA', + `a ${tag} marker on chanB must not steal the default publish locus`); + } + + // A real message still retargets — the skip is tag-scoped, not a freeze. + registry.handleIncoming('discord', incoming('chanB', 'real message on B')); + assert.equal(registry.getDefaultPublishChannel(), 'chanB'); +}); + +test('a marker on a NEVER-conversed channel leaves the locus null, not marker-anchored (#157)', async () => { + const { registry } = makeRegistry(undefined); + seedRegistered(registry, 'discord', 'chanB'); + registry.handleIncoming('discord', incoming('chanB', '[edited] marker', undefined, ['chat:edited'])); + assert.equal(registry.getDefaultPublishChannel(), null, + 'an edit marker alone must not establish a publish locus'); +}); diff --git a/test/present-while-acting.test.ts b/test/present-while-acting.test.ts index c5c3157bc..d6a7852f5 100644 --- a/test/present-while-acting.test.ts +++ b/test/present-while-acting.test.ts @@ -757,6 +757,36 @@ describe('present while acting', () => { await framework.stop(); }); + for (const tag of ['chat:reaction-remove', 'chat:edited', 'chat:deleted']) { + it(`a ${tag} marker injected mid-turn does not clear send suppression (#157)`, async () => { + // Edits, deletions and reaction removals (RFC-001 core tags, delivered + // as synthetic incoming messages by e.g. zulip_mcp#23) are markers + // ABOUT a message the agent has seen — the same category as a + // reaction. Before #157 only `chat:reaction` was exempt, so an + // "[edited] Ann: typo fix" arriving right after an explicit send + // re-opened auto-routing and double-posted the same-round prose. + membrane.pushResponse(createMockResponse([ + { type: 'tool_use', id: 'c1', name: 'robot--send_message', input: { text: 'reply to room4' } }, + ] as ContentBlock[], 'tool_use')); + membrane.pushResponse(createMockResponse([ + { type: 'text', text: 'Sent it.' }, + ] as ContentBlock[])); + + const framework = await createFramework(); + const routed = stubChannelRegistry(framework); + module.interjection = `[${tag}] marker about an earlier message`; + module.interjectionChannelId = null; + module.interjectionMetadata = { channelId: '999888777', tags: [tag] }; + + trigger(framework); + await framework.runUntilIdle(); + + assert.deepEqual(routed, [], `post-send prose stayed suppressed after a ${tag} marker`); + + await framework.stop(); + }); + } + it('text-only turns route to the turn-frozen locus, not a live re-resolution', async () => { // The text-only dispatch runs after the agent is idle; a live resolution // there can read the NEXT turn's trigger state or a post-restart cleared From 4ed84f787691930861e9696858edf9638f69efd5 Mon Sep 17 00:00:00 2001 From: slimepriestess Date: Mon, 28 Sep 2026 10:58:51 -0700 Subject: [PATCH 0002/1112] fix(code-execution): the python interpreter gets the child allowlist too, not the host's secrets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #175 stops stdio MCPL children inheriting the host environment. The other child the framework spawns, the python interpreter behind code_execution, still got `{ ...process.env }` — and it runs model-authored code, so any agent with code execution could read every provider key and bot token on the host with one os.environ. The changelog's "one server's credentials are no longer readable by another" was not yet true on a host with code execution on. PyRunner builds its child env through buildChildEnv: the same operating allowlist (+ LC_*), plus the runner's own `env`, with `inheritEnv: true` as the same escape hatch. CodeExecutionConfig carries both and the framework threads them to both runner sites. PYTHONUNBUFFERED stays. Test: model-authored python asks os.environ for a host secret and gets None (red on the PR head: it printed the secret); declared env reaches the interpreter; inheritEnv restores the host env. Suite 964 / 960 / 0. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01CoQK2cP55YhezE6ajSx58h --- .../fix-mcpl-child-env-scrub.breaking.md | 5 +++ src/code-execution/py-runner.ts | 14 ++++++- src/framework.ts | 4 ++ src/types/framework.ts | 6 +++ test/code-execution-env.test.ts | 42 +++++++++++++++++++ 5 files changed, 70 insertions(+), 1 deletion(-) create mode 100644 test/code-execution-env.test.ts diff --git a/changelog.d/fix-mcpl-child-env-scrub.breaking.md b/changelog.d/fix-mcpl-child-env-scrub.breaking.md index 41b109f83..6c49f6e92 100644 --- a/changelog.d/fix-mcpl-child-env-scrub.breaking.md +++ b/changelog.d/fix-mcpl-child-env-scrub.breaking.md @@ -5,3 +5,8 @@ are no longer readable by another. A server that relied on an inherited variable must declare it in its `env` (recipes can `${VAR}`-substitute), or set `inheritEnv: true` to restore full inheritance. +- **Operators with `codeExecution` enabled:** the python interpreter that runs + model-authored scripts gets the same allowlist plus `codeExecution.env`; + `codeExecution.inheritEnv: true` restores full inheritance. Before this, + any agent with code execution could read every host secret via + `os.environ`. diff --git a/src/code-execution/py-runner.ts b/src/code-execution/py-runner.ts index b9f2cf805..336e92ee3 100644 --- a/src/code-execution/py-runner.ts +++ b/src/code-execution/py-runner.ts @@ -15,6 +15,7 @@ */ import { spawn, type ChildProcessWithoutNullStreams } from 'node:child_process'; +import { buildChildEnv } from '../mcpl/transport.js'; import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; @@ -70,6 +71,13 @@ export interface PyRunnerOptions { onToolCall: ScriptToolCallHandler; /** Log prefix, typically the agent name. */ label?: string; + /** Extra environment for the interpreter (PYTHONPATH, a venv, …). The + * child gets the same operating allowlist stdio MCPL children get + * (CHILD_ENV_ALLOWLIST + LC_*) plus exactly these — never the host's + * secrets, since the code it runs is the model's. */ + env?: Record; + /** Escape hatch: pass the host's entire environment, as before. */ + inheritEnv?: boolean; } const DEFAULT_TOOL_CALL_TIMEOUT_MS = 270_000; @@ -92,6 +100,7 @@ export class PyRunner { private readonly idleReclaimMs: number; private readonly onToolCall: ScriptToolCallHandler; private readonly label: string; + private readonly childEnv: NodeJS.ProcessEnv; private onWake: ((line: number, payload: unknown) => Promise) | null = null; private child: ChildProcessWithoutNullStreams | null = null; @@ -110,6 +119,9 @@ export class PyRunner { this.idleReclaimMs = options.idleReclaimMs ?? DEFAULT_IDLE_RECLAIM_MS; this.onToolCall = options.onToolCall; this.label = options.label ?? 'pytc'; + // The interpreter runs model-authored code: the same env discipline as an + // MCPL connector child (#175), for a child that deserves it more. + this.childEnv = { ...buildChildEnv({ env: options.env, inheritEnv: options.inheritEnv }), PYTHONUNBUFFERED: '1' }; } get busy(): boolean { @@ -244,7 +256,7 @@ export class PyRunner { const child = spawn(this.pythonPath, [runtimePath], { stdio: ['pipe', 'pipe', 'pipe'], - env: { ...process.env, PYTHONUNBUFFERED: '1' }, + env: this.childEnv, }); this.child = child; diff --git a/src/framework.ts b/src/framework.ts index 976d39ac4..ba80dc4b3 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -10246,6 +10246,8 @@ export class AgentFramework { toolCallTimeoutMs: cfg?.toolCallTimeoutMs, scriptTimeoutMs: cfg?.scriptTimeoutMs, idleReclaimMs: 0, // dedicated runner; lifetime is the exec deadline + env: cfg?.env, + inheritEnv: cfg?.inheritEnv, label: `${agentName}:${scriptId}`, onToolCall: (toolName, args) => this.handleScriptToolCall(agentName, toolName, args), }); @@ -10569,6 +10571,8 @@ export class AgentFramework { toolCallTimeoutMs: cfg?.toolCallTimeoutMs, scriptTimeoutMs: cfg?.scriptTimeoutMs, idleReclaimMs: cfg?.idleReclaimMs, + env: cfg?.env, + inheritEnv: cfg?.inheritEnv, label: agentName, onToolCall: (toolName, args) => this.handleScriptToolCall(agentName, toolName, args), }); diff --git a/src/types/framework.ts b/src/types/framework.ts index e7fb509ac..584da99f9 100644 --- a/src/types/framework.ts +++ b/src/types/framework.ts @@ -68,6 +68,12 @@ export interface CodeExecutionConfig { /** Max wake_agent() calls per background script (default 100); further * calls raise RuntimeError inside the script. */ maxWakesPerScript?: number; + /** Environment for the python interpreter on top of the operating + * allowlist (see CHILD_ENV_ALLOWLIST); the host's secrets are not passed + * to model-authored code. `${VAR}`-substitute from .env in a recipe. */ + env?: Record; + /** Escape hatch: the interpreter inherits the host's whole environment. */ + inheritEnv?: boolean; } /** See `FrameworkConfig.providerHold`. */ diff --git a/test/code-execution-env.test.ts b/test/code-execution-env.test.ts new file mode 100644 index 000000000..65eac6ffc --- /dev/null +++ b/test/code-execution-env.test.ts @@ -0,0 +1,42 @@ +/** + * The code-execution child runs MODEL-AUTHORED code. It must not inherit the + * host's secrets any more than an MCPL connector child does (#175): on a + * multi-resident host the .env carries provider keys and every bot's token, + * and `os.environ` would hand all of them to any agent with code_execution. + * Same operating allowlist as stdio MCPL children, plus the runner's own + * declared env; `inheritEnv: true` restores full inheritance. + */ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; + +import { PyRunner } from '../src/index.js'; + +test('model-authored python does not see a host secret', async () => { + const key = 'AF_CODE_EXEC_TEST_HOST_SECRET'; + process.env[key] = 'must-not-leak'; + const runner = new PyRunner({ onToolCall: async () => '' }); + try { + const r = await runner.exec(`import os; print(repr(os.environ.get(${JSON.stringify(key)})))`, []); + assert.equal(r.returnCode, 0, r.stderr); + assert.match(r.stdout, /None/, `the code-execution child inherited a host secret: ${r.stdout.trim()}`); + } finally { + runner.dispose(); + delete process.env[key]; + } +}); + +test('declared env reaches the interpreter; inheritEnv restores the host env', async () => { + const key = 'AF_CODE_EXEC_TEST_HOST_SECRET'; + process.env[key] = 'must-not-leak'; + const declared = new PyRunner({ onToolCall: async () => '', env: { AF_CODE_EXEC_DECLARED: 'yes' } }); + const inheriting = new PyRunner({ onToolCall: async () => '', inheritEnv: true }); + try { + const d = await declared.exec(`import os; print(os.environ.get('AF_CODE_EXEC_DECLARED'), repr(os.environ.get(${JSON.stringify(key)})))`, []); + assert.match(d.stdout, /^yes None/m, d.stdout + d.stderr); + const i = await inheriting.exec(`import os; print(repr(os.environ.get(${JSON.stringify(key)})))`, []); + assert.match(i.stdout, /must-not-leak/, i.stdout + i.stderr); + } finally { + declared.dispose(); inheriting.dispose(); + delete process.env[key]; + } +}); From d8ea631edbdc34c211c19075fa4a784799e5582d Mon Sep 17 00:00:00 2001 From: slimepriestess Date: Tue, 29 Sep 2026 13:15:21 -0700 Subject: [PATCH 0003/1112] fix: rebuild the child env per spawn; match the allowlist case-insensitively on Windows Greptile's two findings on #193, both real: - The runner snapshotted its child env at construction, so a respawn after idle reclaim reused stale host values where the old per-spawn `{ ...process.env }` saw current ones. The env options are stored and buildChildEnv runs at every spawn. - Windows enumerates env names in arbitrary spellings (`Path`, `SystemRoot`, `ComSpec`) and the exact-case allowlist dropped them, starting every stdio/python child without its search path. On win32 the allowlist now matches by upper-cased name, keeping the original spelling; POSIX stays case-sensitive (`Path` is a different variable there). This also closes the #175 review's standing Windows note. Both tests red on the previous head: freshness 0/1, win32 casing 0/1; full suite 976/972/0. Co-Authored-By: Claude Fable 5 --- src/code-execution/py-runner.ts | 11 +++++++---- src/mcpl/transport.ts | 9 ++++++++- test/code-execution-env.test.ts | 15 +++++++++++++++ test/mcpl-child-env.test.ts | 21 +++++++++++++++++++++ 4 files changed, 51 insertions(+), 5 deletions(-) diff --git a/src/code-execution/py-runner.ts b/src/code-execution/py-runner.ts index 336e92ee3..5fed85d2e 100644 --- a/src/code-execution/py-runner.ts +++ b/src/code-execution/py-runner.ts @@ -100,7 +100,7 @@ export class PyRunner { private readonly idleReclaimMs: number; private readonly onToolCall: ScriptToolCallHandler; private readonly label: string; - private readonly childEnv: NodeJS.ProcessEnv; + private readonly envOptions: { env?: Record; inheritEnv?: boolean }; private onWake: ((line: number, payload: unknown) => Promise) | null = null; private child: ChildProcessWithoutNullStreams | null = null; @@ -120,8 +120,11 @@ export class PyRunner { this.onToolCall = options.onToolCall; this.label = options.label ?? 'pytc'; // The interpreter runs model-authored code: the same env discipline as an - // MCPL connector child (#175), for a child that deserves it more. - this.childEnv = { ...buildChildEnv({ env: options.env, inheritEnv: options.inheritEnv }), PYTHONUNBUFFERED: '1' }; + // MCPL connector child (#175), for a child that deserves it more. Stored + // as options, not a snapshot — the env is rebuilt at every spawn so a + // respawn after idle reclaim sees current host values, as the old + // per-spawn `{ ...process.env }` did. + this.envOptions = { env: options.env, inheritEnv: options.inheritEnv }; } get busy(): boolean { @@ -256,7 +259,7 @@ export class PyRunner { const child = spawn(this.pythonPath, [runtimePath], { stdio: ['pipe', 'pipe', 'pipe'], - env: this.childEnv, + env: { ...buildChildEnv(this.envOptions), PYTHONUNBUFFERED: '1' }, }); this.child = child; diff --git a/src/mcpl/transport.ts b/src/mcpl/transport.ts index 2fc8a523e..536e17390 100644 --- a/src/mcpl/transport.ts +++ b/src/mcpl/transport.ts @@ -125,12 +125,19 @@ export const CHILD_ENV_ALLOWLIST = [ export function buildChildEnv( config: Pick, hostEnv: NodeJS.ProcessEnv = process.env, + platform: NodeJS.Platform = process.platform, ): NodeJS.ProcessEnv { if (config.inheritEnv) return { ...hostEnv, ...config.env }; const env: NodeJS.ProcessEnv = {}; + // Windows env var names are case-insensitive and enumerate in arbitrary + // spellings (`Path`, `SystemRoot`, `ComSpec`); match the allowlist by + // upper-cased name there and keep the original spelling. POSIX names are + // case-sensitive, so `Path` staying excluded there is correct. + const win = platform === 'win32'; for (const [key, value] of Object.entries(hostEnv)) { if (value === undefined) continue; - if (CHILD_ENV_ALLOWLIST.includes(key) || key.startsWith('LC_')) env[key] = value; + const probe = win ? key.toUpperCase() : key; + if (CHILD_ENV_ALLOWLIST.includes(probe) || probe.startsWith('LC_')) env[key] = value; } return { ...env, ...config.env }; } diff --git a/test/code-execution-env.test.ts b/test/code-execution-env.test.ts index 65eac6ffc..e18d81e60 100644 --- a/test/code-execution-env.test.ts +++ b/test/code-execution-env.test.ts @@ -40,3 +40,18 @@ test('declared env reaches the interpreter; inheritEnv restores the host env', a delete process.env[key]; } }); + +test('the child env is rebuilt per spawn, not snapshotted at construction', async () => { + const key = 'AF_CODE_EXEC_TEST_LATE_VAR'; + const runner = new PyRunner({ onToolCall: async () => '', inheritEnv: true }); + process.env[key] = 'set-after-construction'; + try { + const r = await runner.exec(`import os; print(repr(os.environ.get(${JSON.stringify(key)})))`, []); + assert.equal(r.returnCode, 0, r.stderr); + assert.match(r.stdout, /set-after-construction/, + `a var set after construction must reach the next spawn: ${r.stdout.trim()}`); + } finally { + runner.dispose(); + delete process.env[key]; + } +}); diff --git a/test/mcpl-child-env.test.ts b/test/mcpl-child-env.test.ts index 9458a5af7..4751227ed 100644 --- a/test/mcpl-child-env.test.ts +++ b/test/mcpl-child-env.test.ts @@ -72,3 +72,24 @@ test('a real spawned stdio child sees only allowlist + declared env', async () = delete process.env[secretKey]; } }); + +test('win32: allowlist matches case-insensitively, original spelling kept', () => { + const env = buildChildEnv({}, { + Path: 'C:\\Windows;C:\\Windows\\System32', + SystemRoot: 'C:\\Windows', + ComSpec: 'C:\\Windows\\System32\\cmd.exe', + lc_ctype: 'UTF-8', + ANTHROPIC_AUTH_TOKEN: 'host-anthropic-secret', + }, 'win32'); + assert.deepEqual(env, { + Path: 'C:\\Windows;C:\\Windows\\System32', + SystemRoot: 'C:\\Windows', + ComSpec: 'C:\\Windows\\System32\\cmd.exe', + lc_ctype: 'UTF-8', + }); +}); + +test('posix: allowlist stays case-sensitive (`Path` is a different variable)', () => { + const env = buildChildEnv({}, { Path: '/spoofed', PATH: '/usr/bin' }, 'linux'); + assert.deepEqual(env, { PATH: '/usr/bin' }); +}); From 36d51d733ff3311828dbe33f3ca645d95aaab335 Mon Sep 17 00:00:00 2001 From: slimepriestess Date: Tue, 29 Sep 2026 18:12:56 -0700 Subject: [PATCH 0004/1112] docs(changelog): fragment for the py-runner env scrub The branch's earlier fragment landed on main with #175, so the check saw no new entry for this PR. Co-Authored-By: Claude Fable 5 --- changelog.d/fix-code-execution-child-env.fixed.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 changelog.d/fix-code-execution-child-env.fixed.md diff --git a/changelog.d/fix-code-execution-child-env.fixed.md b/changelog.d/fix-code-execution-child-env.fixed.md new file mode 100644 index 000000000..9e4be7171 --- /dev/null +++ b/changelog.d/fix-code-execution-child-env.fixed.md @@ -0,0 +1,7 @@ +- The code-execution python interpreter no longer inherits the host + environment: it gets the same operating allowlist as stdio MCPL children + (#175), plus the runner's declared `env`, with `inheritEnv: true` as the + escape hatch — model-authored code can no longer read host secrets via + `os.environ`. The child env is rebuilt at every spawn, and on Windows the + allowlist matches names case-insensitively (`Path`, `SystemRoot`, + `ComSpec` survive with their original spelling). From 17034b1544c94aec6aa158d5b3fcbb88ff18a40c Mon Sep 17 00:00:00 2001 From: Iris-308 Date: Fri, 2 Oct 2026 01:18:09 -0400 Subject: [PATCH 0005/1112] fix(routing): deliver speech failure notices to the speaking agent Use the route failure callback's conversationId to target registered agents through addMessage. This keeps a fork's failure notice in its isolated window and evaluates mid-turn deferral against the speaker instead of the primary. Preserve the primary fallback for unknown conversation IDs and the context-only, no-wake behavior. Add five framework integration tests using the real ChannelRegistry callback and Chronicle contexts. Cover primary and unknown-ID behavior, secondary-agent manager selection, secondary live-turn deferral, and an actual incoming-DM fork whose failed reply is followed by its notice without another inference. Ordinary residents share a message slot, so the secondary test checks the receiving manager while the fork test verifies window isolation. Validation: compiled new tests on original source gave 2 pass / 3 fail (secondary targeting, deferral, and fork isolation). After the fix, the compiled targeting and existing registry routing tests gave 27 pass / 0 fail. Build and git diff --check passed. Full compiled suite is running separately before PR submission. Fixes #188. Co-Authored-By: GPT-6 Astra --- CHANGELOG.md | 4 + src/framework.ts | 5 +- test/route-failure-targeting.test.ts | 141 +++++++++++++++++++++++++++ 3 files changed, 149 insertions(+), 1 deletion(-) create mode 100644 test/route-failure-targeting.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b9a30b4c..3bf98d349 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,10 @@ Releases up to and including 0.7.3 predate this file; for their contents see ## Unreleased +### Fixed + +- Speech-route failure notices now target the speaking agent, including conversation forks, and respect that agent's mid-turn deferral. Unregistered conversation IDs retain the primary-agent fallback; notices remain context-only and do not trigger inference (#188). + ## 0.19.0 — 2026-09-28 ### Added diff --git a/src/framework.ts b/src/framework.ts index 510b9183a..b18147ac1 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -12541,7 +12541,7 @@ export class AgentFramework { // addMessage() alone does not request inference, so this never wakes // her (matching the `discord-send-failed-skip` gate intent: context // yes, wake no). - onRouteFailure: ({ channelId, reason, textLen }) => { + onRouteFailure: ({ conversationId, channelId, reason, textLen }) => { try { // Render a human-readable channel name when we can — a bare // snowflake in the marker is unresolvable for the agent (the @@ -12563,6 +12563,9 @@ export class AgentFramework { text: `[discord-send-failed] Your previous reply (${textLen} chars) could not be delivered to ${where} (${reason}). It was saved to your archive but the human did not receive it.`, }], { system: true, kind: 'discord-send-failed', channelId: channelId ?? '', reason }, + // Keep the notice with its speaker, including mid-turn deferral. + // Unknown conversation IDs retain the primary-agent fallback. + this.agents.has(conversationId) ? { forAgent: conversationId } : undefined, ); } catch (err) { console.error('onRouteFailure: failed to record send-failure marker:', err); diff --git a/test/route-failure-targeting.test.ts b/test/route-failure-targeting.test.ts new file mode 100644 index 000000000..e17e55a24 --- /dev/null +++ b/test/route-failure-targeting.test.ts @@ -0,0 +1,141 @@ +import { describe, it, beforeEach, afterEach } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { AgentFramework } from '../src/index.js'; +import type { ProcessEvent } from '../src/index.js'; +import type { ChannelRegistry } from '../src/mcpl/channel-registry.js'; +import { MockMembrane, createMockResponse } from './helpers/mock-membrane.js'; + +function internals(framework: AgentFramework) { + return framework as unknown as { + initializeMcpl(servers: []): Promise; + channelRegistry: ChannelRegistry; + activeTurnTokens: Map; + deferredMessages: Array<{ forAgent?: string; metadata?: Record }>; + pendingRequests: unknown[]; + }; +} + +describe('speech route failure notice targeting', () => { + let tempDir: string; + let framework: AgentFramework; + let membrane: MockMembrane; + + beforeEach(async () => { + tempDir = mkdtempSync(join(tmpdir(), 'route-failure-targeting-')); + membrane = new MockMembrane(); + framework = await AgentFramework.create({ + storePath: join(tempDir, 'test.chronicle'), + membrane: membrane.asMembrane(), + agents: [ + { name: 'primary', model: 'test-model', systemPrompt: 'Primary.', proseRouting: 'locus' }, + { name: 'secondary', model: 'test-model', systemPrompt: 'Secondary.' }, + ], + modules: [], + conversations: { templateAgent: 'primary' }, + }); + // Install the real framework callback and registry without a live connector. + await internals(framework).initializeMcpl([]); + }); + + afterEach(async () => { + internals(framework).activeTurnTokens.clear(); + await framework.stop(); + rmSync(tempDir, { recursive: true, force: true }); + }); + + it('records a secondary agent failure through that agent\'s context manager', async (t) => { + // Ordinary residents share a Chronicle message slot. Check the receiving + // manager here; the real conversation fork below checks window isolation. + const primary = t.mock.method(framework.getAgent('primary')!.getContextManager(), 'addMessage'); + const secondary = t.mock.method(framework.getAgent('secondary')!.getContextManager(), 'addMessage'); + const result = await internals(framework).channelRegistry.routeSpeech('secondary', 'hello', 'missing-channel'); + + assert.equal(result, null); + assert.equal(primary.mock.callCount(), 0); + assert.equal(secondary.mock.callCount(), 1); + const [participant, content, metadata] = secondary.mock.calls[0]!.arguments; + assert.equal(participant, 'user'); + assert.match(JSON.stringify(content), /\[discord-send-failed\].*5 chars.*missing-channel/); + assert.equal(metadata?.system, true); + assert.equal(metadata?.kind, 'discord-send-failed'); + assert.equal(metadata?.channelId, 'missing-channel'); + assert.match(String(metadata?.reason), /no registered channel/); + assert.equal(internals(framework).pendingRequests.length, 0, 'a failure notice must not request inference'); + assert.equal(membrane.calls.length, 0); + }); + + it('keeps primary failures in the primary context manager', async (t) => { + const primary = t.mock.method(framework.getAgent('primary')!.getContextManager(), 'addMessage'); + const secondary = t.mock.method(framework.getAgent('secondary')!.getContextManager(), 'addMessage'); + + await internals(framework).channelRegistry.routeSpeech('primary', 'hello', null); + + assert.equal(primary.mock.callCount(), 1); + assert.equal(secondary.mock.callCount(), 0); + assert.equal(primary.mock.calls[0]!.arguments[2]?.kind, 'discord-send-failed'); + assert.equal(internals(framework).pendingRequests.length, 0); + }); + + it('falls back to the primary for an unregistered conversation', async (t) => { + const primary = t.mock.method(framework.getAgent('primary')!.getContextManager(), 'addMessage'); + const secondary = t.mock.method(framework.getAgent('secondary')!.getContextManager(), 'addMessage'); + + await internals(framework).channelRegistry.routeSpeech('unknown-conversation', 'hello', null); + + assert.equal(primary.mock.callCount(), 1); + assert.equal(secondary.mock.callCount(), 0); + assert.equal(primary.mock.calls[0]!.arguments[2]?.kind, 'discord-send-failed'); + assert.equal(internals(framework).pendingRequests.length, 0); + }); + + it('defers a failure notice against the speaking agent\'s live turn', async (t) => { + const primary = t.mock.method(framework.getAgent('primary')!.getContextManager(), 'addMessage'); + const secondary = t.mock.method(framework.getAgent('secondary')!.getContextManager(), 'addMessage'); + const state = internals(framework); + state.activeTurnTokens.set('secondary', 1); + + await state.channelRegistry.routeSpeech('secondary', 'hello', null); + + assert.equal(primary.mock.callCount(), 0); + assert.equal(secondary.mock.callCount(), 0, 'the notice waits for the owner\'s safe boundary'); + assert.equal(state.deferredMessages.length, 1); + assert.equal(state.deferredMessages[0]!.forAgent, 'secondary'); + assert.equal(state.deferredMessages[0]!.metadata?.kind, 'discord-send-failed'); + assert.equal(state.pendingRequests.length, 0); + }); + + it('delivers a failed fork reply notice to the fork after its turn, without waking it again', async () => { + const reply = 'Reply from the fork.'; + membrane.pushResponse(createMockResponse([{ type: 'text', text: reply }])); + framework.pushEvent({ + type: 'mcpl:channel-incoming', + serverId: 'srv', + channelId: 'missing-channel', + messageId: 'incoming-1', + author: { id: 'user-1', name: 'User' }, + content: [{ type: 'text', text: 'Hello fork.' }], + timestamp: new Date().toISOString(), + metadata: { channel_type: 'im' }, + triggerInference: true, + } as unknown as ProcessEvent); + await framework.runUntilIdle(); + + const binding = framework.getConversationRouter()!.getBinding('missing-channel'); + assert.ok(binding, 'the incoming DM must spawn a real conversation fork'); + const fork = framework.getAgent(binding.agentName)!; + const { messages: forkMessages } = await fork.getContextManager().compile(); + const { messages: primaryMessages } = await framework.getAgent('primary')!.getContextManager().compile(); + const notices = forkMessages.filter((m) => JSON.stringify(m.content).includes('[discord-send-failed]')); + assert.equal(notices.length, 1, 'the fork must see its own failure notice exactly once'); + assert.ok(!primaryMessages.some((m) => JSON.stringify(m.content).includes('[discord-send-failed]'))); + const replyIndex = forkMessages.findIndex((m) => m.participant === binding.agentName && JSON.stringify(m.content).includes(reply)); + assert.ok(replyIndex >= 0, 'the attempted reply is still archived'); + assert.ok(forkMessages.indexOf(notices[0]!) > replyIndex, 'failure notice follows the attempted reply'); + assert.equal(internals(framework).deferredMessages.length, 0, 'the completed turn flushes its notice'); + assert.equal(internals(framework).pendingRequests.length, 0); + assert.equal(membrane.calls.length, 1, 'failure notice is context only, not another wake'); + }); +}); From ebfc3154487f98b597b7b984441837ea731b3191 Mon Sep 17 00:00:00 2001 From: Felix-299 Date: Fri, 2 Oct 2026 01:20:00 -0400 Subject: [PATCH 0006/1112] fix(mcpl): contain inbound request handler failures Plain EventEmitter listeners discard returned promises. Channel handlers acknowledge requests before reconciliation, so a later lifecycle-store exception used to become an unhandled rejection and could terminate the host. Wrap the inbound RPC listeners once at wireMcplEvents. Invoke handlers immediately to retain control/data-plane ordering, track result and error response attempts, consume synchronous throws and asynchronous rejections, and send -32603 only before any response attempt. A throwing write is outcome-unknown and is never followed by another reply. Emit a typed mcpl:request-handler-error trace, including a secondary error-response failure when present. Apply the same boundary to push, inference, incoming, host-command, and synchronous RPC listeners. Regression tests use the real framework wiring and channel registry with an injected Chronicle lifecycle append failure; cover pre/post-ACK errors, notifications, all async listener families, synchronous throws, failed transport responses, and preserved inference IDs/synchronous success. The suite was 1 pass / 12 fail on original source and 13 pass / 0 fail with the wrapper. Full compiled differential is being collected for the PR. Co-Authored-By: GPT-6 Astra --- CHANGELOG.md | 4 + src/framework.ts | 91 +++++++++--- src/types/trace.ts | 14 ++ test/mcpl-request-handler-errors.test.ts | 174 +++++++++++++++++++++++ 4 files changed, 265 insertions(+), 18 deletions(-) create mode 100644 test/mcpl-request-handler-errors.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b9a30b4c..49668fc0f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,10 @@ Releases up to and including 0.7.3 predate this file; for their contents see ## Unreleased +### Fixed + +- MCPL request and notification handlers contain synchronous throws and async rejections (#186), including channel reconciliation failures after an acknowledgement. Failures emit `mcpl:request-handler-error`; requests receive JSON-RPC `-32603` only when no response write has been attempted. Failed error-response writes are included in the trace, and notifications receive no response. + ## 0.19.0 — 2026-09-28 ### Added diff --git a/src/framework.ts b/src/framework.ts index 510b9183a..9109ba93d 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -13542,6 +13542,66 @@ export class AgentFramework { * Push events and inference requests are deferred to Steps 6/7. */ private wireMcplEvents(connection: McplServerConnection): void { + type Responder = { + id: string | number; + respond: (result: unknown) => void; + respondError: (code: number, message: string, data?: unknown) => void; + }; + + // EventEmitter does not consume promises returned by listeners. Keep + // each RPC handler's rejection at this boundary, including failures after + // channels/register or channels/changed has acknowledged the request. + const onRequest = ( + event: string, + handler: (params: T, responder?: Responder) => void | Promise, + ): void => { + connection.on(event, (params: T, responder?: Responder) => { + let responseAttempted = false; + const tracked = responder && { + ...responder, + respond: (result: unknown) => { + // A throwing transport write may have partially sent the reply. + // Mark before writing so recovery never sends a second response. + responseAttempted = true; + responder.respond(result); + }, + respondError: (code: number, message: string, data?: unknown) => { + responseAttempted = true; + responder.respondError(code, message, data); + }, + }; + const failed = (error: unknown): void => { + const message = error instanceof Error ? error.message : String(error); + const attemptedBeforeFailure = responseAttempted; + let responseError: string | undefined; + if (tracked && !responseAttempted) { + try { + tracked.respondError(-32603, message); + } catch (sendError) { + responseError = sendError instanceof Error ? sendError.message : String(sendError); + } + } + this.emitTrace({ + type: 'mcpl:request-handler-error', + serverId: connection.id, + event, + requestId: responder?.id, + responseAttempted: attemptedBeforeFailure, + error: message, + ...(responseError === undefined ? {} : { responseError }), + }); + }; + try { + // Invoke immediately: control-plane responses must retain their + // ACK-before-reconcile ordering and data-plane barrier semantics. + const result = handler(params, tracked); + if (result) void result.catch(failed); + } catch (error) { + failed(error); + } + }); + }; + // Forward subprocess stderr lines as trace events so consumers (conhost, // log sinks, TUI badges) can persist and surface them. connection.on('stderr', (params: { line: string }) => { @@ -13559,7 +13619,7 @@ export class AgentFramework { // framework; the ScopeManager and config `scopes` wiring went with it. // Handle push events (Step 6) - connection.on('push-event', async ( + onRequest('push-event', async ( params: PushEventParams, responder?: { respond: (result: unknown) => void; respondError: (code: number, message: string) => void }, ) => { @@ -13569,7 +13629,7 @@ export class AgentFramework { }); // Handle server-initiated inference requests (Step 6) - connection.on('inference-request', async ( + onRequest('inference-request', async ( params: McplInferenceRequestParams, responder?: { id: string | number; respond: (result: unknown) => void; respondError: (code: number, message: string) => void }, ) => { @@ -13590,7 +13650,7 @@ export class AgentFramework { }); // Handle channel registration (Step 7) - connection.on('channels-register', async ( + onRequest('channels-register', async ( params: ChannelsRegisterParams, responder?: { respond: (result: unknown) => void }, ) => { @@ -13601,7 +13661,7 @@ export class AgentFramework { // present for the Request form and carries itemized added-descriptor // results; absent for Notifications, where rejection is itemwise // filtering plus a diagnostic trace. - connection.on('channels-changed', async ( + onRequest('channels-changed', async ( params: ChannelsChangedParams, responder?: { respond: (result: unknown) => void }, ) => { @@ -13619,7 +13679,7 @@ export class AgentFramework { }); }); - connection.on('channels-list', ( + onRequest('channels-list', ( _params: unknown, responder?: { respond: (result: unknown) => void }, ) => { @@ -13631,7 +13691,7 @@ export class AgentFramework { // RFC-007 §6: the server's filter over tools/lifecycle. Admission already // required toolLifecycle.observe (-32002 otherwise). Validation failures // leave the previous filter in force (§6.6). - connection.on('tools-observe', ( + onRequest('tools-observe', ( params: unknown, responder?: { respond: (result: unknown) => void; respondError?: (code: number, message: string, data?: unknown) => void }, ) => { @@ -13645,7 +13705,7 @@ export class AgentFramework { responder?.respond({}); }); - connection.on('model-info', ( + onRequest('model-info', ( _params: unknown, responder?: { respond: (result: unknown) => void; respondError?: (code: number, message: string, data?: unknown) => void }, ) => { @@ -13662,7 +13722,7 @@ export class AgentFramework { }); // Handle incoming channel messages (Step 7) - connection.on('channels-incoming', async ( + onRequest('channels-incoming', async ( params: ChannelsIncomingParams, responder?: { respond: (result: unknown) => void }, ) => { @@ -13672,20 +13732,15 @@ export class AgentFramework { }); // Handle host-level admin commands from a surface (e.g. Discord /undo) - connection.on('host-command', async ( + onRequest('host-command', async ( params: HostCommandParams, responder?: { respond: (result: unknown) => void; respondError: (code: number, message: string) => void }, ) => { if (!responder) return; - try { - const barrier = this.discordAwarenessBarrier; - if (barrier) await barrier.promise; - const result = await this.handleHostCommand(connection.id, params ?? {}); - responder.respond(result); - } catch (error) { - const err = error instanceof Error ? error : new Error(String(error)); - responder.respondError(-32603, err.message); - } + const barrier = this.discordAwarenessBarrier; + if (barrier) await barrier.promise; + const result = await this.handleHostCommand(connection.id, params ?? {}); + responder.respond(result); }); // Handle dynamic tool list changes (notifications/tools/list_changed) diff --git a/src/types/trace.ts b/src/types/trace.ts index e4a6bdba9..1fa4bfd55 100644 --- a/src/types/trace.ts +++ b/src/types/trace.ts @@ -428,6 +428,20 @@ export type TraceEvent = serverId: string; error: string; }) + | (TraceEventBase & { + /** An inbound RPC request/notification handler threw or rejected. */ + type: 'mcpl:request-handler-error'; + serverId: string; + /** Connection event name, e.g. channels-register or push-event. */ + event: string; + /** Absent for notifications. */ + requestId?: string | number; + /** True if a response write was attempted before the handler failed. */ + responseAttempted: boolean; + error: string; + /** Failure while trying to send an internal-error response. */ + responseError?: string; + }) | (TraceEventBase & { type: 'mcpl:orphaned-response'; serverId: string; diff --git a/test/mcpl-request-handler-errors.test.ts b/test/mcpl-request-handler-errors.test.ts new file mode 100644 index 000000000..c6c4f84ec --- /dev/null +++ b/test/mcpl-request-handler-errors.test.ts @@ -0,0 +1,174 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { EventEmitter } from 'node:events'; + +import { AgentFramework } from '../src/framework.js'; +import { ChannelRegistry } from '../src/mcpl/channel-registry.js'; +import { CapabilityGrant, ALL_CAPABILITY_PATHS } from '../src/mcpl/capability-grant.js'; + +function harness() { + const traces: any[] = []; + const connection = Object.assign(new EventEmitter(), { id: 'srv' }); + const framework = Object.create(AgentFramework.prototype) as any; + framework.traceListeners = [(event: unknown) => traces.push(event)]; + framework.wireMcplEvents(connection); + return { framework, connection, traces }; +} + +function responder() { + const results: unknown[] = []; + const errors: unknown[] = []; + return { + results, + errors, + id: 17, + respond(result: unknown) { results.push(result); }, + respondError(code: number, message: string, data?: unknown) { + errors.push({ code, message, data }); + }, + }; +} + +const settle = () => new Promise((resolve) => setImmediate(resolve)); +const failureTraces = (traces: any[]) => traces.filter((t) => t.type === 'mcpl:request-handler-error'); + +for (const [event, field] of [['channels-register', 'channels'], ['channels-changed', 'added']] as const) { + test(event + ': lifecycle persistence failure after ACK is traced without another response', async () => { + const { framework, connection, traces } = harness(); + const server = { + grant: new CapabilityGrant(new Set(ALL_CAPABILITY_PATHS), []), + async sendChannelsOpen() { return {}; }, + async sendChannelsClose() { return {}; }, + }; + const registry = new ChannelRegistry( + { getServer: () => server } as never, + {} as never, + () => {}, + () => {}, + ); + // Inject the reported failure at the real Chronicle append boundary. + // A descriptor without a label avoids the unrelated label-history write. + (registry as any).store = { + appendToStateJson() { throw new Error('injected lifecycle-store failure'); }, + }; + registry.setSubscriptionPolicy('srv', 'auto'); + framework.channelRegistry = registry; + const reply = responder(); + connection.emit(event, { [field]: [{ id: 'room', type: 'test', direction: 'bidirectional' }] }, reply); + await settle(); + + assert.equal(reply.results.length, 1, 'ACK still precedes reconciliation'); + assert.equal(reply.errors.length, 0, 'an accepted registration must not get a second response'); + const failures = failureTraces(traces); + assert.equal(failures.length, 1); + assert.equal(failures[0].serverId, 'srv'); + assert.equal(failures[0].event, event); + assert.equal(failures[0].requestId, 17); + assert.equal(failures[0].responseAttempted, true); + assert.match(failures[0].error, /injected lifecycle-store failure/); + }); +} + +for (const [event, component, method] of [ + ['channels-register', 'channelRegistry', 'handleRegister'], + ['channels-changed', 'channelRegistry', 'handleChanged'], + ['channels-incoming', 'channelRegistry', 'handleIncoming'], + ['push-event', 'pushHandler', 'handlePushEvent'], + ['inference-request', 'inferenceRouter', 'handleInferenceRequest'], +] as const) { + test(event + ': rejection before a response gets one JSON-RPC internal error', async () => { + const { framework, connection, traces } = harness(); + framework[component] = { + async [method]() { + await Promise.resolve(); + throw new Error('handler failed before response'); + }, + }; + const reply = responder(); + connection.emit(event, {}, reply); + await settle(); + + assert.equal(reply.results.length, 0); + assert.deepEqual(reply.errors, [{ code: -32603, message: 'handler failed before response', data: undefined }]); + const failures = failureTraces(traces); + assert.equal(failures.length, 1); + assert.equal(failures[0].event, event); + assert.equal(failures[0].responseAttempted, false); + }); +} + +test('notification rejection is traced without inventing a response', async () => { + const { framework, connection, traces } = harness(); + framework.channelRegistry = { async handleChanged() { throw new Error('notification failed'); } }; + connection.emit('channels-changed', {}); + await settle(); + const failures = failureTraces(traces); + assert.equal(failures.length, 1); + assert.equal(failures[0].requestId, undefined); + assert.match(failures[0].error, /notification failed/); +}); + +test('a synchronous handler throw is contained and answered', async () => { + const { framework, connection, traces } = harness(); + framework.channelRegistry = { descriptorsForServer() { throw new Error('list failed'); } }; + const reply = responder(); + assert.doesNotThrow(() => connection.emit('channels-list', {}, reply)); + await settle(); + assert.equal(reply.errors.length, 1); + assert.equal(failureTraces(traces).length, 1); +}); + +test('a throwing success response is not retried as an error response', async () => { + const { framework, connection, traces } = harness(); + framework.channelRegistry = { descriptorsForServer() { return []; } }; + const reply = responder(); + let writes = 0; + reply.respond = () => { writes++; throw new Error('response transport failed'); }; + assert.doesNotThrow(() => connection.emit('channels-list', {}, reply)); + await settle(); + assert.equal(writes, 1); + assert.equal(reply.errors.length, 0); + assert.equal(failureTraces(traces)[0].responseAttempted, true); +}); + +test('a failed error response is traced without an escaping rejection', async () => { + const { framework, connection, traces } = harness(); + framework.channelRegistry = { async handleRegister() { throw new Error('original failure'); } }; + const reply = responder(); + let writes = 0; + reply.respondError = () => { writes++; throw new Error('error response transport failed'); }; + connection.emit('channels-register', {}, reply); + await settle(); + assert.equal(writes, 1); + const failures = failureTraces(traces); + assert.equal(failures.length, 1); + assert.match(failures[0].error, /original failure/); + assert.match(failures[0].responseError, /error response transport failed/); +}); + +test('inference responders retain request IDs and successful handling stays synchronous', async () => { + const { framework, connection, traces } = harness(); + framework.inferenceRouter = { + handleInferenceRequest(_server: string, _params: unknown, reply: any) { + assert.equal(reply.requestId, 17); + reply.respond({ accepted: true }); + }, + }; + const reply = responder(); + connection.emit('inference-request', {}, reply); + assert.deepEqual(reply.results, [{ accepted: true }]); + await settle(); + assert.equal(reply.errors.length, 0); + assert.equal(failureTraces(traces).length, 0); +}); + +test('host command response-write failure is traced without a second reply', async () => { + const { framework, connection, traces } = harness(); + framework.handleHostCommand = async () => ({ done: true }); + const reply = responder(); + reply.respond = () => { throw new Error('host response failed'); }; + connection.emit('host-command', {}, reply); + await settle(); + assert.equal(reply.errors.length, 0); + assert.match(failureTraces(traces)[0].error, /host response failed/); +}); From 6ac4743618d647f584685057937af943fc9d0ee1 Mon Sep 17 00:00:00 2001 From: Jasper Date: Fri, 2 Oct 2026 01:21:16 -0400 Subject: [PATCH 0007/1112] fix(usage): account for tool-ended inference turns Tool results with endTurn bypass the natural complete event, leaving usage totals and completion observers unaware of completed work. Register a stream-scoped, idempotent accounting callback that both terminal paths use. Finalize tool-ended turns after resetting the agent and before asynchronous cancellation can dispose an ephemeral or hand control to another stream. Retain the latest cumulative Membrane usage snapshot rather than summing round snapshots. Preserve cache counts and provider-priced estimated cost. Natural completion continues to use response.details.usage as its authoritative total. Cancellation and MCPL lifecycle behavior stay unchanged. Added five integration checks for one and multiple tool rounds, natural completion, attribution and persistence through reopen, and missing usage. On the original 03c31d9 source these yielded 1 pass / 4 fail; the fix passes all five. Built compiled output and ran the new suite plus ephemeral-settle and deferred-wake-integrity: 18 pass / 0 fail. Bounded full-suite comparison is running separately. Co-Authored-By: GPT-6 Astra --- CHANGELOG.md | 4 + src/framework.ts | 58 +++++++---- test/end-turn-usage.test.ts | 196 ++++++++++++++++++++++++++++++++++++ 3 files changed, 241 insertions(+), 17 deletions(-) create mode 100644 test/end-turn-usage.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b9a30b4c..14488fad5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,10 @@ Releases up to and including 0.7.3 predate this file; for their contents see ## Unreleased +### Fixed + +- Tool-ended turns now emit `inference:completed` and record cumulative token, cache, and estimated-cost usage exactly once, including turns ended by `skip_reply` or other `endTurn` tools. + ## 0.19.0 — 2026-09-28 ### Added diff --git a/src/framework.ts b/src/framework.ts index 510b9183a..b2c66c876 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -3,7 +3,7 @@ import { INLINE_WITHHELD_TEXT, classifyBlock, isInlineContradiction, referenceRe import { ReferenceFetcher, DEFAULT_FETCH_MAX_BYTES, EAGER_FETCH_TIMEOUT_MS } from './mcpl/reference-fetcher.js'; import { appendFileSync, existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs'; import { JsStore } from '@animalabs/chronicle'; -import type { Membrane, ContentBlock, NormalizedRequest, YieldingStream, ToolResult as MembraneToolResult, ToolResultContentBlock } from '@animalabs/membrane'; +import type { Membrane, ContentBlock, NormalizedRequest, YieldingStream, DetailedUsage, ToolResult as MembraneToolResult, ToolResultContentBlock } from '@animalabs/membrane'; import { MembraneError } from '@animalabs/membrane'; import { ContextManager, PassthroughStrategy, WindowedPassthroughStrategy, OverBudgetError, UncoveredDropError } from '@animalabs/context-manager'; import type { CacheWireReceipt } from './kv-unified-wire.js'; @@ -1002,6 +1002,10 @@ export class AgentFramework { private processLoggingPersist: boolean; private processLoggingBroadcast: boolean; private activeStreams: Map> = new Map(); + // Tool results settle outside driveStream. Key accounting by the physical + // stream so an endTurn result can finalize it before cancellation teardown + // or ephemeral disposal, without touching a successor's usage. + private streamCompletionAccounting = new WeakMap void>(); /** Per-agent output locus FROZEN for the CURRENT logical turn. Resolved * eagerly in startAgentStream (home → addressed trigger → global default) @@ -6635,6 +6639,9 @@ export class AgentFramework { // driveStream's finally is the backstop.) this.eventGate?.onInferenceEnded(agent.name); this.settleAgent(agent.name, { stopReason: 'turn_ended', speech: '' }); + if (currentState.stream) { + this.streamCompletionAccounting.get(currentState.stream)?.(); + } this.emitTrace({ type: 'inference:turn_ended', agentName: agent.name }); } else if (overBudget || overPhysical) { // Context budget exceeded: break the stream, let compile() compress. @@ -9072,6 +9079,31 @@ export class AgentFramework { const startTime = Date.now(); const requestId = `${agent.name}-${startTime}-${Math.random().toString(36).slice(2, 8)}`; const myStreamId = agent.streamId; + let latestUsage: DetailedUsage | undefined; + let completionAccounted = false; + const completeAccounting = (usage: DetailedUsage | undefined): void => { + if (completionAccounted) return; + completionAccounted = true; + this.streamCompletionAccounting.delete(stream); + this.emitTrace({ + type: 'inference:completed', + agentName: agent.name, + durationMs: Date.now() - startTime, + tokenUsage: usage ? { + input: usage.inputTokens, + output: usage.outputTokens, + cacheCreation: usage.cacheCreationTokens, + cacheRead: usage.cacheReadTokens, + } : undefined, + }); + if (usage) { + this.usageTracker.onInferenceCompleted(agent.name, usage, usage.estimatedCost + ? { total: usage.estimatedCost.total, currency: usage.estimatedCost.currency } + : undefined); + this.persistUsageState(); + } + }; + this.streamCompletionAccounting.set(stream, () => completeAccounting(latestUsage)); // Membrane usage events are cumulative across the native/XML tool loop. // Keep the previous cumulative sample so consumers that operate at the // physical provider-call boundary (estimator calibration and kv receipt @@ -9609,22 +9641,7 @@ export class AgentFramework { .join('\n'), }); - this.emitTrace({ - type: 'inference:completed', - agentName: agent.name, - durationMs, - tokenUsage, - }); - - if (du) { - this.usageTracker.onInferenceCompleted(agent.name, { - inputTokens: du.inputTokens, - outputTokens: du.outputTokens, - cacheCreationTokens: du.cacheCreationTokens, - cacheReadTokens: du.cacheReadTokens, - }, du.estimatedCost ? { total: du.estimatedCost.total, currency: du.estimatedCost.currency } : undefined); - this.persistUsageState(); - } + completeAccounting(du); // Log inference this.logInference({ @@ -10144,6 +10161,12 @@ export class AgentFramework { } case 'usage': { + // Already cumulative across rounds: replace, rather than sum, and + // retain Membrane's cost at the rates that served those rounds. + latestUsage = { + ...event.usage, + estimatedCost: event.usage.estimatedCost ? { ...event.usage.estimatedCost } : undefined, + }; agent.lastStreamInputTokens = event.usage.inputTokens; agent.lastStreamRealInputTokens = (event.usage.inputTokens ?? 0) + @@ -10361,6 +10384,7 @@ export class AgentFramework { } } this.frameworkCancelledStreams.delete(`${agent.name}:${myStreamId}`); + this.streamCompletionAccounting.delete(stream); if (ownsPhysicalStream) { this.activeStreams.delete(agent.name); this.pendingAssistantBlocks.delete(agent.name); diff --git a/test/end-turn-usage.test.ts b/test/end-turn-usage.test.ts new file mode 100644 index 000000000..eb4e21451 --- /dev/null +++ b/test/end-turn-usage.test.ts @@ -0,0 +1,196 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import type { DetailedUsage, NormalizedResponse } from '@animalabs/membrane'; +import { AgentFramework } from '../src/index.js'; +import type { Module, ToolCall, TraceEvent } from '../src/index.js'; +import { createMockResponse, MockMembrane } from './helpers/mock-membrane.js'; + +class TurnTools implements Module { + readonly name = 'test'; + async start() {} + async stop() {} + async onProcess() { return {}; } + getTools() { + return ['echo', 'finish'].map((name) => ({ + name, + description: name, + inputSchema: { type: 'object' as const, properties: {} }, + })); + } + async handleToolCall(call: ToolCall) { + return { success: true, endTurn: call.name.endsWith('finish'), data: {} }; + } +} + +const firstUsage: DetailedUsage = { + inputTokens: 100, outputTokens: 20, cacheCreationTokens: 30, cacheReadTokens: 40, + estimatedCost: { input: 0.01, output: 0.02, total: 0.03, currency: 'USD' }, +}; +const totalUsage: DetailedUsage = { + inputTokens: 250, outputTokens: 55, cacheCreationTokens: 45, cacheReadTokens: 90, + estimatedCost: { input: 0.03, output: 0.05, total: 0.08, currency: 'USD' }, +}; + +function toolResponse(name: string, id: string, usage?: DetailedUsage): NormalizedResponse { + const response = createMockResponse([{ type: 'tool_use', name: 'test--' + name, id, input: {} }], 'tool_use'); + // Membrane emits cumulative usage across all rounds of a yielding stream. + response.usage = usage!; + return response; +} + +function expectedTokens(usage: DetailedUsage) { + return { + input: usage.inputTokens, output: usage.outputTokens, + cacheCreation: usage.cacheCreationTokens, cacheRead: usage.cacheReadTokens, + }; +} + +function expectedTotals(usage: DetailedUsage) { + return { + inputTokens: usage.inputTokens, outputTokens: usage.outputTokens, + cacheCreationTokens: usage.cacheCreationTokens ?? 0, + cacheReadTokens: usage.cacheReadTokens ?? 0, + estimatedCost: usage.estimatedCost + ? { total: usage.estimatedCost.total, currency: usage.estimatedCost.currency } + : undefined, + }; +} + +async function fixture() { + const dir = mkdtempSync(join(tmpdir(), 'af-end-turn-usage-')); + const storePath = join(dir, 'store.chronicle'); + const membrane = new MockMembrane(); + const framework = await AgentFramework.create({ + storePath, membrane: membrane.asMembrane(), agents: [], + modules: [new TurnTools()], syncIntervalMs: 0, + }); + const traces: TraceEvent[] = []; + const completionStates: Array = []; + framework.onTrace((event) => { + traces.push(event); + if (event.type === 'inference:completed') { + completionStates.push(framework.getAgent(event.agentName)?.state.status); + } + }); + framework.start(); + return { dir, storePath, membrane, framework, traces, completionStates }; +} + +async function run(framework: AgentFramework, name: string) { + const { agent, contextManager } = await framework.createEphemeralAgent({ + name, model: 'test-model', systemPrompt: 'Do the task.', allowedTools: 'all', + proseRouting: 'disabled', + }); + contextManager.addMessage('user', [{ type: 'text', text: 'Run once.' }]); + const result = await framework.runEphemeralToCompletion(agent, contextManager); + // Also observe asynchronous cancellation teardown before checking counts. + await new Promise((resolve) => setImmediate(resolve)); + return result; +} + +describe('tool-ended turn usage accounting', () => { + for (const rounds of [1, 2]) { + it('accounts once for ' + rounds + ' tool round(s), with cumulative cache usage and cost', async () => { + const f = await fixture(); + try { + if (rounds === 2) f.membrane.pushResponse(toolResponse('echo', 'echo-1', firstUsage)); + f.membrane.pushResponse(toolResponse('finish', 'finish-1', totalUsage)); + const result = await run(f.framework, 'worker'); + assert.equal(result.toolCallsCount, rounds); + assert.equal(f.membrane.calls.length, 1, 'endTurn must not request another inference'); + assert.equal(f.membrane.lastStream!.receivedToolResults.length, rounds - 1); + assert.deepEqual(f.completionStates, ['idle']); + const completed = f.traces.filter((e) => e.type === 'inference:completed'); + assert.equal(completed.length, 1); + assert.equal(completed[0].agentName, 'worker'); + assert.deepEqual(completed[0].tokenUsage, expectedTokens(totalUsage)); + assert.ok(completed[0].durationMs >= 0); + assert.equal(f.traces.filter((e) => e.type === 'inference:turn_ended').length, 1); + assert.equal(f.traces.filter((e) => e.type === 'usage:updated').length, 1); + assert.equal(f.traces.filter((e) => e.type === 'inference:exhausted').length, 0); + const snapshot = f.framework.getSessionUsage(); + assert.equal(snapshot.inferenceCount, 1); + assert.deepEqual(snapshot.totals, expectedTotals(totalUsage)); + assert.deepEqual(snapshot.byAgent, [{ + agentName: 'worker', usage: expectedTotals(totalUsage), inferenceCount: 1, + }]); + } finally { + await f.framework.stop(); + rmSync(f.dir, { recursive: true, force: true }); + } + }); + } + + it('keeps natural completion authoritative and counts neither cumulative samples nor final usage twice', async () => { + const f = await fixture(); + try { + f.membrane.pushResponse(toolResponse('echo', 'echo-1', firstUsage)); + const response = createMockResponse([{ type: 'text', text: 'Done.' }]); + response.usage = { ...totalUsage, outputTokens: 50 }; // final details remain authoritative + response.details!.usage = totalUsage; + f.membrane.pushResponse(response); + await run(f.framework, 'ordinary'); + assert.deepEqual(f.completionStates, ['idle']); + const completed = f.traces.filter((e) => e.type === 'inference:completed'); + assert.equal(completed.length, 1); + assert.deepEqual(completed[0].tokenUsage, expectedTokens(totalUsage)); + assert.equal(f.traces.filter((e) => e.type === 'usage:updated').length, 1); + assert.equal(f.traces.filter((e) => e.type === 'inference:turn_ended').length, 0); + assert.equal(f.framework.getSessionUsage().inferenceCount, 1); + assert.deepEqual(f.framework.getSessionUsage().totals, expectedTotals(totalUsage)); + } finally { + await f.framework.stop(); + rmSync(f.dir, { recursive: true, force: true }); + } + }); + + it('attributes successive streams separately and persists their totals through store reopen', async () => { + const f = await fixture(); + let stopped = false; + try { + f.membrane.pushResponse(toolResponse('finish', 'finish-a', firstUsage)); + await run(f.framework, 'alice'); + f.membrane.pushResponse(toolResponse('finish', 'finish-b', totalUsage)); + await run(f.framework, 'bob'); + const snapshot = f.framework.getSessionUsage(); + assert.equal(snapshot.inferenceCount, 2); + assert.deepEqual(snapshot.byAgent, [ + { agentName: 'alice', usage: expectedTotals(firstUsage), inferenceCount: 1 }, + { agentName: 'bob', usage: expectedTotals(totalUsage), inferenceCount: 1 }, + ]); + assert.deepEqual(f.traces.filter((e) => e.type === 'usage:updated').map((e) => e.agentName), ['alice', 'bob']); + await f.framework.stop(); + stopped = true; + const reopened = await AgentFramework.create({ + storePath: f.storePath, membrane: f.membrane.asMembrane(), agents: [], modules: [], syncIntervalMs: 0, + }); + try { + assert.deepEqual(reopened.getSessionUsage(), snapshot); + } finally { + await reopened.stop(); + } + } finally { + if (!stopped) await f.framework.stop(); + rmSync(f.dir, { recursive: true, force: true }); + } + }); + + it('reports completion without inventing usage when no sample is available', async () => { + const f = await fixture(); + try { + f.membrane.pushResponse(toolResponse('finish', 'finish-1')); + await run(f.framework, 'unknown-usage'); + const completed = f.traces.filter((e) => e.type === 'inference:completed'); + assert.equal(completed.length, 1); + assert.equal(completed[0].tokenUsage, undefined); + assert.equal(f.traces.filter((e) => e.type === 'usage:updated').length, 0); + assert.equal(f.framework.getSessionUsage().inferenceCount, 0); + } finally { + await f.framework.stop(); + rmSync(f.dir, { recursive: true, force: true }); + } + }); +}); From 1a4fbd95dc6983e0cf38b99703a5d88a0a87f337 Mon Sep 17 00:00:00 2001 From: Felix-299 Date: Fri, 2 Oct 2026 01:26:20 -0400 Subject: [PATCH 0008/1112] fix(mcpl): enforce a single response attempt inside handlers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wren’s independent review exercised the real InferenceRouter: it catches a failed result write and tries an error response internally, before the outer listener wrapper sees the rejection. Enforce the response-attempt guard in both tracked responder methods, preserve the original transport exception, and report it even when a handler swallows it. The wrapper consumes a subsequent throw without forwarding another response. Add the real inference-handler regression, a swallowed-write regression, and an already-sent error-response case. Against ebfc315 the expanded focused suite is 14 pass / 2 fail; with this correction it is 16 pass / 0 fail. Rebuild succeeds; updated full-suite and independent re-review follow. Co-Authored-By: GPT-6 Astra --- src/framework.ts | 37 +++++++++---- test/mcpl-request-handler-errors.test.ts | 66 ++++++++++++++++++++++++ 2 files changed, 92 insertions(+), 11 deletions(-) diff --git a/src/framework.ts b/src/framework.ts index 9109ba93d..615bd4189 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -13557,20 +13557,30 @@ export class AgentFramework { ): void => { connection.on(event, (params: T, responder?: Responder) => { let responseAttempted = false; + let writeFailure: { error: unknown } | undefined; + const writeResponse = (write: () => void): void => { + if (responseAttempted) { + // Inner handler catches can try to answer a failed result with + // an error. Preserve the first failure and refuse the second write. + throw writeFailure ? writeFailure.error : new Error('MCPL response already attempted'); + } + // A throwing transport write may have partially sent the reply. + responseAttempted = true; + try { + write(); + } catch (error) { + writeFailure = { error }; + throw error; + } + }; const tracked = responder && { ...responder, - respond: (result: unknown) => { - // A throwing transport write may have partially sent the reply. - // Mark before writing so recovery never sends a second response. - responseAttempted = true; - responder.respond(result); - }, - respondError: (code: number, message: string, data?: unknown) => { - responseAttempted = true; - responder.respondError(code, message, data); - }, + respond: (result: unknown) => writeResponse(() => responder.respond(result)), + respondError: (code: number, message: string, data?: unknown) => + writeResponse(() => responder.respondError(code, message, data)), }; const failed = (error: unknown): void => { + if (writeFailure) error = writeFailure.error; const message = error instanceof Error ? error.message : String(error); const attemptedBeforeFailure = responseAttempted; let responseError: string | undefined; @@ -13594,8 +13604,13 @@ export class AgentFramework { try { // Invoke immediately: control-plane responses must retain their // ACK-before-reconcile ordering and data-plane barrier semantics. + const succeeded = (): void => { + // A handler may swallow a transport failure without rethrowing. + if (writeFailure) failed(writeFailure.error); + }; const result = handler(params, tracked); - if (result) void result.catch(failed); + if (result) void result.then(succeeded, failed); + else succeeded(); } catch (error) { failed(error); } diff --git a/test/mcpl-request-handler-errors.test.ts b/test/mcpl-request-handler-errors.test.ts index c6c4f84ec..2a414791f 100644 --- a/test/mcpl-request-handler-errors.test.ts +++ b/test/mcpl-request-handler-errors.test.ts @@ -4,6 +4,7 @@ import { EventEmitter } from 'node:events'; import { AgentFramework } from '../src/framework.js'; import { ChannelRegistry } from '../src/mcpl/channel-registry.js'; +import { InferenceRouter } from '../src/mcpl/inference-router.js'; import { CapabilityGrant, ALL_CAPABILITY_PATHS } from '../src/mcpl/capability-grant.js'; function harness() { @@ -162,6 +163,71 @@ test('inference responders retain request IDs and successful handling stays sync assert.equal(failureTraces(traces).length, 0); }); +test('real inference handler cannot turn a failed result write into a second response', async () => { + const { framework, connection, traces } = harness(); + framework.inferenceRouter = new InferenceRouter( + { + async complete() { + return { + content: [{ type: 'text', text: 'done' }], + stopReason: 'end_turn', + usage: { inputTokens: 1, outputTokens: 1 }, + }; + }, + } as never, + { isInHook: false } as never, + { validateInbound() {} } as never, + null, + (event) => traces.push(event), + ); + const reply = responder(); + let resultWrites = 0; + reply.respond = () => { resultWrites++; throw new Error('partially written result'); }; + connection.emit('inference-request', { + featureSet: 'test', messages: [{ role: 'user', content: 'hello' }], + }, reply); + await settle(); + assert.equal(resultWrites, 1); + assert.equal(reply.errors.length, 0, 'the inner inference catch must not send another reply'); + const failures = failureTraces(traces); + assert.equal(failures.length, 1); + assert.match(failures[0].error, /partially written result/); + assert.equal(failures[0].responseAttempted, true); +}); + +test('a swallowed response-write failure still produces a diagnostic', async () => { + const { framework, connection, traces } = harness(); + framework.channelRegistry = { + async handleRegister(_server: string, _params: unknown, reply: any) { + try { reply.respond({}); } catch { /* Simulate a handler swallowing a transport error. */ } + }, + }; + const reply = responder(); + reply.respond = () => { throw new Error('swallowed write failure'); }; + connection.emit('channels-register', {}, reply); + await settle(); + assert.equal(reply.errors.length, 0); + assert.equal(failureTraces(traces).length, 1); + assert.match(failureTraces(traces)[0].error, /swallowed write failure/); +}); + +test('an error response followed by a handler rejection is not answered again', async () => { + const { framework, connection, traces } = harness(); + framework.channelRegistry = { + async handleRegister(_server: string, _params: unknown, reply: any) { + reply.respondError(-32602, 'bad params'); + throw new Error('after error response'); + }, + }; + const reply = responder(); + connection.emit('channels-register', {}, reply); + await settle(); + assert.equal(reply.errors.length, 1); + assert.equal(failureTraces(traces).length, 1); + assert.match(failureTraces(traces)[0].error, /after error response/); + assert.equal(failureTraces(traces)[0].responseAttempted, true); +}); + test('host command response-write failure is traced without a second reply', async () => { const { framework, connection, traces } = harness(); framework.handleHostCommand = async () => ({ done: true }); From 5fba953e0617ca5ea1b36bf753fea0f55cd0da79 Mon Sep 17 00:00:00 2001 From: Jasper Date: Fri, 2 Oct 2026 01:36:18 -0400 Subject: [PATCH 0009/1112] fix(mcpl): reconcile channel lifecycle operations in order Overlapping registration reconciliation and channel tools could leave the server in an older state than durable intent. A removed/re-added entry made this worse: a late close mutated the deleted entry while the current entry incorrectly stayed open. Serialize transport operations per logical server/channel, keeping registration responses outside the queue. Re-read live registration, server, descriptor target, and desired state at dispatch and after each completion. Apply receipts only to the same live entry/connection, and reconcile again when the target or transport intent changed. Label-only updates do not repeat transport operations. Rejected operations release the queue for later decisions. Record admission intent before waiting on other channels. Explicit operations preserve immediate desired-state persistence and report when newer intent supersedes their requested state. Open/close/send fast paths now respect pending lifecycle work, so provisional live state cannot bypass the queue. Added deterministic deferred-RPC tests for the reported out-of-order reproduction, explicit open/close races, removal, descriptor updates, failure recovery, unrelated-channel progress, response-before-reconcile, and tune-out intent. The initial nine-case original-source run was 2 pass / 7 fail. The fixed expanded focused run is 66 pass / 0 fail without force-exit; full compiled-suite and final ten-case original-source comparison are running. Co-Authored-By: GPT-6 Astra --- CHANGELOG.md | 4 + src/mcpl/channel-registry.ts | 150 +++++++++++++-------- test/channel-lifecycle-order.test.ts | 194 +++++++++++++++++++++++++++ 3 files changed, 294 insertions(+), 54 deletions(-) create mode 100644 test/channel-lifecycle-order.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b9a30b4c..ba3a8f7dd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,10 @@ Releases up to and including 0.7.3 predate this file; for their contents see ## Unreleased +### Fixed + +- Channel lifecycle operations now run in order per channel and reconcile the latest desired state and registration after an in-flight operation completes. Concurrent registration, open, and close requests can no longer leave the server in an older state; a superseded tool request reports that its requested state no longer applies. + ## 0.19.0 — 2026-09-28 ### Added diff --git a/src/mcpl/channel-registry.ts b/src/mcpl/channel-registry.ts index 01c81ec1a..098b036d8 100644 --- a/src/mcpl/channel-registry.ts +++ b/src/mcpl/channel-registry.ts @@ -16,6 +16,7 @@ import type { ContentBlock } from '@animalabs/membrane'; import { INLINE_WITHHELD_TEXT, isInlineContradiction, referenceStubOrNull } from './references.js'; import type { JsStore } from '@animalabs/chronicle'; +import { isDeepStrictEqual } from 'node:util'; import type { ChannelDescriptor, @@ -656,6 +657,15 @@ export class ChannelRegistry { /** One-time migration inputs from the retired recipe auto-open policy. */ private legacyPolicies = new Map(); private migratedLegacyPolicies = new Set(); + + // Keep one transport operation in flight per logical channel, including + // across descriptor replacement. Settled tails release themselves; a + // failed operation must not poison later decisions. + private channelLifecycleTails = new Map>(); + + private hasPendingLifecycle(entry: ChannelEntry): boolean { + return this.channelLifecycleTails.has(this.lifecycleKey(entry.serverId, entry.descriptor.id)); + } private handleCoalescedIncoming?: ChannelRegistryOptions['handleCoalescedIncoming']; constructor( @@ -2065,6 +2075,68 @@ export class ChannelRegistry { return channel.initiallyOpen !== true && byPolicy; } + /** + * Apply current durable intent, serializing the transport rather than the + * server's registration ACK. Re-read after every await: registration, + * descriptor updates, or a newer decision may have replaced our snapshot. + */ + private applyDesiredChannelState( + serverId: string, + channelId: string, + history?: ChannelHistoryRequest, + ): Promise<{ open: boolean; result?: ChannelsOpenResult }> { + const lifecycleKey = this.lifecycleKey(serverId, channelId); + const channelKey = `${serverId}:${channelId}`; + const previous = this.channelLifecycleTails.get(lifecycleKey) ?? Promise.resolve(); + const operation = previous.then(async () => { + for (;;) { + const entry = this.channels.get(channelKey); + if (!entry) throw new Error(`Channel is no longer registered: ${channelId}`); + const server = this.serverRegistry.getServer(serverId); + if (!server) throw new Error(`Server not found: ${serverId}`); + if (!CapabilityGrant.of(server).has('channels.lifecycle')) { + throw new Error(`channels.lifecycle not in "${serverId}"'s effective grant (§14.1)`); + } + const descriptor = entry.descriptor; + const desired = this.getDesiredState(serverId, channelId); + const open = desired === 'open' || desired === 'tuned-out'; + let result: ChannelsOpenResult | undefined; + try { + if (open) { + result = await server.sendChannelsOpen({ + channelId, type: descriptor.type, address: descriptor.address, + ...(history ? { history } : {}), + }); + } else { + await server.sendChannelsClose({ channelId }); + } + } catch (error) { + if (open && this.channels.get(channelKey) === entry && + this.serverRegistry.getServer(serverId) === server) entry.open = false; + throw error; + } + // Never apply a receipt to a removed entry or a replacement connection. + // Reconcile their current state before releasing this channel's queue. + if (this.channels.get(channelKey) !== entry || + this.serverRegistry.getServer(serverId) !== server) continue; + entry.open = open; + const now = this.getDesiredState(serverId, channelId); + const targetChanged = entry.descriptor.type !== descriptor.type || + !isDeepStrictEqual(entry.descriptor.address, descriptor.address); + if (targetChanged || open !== (now === 'open' || now === 'tuned-out')) continue; + return { open, result }; + } + }); + const tail = operation.then(() => {}, () => {}); + this.channelLifecycleTails.set(lifecycleKey, tail); + void tail.then(() => { + if (this.channelLifecycleTails.get(lifecycleKey) === tail) { + this.channelLifecycleTails.delete(lifecycleKey); + } + }); + return operation; + } + private async reconcileChannels( serverId: string, channels: ChannelDescriptor[], @@ -2090,46 +2162,26 @@ export class ChannelRegistry { // first boot on a new policy doesn't produce one notice per channel. const policyOpened: Array<{ channelId: string; label?: string }> = []; + // Record all admissions before waiting on any channel. A slow first RPC + // must not delay intent for the rest of this registration batch. for (const channel of channels) { - if (this.ensureInitialDesiredState(serverId, channel)) { - policyOpened.push({ channelId: channel.id, label: channel.label }); + const entry = this.channels.get(`${serverId}:${channel.id}`); + if (entry && this.ensureInitialDesiredState(serverId, entry.descriptor)) { + policyOpened.push({ channelId: channel.id, label: entry.descriptor.label }); } - const key = `${serverId}:${channel.id}`; - const desired = this.getDesiredState(serverId, channel.id); - const entry = this.channels.get(key); - // Tuned-out sits on the OPEN side of reconcile: traffic must keep - // arriving (the subconscious reads it); only main's wake/visibility - // is diverted, downstream at ingestion. - if (desired !== 'open' && desired !== 'tuned-out') { - try { - await server.sendChannelsClose({ channelId: channel.id }); - if (entry) entry.open = false; - } catch (err) { - this.emitTraceFn({ - type: 'mcpl:channel-reconcile-failed', - serverId, - channelId: channel.id, - desired, - error: (err as Error).message, - }); - } - continue; - } - + } + for (const channel of channels) { + if (!this.channels.has(`${serverId}:${channel.id}`)) continue; try { - await server.sendChannelsOpen({ - channelId: channel.id, - type: channel.type, - address: channel.address, - }); - if (entry) entry.open = true; + await this.applyDesiredChannelState(serverId, channel.id); } catch (err) { - if (entry) entry.open = false; + // Removal while queued is cancellation, not a failed subscription. + if (!this.channels.has(`${serverId}:${channel.id}`)) continue; this.emitTraceFn({ type: 'mcpl:channel-reconcile-failed', serverId, channelId: channel.id, - desired, + desired: this.getDesiredState(serverId, channel.id), error: (err as Error).message, }); } @@ -2284,23 +2336,11 @@ export class ChannelRegistry { history?: ChannelHistoryRequest, ): Promise { this.setDesiredState(entry.serverId, entry.descriptor.id, 'open', source); - const server = this.serverRegistry.getServer(entry.serverId); - if (!server) { - throw new Error(`Server not found: ${entry.serverId}`); + const applied = await this.applyDesiredChannelState(entry.serverId, entry.descriptor.id, history); + if (!applied.open || this.getDesiredState(entry.serverId, entry.descriptor.id) !== 'open') { + throw new Error('Channel open was superseded by a newer lifecycle decision'); } - if (!CapabilityGrant.of(server).has('channels.lifecycle')) { - // Desired state is already recorded — intent sticks; reconciliation - // will retry if the grant later widens (§6.7 expansion-on-receipt). - throw new Error(`channels.lifecycle not in "${entry.serverId}"'s effective grant (§14.1)`); - } - const result: ChannelsOpenResult = await server.sendChannelsOpen({ - channelId: entry.descriptor.id, - type: entry.descriptor.type, - address: entry.descriptor.address, - ...(history ? { history } : {}), - }); - entry.open = true; - return result; + return applied.result!; } /** @@ -2475,7 +2515,7 @@ export class ChannelRegistry { if (matches.length > 1) return { status: 'ambiguous' }; const entry = matches[0]!; const resolved = { channelId: entry.descriptor.id, label: entry.descriptor.label }; - if (entry.open && this.getDesiredState(entry.serverId, entry.descriptor.id) === 'open') { + if (entry.open && this.getDesiredState(entry.serverId, entry.descriptor.id) === 'open' && !this.hasPendingLifecycle(entry)) { return { status: 'already-open', ...resolved }; } try { @@ -2530,7 +2570,7 @@ export class ChannelRegistry { } const alreadyDesiredOpen = this.getDesiredState(entry.serverId, input.channelId) === 'open'; - if (entry.open && alreadyDesiredOpen && !input.backscroll) { + if (entry.open && alreadyDesiredOpen && !input.backscroll && !this.hasPendingLifecycle(entry)) { return { success: true, data: { channelId: input.channelId, status: 'already open' }, @@ -2593,7 +2633,7 @@ export class ChannelRegistry { } const alreadyDesiredClosed = this.getDesiredState(entry.serverId, input.channelId) === 'closed'; - if (!entry.open && alreadyDesiredClosed) { + if (!entry.open && alreadyDesiredClosed && !this.hasPendingLifecycle(entry)) { return { success: true, data: { channelId: input.channelId, status: 'already closed' }, @@ -2662,8 +2702,10 @@ export class ChannelRegistry { isError: true, }; } - await server.sendChannelsClose({ channelId: input.channelId }); - entry.open = false; + const applied = await this.applyDesiredChannelState(entry.serverId, input.channelId); + if (applied.open || this.getDesiredState(entry.serverId, input.channelId) !== 'closed') { + throw new Error('Channel close was superseded by a newer lifecycle decision'); + } return { success: true, data: { channelId: input.channelId, status: 'closed' }, @@ -2955,7 +2997,7 @@ export class ChannelRegistry { // so typing indicators, reaction machinery, and inbound forwarding come // alive with the reply. If the open fails, the speech does NOT go out: // a half-alive delivery is worse than a loud marker. - if (!entry.open) { + if (!entry.open || this.hasPendingLifecycle(entry)) { try { await this.openChannelNow(entry, 'opened-by-delivery'); console.error( diff --git a/test/channel-lifecycle-order.test.ts b/test/channel-lifecycle-order.test.ts new file mode 100644 index 000000000..ea084bd62 --- /dev/null +++ b/test/channel-lifecycle-order.test.ts @@ -0,0 +1,194 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { ChannelRegistry } from '../src/mcpl/channel-registry.js'; +import { CapabilityGrant, ALL_CAPABILITY_PATHS } from '../src/mcpl/capability-grant.js'; +import type { McplServerRegistry } from '../src/mcpl/server-registry.js'; +import type { FeatureSetManager } from '../src/mcpl/feature-set-manager.js'; +import type { ChannelDescriptor, ChannelsOpenResult } from '../src/mcpl/types.js'; + +const tick = () => new Promise((resolve) => setImmediate(resolve)); +function descriptor(id = 'x', initiallyOpen = false, address = 'original'): ChannelDescriptor { + return { id, type: 'test', label: id, direction: 'bidirectional', initiallyOpen, address: { value: address } }; +} + +function fixture() { + const actual = new Map(); + const calls: Array<{ kind: 'open' | 'close'; id: string; address?: unknown }> = []; + const pending: Array<{ finish: () => void; fail: () => void }> = []; + const held = new Set(); + const serve = (kind: 'open' | 'close', params: { channelId?: string; address?: unknown }) => { + const id = params.channelId!; + calls.push({ kind, id, address: params.address }); + return new Promise((resolve, reject) => { + const finish = () => { + actual.set(id, kind === 'open'); + resolve(kind === 'open' ? { channel: descriptor(id) } : { closed: true }); + }; + if (held.has(id)) pending.push({ finish, fail: () => reject(new Error('fixture lifecycle failure')) }); + else finish(); + }); + }; + const server = { + grant: new CapabilityGrant(new Set(ALL_CAPABILITY_PATHS), []), + sendChannelsOpen: (params: { channelId?: string; address?: unknown }) => serve('open', params) as Promise, + sendChannelsClose: (params: { channelId: string }) => serve('close', params), + }; + const traces: Array<{ type: string; [key: string]: unknown }> = []; + const registry = new ChannelRegistry( + { getServer: () => server } as unknown as McplServerRegistry, + {} as FeatureSetManager, () => {}, (event) => traces.push(event), + ); + const isOpen = (id = 'x') => registry.getOpenChannels().some((e) => e.descriptor.id === id); + const tool = (kind: 'open' | 'close') => registry.handleChannelToolCall('channel_' + kind, { serverId: 'test', channelId: 'x' }); + // An out-of-order server serves the most recently received operation first. + // The fixed host should never have two in flight for this channel. + const drain = async (...work: Promise[]) => { + let settled = false; + const result = Promise.all(work).finally(() => { settled = true; }); + for (let i = 0; i < 100 && !settled; i++) { + await tick(); + pending.pop()?.finish(); + } + assert.equal(settled, true, 'lifecycle operations must settle'); + await result; + }; + return { registry, actual, calls, pending, held, traces, isOpen, tool, drain }; +} + +test('remove/re-add during reconcile converges on the replacement desired state, with ACK first (#185)', async () => { + const f = fixture(); + f.held.add('x'); + const first = f.registry.handleChanged('test', { added: [descriptor()] }); + await tick(); + assert.equal(f.pending.length, 1); + let acknowledged = false; + const replacement = f.registry.handleChanged('test', { + removed: ['x'], added: [descriptor('x', true, 'replacement')], + }, { respond: () => { acknowledged = true; } }); + assert.equal(acknowledged, true, 'registration ACK must not wait for the lifecycle queue'); + await f.drain(first, replacement); + assert.equal(f.registry.getDesiredState('test', 'x'), 'open'); + assert.equal(f.isOpen(), true); + assert.equal(f.actual.get('x'), true, 'older close must not be the last server-side operation'); + assert.deepEqual(f.calls.filter((c) => c.kind === 'open').at(-1)?.address, { value: 'replacement' }); +}); + +test('explicit open racing a default-close reconcile uses the latest intent', async () => { + const f = fixture(); + f.held.add('x'); + const reconcile = f.registry.handleChanged('test', { added: [descriptor()] }); + await tick(); + const opened = f.tool('open'); + await f.drain(reconcile, opened); + assert.equal((await opened).success, true); + assert.equal(f.registry.getDesiredState('test', 'x'), 'open'); + assert.equal(f.isOpen(), true); + assert.equal(f.actual.get('x'), true); +}); + +for (const firstKind of ['open', 'close'] as const) { + test('overlapping explicit ' + firstKind + ' then opposite intent leaves the newest decision in force', async () => { + const f = fixture(); + await f.registry.handleChanged('test', { added: [descriptor('x', firstKind === 'close')] }); + f.held.add('x'); + const first = f.tool(firstKind); + await tick(); + const lastKind = firstKind === 'open' ? 'close' : 'open'; + const last = f.tool(lastKind); + await f.drain(first, last); + assert.equal((await last).success, true); + const desired = lastKind === 'open'; + assert.equal(f.registry.getDesiredState('test', 'x'), lastKind === 'open' ? 'open' : 'closed'); + assert.equal(f.isOpen(), desired); + assert.equal(f.actual.get('x'), desired); + }); +} + +test('an already-closed shortcut waits for pending transport work', async () => { + const f = fixture(); + f.actual.set('x', true); + f.held.add('x'); + const reconcile = f.registry.handleChanged('test', { added: [descriptor()] }); + await tick(); + let closeSettled = false; + const close = f.tool('close').then((result) => { closeSettled = true; return result; }); + await tick(); + assert.equal(closeSettled, false, 'entry.open is provisional while a lifecycle RPC is pending'); + await f.drain(reconcile, close); + assert.equal((await close).success, true); + assert.equal(f.actual.get('x'), false); +}); + +test('removal cancels queued work rather than recreating or mutating an unregistered channel', async () => { + const f = fixture(); + f.held.add('x'); + const reconcile = f.registry.handleChanged('test', { added: [descriptor()] }); + await tick(); + const open = f.tool('open'); + await f.registry.handleChanged('test', { removed: ['x'] }); + await f.drain(reconcile, open); + assert.equal((await open).success, false); + assert.equal(f.registry.listChannelsRaw().length, 0); + assert.equal(f.calls.filter((c) => c.kind === 'open').length, 0); +}); + +test('a failed operation releases the queue for a later decision', async () => { + const f = fixture(); + f.held.add('x'); + const reconcile = f.registry.handleChanged('test', { added: [descriptor()] }); + await tick(); + const open = f.tool('open'); + f.pending.shift()!.fail(); + await f.drain(reconcile, open); + assert.equal((await open).success, true); + assert.equal(f.actual.get('x'), true); + assert.equal(f.isOpen(), true); + assert.ok(f.traces.some((e) => e.type === 'mcpl:channel-reconcile-failed')); +}); + +test('a blocked channel does not delay another channel lifecycle or its ACK', async () => { + const f = fixture(); + f.held.add('x'); + const blocked = f.registry.handleChanged('test', { added: [descriptor()] }); + await tick(); + let acknowledged = false; + await f.registry.handleChanged('test', { added: [descriptor('y', true)] }, { + respond: () => { acknowledged = true; }, + }); + assert.equal(acknowledged, true); + assert.equal(f.actual.get('y'), true); + assert.equal(f.pending.length, 1); + await f.drain(blocked); +}); + +test('descriptor updates during an open refresh its target, but labels alone do not reopen', async () => { + for (const changesAddress of [false, true]) { + const f = fixture(); + f.held.add('x'); + const opening = f.registry.handleChanged('test', { added: [descriptor('x', true)] }); + await tick(); + await f.registry.handleChanged('test', { + updated: [{ ...descriptor('x', true, changesAddress ? 'new-target' : 'original'), label: 'new label' }], + }); + await f.drain(opening); + assert.equal(f.calls.length, changesAddress ? 2 : 1); + assert.deepEqual(f.calls.at(-1)?.address, { value: changesAddress ? 'new-target' : 'original' }); + assert.equal(f.isOpen(), true); + } +}); + +test('a newer tune-out decision keeps transport open when an older close completes', async () => { + const f = fixture(); + await f.registry.handleChanged('test', { added: [descriptor('x', true)] }); + f.held.add('x'); + const close = f.tool('close'); + await tick(); + f.registry.enterTuneOut('test', 'x', { + epochId: 'new-epoch', cadenceSeconds: 60, backlogCap: 20, maxWakes: 3, startedAtSequence: 1, + }, 'agent-tool'); + await f.drain(close); + assert.equal(f.registry.getDesiredState('test', 'x'), 'tuned-out'); + assert.equal(f.actual.get('x'), true); + assert.equal(f.isOpen(), true); + assert.equal((await close).success, false, 'superseded close must not report current state as closed'); +}); From cd2638d882c85c50fac2e683e874ebfe4a9c0772 Mon Sep 17 00:00:00 2001 From: Jasper Date: Fri, 2 Oct 2026 01:42:07 -0400 Subject: [PATCH 0010/1112] fix(mcpl): preserve tune-out while settling delivery transport Felix independently reproduced a regression in the new routeSpeech pending-operation path: waiting for transport called openChannelNow, whose desired-open write erased a newer tune-out epoch. The original focused and full suites did not cover that interaction. Automatic reply and delivery opens now preserve an existing tuned-out decision, and accept a tune-out established while their RPC is pending. They need open transport, not normal resident attention. Explicit channel_open retains the requirement to cancel tune-out first. Narrow the source type to the three real callers so this distinction stays visible. Added three regressions from the review: pending speech, pending reply preparation, and a new tune-out during automatic open. All three fail at 5fba953. The final 13-case file gives 3 pass / 10 fail on original main and 13 pass / 0 fail here. Expanded compiled focused tests pass 69/69 without force-exit. The full suite on 5fba953 was 1111/0; the revised full suite is running and will be reported independently. Co-Authored-By: GPT-6 Astra --- CHANGELOG.md | 2 +- src/mcpl/channel-registry.ts | 12 +++++-- test/channel-lifecycle-order.test.ts | 47 +++++++++++++++++++++++++++- 3 files changed, 56 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ba3a8f7dd..b02aec4b1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,7 @@ Releases up to and including 0.7.3 predate this file; for their contents see ### Fixed -- Channel lifecycle operations now run in order per channel and reconcile the latest desired state and registration after an in-flight operation completes. Concurrent registration, open, and close requests can no longer leave the server in an older state; a superseded tool request reports that its requested state no longer applies. +- Channel lifecycle operations now run in order per channel and reconcile the latest desired state and registration after an in-flight operation completes. Concurrent registration, open, and close requests can no longer leave the server in an older state; a superseded tool request reports that its requested state no longer applies. Automatic delivery preserves active tune-out attention state while opening or awaiting transport. ## 0.19.0 — 2026-09-28 diff --git a/src/mcpl/channel-registry.ts b/src/mcpl/channel-registry.ts index 098b036d8..d7b8be6a5 100644 --- a/src/mcpl/channel-registry.ts +++ b/src/mcpl/channel-registry.ts @@ -2329,15 +2329,21 @@ export class ChannelRegistry { * events, desired state, and live state can never diverge by path. * Desired state is recorded BEFORE the server round-trip: intent sticks * even if the subscribe fails, and reconciliation retries later. + * Automatic delivery needs open transport, not a change of attention: + * preserve tune-out epochs both before and during the round-trip. */ private async openChannelNow( entry: ChannelEntry, - source: string, + source: 'agent-tool' | 'opened-by-reply' | 'opened-by-delivery', history?: ChannelHistoryRequest, ): Promise { - this.setDesiredState(entry.serverId, entry.descriptor.id, 'open', source); + const transportOnly = source !== 'agent-tool'; + if (!transportOnly || this.getDesiredState(entry.serverId, entry.descriptor.id) !== 'tuned-out') { + this.setDesiredState(entry.serverId, entry.descriptor.id, 'open', source); + } const applied = await this.applyDesiredChannelState(entry.serverId, entry.descriptor.id, history); - if (!applied.open || this.getDesiredState(entry.serverId, entry.descriptor.id) !== 'open') { + const desired = this.getDesiredState(entry.serverId, entry.descriptor.id); + if (!applied.open || (desired !== 'open' && !(transportOnly && desired === 'tuned-out'))) { throw new Error('Channel open was superseded by a newer lifecycle decision'); } return applied.result!; diff --git a/test/channel-lifecycle-order.test.ts b/test/channel-lifecycle-order.test.ts index ea084bd62..3d63f6116 100644 --- a/test/channel-lifecycle-order.test.ts +++ b/test/channel-lifecycle-order.test.ts @@ -16,6 +16,7 @@ function fixture() { const calls: Array<{ kind: 'open' | 'close'; id: string; address?: unknown }> = []; const pending: Array<{ finish: () => void; fail: () => void }> = []; const held = new Set(); + const publishedWhileOpen: boolean[] = []; const serve = (kind: 'open' | 'close', params: { channelId?: string; address?: unknown }) => { const id = params.channelId!; calls.push({ kind, id, address: params.address }); @@ -32,6 +33,10 @@ function fixture() { grant: new CapabilityGrant(new Set(ALL_CAPABILITY_PATHS), []), sendChannelsOpen: (params: { channelId?: string; address?: unknown }) => serve('open', params) as Promise, sendChannelsClose: (params: { channelId: string }) => serve('close', params), + sendChannelsPublish: async (params: { channelId: string }) => { + publishedWhileOpen.push(actual.get(params.channelId) === true); + return { delivered: true }; + }, }; const traces: Array<{ type: string; [key: string]: unknown }> = []; const registry = new ChannelRegistry( @@ -52,7 +57,7 @@ function fixture() { assert.equal(settled, true, 'lifecycle operations must settle'); await result; }; - return { registry, actual, calls, pending, held, traces, isOpen, tool, drain }; + return { registry, actual, calls, pending, held, traces, isOpen, tool, drain, publishedWhileOpen }; } test('remove/re-add during reconcile converges on the replacement desired state, with ACK first (#185)', async () => { @@ -177,6 +182,46 @@ test('descriptor updates during an open refresh its target, but labels alone do } }); +for (const delivery of ['speech', 'reply'] as const) { + test(delivery + ' during pending transport work preserves the newer tune-out epoch', async () => { + // Felix's review reproduction: merely waiting for transport must not + // turn a delivery into a decision to cancel attention diversion. + const f = fixture(); + await f.registry.handleChanged('test', { added: [descriptor('x', true)] }); + f.held.add('x'); + const close = f.tool('close'); + await tick(); + f.registry.enterTuneOut('test', 'x', { + epochId: 'new-epoch', cadenceSeconds: 60, backlogCap: 20, maxWakes: 3, startedAtSequence: 1, + }, 'agent-tool'); + const sent = delivery === 'speech' + ? f.registry.routeSpeech('resident', 'hello', 'x').then((result) => result?.delivered === true) + : f.registry.openIfClosedForSend('x', 'test').then((result) => result.status === 'opened'); + await f.drain(close, sent); + assert.equal(await sent, true); + assert.equal(f.registry.getTuneOutState('test', 'x')?.params.epochId, 'new-epoch'); + assert.equal(f.actual.get('x'), true); + if (delivery === 'speech') assert.deepEqual(f.publishedWhileOpen, [true]); + assert.equal((await f.tool('open')).success, false, 'explicit channel_open still requires tune-out cancellation'); + assert.equal(f.registry.getTuneOutState('test', 'x')?.params.epochId, 'new-epoch'); + }); +} + +test('automatic opening accepts a newer tune-out established during its RPC without clearing it', async () => { + const f = fixture(); + await f.registry.handleChanged('test', { added: [descriptor()] }); + f.held.add('x'); + const speech = f.registry.routeSpeech('resident', 'hello', 'x'); + await tick(); + f.registry.enterTuneOut('test', 'x', { + epochId: 'mid-open-epoch', cadenceSeconds: 60, backlogCap: 20, maxWakes: 3, startedAtSequence: 1, + }, 'agent-tool'); + await f.drain(speech); + assert.equal((await speech)?.delivered, true); + assert.equal(f.registry.getTuneOutState('test', 'x')?.params.epochId, 'mid-open-epoch'); + assert.deepEqual(f.publishedWhileOpen, [true]); +}); + test('a newer tune-out decision keeps transport open when an older close completes', async () => { const f = fixture(); await f.registry.handleChanged('test', { added: [descriptor('x', true)] }); From b71587726b9ba024b40d4f7aa50afb58d268193a Mon Sep 17 00:00:00 2001 From: Jasper Date: Fri, 2 Oct 2026 02:04:05 -0400 Subject: [PATCH 0011/1112] fix(usage): log tool-ended completions for health accounting Address the source-confirmed Greptile finding on PR #207: restored usage/completion telemetry still bypassed the successful inference log, which HealthModule uses for inference and token totals. Write the terminal log inside the same per-stream idempotent completion callback. Natural completion retains its raw provider response, stop reason, and duration; tool-ended completion records turn_ended and an explicit note that the stream ended by a tool result rather than inventing a final provider response. Missing usage remains absent. Extend regression coverage for one log per one/multiple-tool-round stream, normal completion without duplication, agent attribution, unique request IDs, store reopen, missing usage, and the real log-backed HealthModule snapshot. On 6ac4743 the six-case suite gives 1 pass / 5 fail; the fix passes 6/6. Expanded compiled focused tests pass 32/32 without force-exit. The revised full compiled suite is running separately before push. Co-Authored-By: GPT-6 Astra --- CHANGELOG.md | 2 +- src/framework.ts | 52 +++++++++++++++++++++---------------- src/types/framework.ts | 2 +- test/end-turn-usage.test.ts | 49 ++++++++++++++++++++++++++++++++++ 4 files changed, 80 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 14488fad5..66f48f1a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,7 @@ Releases up to and including 0.7.3 predate this file; for their contents see ### Fixed -- Tool-ended turns now emit `inference:completed` and record cumulative token, cache, and estimated-cost usage exactly once, including turns ended by `skip_reply` or other `endTurn` tools. +- Tool-ended turns now emit `inference:completed`, record a successful inference-log entry, and record cumulative token, cache, and estimated-cost usage exactly once, including turns ended by `skip_reply` or other `endTurn` tools. Log-backed health totals now include these turns. ## 0.19.0 — 2026-09-28 diff --git a/src/framework.ts b/src/framework.ts index b2c66c876..4a9cc7757 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -3,7 +3,7 @@ import { INLINE_WITHHELD_TEXT, classifyBlock, isInlineContradiction, referenceRe import { ReferenceFetcher, DEFAULT_FETCH_MAX_BYTES, EAGER_FETCH_TIMEOUT_MS } from './mcpl/reference-fetcher.js'; import { appendFileSync, existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs'; import { JsStore } from '@animalabs/chronicle'; -import type { Membrane, ContentBlock, NormalizedRequest, YieldingStream, DetailedUsage, ToolResult as MembraneToolResult, ToolResultContentBlock } from '@animalabs/membrane'; +import type { Membrane, ContentBlock, NormalizedRequest, NormalizedResponse, YieldingStream, DetailedUsage, ToolResult as MembraneToolResult, ToolResultContentBlock } from '@animalabs/membrane'; import { MembraneError } from '@animalabs/membrane'; import { ContextManager, PassthroughStrategy, WindowedPassthroughStrategy, OverBudgetError, UncoveredDropError } from '@animalabs/context-manager'; import type { CacheWireReceipt } from './kv-unified-wire.js'; @@ -9081,20 +9081,25 @@ export class AgentFramework { const myStreamId = agent.streamId; let latestUsage: DetailedUsage | undefined; let completionAccounted = false; - const completeAccounting = (usage: DetailedUsage | undefined): void => { + const completeAccounting = ( + usage: DetailedUsage | undefined, + response?: NormalizedResponse, + durationMs = Date.now() - startTime, + ): void => { if (completionAccounted) return; completionAccounted = true; this.streamCompletionAccounting.delete(stream); + const tokenUsage = usage ? { + input: usage.inputTokens, + output: usage.outputTokens, + cacheCreation: usage.cacheCreationTokens, + cacheRead: usage.cacheReadTokens, + } : undefined; this.emitTrace({ type: 'inference:completed', agentName: agent.name, - durationMs: Date.now() - startTime, - tokenUsage: usage ? { - input: usage.inputTokens, - output: usage.outputTokens, - cacheCreation: usage.cacheCreationTokens, - cacheRead: usage.cacheReadTokens, - } : undefined, + durationMs, + tokenUsage, }); if (usage) { this.usageTracker.onInferenceCompleted(agent.name, usage, usage.estimatedCost @@ -9102,6 +9107,20 @@ export class AgentFramework { : undefined); this.persistUsageState(); } + // Health snapshots and log queries read this same terminal accounting. + // A tool-ended stream has no final provider response; record that fact + // rather than fabricating response content or waiting for cancellation. + this.logInference({ + timestamp: startTime, + agentName: agent.name, + requestId, + success: true, + request: compiledRequest ?? { note: 'streaming request' }, + response: response?.raw ?? { note: response ? 'streaming response' : 'stream ended by tool result' }, + durationMs, + tokenUsage, + stopReason: response?.stopReason ?? 'turn_ended', + }); }; this.streamCompletionAccounting.set(stream, () => completeAccounting(latestUsage)); // Membrane usage events are cumulative across the native/XML tool loop. @@ -9641,20 +9660,7 @@ export class AgentFramework { .join('\n'), }); - completeAccounting(du); - - // Log inference - this.logInference({ - timestamp: startTime, - agentName: agent.name, - requestId, - success: true, - request: compiledRequest ?? { note: 'streaming request' }, - response: response.raw ?? { note: 'streaming response' }, - durationMs, - tokenUsage, - stopReason: response.stopReason, - }); + completeAccounting(du, response, durationMs); // Surface refusals instead of going silently mute: stderr line // (headless inference failures are otherwise under-logged) + an diff --git a/src/types/framework.ts b/src/types/framework.ts index 3ff4ba9dc..8d4ee3636 100644 --- a/src/types/framework.ts +++ b/src/types/framework.ts @@ -396,7 +396,7 @@ export interface InferenceLogEntry { cacheCreation?: number; cacheRead?: number; }; - /** Stop reason from the model */ + /** Stop reason from the model, or 'turn_ended' for a tool-ended stream. */ stopReason?: string; } diff --git a/test/end-turn-usage.test.ts b/test/end-turn-usage.test.ts index eb4e21451..276c6dd63 100644 --- a/test/end-turn-usage.test.ts +++ b/test/end-turn-usage.test.ts @@ -5,6 +5,7 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import type { DetailedUsage, NormalizedResponse } from '@animalabs/membrane'; import { AgentFramework } from '../src/index.js'; +import { HealthModule } from '../src/modules/health/index.js'; import type { Module, ToolCall, TraceEvent } from '../src/index.js'; import { createMockResponse, MockMembrane } from './helpers/mock-membrane.js'; @@ -108,6 +109,13 @@ describe('tool-ended turn usage accounting', () => { assert.equal(completed[0].agentName, 'worker'); assert.deepEqual(completed[0].tokenUsage, expectedTokens(totalUsage)); assert.ok(completed[0].durationMs >= 0); + const logs = f.framework.queryInferenceLogs({ agentName: 'worker' }); + assert.equal(logs.total, 1); + assert.equal(logs.entries[0].entry.success, true); + assert.equal(logs.entries[0].entry.stopReason, 'turn_ended'); + assert.deepEqual(logs.entries[0].entry.tokenUsage, expectedTokens(totalUsage)); + assert.deepEqual(logs.entries[0].entry.response, { note: 'stream ended by tool result' }); + assert.equal(logs.entries[0].entry.durationMs, completed[0].durationMs); assert.equal(f.traces.filter((e) => e.type === 'inference:turn_ended').length, 1); assert.equal(f.traces.filter((e) => e.type === 'usage:updated').length, 1); assert.equal(f.traces.filter((e) => e.type === 'inference:exhausted').length, 0); @@ -141,6 +149,10 @@ describe('tool-ended turn usage accounting', () => { assert.equal(f.traces.filter((e) => e.type === 'inference:turn_ended').length, 0); assert.equal(f.framework.getSessionUsage().inferenceCount, 1); assert.deepEqual(f.framework.getSessionUsage().totals, expectedTotals(totalUsage)); + const logs = f.framework.queryInferenceLogs({ agentName: 'ordinary' }); + assert.equal(logs.total, 1, 'natural completion logs exactly once'); + assert.equal(logs.entries[0].entry.stopReason, 'end_turn'); + assert.deepEqual(logs.entries[0].entry.tokenUsage, expectedTokens(totalUsage)); } finally { await f.framework.stop(); rmSync(f.dir, { recursive: true, force: true }); @@ -162,6 +174,10 @@ describe('tool-ended turn usage accounting', () => { { agentName: 'bob', usage: expectedTotals(totalUsage), inferenceCount: 1 }, ]); assert.deepEqual(f.traces.filter((e) => e.type === 'usage:updated').map((e) => e.agentName), ['alice', 'bob']); + const logs = f.framework.queryInferenceLogs(); + assert.equal(logs.total, 2); + assert.equal(new Set(logs.entries.map((e) => e.entry.requestId)).size, 2); + assert.deepEqual(logs.entries.map((e) => e.entry.agentName).sort(), ['alice', 'bob']); await f.framework.stop(); stopped = true; const reopened = await AgentFramework.create({ @@ -169,6 +185,7 @@ describe('tool-ended turn usage accounting', () => { }); try { assert.deepEqual(reopened.getSessionUsage(), snapshot); + assert.deepEqual(reopened.queryInferenceLogs(), logs); } finally { await reopened.stop(); } @@ -178,6 +195,34 @@ describe('tool-ended turn usage accounting', () => { } }); + it('includes tool-ended turns in the log-backed health snapshot', async () => { + const f = await fixture(); + try { + f.membrane.pushResponse(toolResponse('echo', 'echo-1', firstUsage)); + f.membrane.pushResponse(toolResponse('finish', 'finish-1', totalUsage)); + await run(f.framework, 'worker'); + const health = new HealthModule({ timeZone: 'UTC' }); + health.bind(f.framework, f.framework.getStore()); + const result = await health.handleToolCall({ + id: 'health-snapshot', name: 'snapshot', input: { includeSubagents: false }, + }); + assert.equal(result.success, true); + const data = result.data as { + inferences: { successCount: number; errorCount: number }; + tokenTotalsByAgent: Record; + }; + assert.equal(data.inferences.successCount, 1); + assert.equal(data.inferences.errorCount, 0); + assert.deepEqual(data.tokenTotalsByAgent.worker, { + input: totalUsage.inputTokens, output: totalUsage.outputTokens, + cacheRead: totalUsage.cacheReadTokens, inferences: 1, + }); + } finally { + await f.framework.stop(); + rmSync(f.dir, { recursive: true, force: true }); + } + }); + it('reports completion without inventing usage when no sample is available', async () => { const f = await fixture(); try { @@ -188,6 +233,10 @@ describe('tool-ended turn usage accounting', () => { assert.equal(completed[0].tokenUsage, undefined); assert.equal(f.traces.filter((e) => e.type === 'usage:updated').length, 0); assert.equal(f.framework.getSessionUsage().inferenceCount, 0); + const logs = f.framework.queryInferenceLogs({ agentName: 'unknown-usage' }); + assert.equal(logs.total, 1); + assert.equal(logs.entries[0].entry.success, true); + assert.equal(logs.entries[0].entry.tokenUsage, undefined); } finally { await f.framework.stop(); rmSync(f.dir, { recursive: true, force: true }); From 0b417696aa31e92fe315404899fa2e77a57ca0b1 Mon Sep 17 00:00:00 2001 From: Iris-308 Date: Fri, 2 Oct 2026 02:04:30 -0400 Subject: [PATCH 0012/1112] fix(routing): retain closure failure notices before disposing forks Greptile's review of PR #204 identified a real teardown ordering bug: a failed TTL-closure publish queues its owner-targeted notice, but driveStream disposed the fork before the deferred flush. The subsequent addMessage call dropped the notice because its target was already unregistered. Release the turn token and flush owner-scoped deferred writes before disposing a terminal closure fork. Scope the pending-tool check to that agent so another agent's tool cycle cannot prevent its final notice from landing. Disposal now retains an agent with an active turn/stream/tool cycle or pending/unacknowledged context writes. This also protects the fallback reaper: a quiesced closure stays registered until its deferred notice is flushed, then the reaper disposes it safely. Add three real TTL-closure regressions with a server-reported delivered:false response: ordinary closure, another agent's tool cycle, and a quiesced closure plus fallback reaping after flush. Each opens a fresh ContextManager on the archived fork namespace and checks final-reply/notice order, exactly one notice, primary isolation, eventual disposal, and no additional inference. All three fail on PR head 17034b1 (5 pass / 3 fail in the extended file). Corrected targeting, per-agent delivery, deferred-wake, and host-quiesce tests pass 47/47 without force-exit; build and diff check pass. Full bounded compiled suite is running before publication. Addresses review https://github.com/anima-research/agent-framework/pull/204#discussion_r4163104814. Co-Authored-By: GPT-6 Astra --- CHANGELOG.md | 2 +- src/framework.ts | 27 ++++++--- test/route-failure-targeting.test.ts | 86 ++++++++++++++++++++++++++++ 3 files changed, 107 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3bf98d349..155762653 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,7 @@ Releases up to and including 0.7.3 predate this file; for their contents see ### Fixed -- Speech-route failure notices now target the speaking agent, including conversation forks, and respect that agent's mid-turn deferral. Unregistered conversation IDs retain the primary-agent fallback; notices remain context-only and do not trigger inference (#188). +- Speech-route failure notices now target the speaking agent, including conversation forks, and respect that agent's mid-turn deferral. Unregistered conversation IDs retain the primary-agent fallback; notices remain context-only and do not trigger inference. A fork's final closure notice is flushed before disposal, and deferred writes retain their target through quiesce until they can be stored (#188). ## 0.19.0 — 2026-09-28 diff --git a/src/framework.ts b/src/framework.ts index b18147ac1..c761b2cb9 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -7169,6 +7169,18 @@ export class AgentFramework { * forever. The fork's context stays in Chronicle for investigation. */ private disposeConversationAgent(agentName: string): void { + // A closure can finish while writes are still deferred (e.g. quiesce), + // or while a successor owns the turn token. Keep the target registered + // until those writes land; the fallback reaper can dispose it afterward. + const belongsToAgent = (msg: { forAgent?: string }) => + (msg.forAgent ?? this.primaryAgentName) === agentName; + if ( + this.activeTurnTokens.has(agentName) || + this.activeStreams.has(agentName) || + this.pendingAssistantBlocks.has(agentName) || + this.deferredMessages.some(belongsToAgent) || + this.unackedDeferredWrites.some(belongsToAgent) + ) return; this.closingConversationAgents.delete(agentName); const channelId = this.conversationAgentHomes.get(agentName); const agent = this.agents.get(agentName); @@ -10366,12 +10378,6 @@ export class AgentFramework { this.pendingAssistantBlocks.delete(agent.name); } - // A conversation fork whose TTL closure turn just finished is done for - // good — dispose it so the agent map doesn't grow monotonically. - if (ownsPhysicalStream && this.closingConversationAgents.has(agent.name)) { - this.disposeConversationAgent(agent.name); - } - // The turn is torn down — release the turn-alive marker BEFORE the // deferred flush below, so the flush appends at the settled tail. // Token-matched: if a successor turn already owns the agent (endTurn @@ -10388,7 +10394,7 @@ export class AgentFramework { // pending). Only THIS agent's messages: other targets' entries wait // for their own boundaries (re-adding via addMessage re-defers if the // target has meanwhile started a turn). - if (frameReachedTerminal && this.deferredMessages.length > 0 && this.pendingAssistantBlocks.size === 0) { + if (frameReachedTerminal && this.deferredMessages.length > 0 && !this.pendingAssistantBlocks.has(agent.name)) { const deferred = this.drainDeferredFor(agent.name); for (const msg of deferred) { this.addMessage(msg.participant, msg.content, msg.metadata, { @@ -10398,6 +10404,13 @@ export class AgentFramework { } this.ackDeferredWrites(); } + + // Flush the closure turn's notices while its owner is still registered. + // If quiesce or a successor re-deferred them, disposal waits for the + // fallback reaper after their eventual safe write boundary. + if (frameReachedTerminal && ownsPhysicalStream && this.closingConversationAgents.has(agent.name)) { + this.disposeConversationAgent(agent.name); + } } } diff --git a/test/route-failure-targeting.test.ts b/test/route-failure-targeting.test.ts index e17e55a24..2b8890de3 100644 --- a/test/route-failure-targeting.test.ts +++ b/test/route-failure-targeting.test.ts @@ -4,6 +4,9 @@ import { mkdtempSync, rmSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { AgentFramework } from '../src/index.js'; +import { ContextManager, PassthroughStrategy } from '@animalabs/context-manager'; +import { CapabilityGrant } from '../src/mcpl/capability-grant.js'; +import type { McplServerRegistry } from '../src/mcpl/server-registry.js'; import type { ProcessEvent } from '../src/index.js'; import type { ChannelRegistry } from '../src/mcpl/channel-registry.js'; import { MockMembrane, createMockResponse } from './helpers/mock-membrane.js'; @@ -15,6 +18,12 @@ function internals(framework: AgentFramework) { activeTurnTokens: Map; deferredMessages: Array<{ forAgent?: string; metadata?: Record }>; pendingRequests: unknown[]; + pendingAssistantBlocks: Map; + mcplServerRegistry: McplServerRegistry; + lastConversationSweep: number; + sweepExpiredConversations(): void; + quiesced: boolean; + flushDeferredWrites(label: string): Promise; }; } @@ -42,6 +51,8 @@ describe('speech route failure notice targeting', () => { afterEach(async () => { internals(framework).activeTurnTokens.clear(); + internals(framework).pendingAssistantBlocks.clear(); + internals(framework).quiesced = false; await framework.stop(); rmSync(tempDir, { recursive: true, force: true }); }); @@ -107,6 +118,81 @@ describe('speech route failure notice targeting', () => { assert.equal(state.pendingRequests.length, 0); }); + for (const condition of ['ordinary', 'other-tool-cycle', 'quiesced'] as const) { + it('retains a failed TTL-closure reply notice before disposal: ' + condition, async (t) => { + const state = internals(framework); + const channelId = 'closure-channel'; + let failPublishing = false; + const publisher = { + grant: new CapabilityGrant(new Set(['channels.publish']), []), + sendChannelsTyping() {}, + async sendChannelsPublish() { + if (failPublishing && condition === 'quiesced') state.quiesced = true; + return { delivered: !failPublishing }; + }, + }; + t.mock.method(state.mcplServerRegistry, 'getServer', () => publisher as never); + (state.channelRegistry as unknown as { channels: Map }).channels.set('srv:' + channelId, { + serverId: 'srv', descriptor: { id: channelId, type: 'srv', label: 'closure test' }, open: true, + }); + membrane.pushResponse(createMockResponse([{ type: 'text', text: 'Initial reply.' }])); + framework.pushEvent({ + type: 'mcpl:channel-incoming', serverId: 'srv', channelId, messageId: 'closure-incoming', + author: { id: 'user', name: 'User' }, + content: [{ type: 'text', text: 'Please help.' }], + timestamp: new Date().toISOString(), + metadata: { mentioned: true }, + triggerInference: true, + } as unknown as ProcessEvent); + await framework.runUntilIdle(); + const binding = framework.getConversationRouter()!.getBinding(channelId)!; + assert.ok(binding); + const fork = framework.getAgent(binding.agentName)!; + + failPublishing = true; + const finalReply = 'Final reply that the server cannot deliver.'; + membrane.pushResponse(createMockResponse([{ type: 'text', text: finalReply }])); + if (condition === 'other-tool-cycle') state.pendingAssistantBlocks.set('secondary', []); + binding.lastActivity = 0; + state.lastConversationSweep = 0; + state.sweepExpiredConversations(); + await framework.runUntilIdle(); + state.pendingAssistantBlocks.clear(); + + if (condition === 'quiesced') { + assert.ok(framework.getAgent(fork.name) === fork, 'a deferred write keeps its owner registered while quiesced'); + assert.equal(state.deferredMessages.length, 1); + state.lastConversationSweep = 0; + state.sweepExpiredConversations(); + assert.ok(framework.getAgent(fork.name) === fork, 'the fallback reaper must also retain the pending write target'); + state.quiesced = false; + await state.flushDeferredWrites('closure test resume'); + state.lastConversationSweep = 0; + state.sweepExpiredConversations(); + } + + assert.equal(framework.getAgent(fork.name), null, 'the settled fork is eventually disposed'); + assert.equal(state.deferredMessages.length, 0); + // Open a fresh manager over the retained fork namespace, rather than + // relying only on the disposed Agent's in-memory context manager. + const archive = await ContextManager.open({ + store: framework.getStore(), namespace: 'conversations/' + fork.name, + isolate: true, strategy: new PassthroughStrategy(), + }); + const { messages } = await archive.compile(); + const noticeIndex = messages.findIndex((m) => JSON.stringify(m.content).includes('[discord-send-failed]')); + const replyIndex = messages.findIndex((m) => m.participant === fork.name && JSON.stringify(m.content).includes(finalReply)); + assert.ok(replyIndex >= 0, 'the failed closure reply remains archived'); + assert.ok(noticeIndex > replyIndex, 'the archived failure notice follows the closure reply'); + assert.equal(messages.filter((m) => JSON.stringify(m.content).includes('[discord-send-failed]')).length, 1); + assert.match(JSON.stringify(messages[noticeIndex]!.content), /delivered:false/); + const primary = await framework.getAgent('primary')!.getContextManager().compile(); + assert.ok(!primary.messages.some((m) => JSON.stringify(m.content).includes('[discord-send-failed]'))); + assert.equal(membrane.calls.length, 2, 'only the initial and closure turns ran'); + assert.equal(state.pendingRequests.length, 0, 'a failed closure reply does not wake another turn'); + }); + } + it('delivers a failed fork reply notice to the fork after its turn, without waking it again', async () => { const reply = 'Reply from the fork.'; membrane.pushResponse(createMockResponse([{ type: 'text', text: reply }])); From d64c8034c35e27b09b59e28a8107ea61a15eca58 Mon Sep 17 00:00:00 2001 From: Jasper Date: Fri, 2 Oct 2026 02:42:59 -0400 Subject: [PATCH 0013/1112] fix(mcpl): distinguish delivery joins from opening intent Address the five source-confirmed review findings on PR #210. The prior pending-work check turned a delivery wait into a new open decision, overriding an explicit close and repeating opens/notices for backscroll. Its open-error catch also erased known-open state, and callers could describe an obsolete registration. Delivery on an already-open channel now joins its per-channel lifecycle tail without changing desired intent. If the join settles closed or loses registration, delivery fails visibly. Fresh delivery into a settled closed channel retains the existing engagement behavior. Concurrent delivery-forced opens skip redundant RPCs and report a transition only once. Re-read the entry after waiting so notices and reply preparation report the current label. Failed redundant open requests preserve known-live state rather than inventing a close. Policy notices now report durable admission intent before transport work rather than delayed Now open confirmations. That preserves once-per-admission/reboot semantics and informs the resident even when the first RPC fails; later reconciliation needs no second ledger or notice. Explicitly document the callback timing/meaning change for direct ChannelRegistry consumers. Built-in text distinguishes intent from confirmed delivery opens and accurately names the fresh-send engagement contract. Wren independently corroborated pending-close and stale-admission behaviors and helped clarify the intent/transport boundary. The final 23-case regression file yields 15 pass / 8 fail on published cd2638d and 23/23 here. It includes current replacement labels, failed backscroll, close during a join, duplicate delivery joins, fresh settled-close engagement, and real resident notice text through first-open failure and registry reconstruction. Expanded compiled focused tests pass 79/79 without force-exit. The final full compiled suite is running before push; independent review will cover this committed head. Co-Authored-By: GPT-6 Astra --- CHANGELOG.md | 6 + src/framework.ts | 17 ++- src/mcpl/channel-registry.ts | 194 +++++++++++++++++---------- test/channel-lifecycle-order.test.ts | 190 +++++++++++++++++++++++++- 4 files changed, 322 insertions(+), 85 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b02aec4b1..9dbbe6281 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,8 +12,14 @@ Releases up to and including 0.7.3 predate this file; for their contents see ## Unreleased +### Breaking (direct ChannelRegistry consumers) + +- `onChannelAutoOpened` with `source: 'subscription-policy'` now reports a durable admission decision before transport reconciliation. Treat it as intent; await the registration call and inspect `getOpenChannels()` when confirmed live state is needed. `opened-by-delivery` still reports a confirmed transition. The built-in host notice now distinguishes selected-for-opening from currently open. + ### Fixed +- Failed backscroll requests preserve the registry's known-open state rather than claiming a close. Automatic delivery joins pending lifecycle work without reopening a channel closed during the wait, and reports only actual delivery-forced opens using the current registration label. + - Channel lifecycle operations now run in order per channel and reconcile the latest desired state and registration after an in-flight operation completes. Concurrent registration, open, and close requests can no longer leave the server in an older state; a superseded tool request reports that its requested state no longer applies. Automatic delivery preserves active tune-out attention state while opening or awaiting transport. ## 0.19.0 — 2026-09-28 diff --git a/src/framework.ts b/src/framework.ts index 510b9183a..92b652791 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -8324,15 +8324,14 @@ export class AgentFramework { .map((c) => (c.label && c.label !== c.channelId ? `${c.label} (${c.channelId})` : c.channelId)) .join(', '); const plural = channels.length > 1; - const causeText = - cause === 'subscription-policy' - ? 'your subscription policy just admitted ' + (plural ? 'them' : 'it') - : 'your own message into the closed channel engaged it'; - const text = - `[channels] Now open: ${shown} — ${causeText}. Ongoing traffic from ` + - (plural ? 'these channels' : 'this channel') + - ` will reach you. If you don't want ${plural ? 'one of them' : 'it'}, use channel_close — ` + - 'your choice sticks; neither policy nor a later send will silently reopen it.'; + const text = (cause === 'subscription-policy' + ? `[channels] Selected for opening by subscription policy: ${shown}. ` + + 'This records the admission decision; channel_list shows current transport state. ' + + 'When open, ' + (plural ? 'their' : 'its') + ' ongoing traffic will reach you. ' + : `[channels] Now open: ${shown} — your own message into the closed channel engaged it. ` + + 'Ongoing traffic from ' + (plural ? 'these channels' : 'this channel') + ' will reach you. ') + + `If you don't want ${plural ? 'one of them' : 'it'}, use channel_close; your choice outranks policy. ` + + 'A fresh message into a closed channel engages it again.'; try { // Through framework.addMessage for mid-turn safety (deferral + // hear-while-acting injection) — a delivery-forced open happens inside diff --git a/src/mcpl/channel-registry.ts b/src/mcpl/channel-registry.ts index d7b8be6a5..7ef70c9cb 100644 --- a/src/mcpl/channel-registry.ts +++ b/src/mcpl/channel-registry.ts @@ -485,13 +485,12 @@ interface ChannelRegistryOptions { textLen: number; }) => void; /** - * Called when channels were opened WITHOUT the agent asking (subscription - * policy admitting a newly discovered channel, or delivery into a closed - * locus). The host wires this to drop a durable notice into the agent's - * window — the agent must always learn that new traffic will start - * flowing, and that `channel_close` opts out (their decision outranks - * policy). Fires ONCE per channel ever: the desired-state decision is - * durable, so reboots do not re-announce. Must not trigger inference. + * Announces either a durable subscription-policy admission (before its + * transport RPC; not an open confirmation), or a confirmed delivery-forced + * closed-to-open transition. The host records a durable, non-waking notice. + * Policy admissions fire once per channel: the persisted desired-state + * decision prevents re-announcement on retry or reboot. channel_close + * supersedes policy; a fresh delivery into a closed locus engages it again. */ onChannelAutoOpened?: (info: { /** Agent whose action caused the open (delivery); absent for policy opens. */ @@ -2083,12 +2082,14 @@ export class ChannelRegistry { private applyDesiredChannelState( serverId: string, channelId: string, - history?: ChannelHistoryRequest, - ): Promise<{ open: boolean; result?: ChannelsOpenResult }> { + options?: { history?: ChannelHistoryRequest; onlyIfClosed?: boolean }, + ): Promise<{ open: boolean; opened: boolean; result?: ChannelsOpenResult }> { const lifecycleKey = this.lifecycleKey(serverId, channelId); const channelKey = `${serverId}:${channelId}`; const previous = this.channelLifecycleTails.get(lifecycleKey) ?? Promise.resolve(); const operation = previous.then(async () => { + let issued = false; + let opened = false; for (;;) { const entry = this.channels.get(channelKey); if (!entry) throw new Error(`Channel is no longer registered: ${channelId}`); @@ -2100,31 +2101,33 @@ export class ChannelRegistry { const descriptor = entry.descriptor; const desired = this.getDesiredState(serverId, channelId); const open = desired === 'open' || desired === 'tuned-out'; + if (!issued && options?.onlyIfClosed && open && entry.open) { + return { open: true, opened: false }; + } + issued = true; let result: ChannelsOpenResult | undefined; - try { - if (open) { - result = await server.sendChannelsOpen({ - channelId, type: descriptor.type, address: descriptor.address, - ...(history ? { history } : {}), - }); - } else { - await server.sendChannelsClose({ channelId }); - } - } catch (error) { - if (open && this.channels.get(channelKey) === entry && - this.serverRegistry.getServer(serverId) === server) entry.open = false; - throw error; + if (open) { + result = await server.sendChannelsOpen({ + channelId, type: descriptor.type, address: descriptor.address, + ...(options?.history ? { history: options.history } : {}), + }); + } else { + await server.sendChannelsClose({ channelId }); } + // Failure is not a confirmed transition. In particular, a failed + // backscroll request must not erase an already-live subscription. // Never apply a receipt to a removed entry or a replacement connection. // Reconcile their current state before releasing this channel's queue. if (this.channels.get(channelKey) !== entry || this.serverRegistry.getServer(serverId) !== server) continue; + if (open && !entry.open) opened = true; + if (!open) opened = false; entry.open = open; const now = this.getDesiredState(serverId, channelId); const targetChanged = entry.descriptor.type !== descriptor.type || !isDeepStrictEqual(entry.descriptor.address, descriptor.address); if (targetChanged || open !== (now === 'open' || now === 'tuned-out')) continue; - return { open, result }; + return { open, opened, result }; } }); const tail = operation.then(() => {}, () => {}); @@ -2137,6 +2140,26 @@ export class ChannelRegistry { return operation; } + /** + * Join work on an already-open channel without recording new intent. + * A close that settles during the wait remains authoritative; waiting for + * it is not an instruction to open the channel again. + */ + private async waitForOpenChannel(entry: ChannelEntry): Promise { + const lifecycleKey = this.lifecycleKey(entry.serverId, entry.descriptor.id); + for (;;) { + const pending = this.channelLifecycleTails.get(lifecycleKey); + if (!pending) break; + await pending; + } + const current = this.channels.get(`${entry.serverId}:${entry.descriptor.id}`); + if (!current) throw new Error('Channel registration was removed while delivery waited'); + if (!current.open || this.getDesiredState(current.serverId, current.descriptor.id) === 'closed') { + throw new Error('Channel was closed or selected for closure while delivery waited'); + } + return current; + } + private async reconcileChannels( serverId: string, channels: ChannelDescriptor[], @@ -2170,6 +2193,28 @@ export class ChannelRegistry { policyOpened.push({ channelId: channel.id, label: entry.descriptor.label }); } } + + // Announce the durable admission decision before awaiting transport. + // This is intent, not confirmation that an open RPC succeeded. It stays + // truthful if the first attempt fails, and a retry needs no new admission. + // channel_close supersedes policy; use current entries at this boundary. + const currentAdmissions = policyOpened.flatMap(({ channelId }) => { + const entry = this.channels.get(`${serverId}:${channelId}`); + if (!entry || this.getDesiredState(serverId, channelId) !== 'open') return []; + return [{ channelId, label: entry.descriptor.label }]; + }); + if (currentAdmissions.length > 0) { + try { + this.onChannelAutoOpened?.({ + serverId, + source: 'subscription-policy', + channels: currentAdmissions, + }); + } catch (err) { + console.error('onChannelAutoOpened (policy) failed:', err); + } + } + for (const channel of channels) { if (!this.channels.has(`${serverId}:${channel.id}`)) continue; try { @@ -2186,21 +2231,6 @@ export class ChannelRegistry { }); } } - - // Announce policy admissions to the agent — nothing may start flowing - // traffic into their window without them being told, and told how to - // opt out (channel_close; their decision outranks policy). - if (policyOpened.length > 0) { - try { - this.onChannelAutoOpened?.({ - serverId, - source: 'subscription-policy', - channels: policyOpened, - }); - } catch (err) { - console.error('onChannelAutoOpened (policy) failed:', err); - } - } } // ========================================================================== @@ -2336,17 +2366,17 @@ export class ChannelRegistry { entry: ChannelEntry, source: 'agent-tool' | 'opened-by-reply' | 'opened-by-delivery', history?: ChannelHistoryRequest, - ): Promise { + ): Promise<{ result: ChannelsOpenResult; opened: boolean }> { const transportOnly = source !== 'agent-tool'; if (!transportOnly || this.getDesiredState(entry.serverId, entry.descriptor.id) !== 'tuned-out') { this.setDesiredState(entry.serverId, entry.descriptor.id, 'open', source); } - const applied = await this.applyDesiredChannelState(entry.serverId, entry.descriptor.id, history); + const applied = await this.applyDesiredChannelState(entry.serverId, entry.descriptor.id, { history, onlyIfClosed: transportOnly }); const desired = this.getDesiredState(entry.serverId, entry.descriptor.id); if (!applied.open || (desired !== 'open' && !(transportOnly && desired === 'tuned-out'))) { throw new Error('Channel open was superseded by a newer lifecycle decision'); } - return applied.result!; + return { result: applied.result!, opened: applied.opened }; } /** @@ -2519,19 +2549,25 @@ export class ChannelRegistry { } if (matches.length === 0) return { status: 'unknown-channel' }; if (matches.length > 1) return { status: 'ambiguous' }; - const entry = matches[0]!; - const resolved = { channelId: entry.descriptor.id, label: entry.descriptor.label }; - if (entry.open && this.getDesiredState(entry.serverId, entry.descriptor.id) === 'open' && !this.hasPendingLifecycle(entry)) { - return { status: 'already-open', ...resolved }; - } + let entry = matches[0]!; try { - await this.openChannelNow(entry, 'opened-by-reply'); - this.emitTraceFn({ - type: 'mcpl:channel-opened-by-send', - serverId: entry.serverId, - channelId: entry.descriptor.id, - }); - return { status: 'opened', ...resolved }; + if (entry.open && this.hasPendingLifecycle(entry)) { + entry = await this.waitForOpenChannel(entry); + } + const desired = this.getDesiredState(entry.serverId, entry.descriptor.id); + if (entry.open && (desired === 'open' || desired === 'tuned-out')) { + return { status: 'already-open', channelId: entry.descriptor.id, label: entry.descriptor.label }; + } + const { opened } = await this.openChannelNow(entry, 'opened-by-reply'); + entry = await this.waitForOpenChannel(entry); + if (opened) { + this.emitTraceFn({ + type: 'mcpl:channel-opened-by-send', + serverId: entry.serverId, + channelId: entry.descriptor.id, + }); + } + return { status: opened ? 'opened' : 'already-open', channelId: entry.descriptor.id, label: entry.descriptor.label }; } catch (err) { this.emitTraceFn({ type: 'mcpl:channel-open-failed', @@ -2539,7 +2575,9 @@ export class ChannelRegistry { channelId: entry.descriptor.id, error: (err as Error).message, }); - return { status: 'open-failed', ...resolved }; + const current = this.channels.get(`${entry.serverId}:${entry.descriptor.id}`); + return { status: 'open-failed', channelId: entry.descriptor.id, + ...(current ? { label: current.descriptor.label } : {}) }; } } @@ -2590,7 +2628,7 @@ export class ChannelRegistry { ...(input.beforeMessageId ? { beforeMessageId: input.beforeMessageId } : {}), } : undefined; - const result = await this.openChannelNow(entry, 'agent-tool', history); + const { result } = await this.openChannelNow(entry, 'agent-tool', history); return { success: true, data: { @@ -2992,10 +3030,17 @@ export class ChannelRegistry { return fail(null, 'turn has no locus (no home/trigger channel; nothing to deliver into)'); } - const entry = this.findChannelEntry(channelId); + let entry = this.findChannelEntry(channelId); if (!entry) { return fail(channelId, `no registered channel for locus "${channelId}"`); } + if (entry.open && this.hasPendingLifecycle(entry)) { + try { + entry = await this.waitForOpenChannel(entry); + } catch (err) { + return fail(channelId, `lifecycle work prevented delivery: ${(err as Error).message}`); + } + } // INVARIANT: it is not possible to send into a closed channel. Speech // routed into a closed-but-registered locus (the canonical case: a DM @@ -3003,26 +3048,29 @@ export class ChannelRegistry { // so typing indicators, reaction machinery, and inbound forwarding come // alive with the reply. If the open fails, the speech does NOT go out: // a half-alive delivery is worse than a loud marker. - if (!entry.open || this.hasPendingLifecycle(entry)) { + if (!entry.open) { try { - await this.openChannelNow(entry, 'opened-by-delivery'); - console.error( - `[routeSpeech] ${conversationId}: locus ${channelId} was closed — opened by delivery (speech implies engagement)`, - ); - this.emitTraceFn({ - type: 'mcpl:channel-opened-by-send', - serverId: entry.serverId, - channelId, - }); - try { - this.onChannelAutoOpened?.({ - conversationId, + const { opened } = await this.openChannelNow(entry, 'opened-by-delivery'); + entry = await this.waitForOpenChannel(entry); + if (opened) { + console.error( + `[routeSpeech] ${conversationId}: locus ${channelId} was closed — opened by delivery (speech implies engagement)`, + ); + this.emitTraceFn({ + type: 'mcpl:channel-opened-by-send', serverId: entry.serverId, - source: 'opened-by-delivery', - channels: [{ channelId, label: entry.descriptor.label }], + channelId, }); - } catch (err) { - console.error('onChannelAutoOpened (delivery) failed:', err); + try { + this.onChannelAutoOpened?.({ + conversationId, + serverId: entry.serverId, + source: 'opened-by-delivery', + channels: [{ channelId, label: entry.descriptor.label }], + }); + } catch (err) { + console.error('onChannelAutoOpened (delivery) failed:', err); + } } } catch (err) { return fail(channelId, `locus channel is closed and open failed: ${(err as Error).message}`); diff --git a/test/channel-lifecycle-order.test.ts b/test/channel-lifecycle-order.test.ts index 3d63f6116..e7a3ff984 100644 --- a/test/channel-lifecycle-order.test.ts +++ b/test/channel-lifecycle-order.test.ts @@ -1,4 +1,10 @@ import { test } from 'node:test'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import type { JsStore } from '@animalabs/chronicle'; +import { AgentFramework } from '../src/index.js'; +import { MockMembrane } from './helpers/mock-membrane.js'; import assert from 'node:assert/strict'; import { ChannelRegistry } from '../src/mcpl/channel-registry.js'; import { CapabilityGrant, ALL_CAPABILITY_PATHS } from '../src/mcpl/capability-grant.js'; @@ -11,12 +17,15 @@ function descriptor(id = 'x', initiallyOpen = false, address = 'original'): Chan return { id, type: 'test', label: id, direction: 'bidirectional', initiallyOpen, address: { value: address } }; } -function fixture() { +type NoticeHandler = NonNullable[4]>['onChannelAutoOpened']>; + +function fixture(onNotice?: NoticeHandler, store?: JsStore) { const actual = new Map(); const calls: Array<{ kind: 'open' | 'close'; id: string; address?: unknown }> = []; const pending: Array<{ finish: () => void; fail: () => void }> = []; const held = new Set(); const publishedWhileOpen: boolean[] = []; + const notices: Array<{ source: string; channels: Array<{ channelId: string; label?: string }> }> = []; const serve = (kind: 'open' | 'close', params: { channelId?: string; address?: unknown }) => { const id = params.channelId!; calls.push({ kind, id, address: params.address }); @@ -42,6 +51,7 @@ function fixture() { const registry = new ChannelRegistry( { getServer: () => server } as unknown as McplServerRegistry, {} as FeatureSetManager, () => {}, (event) => traces.push(event), + { store, onChannelAutoOpened: (info) => { notices.push(info); onNotice?.(info); } }, ); const isOpen = (id = 'x') => registry.getOpenChannels().some((e) => e.descriptor.id === id); const tool = (kind: 'open' | 'close') => registry.handleChannelToolCall('channel_' + kind, { serverId: 'test', channelId: 'x' }); @@ -57,7 +67,7 @@ function fixture() { assert.equal(settled, true, 'lifecycle operations must settle'); await result; }; - return { registry, actual, calls, pending, held, traces, isOpen, tool, drain, publishedWhileOpen }; + return { registry, actual, calls, pending, held, traces, isOpen, tool, drain, publishedWhileOpen, notices }; } test('remove/re-add during reconcile converges on the replacement desired state, with ACK first (#185)', async () => { @@ -196,7 +206,7 @@ for (const delivery of ['speech', 'reply'] as const) { }, 'agent-tool'); const sent = delivery === 'speech' ? f.registry.routeSpeech('resident', 'hello', 'x').then((result) => result?.delivered === true) - : f.registry.openIfClosedForSend('x', 'test').then((result) => result.status === 'opened'); + : f.registry.openIfClosedForSend('x', 'test').then((result) => result.status === 'opened' || result.status === 'already-open'); await f.drain(close, sent); assert.equal(await sent, true); assert.equal(f.registry.getTuneOutState('test', 'x')?.params.epochId, 'new-epoch'); @@ -222,6 +232,180 @@ test('automatic opening accepts a newer tune-out established during its RPC with assert.deepEqual(f.publishedWhileOpen, [true]); }); +test('a failed backscroll open preserves an already-live subscription', async () => { + const f = fixture(); + await f.registry.handleChanged('test', { added: [descriptor('x', true)] }); + f.calls.length = 0; + f.held.add('x'); + const backscroll = f.registry.handleChannelToolCall('channel_open', { serverId: 'test', channelId: 'x', backscroll: 5 }); + await tick(); + f.pending.shift()!.fail(); + assert.equal((await backscroll).success, false); + assert.equal(f.isOpen(), true, 'a failed redundant open is not a confirmed close'); + const speech = f.registry.routeSpeech('resident', 'hello', 'x'); + await f.drain(speech); + assert.equal((await speech)?.delivered, true); + assert.equal(f.calls.length, 1, 'delivery must not need another open after a failed backscroll'); + assert.deepEqual(f.publishedWhileOpen, [true]); +}); + +test('speech does not replace a pending explicit close with a new open decision', async () => { + const f = fixture(); + await f.registry.handleChanged('test', { added: [descriptor('x', true)] }); + f.held.add('x'); + const close = f.tool('close'); + await tick(); + const speech = f.registry.routeSpeech('resident', 'tail of the turn', 'x'); + await f.drain(close, speech); + assert.equal((await close).success, true); + assert.equal(f.registry.getDesiredState('test', 'x'), 'closed'); + assert.equal(f.actual.get('x'), false); + assert.equal(f.isOpen(), false); + assert.equal(f.notices.length, 0); +}); + +test('waiting for a backscroll does not issue new opens or auto-open notices for each speech', async () => { + const f = fixture(); + await f.registry.handleChanged('test', { added: [descriptor('x', true)] }); + f.calls.length = 0; + f.held.add('x'); + const backscroll = f.registry.handleChannelToolCall('channel_open', { serverId: 'test', channelId: 'x', backscroll: 5 }); + await tick(); + const speech1 = f.registry.routeSpeech('resident', 'one', 'x'); + const speech2 = f.registry.routeSpeech('resident', 'two', 'x'); + await f.drain(backscroll, speech1, speech2); + assert.equal((await speech1)?.delivered, true); + assert.equal((await speech2)?.delivered, true); + assert.equal(f.calls.length, 1, 'only the explicit backscroll needs an open RPC'); + assert.equal(f.notices.length, 0, 'waiting is not a closed-to-open transition'); +}); + +for (const delivery of ['speech', 'reply'] as const) { + test(delivery + ' opening reports the replacement registration label', async () => { + const f = fixture(); + await f.registry.handleChanged('test', { added: [descriptor()] }); + f.held.add('x'); + const opening = delivery === 'speech' + ? f.registry.routeSpeech('resident', 'hello', 'x') + : f.registry.openIfClosedForSend('x', 'test'); + await tick(); + const replacement = f.registry.handleChanged('test', { + removed: ['x'], added: [{ ...descriptor('x', true, 'replacement'), label: 'Replacement label' }], + }); + await f.drain(opening, replacement); + if (delivery === 'speech') { + assert.equal(f.notices.length, 1); + assert.equal(f.notices[0].channels[0].label, 'Replacement label'); + } else { + const result = await opening; + assert.ok(result && 'label' in result); + assert.equal(result.label, 'Replacement label'); + } + }); +} + +test('policy announces intent before transport rather than a stale open after supersession', async () => { + const f = fixture(); + f.registry.setSubscriptionPolicy('test', 'auto'); + f.held.add('x'); + const batch = f.registry.handleChanged('test', { added: [descriptor(), descriptor('y')] }); + assert.equal(f.notices.length, 1, 'the decision is announced before transport can stall'); + assert.deepEqual(f.notices[0].channels, [{ channelId: 'x', label: 'x' }, { channelId: 'y', label: 'y' }]); + await tick(); + const closeY = await f.registry.handleChannelToolCall('channel_close', { serverId: 'test', channelId: 'y' }); + assert.equal(closeY.success, true); + await f.drain(batch); + assert.equal(f.registry.getDesiredState('test', 'y'), 'closed'); + assert.equal(f.notices.length, 1, 'no stale confirmation is emitted after the later close'); +}); + +test('a fresh delivery into a settled closed channel remains a new engagement', async () => { + const f = fixture(); + await f.registry.handleChanged('test', { added: [descriptor('x', true)] }); + assert.equal((await f.tool('close')).success, true); + assert.equal(f.actual.get('x'), false); + const delivered = await f.registry.routeSpeech('resident', 'a new engagement', 'x'); + assert.equal(delivered?.delivered, true); + assert.equal(f.registry.getDesiredState('test', 'x'), 'open'); + assert.equal(f.notices.length, 1); +}); + +test('a close that completes while delivery joins a backscroll remains authoritative', async () => { + const f = fixture(); + await f.registry.handleChanged('test', { added: [descriptor('x', true)] }); + f.held.add('x'); + const backscroll = f.registry.handleChannelToolCall('channel_open', { serverId: 'test', channelId: 'x', backscroll: 5 }); + await tick(); + const speech = f.registry.routeSpeech('resident', 'tail', 'x'); + const close = f.tool('close'); + await f.drain(backscroll, speech, close); + assert.equal((await close).success, true); + assert.equal(await speech, null); + assert.equal(f.registry.getDesiredState('test', 'x'), 'closed'); + assert.deepEqual(f.publishedWhileOpen, []); + assert.equal(f.notices.length, 0); +}); + +test('concurrent delivery into a closed channel shares one opening and one notice', async () => { + const f = fixture(); + await f.registry.handleChanged('test', { added: [descriptor()] }); + f.calls.length = 0; + f.held.add('x'); + const first = f.registry.routeSpeech('resident', 'one', 'x'); + const second = f.registry.routeSpeech('resident', 'two', 'x'); + await f.drain(first, second); + assert.equal((await first)?.delivered, true); + assert.equal((await second)?.delivered, true); + assert.equal(f.calls.length, 1); + assert.equal(f.notices.length, 1); + assert.deepEqual(f.publishedWhileOpen, [true, true]); +}); + +test('failed policy opening records intent once, with truthful resident text through a retry', async () => { + const dir = mkdtempSync(join(tmpdir(), 'policy-intent-notice-')); + const framework = await AgentFramework.create({ + storePath: join(dir, 'store'), membrane: new MockMembrane().asMembrane(), + agents: [{ name: 'resident', model: 'test-model', systemPrompt: 'Test.' }], + modules: [], syncIntervalMs: 0, + }); + try { + const recordNotice = (framework as unknown as { + recordChannelAutoOpenNotice(agent: undefined, channels: Array<{ channelId: string; label?: string }>, cause: 'subscription-policy' | 'opened-by-delivery'): void; + }).recordChannelAutoOpenNotice.bind(framework); + const onNotice: NoticeHandler = (info) => recordNotice(undefined, info.channels, info.source); + const f = fixture(onNotice, framework.getStore()); + f.registry.setSubscriptionPolicy('test', 'auto'); + f.held.add('x'); + const first = f.registry.handleChanged('test', { added: [descriptor()] }); + assert.equal(f.notices.length, 1); + await tick(); + f.pending.shift()!.fail(); + await first; + assert.equal(f.isOpen(), false); + + const noticeTexts = () => framework.getAgent('resident')!.getContextManager().getAllMessages() + .flatMap((m) => m.content) + .filter((b) => b.type === 'text' && b.text.startsWith('[channels]')) + .map((b) => (b as { text: string }).text); + assert.equal(noticeTexts().length, 1); + assert.match(noticeTexts()[0], /Selected for opening by subscription policy/); + assert.match(noticeTexts()[0], /admission decision; channel_list shows current transport state/); + assert.doesNotMatch(noticeTexts()[0], /Now open/); + + // Reconstruct the registry from the same Chronicle state before retrying. + // The admission persists even though its first transport attempt failed. + const retry = fixture(onNotice, framework.getStore()); + retry.registry.setSubscriptionPolicy('test', 'auto'); + await retry.registry.handleChanged('test', { added: [descriptor()] }); + assert.equal(retry.isOpen(), true); + assert.equal(retry.notices.length, 0); + assert.equal(noticeTexts().length, 1); + } finally { + await framework.stop(); + rmSync(dir, { recursive: true, force: true }); + } +}); + test('a newer tune-out decision keeps transport open when an older close completes', async () => { const f = fixture(); await f.registry.handleChanged('test', { added: [descriptor('x', true)] }); From c5b1217845e439e575e3c51efe9cfaac71bd0a2d Mon Sep 17 00:00:00 2001 From: Felix-299 Date: Fri, 2 Oct 2026 02:43:29 -0400 Subject: [PATCH 0014/1112] fix(mcpl): distinguish response encoding failures from writes Upstream review of PR #208 identified that unserializable command progress (BigInt or circular data) fails before the transport sees bytes. Treating that as a potentially partial write prevented an internal-error response and left the request unanswered. Wrap JSON encoding failures in McplResponseSerializationError at the real connection response boundary, without retrying or pre-encoding the value. The request guard releases its attempt flag only for that explicit pre-write class, allowing one encodable error reply. Generic transport exceptions remain outcome-unknown and single-attempt. Preserve encoding failures even when swallowed by handlers, and report failures to stderr as well as traces for headless deployments. Real connection-routing regressions exercise BigInt, circular and throwing-toJSON results, invalid error data, swallowed encoding failures, and transport failures after encoding. A headless diagnostic test covers post-response failures without listeners. The expanded focused suite is 17 pass / 6 fail on 1a4fbd9, then 23 pass / 0 fail after the correction. Build passes after adding the public test-transport constructor; full suite and independent review are underway. Co-Authored-By: GPT-6 Astra --- CHANGELOG.md | 2 +- src/framework.ts | 20 ++-- src/mcpl/errors.ts | 8 ++ src/mcpl/server-connection.ts | 15 ++- test/mcpl-request-handler-errors.test.ts | 115 ++++++++++++++++++++++- 5 files changed, 147 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 49668fc0f..fcb866ca7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,7 @@ Releases up to and including 0.7.3 predate this file; for their contents see ### Fixed -- MCPL request and notification handlers contain synchronous throws and async rejections (#186), including channel reconciliation failures after an acknowledgement. Failures emit `mcpl:request-handler-error`; requests receive JSON-RPC `-32603` only when no response write has been attempted. Failed error-response writes are included in the trace, and notifications receive no response. +- MCPL request and notification handlers contain synchronous throws and async rejections (#186), including channel reconciliation failures after an acknowledgement. Failures emit `mcpl:request-handler-error`; requests receive JSON-RPC `-32603` only when no response write has been attempted. Pre-write JSON encoding failures can receive an internal-error reply; possibly partial transport writes remain single-attempt. Failed error-response writes are included in the trace, failures also reach stderr for headless hosts, and notifications receive no response. ## 0.19.0 — 2026-09-28 diff --git a/src/framework.ts b/src/framework.ts index 615bd4189..e52f08e58 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -385,6 +385,7 @@ import { EventGate, formatShadowWarning } from './gate/event-gate.js'; import { UsageTracker, type PersistedUsageState } from './usage/usage-tracker.js'; import type { SessionUsageSnapshot, UsageUpdatedEvent } from './usage/types.js'; import type { McplServerConnection } from './mcpl/server-connection.js'; +import { McplResponseSerializationError } from './mcpl/errors.js'; import type { McplServerConfig, McplHostCapabilities, @@ -13557,19 +13558,22 @@ export class AgentFramework { ): void => { connection.on(event, (params: T, responder?: Responder) => { let responseAttempted = false; - let writeFailure: { error: unknown } | undefined; + let responseFailure: { error: unknown; responseAttempted: boolean } | undefined; const writeResponse = (write: () => void): void => { if (responseAttempted) { // Inner handler catches can try to answer a failed result with // an error. Preserve the first failure and refuse the second write. - throw writeFailure ? writeFailure.error : new Error('MCPL response already attempted'); + throw responseFailure ? responseFailure.error : new Error('MCPL response already attempted'); } // A throwing transport write may have partially sent the reply. responseAttempted = true; try { write(); } catch (error) { - writeFailure = { error }; + // Only the encoder can establish that no write was attempted. + // Transport exceptions stay outcome-unknown and cannot be retried. + if (error instanceof McplResponseSerializationError) responseAttempted = false; + responseFailure = { error, responseAttempted }; throw error; } }; @@ -13580,9 +13584,9 @@ export class AgentFramework { writeResponse(() => responder.respondError(code, message, data)), }; const failed = (error: unknown): void => { - if (writeFailure) error = writeFailure.error; + if (responseFailure) error = responseFailure.error; const message = error instanceof Error ? error.message : String(error); - const attemptedBeforeFailure = responseAttempted; + const attemptedBeforeFailure = responseFailure?.responseAttempted ?? responseAttempted; let responseError: string | undefined; if (tracked && !responseAttempted) { try { @@ -13591,6 +13595,8 @@ export class AgentFramework { responseError = sendError instanceof Error ? sendError.message : String(sendError); } } + // Keep headless hosts diagnosable even without a trace subscriber. + console.error(`[mcpl] ${connection.id} ${event} handler failed:`, error); this.emitTrace({ type: 'mcpl:request-handler-error', serverId: connection.id, @@ -13605,8 +13611,8 @@ export class AgentFramework { // Invoke immediately: control-plane responses must retain their // ACK-before-reconcile ordering and data-plane barrier semantics. const succeeded = (): void => { - // A handler may swallow a transport failure without rethrowing. - if (writeFailure) failed(writeFailure.error); + // A handler may swallow an encoding/transport failure without rethrowing. + if (responseFailure) failed(responseFailure.error); }; const result = handler(params, tracked); if (result) void result.then(succeeded, failed); diff --git a/src/mcpl/errors.ts b/src/mcpl/errors.ts index 7bbb90297..b6b01bad8 100644 --- a/src/mcpl/errors.ts +++ b/src/mcpl/errors.ts @@ -40,6 +40,14 @@ export const CHANNEL_OPEN_FAILED = -32024; // Error Factories // ============================================================================ +/** Response encoding failed before the transport was given any bytes. */ +export class McplResponseSerializationError extends Error { + constructor(cause: unknown) { + super('MCPL response could not be serialized', { cause }); + this.name = 'McplResponseSerializationError'; + } +} + /** * Create a "feature set not enabled" error. * Returned when a server sends a message tagged with a feature set diff --git a/src/mcpl/server-connection.ts b/src/mcpl/server-connection.ts index f29fea22d..fb2ce88d6 100644 --- a/src/mcpl/server-connection.ts +++ b/src/mcpl/server-connection.ts @@ -14,7 +14,7 @@ import { EventEmitter } from 'node:events'; import { openTransport, type McplTransport, type TransportCloseInfo } from './transport.js'; import { maskNegotiatedCapabilities } from './capability-mask.js'; import { CapabilityGrant, expandAdvertisementShorthand } from './capability-grant.js'; -import { CAPABILITY_DISABLED } from './errors.js'; +import { CAPABILITY_DISABLED, McplResponseSerializationError } from './errors.js'; import type { McplServerConfig, @@ -1222,13 +1222,22 @@ export class McplServerConnection extends EventEmitter { } } + /** Keep definite pre-write encoding failures distinct from transport errors. */ + private serializeResponse(response: JsonRpcResponse): string { + try { + return JSON.stringify(response); + } catch (cause) { + throw new McplResponseSerializationError(cause); + } + } + /** * Send a successful JSON-RPC response back to the server. */ private sendResponse(id: string | number, result: unknown): void { if (this.closed) return; const response: JsonRpcResponse = { jsonrpc: '2.0', id, result }; - this.transport?.writeLine(JSON.stringify(response)); + this.transport?.writeLine(this.serializeResponse(response)); } /** @@ -1241,7 +1250,7 @@ export class McplServerConnection extends EventEmitter { id, error: { code, message, data }, }; - this.transport?.writeLine(JSON.stringify(response)); + this.transport?.writeLine(this.serializeResponse(response)); } // ========================================================================== diff --git a/test/mcpl-request-handler-errors.test.ts b/test/mcpl-request-handler-errors.test.ts index 2a414791f..6f9ec2fc0 100644 --- a/test/mcpl-request-handler-errors.test.ts +++ b/test/mcpl-request-handler-errors.test.ts @@ -5,11 +5,12 @@ import { EventEmitter } from 'node:events'; import { AgentFramework } from '../src/framework.js'; import { ChannelRegistry } from '../src/mcpl/channel-registry.js'; import { InferenceRouter } from '../src/mcpl/inference-router.js'; +import { McplServerConnection } from '../src/mcpl/server-connection.js'; +import { McplTransport } from '../src/mcpl/transport.js'; import { CapabilityGrant, ALL_CAPABILITY_PATHS } from '../src/mcpl/capability-grant.js'; -function harness() { +function harness(connection = Object.assign(new EventEmitter(), { id: 'srv' })) { const traces: any[] = []; - const connection = Object.assign(new EventEmitter(), { id: 'srv' }); const framework = Object.create(AgentFramework.prototype) as any; framework.traceListeners = [(event: unknown) => traces.push(event)]; framework.wireMcplEvents(connection); @@ -238,3 +239,113 @@ test('host command response-write failure is traced without a second reply', asy assert.equal(reply.errors.length, 0); assert.match(failureTraces(traces)[0].error, /host response failed/); }); + +class ResponseTransport extends McplTransport { + constructor() { super(); } + readonly kind = 'stdio' as const; + readonly lines: string[] = []; + failWrite = false; + writeLine(line: string): void { + this.lines.push(line); + if (this.failWrite) throw new TypeError('transport failed after a partial write'); + } + async close(): Promise {} +} + +function wireHarness() { + const transport = new ResponseTransport(); + // Use real connection routing and response encoding without launching a server. + const connection = new (McplServerConnection as any)('srv', null, transport) as McplServerConnection; + (connection as any).allowHostCommands = true; + connection.establishGrant(new CapabilityGrant(new Set(ALL_CAPABILITY_PATHS), [])); + const h = harness(connection); + connection.ready(); + const request = (method = 'host/command') => transport.emit('line', JSON.stringify({ + jsonrpc: '2.0', id: 17, method, params: { command: 'maintain' }, + })); + return { ...h, transport, request }; +} + +for (const kind of ['bigint', 'circular', 'throwing toJSON'] as const) { + test('host-command ' + kind + ' serialization failure receives one internal-error response', async () => { + const { framework, traces, transport, request } = wireHarness(); + let calls = 0; + const circular: any = {}; + circular.self = circular; + const progress = kind === 'bigint' ? { count: 1n } : kind === 'circular' ? circular : { + toJSON() { calls++; throw new Error('snapshot serialization failed'); }, + }; + framework.handleHostCommand = async () => ({ progress }); + request(); + await settle(); + assert.equal(transport.lines.length, 1, 'encoding failed before any result bytes were written'); + const reply = JSON.parse(transport.lines[0]!); + assert.equal(reply.id, 17); + assert.equal(reply.error.code, -32603); + assert.equal(reply.result, undefined); + const failures = failureTraces(traces); + assert.equal(failures.length, 1); + assert.equal(failures[0].responseAttempted, false); + if (kind === 'throwing toJSON') assert.equal(calls, 1, 'results are encoded only once'); + }); +} + +test('unserializable error data also permits one encodable error response', async () => { + const { framework, traces, transport, request } = wireHarness(); + framework.channelRegistry = { + async handleRegister(_server: string, _params: unknown, reply: any) { + reply.respondError(-32602, 'bad input', { count: 1n }); + }, + }; + request('channels/register'); + await settle(); + assert.equal(transport.lines.length, 1); + assert.equal(JSON.parse(transport.lines[0]!).error.code, -32603); + assert.equal(failureTraces(traces)[0].responseAttempted, false); +}); + +test('a swallowed pre-write encoding failure still receives a fallback response', async () => { + const { framework, traces, transport, request } = wireHarness(); + framework.channelRegistry = { + async handleRegister(_server: string, _params: unknown, reply: any) { + try { reply.respond({ count: 1n }); } catch { /* handler swallowed the error */ } + }, + }; + request('channels/register'); + await settle(); + assert.equal(transport.lines.length, 1); + assert.equal(JSON.parse(transport.lines[0]!).error.code, -32603); + assert.equal(failureTraces(traces)[0].responseAttempted, false); +}); + +test('a real transport write failure remains single-attempt after serialization', async () => { + const { framework, traces, transport, request } = wireHarness(); + framework.handleHostCommand = async () => ({ maintained: true }); + transport.failWrite = true; + request(); + await settle(); + assert.equal(transport.lines.length, 1, 'a possibly partial write must not get a second reply'); + assert.deepEqual(JSON.parse(transport.lines[0]!).result, { maintained: true }); + assert.equal(failureTraces(traces)[0].responseAttempted, true); +}); + +test('post-response failures reach stderr when no trace subscriber is attached', async (t) => { + const { framework, connection } = harness(); + framework.traceListeners = []; + framework.channelRegistry = { + async handleRegister(_server: string, _params: unknown, reply: any) { + reply.respond({ registered: [] }); + throw new Error('headless reconciliation failed'); + }, + }; + const errors: unknown[][] = []; + t.mock.method(console, 'error', (...args: unknown[]) => { errors.push(args); }); + const reply = responder(); + connection.emit('channels-register', {}, reply); + await settle(); + assert.equal(reply.results.length, 1); + assert.equal(reply.errors.length, 0); + assert.equal(errors.length, 1); + assert.match(String(errors[0]?.[0]), /srv.*channels-register/); + assert.match(String(errors[0]?.[1]), /headless reconciliation failed/); +}); From b31b7fc3040a914a6c42784b5046519881b079d6 Mon Sep 17 00:00:00 2001 From: Felix-299 Date: Fri, 2 Oct 2026 14:38:57 -0400 Subject: [PATCH 0015/1112] fix(mcpl): contain malformed inbound lines and sync dispatch faults MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Weft’s firsthand review of PR #208 found an adjacent pre-existing host-kill: JSON null and primitive values reached property access outside JSON.parse’s catch. The initialize handshake had the same null-access gap. A synchronous raw notification listener could also escape, including when buffered events were released later. Require object messages before accessing fields in both readers. Route post-parse dispatch through a containment boundary, and use the same safe event dispatch for live input and both ready flushes. Diagnostics reach stderr and registered connection error listeners without creating an unhandled error event when no listener is attached. Restrict method dispatch to own string keys rather than inherited object properties. Existing async handler containment and single-response semantics are unchanged. A separate native host process uses a real stdio child to emit malformed lines during handshake/after connect, or trigger a throwing notification listener, then proves valid notifications and a request/response still work. Direct cases cover all primitive shapes plus live/data/control-plane exception containment and framework traces. Expanded regression suite is 24 pass / 10 fail before, 34 pass / 0 fail after; build and diff checks pass. Full validation and independent review follow. Move this PR’s release note into changelog.d/208-inbound-handler-containment.fixed.md as requested by coordinating reviewer Iris in Commons #20024 and by the maintainer’s concurrent-PR convention feedback. Preserve every other changelog entry. Co-Authored-By: GPT-6 Astra --- CHANGELOG.md | 4 - .../208-inbound-handler-containment.fixed.md | 1 + src/mcpl/server-connection.ts | 134 +++++++++++------- test/fixtures/mcpl-inbound-host-probe.mjs | 48 +++++++ test/mcpl-request-handler-errors.test.ts | 50 +++++++ 5 files changed, 183 insertions(+), 54 deletions(-) create mode 100644 changelog.d/208-inbound-handler-containment.fixed.md create mode 100644 test/fixtures/mcpl-inbound-host-probe.mjs diff --git a/CHANGELOG.md b/CHANGELOG.md index fcb866ca7..6b9a30b4c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,10 +12,6 @@ Releases up to and including 0.7.3 predate this file; for their contents see ## Unreleased -### Fixed - -- MCPL request and notification handlers contain synchronous throws and async rejections (#186), including channel reconciliation failures after an acknowledgement. Failures emit `mcpl:request-handler-error`; requests receive JSON-RPC `-32603` only when no response write has been attempted. Pre-write JSON encoding failures can receive an internal-error reply; possibly partial transport writes remain single-attempt. Failed error-response writes are included in the trace, failures also reach stderr for headless hosts, and notifications receive no response. - ## 0.19.0 — 2026-09-28 ### Added diff --git a/changelog.d/208-inbound-handler-containment.fixed.md b/changelog.d/208-inbound-handler-containment.fixed.md new file mode 100644 index 000000000..c37e96a8c --- /dev/null +++ b/changelog.d/208-inbound-handler-containment.fixed.md @@ -0,0 +1 @@ +- MCPL request and notification handlers contain synchronous throws and async rejections (#186), including channel reconciliation failures after an acknowledgement. Failures emit `mcpl:request-handler-error`; requests receive JSON-RPC `-32603` only when no response write has been attempted. Pre-write JSON encoding failures can receive an internal-error reply; possibly partial transport writes remain single-attempt. Failed error-response writes are included in the trace, failures also reach stderr for headless hosts, and notifications receive no response. Non-object JSON is ignored before handshake/dispatch property access; synchronous inbound dispatch failures are reported without interrupting later messages, including buffered plane flushes. diff --git a/src/mcpl/server-connection.ts b/src/mcpl/server-connection.ts index fb2ce88d6..2d81de9dc 100644 --- a/src/mcpl/server-connection.ts +++ b/src/mcpl/server-connection.ts @@ -292,7 +292,7 @@ export class McplServerConnection extends EventEmitter { // Re-enter the gate for every buffered item. A control handler (notably // tools/list_changed) may install a newer data-plane barrier while this // snapshot is being flushed; later data items must observe that pause. - this.emit(event, ...args); + this.dispatchInboundEvent(event, ...args); } } finally { this.flushDepth--; @@ -317,7 +317,7 @@ export class McplServerConnection extends EventEmitter { // the old direct flush bypassed positive-grant enforcement entirely // (PR #79 review blocker 3). Data-plane items re-buffer in order via // the same call, since dataPlaneReady is still false. - this.emit(item.event, ...item.args); + this.dispatchInboundEvent(item.event, ...item.args); } } finally { this.flushDepth--; @@ -533,6 +533,7 @@ export class McplServerConnection extends EventEmitter { } catch { return; // Ignore non-JSON lines (e.g. logback output from Java servers) } + if (msg === null || typeof msg !== 'object' || Array.isArray(msg)) return; if (msg.id !== initId) return; cleanup(); if (msg.error) { @@ -1113,61 +1114,94 @@ export class McplServerConnection extends EventEmitter { return; } - // Is this a response to one of our outbound requests? - if ('id' in msg && msg.id != null && !('method' in msg)) { - this.handleResponse(msg as JsonRpcResponse); - return; + // JSON-RPC messages must be objects. Ignore unrelated stdout/log values + // before property access, just as malformed JSON lines are ignored. + if (msg === null || typeof msg !== 'object' || Array.isArray(msg)) return; + try { + this.dispatchMessage(msg); + } catch (error) { + this.reportInboundFailure(error); } + }); + } - // It is an inbound request or notification from the server - const request = msg as JsonRpcRequest; - const eventName = McplServerConnection.METHOD_TO_EVENT[request.method]; - - // §7 is removed in 0.5.0: scope/elevate is no longer a method. Answer - // rather than hang (§6.6 — a method that will never be answered MUST - // return an error). - // featureSets/changed is REMOVED in 0.5.0 (§6.7 — it carried a - // server-authored change payload, the self-attestation defect; the - // need is met by §17's host-diffed manifest fetch). It was still - // routed, control-plane classified, and mutating declarations — a - // live ungated self-attestation path (PR #79 review blocker 4). - if (request.method === McplMethod.FeatureSetsChanged) { - if (request.id != null) { - this.sendErrorResponse(request.id, -32601, 'featureSets/changed removed in MCPL 0.5.0 (superseded by mcpl/manifestChanged, SPEC §17)'); - } else { - console.error(`[mcpl] ${this.id}: discarded featureSets/changed — removed in 0.5.0 (§6.7/§17)`); - } - return; + /** Contain synchronous dispatch failures; async RPC handlers own their promises. */ + private dispatchMessage(msg: JsonRpcRequest | JsonRpcResponse): void { + // Is this a response to one of our outbound requests? + if ('id' in msg && msg.id != null && !('method' in msg)) { + this.handleResponse(msg as JsonRpcResponse); + return; + } + + // It is an inbound request or notification from the server + const request = msg as JsonRpcRequest; + const eventName = typeof request.method === 'string' && Object.hasOwn(McplServerConnection.METHOD_TO_EVENT, request.method) + ? McplServerConnection.METHOD_TO_EVENT[request.method] : undefined; + + // §7 is removed in 0.5.0: scope/elevate is no longer a method. Answer + // rather than hang (§6.6 — a method that will never be answered MUST + // return an error). + // featureSets/changed is REMOVED in 0.5.0 (§6.7 — it carried a + // server-authored change payload, the self-attestation defect; the + // need is met by §17's host-diffed manifest fetch). It was still + // routed, control-plane classified, and mutating declarations — a + // live ungated self-attestation path (PR #79 review blocker 4). + if (request.method === McplMethod.FeatureSetsChanged) { + if (request.id != null) { + this.sendErrorResponse(request.id, -32601, 'featureSets/changed removed in MCPL 0.5.0 (superseded by mcpl/manifestChanged, SPEC §17)'); + } else { + console.error(`[mcpl] ${this.id}: discarded featureSets/changed — removed in 0.5.0 (§6.7/§17)`); } + return; + } - if (request.method === McplMethod.ScopeElevate) { - if (request.id != null) { - this.sendErrorResponse(request.id, -32601, 'scope/elevate removed in MCPL 0.5.0 (SPEC §7)'); - } - return; + if (request.method === McplMethod.ScopeElevate) { + if (request.id != null) { + this.sendErrorResponse(request.id, -32601, 'scope/elevate removed in MCPL 0.5.0 (SPEC §7)'); } + return; + } - // Positive-grant enforcement (§5.4, §14.1) happens at ADMISSION, in - // the emit() override below — the one choke-point that live routing - // and staged-buffer flushes share. See the comment there for why - // rejecting here, at line receipt, was wrong for staged connections. - - if (eventName) { - // Emit the typed event with params and (for requests) a respond callback - if (request.id != null) { - // Server expects a response — provide a respond helper - this.emit(eventName, request.params, { - id: request.id, - respond: (result: unknown) => this.sendResponse(request.id!, result), - respondError: (code: number, message: string, data?: unknown) => - this.sendErrorResponse(request.id!, code, message, data), - }); - } else { - // Notification — no response expected - this.emit(eventName, request.params); - } + // Positive-grant enforcement (§5.4, §14.1) happens at ADMISSION, in + // the emit() override below — the one choke-point that live routing + // and staged-buffer flushes share. See the comment there for why + // rejecting here, at line receipt, was wrong for staged connections. + + if (eventName) { + // Emit the typed event with params and (for requests) a respond callback + if (request.id != null) { + // Server expects a response — provide a respond helper + this.dispatchInboundEvent(eventName, request.params, { + id: request.id, + respond: (result: unknown) => this.sendResponse(request.id!, result), + respondError: (code: number, message: string, data?: unknown) => + this.sendErrorResponse(request.id!, code, message, data), + }); + } else { + // Notification — no response expected + this.dispatchInboundEvent(eventName, request.params); } - }); + } + } + + /** Live delivery and delayed plane flushes share the same containment boundary. */ + private dispatchInboundEvent(event: string, ...args: unknown[]): void { + try { + this.emit(event, ...args); + } catch (error) { + this.reportInboundFailure(error); + } + } + + private reportInboundFailure(cause: unknown): void { + const error = new Error(`MCPL server "${this.id}" inbound dispatch failed`, { cause }); + console.error(error.message, cause); + // EventEmitter's unhandled 'error' behavior would defeat containment before + // framework wiring, so stderr remains the fallback when no listener exists. + if (this.listenerCount('error') > 0) { + try { this.emit('error', error); } + catch (reportError) { console.error('MCPL error listener failed:', reportError); } + } } /** diff --git a/test/fixtures/mcpl-inbound-host-probe.mjs b/test/fixtures/mcpl-inbound-host-probe.mjs new file mode 100644 index 000000000..888e39248 --- /dev/null +++ b/test/fixtures/mcpl-inbound-host-probe.mjs @@ -0,0 +1,48 @@ +// Runs as a separate host process: an uncaught inbound exception fails the test. +import assert from 'node:assert/strict'; +import { McplServerConnection } from '../../src/mcpl/server-connection.js'; + +const mode = process.argv[2]; +const server = String.raw` +let input = ''; +const values = ['null', '42', '"hi"', 'true', '[]']; +const send = (value) => process.stdout.write(JSON.stringify(value) + '\n'); +process.stdin.on('data', (chunk) => { + input += chunk; + let end; + while ((end = input.indexOf('\n')) >= 0) { + const line = input.slice(0, end); input = input.slice(end + 1); + const request = JSON.parse(line); + if (request.method === 'initialize') { + if (process.argv[1] === 'handshake') { + for (const value of values) process.stdout.write(value + '\n'); + } + send({jsonrpc:'2.0', id:request.id, result:{capabilities:{}}}); + } else if (request.method === 'tools/list') { + if (process.argv[1] === 'connected') { + for (const value of values) process.stdout.write(value + '\n'); + } + send({jsonrpc:'2.0', method:'notifications/tools/list_changed', params:{sequence:1}}); + send({jsonrpc:'2.0', method:'notifications/tools/list_changed', params:{sequence:2}}); + send({jsonrpc:'2.0', id:request.id, result:{tools:[]}}); + } + } +}); +`; +const connection = await McplServerConnection.connect({ + id: 'probe', command: process.execPath, args: ['-e', server, mode], requestTimeoutMs: 1000, +}, { version: '0.5' }); +try { + const received = []; + connection.on('tools-list-changed', (params) => { + if (mode === 'throwing-listener' && params.sequence === 1) throw new Error('injected sync notification failure'); + received.push(params.sequence); + }); + // Deliberately no connection error subscriber: diagnostics must not throw. + connection.ready(); + assert.deepEqual(await connection.sendToolsList(), { tools: [] }); + assert.deepEqual(received, mode === 'throwing-listener' ? [2] : [1, 2]); + console.log('continued'); +} finally { + await connection.close(); +} diff --git a/test/mcpl-request-handler-errors.test.ts b/test/mcpl-request-handler-errors.test.ts index 6f9ec2fc0..d6f76a81c 100644 --- a/test/mcpl-request-handler-errors.test.ts +++ b/test/mcpl-request-handler-errors.test.ts @@ -1,6 +1,8 @@ import { test } from 'node:test'; import assert from 'node:assert/strict'; import { EventEmitter } from 'node:events'; +import { spawnSync } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; import { AgentFramework } from '../src/framework.js'; import { ChannelRegistry } from '../src/mcpl/channel-registry.js'; @@ -349,3 +351,51 @@ test('post-response failures reach stderr when no trace subscriber is attached', assert.match(String(errors[0]?.[0]), /srv.*channels-register/); assert.match(String(errors[0]?.[1]), /headless reconciliation failed/); }); + +for (const mode of ['handshake', 'connected', 'throwing-listener']) { + test('separate stdio host survives inbound ' + mode + ' faults and processes later messages', () => { + const probe = spawnSync(process.execPath, [ + '--import', 'tsx', fileURLToPath(new URL('./fixtures/mcpl-inbound-host-probe.mjs', import.meta.url)), mode, + ], { encoding: 'utf8', timeout: 5000 }); + assert.equal(probe.error, undefined); + assert.equal(probe.status, 0, probe.stderr); + assert.match(probe.stdout, /continued/); + }); +} + +for (const value of [null, 42, 'hi', true, []]) { + test('non-object inbound ' + JSON.stringify(value) + ' leaves the next notification deliverable', () => { + const { connection, transport } = wireHarness(); + let received = false; + connection.removeAllListeners('tools-list-changed'); + connection.on('tools-list-changed', () => { received = true; }); + assert.doesNotThrow(() => transport.emit('line', JSON.stringify(value))); + transport.emit('line', JSON.stringify({ jsonrpc: '2.0', method: 'notifications/tools/list_changed' })); + assert.equal(received, true); + }); +} + +for (const release of ['live', 'ready', 'readyControlPlane'] as const) { + test('synchronous notification failure is contained during ' + release + ' dispatch and traced', (t) => { + const transport = new ResponseTransport(); + const connection = new (McplServerConnection as any)('srv', null, transport) as McplServerConnection; + const { traces } = harness(connection); + const errors: unknown[][] = []; + t.mock.method(console, 'error', (...args: unknown[]) => { errors.push(args); }); + const seen: number[] = []; + connection.removeAllListeners('tools-list-changed'); + connection.on('tools-list-changed', (params: { sequence: number }) => { + if (params.sequence === 1) throw new Error('raw notification failed'); + seen.push(params.sequence); + }); + if (release === 'live') connection.ready(); + const send = (sequence: number) => transport.emit('line', JSON.stringify({ + jsonrpc: '2.0', method: 'notifications/tools/list_changed', params: { sequence }, + })); + assert.doesNotThrow(() => { send(1); send(2); }); + if (release !== 'live') assert.doesNotThrow(() => connection[release]()); + assert.deepEqual(seen, [2]); + assert.equal(traces.filter((e) => e.type === 'mcpl:server-error').length, 1); + assert.equal(errors.length, 1); + }); +} From 9c94d0a33bc84e6a7fcbf22645584487967d0845 Mon Sep 17 00:00:00 2001 From: Jasper Date: Fri, 2 Oct 2026 14:53:49 -0400 Subject: [PATCH 0016/1112] fix(mcpl): bound channel lifecycle convergence MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address Weft’s blocking review of PR #210: a server could change its descriptor after each successful open and keep the convergence loop plus joined delivery pending indefinitely. Per-RPC timeouts did not bound this sequence of completed requests. Limit one operation to five transport attempts. Exhaustion emits a diagnostic, invalidates the unconfirmed current-target open flag, and rejects the operation. Keep that rejection visible to existing delivery joiners while observing it for unhandled-rejection safety and allowing queued successors to proceed. A failed ordinary backscroll still preserves known-open state; a later stable operation gets a fresh convergence budget. New public-API regressions hold each RPC and change descriptors before release. They prove initiating-tool failure, route-failure notification, reply-preparation failure, no publication while unconverged, queue release and stable subsequent retry, unrelated-channel progress, and success on the fifth attempt. On published d64c803, the first two new cases fail to settle within the bounded probe while the finite control passes; the original 23 tests also remain green after the fix. Move only this PR’s two changelog categories to unique fixed/breaking fragments, explicitly authorized by Felix in Commons message20050, avoiding shared Unreleased conflicts. Preserve the compatibility notice verbatim and add the convergence bound. Current-main integration and independent/full validation follow before push. Co-Authored-By: GPT-6 Astra --- CHANGELOG.md | 10 -- .../210-channel-lifecycle-order.fixed.md | 5 + .../210-policy-admission-notice.breaking.md | 1 + src/mcpl/channel-registry.ts | 50 ++++++++-- test/channel-lifecycle-order.test.ts | 95 ++++++++++++++++++- 5 files changed, 140 insertions(+), 21 deletions(-) create mode 100644 changelog.d/210-channel-lifecycle-order.fixed.md create mode 100644 changelog.d/210-policy-admission-notice.breaking.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 9dbbe6281..6b9a30b4c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,16 +12,6 @@ Releases up to and including 0.7.3 predate this file; for their contents see ## Unreleased -### Breaking (direct ChannelRegistry consumers) - -- `onChannelAutoOpened` with `source: 'subscription-policy'` now reports a durable admission decision before transport reconciliation. Treat it as intent; await the registration call and inspect `getOpenChannels()` when confirmed live state is needed. `opened-by-delivery` still reports a confirmed transition. The built-in host notice now distinguishes selected-for-opening from currently open. - -### Fixed - -- Failed backscroll requests preserve the registry's known-open state rather than claiming a close. Automatic delivery joins pending lifecycle work without reopening a channel closed during the wait, and reports only actual delivery-forced opens using the current registration label. - -- Channel lifecycle operations now run in order per channel and reconcile the latest desired state and registration after an in-flight operation completes. Concurrent registration, open, and close requests can no longer leave the server in an older state; a superseded tool request reports that its requested state no longer applies. Automatic delivery preserves active tune-out attention state while opening or awaiting transport. - ## 0.19.0 — 2026-09-28 ### Added diff --git a/changelog.d/210-channel-lifecycle-order.fixed.md b/changelog.d/210-channel-lifecycle-order.fixed.md new file mode 100644 index 000000000..707a1bda3 --- /dev/null +++ b/changelog.d/210-channel-lifecycle-order.fixed.md @@ -0,0 +1,5 @@ +- Failed backscroll requests preserve the registry's known-open state rather than claiming a close. Automatic delivery joins pending lifecycle work without reopening a channel closed during the wait, and reports only actual delivery-forced opens using the current registration label. + +- Channel lifecycle operations now run in order per channel and reconcile the latest desired state and registration after an in-flight operation completes. Concurrent registration, open, and close requests can no longer leave the server in an older state; a superseded tool request reports that its requested state no longer applies. Automatic delivery preserves active tune-out attention state while opening or awaiting transport. + +- Lifecycle convergence makes at most five transport attempts per operation. Continued descriptor/intent supersession fails with a diagnostic, releases the queue, and gives joined speech or reply preparation a visible failure instead of hanging. Later stable operations can retry. diff --git a/changelog.d/210-policy-admission-notice.breaking.md b/changelog.d/210-policy-admission-notice.breaking.md new file mode 100644 index 000000000..f47b54371 --- /dev/null +++ b/changelog.d/210-policy-admission-notice.breaking.md @@ -0,0 +1 @@ +- `onChannelAutoOpened` with `source: 'subscription-policy'` now reports a durable admission decision before transport reconciliation. Treat it as intent; await the registration call and inspect `getOpenChannels()` when confirmed live state is needed. `opened-by-delivery` still reports a confirmed transition. The built-in host notice now distinguishes selected-for-opening from currently open. diff --git a/src/mcpl/channel-registry.ts b/src/mcpl/channel-registry.ts index 7ef70c9cb..c51552400 100644 --- a/src/mcpl/channel-registry.ts +++ b/src/mcpl/channel-registry.ts @@ -46,6 +46,10 @@ import { CapabilityGrant } from './capability-grant.js'; // ============================================================================ const TYPING_INTERVAL_MS = 7_000; +const MAX_CHANNEL_LIFECYCLE_ATTEMPTS = 5; + +/** A successful RPC can still fail to establish the latest channel target. */ +class ChannelLifecycleConvergenceError extends Error {} const CHANNEL_LIFECYCLE_LOG_ID = 'mcpl/channel-lifecycle'; /** @@ -2087,8 +2091,8 @@ export class ChannelRegistry { const lifecycleKey = this.lifecycleKey(serverId, channelId); const channelKey = `${serverId}:${channelId}`; const previous = this.channelLifecycleTails.get(lifecycleKey) ?? Promise.resolve(); - const operation = previous.then(async () => { - let issued = false; + const operation = previous.catch(() => {}).then(async () => { + let attempts = 0; let opened = false; for (;;) { const entry = this.channels.get(channelKey); @@ -2101,10 +2105,26 @@ export class ChannelRegistry { const descriptor = entry.descriptor; const desired = this.getDesiredState(serverId, channelId); const open = desired === 'open' || desired === 'tuned-out'; - if (!issued && options?.onlyIfClosed && open && entry.open) { + if (attempts === 0 && options?.onlyIfClosed && open && entry.open) { return { open: true, opened: false }; } - issued = true; + // Per-RPC timeouts do not bound a stream of promptly successful but + // continually superseded receipts. Bound this whole operation, then + // release its queue slot and surface the failure to delivery joiners. + if (attempts >= MAX_CHANNEL_LIFECYCLE_ATTEMPTS) { + // No receipt confirmed the current target/intent. A later delivery + // must reconcile it rather than reuse an earlier target's open flag. + entry.open = false; + const error = new ChannelLifecycleConvergenceError( + `Channel ${channelId} did not converge after ${attempts} lifecycle attempts; its target or desired state kept changing`, + ); + this.emitTraceFn({ + type: 'mcpl:channel-reconcile-failed', + serverId, channelId, desired, attempts, error: error.message, + }); + throw error; + } + attempts++; let result: ChannelsOpenResult | undefined; if (open) { result = await server.sendChannelsOpen({ @@ -2130,13 +2150,16 @@ export class ChannelRegistry { return { open, opened, result }; } }); - const tail = operation.then(() => {}, () => {}); + // Keep the rejection visible to existing joiners, while observing it + // immediately and letting each queued successor start after either result. + const tail = operation.then(() => {}); this.channelLifecycleTails.set(lifecycleKey, tail); - void tail.then(() => { + const cleanup = () => { if (this.channelLifecycleTails.get(lifecycleKey) === tail) { this.channelLifecycleTails.delete(lifecycleKey); } - }); + }; + void tail.then(cleanup, cleanup); return operation; } @@ -2150,7 +2173,14 @@ export class ChannelRegistry { for (;;) { const pending = this.channelLifecycleTails.get(lifecycleKey); if (!pending) break; - await pending; + try { + await pending; + } catch (error) { + // A redundant backscroll failure can leave a known-open channel + // usable. Exhausted convergence is different: it never established + // the current target, so joined delivery must fail visibly. + if (error instanceof ChannelLifecycleConvergenceError) throw error; + } } const current = this.channels.get(`${entry.serverId}:${entry.descriptor.id}`); if (!current) throw new Error('Channel registration was removed while delivery waited'); @@ -2221,7 +2251,9 @@ export class ChannelRegistry { await this.applyDesiredChannelState(serverId, channel.id); } catch (err) { // Removal while queued is cancellation, not a failed subscription. - if (!this.channels.has(`${serverId}:${channel.id}`)) continue; + // Exhausted convergence already emitted its diagnostic at the bound. + if (!this.channels.has(`${serverId}:${channel.id}`) || + err instanceof ChannelLifecycleConvergenceError) continue; this.emitTraceFn({ type: 'mcpl:channel-reconcile-failed', serverId, diff --git a/test/channel-lifecycle-order.test.ts b/test/channel-lifecycle-order.test.ts index e7a3ff984..33960de20 100644 --- a/test/channel-lifecycle-order.test.ts +++ b/test/channel-lifecycle-order.test.ts @@ -26,6 +26,7 @@ function fixture(onNotice?: NoticeHandler, store?: JsStore) { const held = new Set(); const publishedWhileOpen: boolean[] = []; const notices: Array<{ source: string; channels: Array<{ channelId: string; label?: string }> }> = []; + const routeFailures: Array<{ reason: string }> = []; const serve = (kind: 'open' | 'close', params: { channelId?: string; address?: unknown }) => { const id = params.channelId!; calls.push({ kind, id, address: params.address }); @@ -51,7 +52,8 @@ function fixture(onNotice?: NoticeHandler, store?: JsStore) { const registry = new ChannelRegistry( { getServer: () => server } as unknown as McplServerRegistry, {} as FeatureSetManager, () => {}, (event) => traces.push(event), - { store, onChannelAutoOpened: (info) => { notices.push(info); onNotice?.(info); } }, + { store, onRouteFailure: (info) => routeFailures.push(info), + onChannelAutoOpened: (info) => { notices.push(info); onNotice?.(info); } }, ); const isOpen = (id = 'x') => registry.getOpenChannels().some((e) => e.descriptor.id === id); const tool = (kind: 'open' | 'close') => registry.handleChannelToolCall('channel_' + kind, { serverId: 'test', channelId: 'x' }); @@ -67,7 +69,7 @@ function fixture(onNotice?: NoticeHandler, store?: JsStore) { assert.equal(settled, true, 'lifecycle operations must settle'); await result; }; - return { registry, actual, calls, pending, held, traces, isOpen, tool, drain, publishedWhileOpen, notices }; + return { registry, actual, calls, pending, held, traces, isOpen, tool, drain, publishedWhileOpen, notices, routeFailures }; } test('remove/re-add during reconcile converges on the replacement desired state, with ACK first (#185)', async () => { @@ -406,6 +408,95 @@ test('failed policy opening records intent once, with truthful resident text thr } }); +test('descriptor churn exhausts convergence and fails the tool plus joined deliveries instead of hanging', async () => { + const f = fixture(); + await f.registry.handleChanged('test', { added: [descriptor('x', true)] }); + f.calls.length = 0; + f.held.add('x'); + let toolSettled = false, speechSettled = false, replySettled = false; + const opening = f.registry.handleChannelToolCall('channel_open', { + serverId: 'test', channelId: 'x', backscroll: 5, + }).then((result) => { toolSettled = true; return result; }); + await tick(); + const speech = f.registry.routeSpeech('resident', 'waiting reply', 'x') + .then((result) => { speechSettled = true; return result; }); + const reply = f.registry.openIfClosedForSend('x', 'test') + .then((result) => { replySettled = true; return result; }); + try { + // Every RPC succeeds promptly, but its target is superseded before the + // receipt arrives. Bound the test itself without a hanging provider. + for (let i = 0; i < 12 && !toolSettled; i++) { + await f.registry.handleChanged('test', { updated: [descriptor('x', true, 'churn-' + i)] }); + f.pending.shift()?.finish(); + await tick(); + } + assert.equal(toolSettled, true, 'the whole convergence operation must be bounded'); + assert.equal(speechSettled, true, 'joined speech must receive a terminal result'); + assert.equal(replySettled, true, 'reply preparation must receive a terminal result'); + assert.equal(f.calls.length, 5, 'at most five transport attempts per operation'); + assert.equal((await opening).success, false); + assert.match((await opening).error!, /did not converge/); + assert.equal(await speech, null); + assert.equal((await reply).status, 'open-failed'); + assert.equal(f.isOpen(), false, 'an unconfirmed current target is not reported open'); + assert.deepEqual(f.publishedWhileOpen, []); + assert.equal(f.routeFailures.length, 1); + assert.match(f.routeFailures[0].reason, /did not converge/); + assert.ok(f.traces.some((e) => e.type === 'mcpl:channel-reconcile-failed' && + String(e.error).includes('did not converge'))); + } finally { + // Original code needs one stable receipt to release its unbounded loop. + await f.drain(opening, speech, reply); + } + f.held.delete('x'); + const retry = await f.registry.routeSpeech('resident', 'a later stable retry', 'x'); + assert.equal(retry?.delivered, true); + assert.equal(f.calls.length, 6, 'later delivery reconfirms the latest target before publishing'); + assert.equal((await f.tool('close')).success, true, 'exhaustion releases the queue'); + assert.equal((await f.tool('open')).success, true, 'a later stable decision gets a fresh retry budget'); +}); + +test('background reconciliation reports supersession exhaustion and still permits other channels', async () => { + const f = fixture(); + f.held.add('x'); + let settled = false; + const reconcile = f.registry.handleChanged('test', { added: [descriptor('x', true)] }) + .then(() => { settled = true; }); + await tick(); + try { + await f.registry.handleChanged('test', { added: [descriptor('y', true)] }); + assert.equal(f.actual.get('y'), true); + for (let i = 0; i < 12 && !settled; i++) { + await f.registry.handleChanged('test', { updated: [descriptor('x', true, 'churn-' + i)] }); + f.pending.shift()?.finish(); + await tick(); + } + assert.equal(settled, true); + assert.equal(f.calls.filter((call) => call.id === 'x').length, 5); + const failures = f.traces.filter((e) => e.type === 'mcpl:channel-reconcile-failed'); + assert.equal(failures.length, 1); + assert.match(String(failures[0].error), /did not converge/); + assert.equal(f.registry.getDesiredState('test', 'x'), 'open', 'failure does not erase desired intent'); + } finally { await f.drain(reconcile); } +}); + +test('finite descriptor supersession can converge on its fifth transport attempt', async () => { + const f = fixture(); + f.held.add('x'); + const reconcile = f.registry.handleChanged('test', { added: [descriptor('x', true)] }); + await tick(); + for (let i = 0; i < 4; i++) { + await f.registry.handleChanged('test', { updated: [descriptor('x', true, 'target-' + i)] }); + f.pending.shift()!.finish(); + await tick(); + } + await f.drain(reconcile); + assert.equal(f.calls.length, 5); + assert.equal(f.isOpen(), true); + assert.deepEqual(f.calls.at(-1)?.address, { value: 'target-3' }); + assert.equal(f.traces.filter((e) => e.type === 'mcpl:channel-reconcile-failed').length, 0); +}); + test('a newer tune-out decision keeps transport open when an older close completes', async () => { const f = fixture(); await f.registry.handleChanged('test', { added: [descriptor('x', true)] }); From 8af55988e441262ffc7f083e624dbd5110b2df70 Mon Sep 17 00:00:00 2001 From: Jasper Date: Fri, 2 Oct 2026 14:57:56 -0400 Subject: [PATCH 0017/1112] docs(changelog): qualify lifecycle convergence at success The new retry bound deliberately returns a visible failure when the latest target cannot be confirmed. Qualify the earlier blanket state-convergence wording so it does not claim exhausted operations synchronize remote state. Source and tests remain identical to 541a339. Co-Authored-By: GPT-6 Astra --- changelog.d/210-channel-lifecycle-order.fixed.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.d/210-channel-lifecycle-order.fixed.md b/changelog.d/210-channel-lifecycle-order.fixed.md index 707a1bda3..7c4db23a5 100644 --- a/changelog.d/210-channel-lifecycle-order.fixed.md +++ b/changelog.d/210-channel-lifecycle-order.fixed.md @@ -1,5 +1,5 @@ - Failed backscroll requests preserve the registry's known-open state rather than claiming a close. Automatic delivery joins pending lifecycle work without reopening a channel closed during the wait, and reports only actual delivery-forced opens using the current registration label. -- Channel lifecycle operations now run in order per channel and reconcile the latest desired state and registration after an in-flight operation completes. Concurrent registration, open, and close requests can no longer leave the server in an older state; a superseded tool request reports that its requested state no longer applies. Automatic delivery preserves active tune-out attention state while opening or awaiting transport. +- Channel lifecycle operations now run in order per channel and reconcile the latest desired state and registration after an in-flight operation completes. Successful reconciliation confirms the current target and desired state at its receipt boundary; a superseded tool request reports that its requested state no longer applies. Automatic delivery preserves active tune-out attention state while opening or awaiting transport. - Lifecycle convergence makes at most five transport attempts per operation. Continued descriptor/intent supersession fails with a diagnostic, releases the queue, and gives joined speech or reply preparation a visible failure instead of hanging. Later stable operations can retry. From de1a9f36579ccef2e30339cd9677128dc8b5c087 Mon Sep 17 00:00:00 2001 From: Jasper Date: Fri, 2 Oct 2026 15:02:58 -0400 Subject: [PATCH 0018/1112] fix(mcpl): validate open receipts against the current target Wren independently reproduced a receipt-validity hole after the convergence cap: successful A receipt marked the entry open even after its descriptor became B, then a failed corrective B open preserved that stale bit and allowed joined speech to publish. Invalidate known-open on type/address retargeting while preserving label-only updates. Check entry, connection, and transport target before applying a successful receipt; a stale receipt triggers reconciliation without transiently confirming the new target. On ordinary RPC failure, retain open state only when the captured connection and target still match. Joined delivery reports an unconfirmed target honestly rather than claiming a close happened. Three added regressions fail on 8af5598 (old-address, old-type, replaced-connection confirmation) and pass after this change. The same-target backscroll and finite/exhausted convergence controls remain green. Runtime source is now ready for independent re-review before serialized full validation and push. Co-Authored-By: GPT-6 Astra --- .../210-channel-lifecycle-order.fixed.md | 2 + src/mcpl/channel-registry.ts | 50 +++++++++++------ test/channel-lifecycle-order.test.ts | 53 ++++++++++++++++++- 3 files changed, 86 insertions(+), 19 deletions(-) diff --git a/changelog.d/210-channel-lifecycle-order.fixed.md b/changelog.d/210-channel-lifecycle-order.fixed.md index 7c4db23a5..94e63d742 100644 --- a/changelog.d/210-channel-lifecycle-order.fixed.md +++ b/changelog.d/210-channel-lifecycle-order.fixed.md @@ -2,4 +2,6 @@ - Channel lifecycle operations now run in order per channel and reconcile the latest desired state and registration after an in-flight operation completes. Successful reconciliation confirms the current target and desired state at its receipt boundary; a superseded tool request reports that its requested state no longer applies. Automatic delivery preserves active tune-out attention state while opening or awaiting transport. +- Open receipts confirm only the current registration, connection, type, and address. Retargeting invalidates an older open flag; failed corrective opens leave the new target unconfirmed and fail joined delivery, while same-target backscroll failures retain known-open state. + - Lifecycle convergence makes at most five transport attempts per operation. Continued descriptor/intent supersession fails with a diagnostic, releases the queue, and gives joined speech or reply preparation a visible failure instead of hanging. Later stable operations can retry. diff --git a/src/mcpl/channel-registry.ts b/src/mcpl/channel-registry.ts index c51552400..9abbd0851 100644 --- a/src/mcpl/channel-registry.ts +++ b/src/mcpl/channel-registry.ts @@ -212,9 +212,14 @@ function shallowEqualRecord( interface ChannelEntry { serverId: string; descriptor: ChannelDescriptor; + /** Last confirmed state for this entry's current transport target. */ open: boolean; } +function sameChannelTarget(a: ChannelDescriptor, b: ChannelDescriptor): boolean { + return a.type === b.type && isDeepStrictEqual(a.address, b.address); +} + /** Minimal responder interface for sending JSON-RPC results back. */ interface Responder { respond(result: unknown): void; @@ -813,6 +818,9 @@ export class ChannelRegistry { this.emitTraceFn({ type: 'mcpl:channel-descriptor-rejected', serverId, channelId: channel.id, reason: 'update for unregistered channel' }); continue; } + // A live subscription to the old target does not confirm the new + // type/address. Label-only updates preserve that confirmation. + if (!sameChannelTarget(existing.descriptor, channel)) existing.open = false; existing.descriptor = channel; this.appendLabelSighting(channel.id, channel.label, extractDmMeta(channel.metadata)); addedResults.push({ id: channel.id, accepted: true }); @@ -2125,28 +2133,36 @@ export class ChannelRegistry { throw error; } attempts++; + const sameTarget = () => this.serverRegistry.getServer(serverId) === server && + sameChannelTarget(entry.descriptor, descriptor); let result: ChannelsOpenResult | undefined; - if (open) { - result = await server.sendChannelsOpen({ - channelId, type: descriptor.type, address: descriptor.address, - ...(options?.history ? { history: options.history } : {}), - }); - } else { - await server.sendChannelsClose({ channelId }); + try { + if (open) { + result = await server.sendChannelsOpen({ + channelId, type: descriptor.type, address: descriptor.address, + ...(options?.history ? { history: options.history } : {}), + }); + } else { + await server.sendChannelsClose({ channelId }); + } + } catch (error) { + // A failed same-target backscroll can retain its known-open state. + // A receipt/failure from an older connection or target cannot. + if (this.channels.get(channelKey) === entry && !sameTarget()) entry.open = false; + throw error; + } + // Never apply an old receipt to the current target, even transiently: + // a corrective open might fail before that stale bit is replaced. + if (this.channels.get(channelKey) !== entry) continue; + if (!sameTarget()) { + entry.open = false; + continue; } - // Failure is not a confirmed transition. In particular, a failed - // backscroll request must not erase an already-live subscription. - // Never apply a receipt to a removed entry or a replacement connection. - // Reconcile their current state before releasing this channel's queue. - if (this.channels.get(channelKey) !== entry || - this.serverRegistry.getServer(serverId) !== server) continue; if (open && !entry.open) opened = true; if (!open) opened = false; entry.open = open; const now = this.getDesiredState(serverId, channelId); - const targetChanged = entry.descriptor.type !== descriptor.type || - !isDeepStrictEqual(entry.descriptor.address, descriptor.address); - if (targetChanged || open !== (now === 'open' || now === 'tuned-out')) continue; + if (open !== (now === 'open' || now === 'tuned-out')) continue; return { open, opened, result }; } }); @@ -2185,7 +2201,7 @@ export class ChannelRegistry { const current = this.channels.get(`${entry.serverId}:${entry.descriptor.id}`); if (!current) throw new Error('Channel registration was removed while delivery waited'); if (!current.open || this.getDesiredState(current.serverId, current.descriptor.id) === 'closed') { - throw new Error('Channel was closed or selected for closure while delivery waited'); + throw new Error('Channel is not confirmed open for delivery, or was selected for closure while delivery waited'); } return current; } diff --git a/test/channel-lifecycle-order.test.ts b/test/channel-lifecycle-order.test.ts index 33960de20..a26f7be60 100644 --- a/test/channel-lifecycle-order.test.ts +++ b/test/channel-lifecycle-order.test.ts @@ -48,9 +48,11 @@ function fixture(onNotice?: NoticeHandler, store?: JsStore) { return { delivered: true }; }, }; + let currentServer = server; + const replaceServer = () => { currentServer = { ...server }; }; const traces: Array<{ type: string; [key: string]: unknown }> = []; const registry = new ChannelRegistry( - { getServer: () => server } as unknown as McplServerRegistry, + { getServer: () => currentServer } as unknown as McplServerRegistry, {} as FeatureSetManager, () => {}, (event) => traces.push(event), { store, onRouteFailure: (info) => routeFailures.push(info), onChannelAutoOpened: (info) => { notices.push(info); onNotice?.(info); } }, @@ -69,7 +71,7 @@ function fixture(onNotice?: NoticeHandler, store?: JsStore) { assert.equal(settled, true, 'lifecycle operations must settle'); await result; }; - return { registry, actual, calls, pending, held, traces, isOpen, tool, drain, publishedWhileOpen, notices, routeFailures }; + return { registry, actual, calls, pending, held, traces, isOpen, tool, drain, publishedWhileOpen, notices, routeFailures, replaceServer }; } test('remove/re-add during reconcile converges on the replacement desired state, with ACK first (#185)', async () => { @@ -408,6 +410,53 @@ test('failed policy opening records intent once, with truthful resident text thr } }); +for (const change of ['address', 'type'] as const) { + test('failed corrective open does not reuse the old ' + change + ' confirmation', async () => { + const f = fixture(); + await f.registry.handleChanged('test', { added: [descriptor('x', true, 'A')] }); + f.calls.length = 0; + f.held.add('x'); + const backscroll = f.registry.handleChannelToolCall('channel_open', { + serverId: 'test', channelId: 'x', backscroll: 5, + }); + await tick(); + const speech = f.registry.routeSpeech('resident', 'waiting reply', 'x'); + const updated = change === 'address' + ? descriptor('x', true, 'B') + : { ...descriptor('x', true, 'A'), type: 'new-type' }; + await f.registry.handleChanged('test', { updated: [updated] }); + assert.equal(f.isOpen(), false, 'old target confirmation is invalid after retargeting'); + f.pending.shift()!.finish(); // successful receipt for A + await tick(); + f.pending.shift()!.fail(); // corrective open for the new target fails + await f.drain(backscroll, speech); + assert.equal((await backscroll).success, false); + assert.equal(await speech, null); + assert.equal(f.isOpen(), false); + assert.deepEqual(f.publishedWhileOpen, []); + assert.equal(f.routeFailures.length, 1); + }); +} + +test('a failed RPC from a replaced connection cannot preserve its old open confirmation', async () => { + const f = fixture(); + await f.registry.handleChanged('test', { added: [descriptor('x', true)] }); + f.held.add('x'); + const backscroll = f.registry.handleChannelToolCall('channel_open', { + serverId: 'test', channelId: 'x', backscroll: 5, + }); + await tick(); + const speech = f.registry.routeSpeech('resident', 'waiting reply', 'x'); + f.replaceServer(); + f.pending.shift()!.fail(); + await f.drain(backscroll, speech); + assert.equal((await backscroll).success, false); + assert.equal(await speech, null); + assert.equal(f.isOpen(), false); + assert.deepEqual(f.publishedWhileOpen, []); + assert.equal(f.routeFailures.length, 1); +}); + test('descriptor churn exhausts convergence and fails the tool plus joined deliveries instead of hanging', async () => { const f = fixture(); await f.registry.handleChanged('test', { added: [descriptor('x', true)] }); From cd726dd915bb510f8ec2f7a26d83f4c8c918122f Mon Sep 17 00:00:00 2001 From: Jasper Date: Fri, 2 Oct 2026 15:07:59 -0400 Subject: [PATCH 0019/1112] fix(mcpl): preserve pending decisions after target invalidation Felix reproduced a delivery regression in the receipt correction: an address update cleared the open flag during a pending explicit close, causing speech/reply preparation to treat the wait as a fresh opening decision and supersede the close. Both automatic delivery paths now join any outstanding lifecycle operation, independent of transport-confirmation state. Fresh engagement is possible when no decision is pending; waiting itself records no new intent. This separates three facts that the open bit alone cannot represent: desired intent, outstanding decision, and confirmed transport. Two public-API retargeted-close regressions fail on de1a9f3 for speech and reply preparation. They now preserve desired closed, publish nothing, and return a visible failure, while the expanded lifecycle/provenance/routing/tune-out/release set remains green. The convergence budget and target-validity helper are unchanged; re-review precedes full validation and push. Co-Authored-By: GPT-6 Astra --- src/mcpl/channel-registry.ts | 7 ++++--- test/channel-lifecycle-order.test.ts | 28 ++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/src/mcpl/channel-registry.ts b/src/mcpl/channel-registry.ts index 9abbd0851..44b368537 100644 --- a/src/mcpl/channel-registry.ts +++ b/src/mcpl/channel-registry.ts @@ -2180,7 +2180,8 @@ export class ChannelRegistry { } /** - * Join work on an already-open channel without recording new intent. + * Join outstanding lifecycle work without recording new intent. Retargeting + * may have invalidated open confirmation while the decision remains pending. * A close that settles during the wait remains authoritative; waiting for * it is not an instruction to open the channel again. */ @@ -2599,7 +2600,7 @@ export class ChannelRegistry { if (matches.length > 1) return { status: 'ambiguous' }; let entry = matches[0]!; try { - if (entry.open && this.hasPendingLifecycle(entry)) { + if (this.hasPendingLifecycle(entry)) { entry = await this.waitForOpenChannel(entry); } const desired = this.getDesiredState(entry.serverId, entry.descriptor.id); @@ -3082,7 +3083,7 @@ export class ChannelRegistry { if (!entry) { return fail(channelId, `no registered channel for locus "${channelId}"`); } - if (entry.open && this.hasPendingLifecycle(entry)) { + if (this.hasPendingLifecycle(entry)) { try { entry = await this.waitForOpenChannel(entry); } catch (err) { diff --git a/test/channel-lifecycle-order.test.ts b/test/channel-lifecycle-order.test.ts index a26f7be60..29599a5cc 100644 --- a/test/channel-lifecycle-order.test.ts +++ b/test/channel-lifecycle-order.test.ts @@ -410,6 +410,34 @@ test('failed policy opening records intent once, with truthful resident text thr } }); +for (const kind of ['speech', 'reply'] as const) { + test(kind + ' joins a pending close even after retargeting clears open confirmation', async () => { + const f = fixture(); + await f.registry.handleChanged('test', { added: [descriptor('x', true, 'A')] }); + f.calls.length = 0; + f.held.add('x'); + const closing = f.tool('close'); + await tick(); + await f.registry.handleChanged('test', { updated: [descriptor('x', true, 'B')] }); + assert.equal(f.isOpen(), false); + const delivery = kind === 'speech' + ? f.registry.routeSpeech('resident', 'waiting reply', 'x') + : f.registry.openIfClosedForSend('x', 'test'); + await f.drain(closing, delivery); + assert.equal(f.registry.getDesiredState('test', 'x'), 'closed'); + assert.equal((await closing).success, true); + assert.equal(f.isOpen(), false); + assert.deepEqual(f.publishedWhileOpen, []); + assert.equal(f.calls.every((call) => call.kind === 'close'), true); + if (kind === 'speech') { + assert.equal(await delivery, null); + assert.equal(f.routeFailures.length, 1); + } else { + assert.equal((await delivery as { status: string }).status, 'open-failed'); + } + }); +} + for (const change of ['address', 'type'] as const) { test('failed corrective open does not reuse the old ' + change + ' confirmation', async () => { const f = fixture(); From 24a84f59d4acb7d3867138de2fff8063f05ab979 Mon Sep 17 00:00:00 2001 From: Jasper Date: Fri, 2 Oct 2026 15:11:29 -0400 Subject: [PATCH 0020/1112] fix(mcpl): distinguish unconfirmed targets from confirmed closure A bounded public-API probe reproduced another reader of the invalidated open flag: open A, start close, retarget B, fail the close, then explicitly close again. The second tool call claimed success through the already-closed shortcut while the simulated server remained open. Track invalidated target confirmation privately with a WeakSet keyed by the current registration entry. Invalidation still projects open:false, but the explicit-close shortcut requires confirmed closure. A successful current-target lifecycle receipt or accepted open-channel incoming message restores confirmation. Removed registrations need no separate cleanup and the public entry shape stays stable. The new regression fails on cd726dd and passes with the fix; its final repeated close also proves genuinely confirmed closure still avoids another RPC. Expanded focused lifecycle/provenance/routing/tune-out/release tests pass112/112. Final independent review and one bounded serialized full run precede publication. Co-Authored-By: GPT-6 Astra --- .../210-channel-lifecycle-order.fixed.md | 2 +- src/mcpl/channel-registry.ts | 26 ++++++++++++++----- test/channel-lifecycle-order.test.ts | 20 ++++++++++++++ 3 files changed, 40 insertions(+), 8 deletions(-) diff --git a/changelog.d/210-channel-lifecycle-order.fixed.md b/changelog.d/210-channel-lifecycle-order.fixed.md index 94e63d742..50b39afd1 100644 --- a/changelog.d/210-channel-lifecycle-order.fixed.md +++ b/changelog.d/210-channel-lifecycle-order.fixed.md @@ -2,6 +2,6 @@ - Channel lifecycle operations now run in order per channel and reconcile the latest desired state and registration after an in-flight operation completes. Successful reconciliation confirms the current target and desired state at its receipt boundary; a superseded tool request reports that its requested state no longer applies. Automatic delivery preserves active tune-out attention state while opening or awaiting transport. -- Open receipts confirm only the current registration, connection, type, and address. Retargeting invalidates an older open flag; failed corrective opens leave the new target unconfirmed and fail joined delivery, while same-target backscroll failures retain known-open state. +- Open receipts confirm only the current registration, connection, type, and address. Retargeting invalidates an older open flag; failed corrective opens leave the new target unconfirmed and fail joined delivery, while same-target backscroll failures retain known-open state. Explicit close retries an unconfirmed target instead of reporting it already closed. - Lifecycle convergence makes at most five transport attempts per operation. Continued descriptor/intent supersession fails with a diagnostic, releases the queue, and gives joined speech or reply preparation a visible failure instead of hanging. Later stable operations can retry. diff --git a/src/mcpl/channel-registry.ts b/src/mcpl/channel-registry.ts index 44b368537..0873fd438 100644 --- a/src/mcpl/channel-registry.ts +++ b/src/mcpl/channel-registry.ts @@ -212,7 +212,7 @@ function shallowEqualRecord( interface ChannelEntry { serverId: string; descriptor: ChannelDescriptor; - /** Last confirmed state for this entry's current transport target. */ + /** Whether the current target is confirmed open; false can be unconfirmed. */ open: boolean; } @@ -670,6 +670,14 @@ export class ChannelRegistry { // across descriptor replacement. Settled tails release themselves; a // failed operation must not poison later decisions. private channelLifecycleTails = new Map>(); + // Losing an open receipt does not confirm closure. Keep that distinction + // private while preserving the public open:boolean availability projection. + private unconfirmedChannelTargets = new WeakSet(); + + private invalidateTransportConfirmation(entry: ChannelEntry): void { + entry.open = false; + this.unconfirmedChannelTargets.add(entry); + } private hasPendingLifecycle(entry: ChannelEntry): boolean { return this.channelLifecycleTails.has(this.lifecycleKey(entry.serverId, entry.descriptor.id)); @@ -820,7 +828,7 @@ export class ChannelRegistry { } // A live subscription to the old target does not confirm the new // type/address. Label-only updates preserve that confirmation. - if (!sameChannelTarget(existing.descriptor, channel)) existing.open = false; + if (!sameChannelTarget(existing.descriptor, channel)) this.invalidateTransportConfirmation(existing); existing.descriptor = channel; this.appendLabelSighting(channel.id, channel.label, extractDmMeta(channel.metadata)); addedResults.push({ id: channel.id, accepted: true }); @@ -963,7 +971,9 @@ export class ChannelRegistry { // the transport is actually open. This repairs transient status only; // durable desired state still changes exclusively through lifecycle // operations. - this.channels.get(incomingKey)!.open = true; + const incomingEntry = this.channels.get(incomingKey)!; + incomingEntry.open = true; + this.unconfirmedChannelTargets.delete(incomingEntry); }; if (!coalesced) markAccepted(); @@ -2122,7 +2132,7 @@ export class ChannelRegistry { if (attempts >= MAX_CHANNEL_LIFECYCLE_ATTEMPTS) { // No receipt confirmed the current target/intent. A later delivery // must reconcile it rather than reuse an earlier target's open flag. - entry.open = false; + this.invalidateTransportConfirmation(entry); const error = new ChannelLifecycleConvergenceError( `Channel ${channelId} did not converge after ${attempts} lifecycle attempts; its target or desired state kept changing`, ); @@ -2148,19 +2158,20 @@ export class ChannelRegistry { } catch (error) { // A failed same-target backscroll can retain its known-open state. // A receipt/failure from an older connection or target cannot. - if (this.channels.get(channelKey) === entry && !sameTarget()) entry.open = false; + if (this.channels.get(channelKey) === entry && !sameTarget()) this.invalidateTransportConfirmation(entry); throw error; } // Never apply an old receipt to the current target, even transiently: // a corrective open might fail before that stale bit is replaced. if (this.channels.get(channelKey) !== entry) continue; if (!sameTarget()) { - entry.open = false; + this.invalidateTransportConfirmation(entry); continue; } if (open && !entry.open) opened = true; if (!open) opened = false; entry.open = open; + this.unconfirmedChannelTargets.delete(entry); const now = this.getDesiredState(serverId, channelId); if (open !== (now === 'open' || now === 'tuned-out')) continue; return { open, opened, result }; @@ -2726,7 +2737,8 @@ export class ChannelRegistry { } const alreadyDesiredClosed = this.getDesiredState(entry.serverId, input.channelId) === 'closed'; - if (!entry.open && alreadyDesiredClosed && !this.hasPendingLifecycle(entry)) { + if (!entry.open && alreadyDesiredClosed && !this.hasPendingLifecycle(entry) && + !this.unconfirmedChannelTargets.has(entry)) { return { success: true, data: { channelId: input.channelId, status: 'already closed' }, diff --git a/test/channel-lifecycle-order.test.ts b/test/channel-lifecycle-order.test.ts index 29599a5cc..cb7c25b9b 100644 --- a/test/channel-lifecycle-order.test.ts +++ b/test/channel-lifecycle-order.test.ts @@ -410,6 +410,26 @@ test('failed policy opening records intent once, with truthful resident text thr } }); +test('an unconfirmed target retries explicit close instead of claiming already closed', async () => { + const f = fixture(); + await f.registry.handleChanged('test', { added: [descriptor('x', true, 'A')] }); + f.calls.length = 0; + f.held.add('x'); + const closing = f.tool('close'); + await tick(); + await f.registry.handleChanged('test', { updated: [descriptor('x', true, 'B')] }); + f.pending.shift()!.fail(); + assert.equal((await closing).success, false); + assert.equal(f.isOpen(), false); + assert.equal(f.actual.get('x'), true, 'failed close did not change the server state'); + f.held.delete('x'); + assert.equal((await f.tool('close')).success, true); + assert.equal(f.actual.get('x'), false, 'explicit retry must actually close the channel'); + assert.equal(f.calls.length, 2, 'unconfirmed is different from confirmed closed'); + assert.equal((await f.tool('close')).success, true); + assert.equal(f.calls.length, 2, 'the confirmed-closed fast path remains available'); +}); + for (const kind of ['speech', 'reply'] as const) { test(kind + ' joins a pending close even after retargeting clears open confirmation', async () => { const f = fixture(); From a2eb50622a2b6bd6159ed67739fc107afe0a5175 Mon Sep 17 00:00:00 2001 From: Aleksandr Markov Date: Sat, 3 Oct 2026 13:53:01 +0200 Subject: [PATCH 0021/1112] fix(tools): enforce registered agent allowlists before dispatch Co-Authored-By: GPT-6.1 Sol (Codex) --- changelog.d/agent-allowed-tools.fixed.md | 1 + src/framework.ts | 37 ++- test/agent-allowed-tools.test.ts | 285 +++++++++++++++++++++++ 3 files changed, 321 insertions(+), 2 deletions(-) create mode 100644 changelog.d/agent-allowed-tools.fixed.md create mode 100644 test/agent-allowed-tools.test.ts diff --git a/changelog.d/agent-allowed-tools.fixed.md b/changelog.d/agent-allowed-tools.fixed.md new file mode 100644 index 000000000..f4593d159 --- /dev/null +++ b/changelog.d/agent-allowed-tools.fixed.md @@ -0,0 +1 @@ +- Enforce registered agents' allowedTools before model, script, and public agent-origin tool dispatch, with paired error results and refusal tracing. Denied tools cannot silence or privatize public prose. Preserve trusted module dispatch, unregistered legacy ephemeral callers, and the implicit prose_help tool for explicit prose agents. diff --git a/src/framework.ts b/src/framework.ts index 25cdb419b..365891f5c 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -9587,7 +9587,9 @@ export class AgentFramework { // with verbatim roundContent are live-routed (see liveProseRouting // note above — the fallback preamble is cumulative in XML mode). if (this.channelRegistry) { - const roundToolNames = event.calls.map((c) => c.name); + const roundToolNames = event.calls + .filter((c) => this.canAgentUseTool(agent.name, c.name)) + .map((c) => c.name); const hasSameRoundPrivateThink = roundToolNames.includes('think') && requestSnapshot.sameRoundThinkTextPolicy === 'private'; @@ -10133,7 +10135,7 @@ export class AgentFramework { const toolNames = response.content .filter((b) => b.type === 'tool_use') .map((b) => (b as unknown as { name?: string }).name) - .filter((n): n is string => typeof n === 'string'); + .filter((n): n is string => typeof n === 'string' && this.canAgentUseTool(agent.name, n)); const silenced = liveProseRouting ? turnSilenced : turnSilenced || toolNames.some(isSilencingTool); @@ -10893,6 +10895,11 @@ export class AgentFramework { } private async executeToolCallFrom(call: ToolCall, origin: ChannelToolOrigin): Promise { + if (origin.kind === 'agent') { + const refusal = this.agentToolPermissionFailure(origin.agentName, call); + if (refusal) return refusal; + } + // Client-side programmatic tool calling for promise-based callers // (SubagentModule ephemerals). Keyed by callerAgentName so each ephemeral // gets its own interpreter state. @@ -11964,7 +11971,33 @@ export class AgentFramework { return best; } + /** Registered agent policy, including the implicit explicit-prose helper. + * Unregistered legacy ephemerals retain their full programmatic surface. */ + private canAgentUseTool(agentName: string, toolName: string): boolean { + const agent = this.agents.get(agentName); + return !agent || agent.canUseTool(toolName) || + (toolName === 'prose_help' && agent.proseRouting === 'explicit'); + } + + private agentToolPermissionFailure(agentName: string, call: ToolCall): ToolResult | undefined { + if (this.canAgentUseTool(agentName, call.name)) return undefined; + const error = `Tool '${call.name}' is not permitted by this agent's allowedTools.`; + this.toolLifecycleEmitter?.refuse(agentName, call.id); + this.emitTrace({ + type: 'tool:failed', module: 'framework', tool: call.name, callId: call.id, error, + }); + return { success: false, error, isError: true }; + } + private dispatchToolCall(agentName: string, call: ToolCall): void { + const refusal = this.agentToolPermissionFailure(agentName, call); + if (refusal) { + this.pushEvent({ + type: 'tool-result', callId: call.id, agentName, moduleName: 'framework', result: refusal, + }); + return; + } + // Enrich call with caller identity so modules can resolve the calling agent const enrichedCall: ToolCall = { ...call, callerAgentName: agentName }; diff --git a/test/agent-allowed-tools.test.ts b/test/agent-allowed-tools.test.ts new file mode 100644 index 000000000..6bbe320fd --- /dev/null +++ b/test/agent-allowed-tools.test.ts @@ -0,0 +1,285 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { AgentFramework } from '../src/framework.js'; +import type { AgentConfig, Module, ModuleContext, ToolCall, ToolResult } from '../src/types/index.js'; +import { CapabilityGrant } from '../src/mcpl/capability-grant.js'; +import { ToolLifecycleEmitter, type ToolLifecycleParams } from '../src/mcpl/tool-lifecycle.js'; +import { createMockResponse, MockMembrane } from './helpers/mock-membrane.js'; + +async function harness(config: Partial) { + const dir = mkdtempSync(join(tmpdir(), 'agent-allowed-tools-')); + const membrane = new MockMembrane(); + const effects: ToolCall[] = []; + let moduleContext: ModuleContext; + const module: Module = { + name: 'probe', + async start(ctx) { moduleContext = ctx; }, + async stop() {}, + getTools: () => ['allowed', 'denied'].map((name) => ({ + name, description: 'Inert dispatch counter', inputSchema: { type: 'object', properties: {} }, + })), + async handleToolCall(call) { + effects.push(call); + return { success: true, data: 'executed' }; + }, + async onProcess(event) { + return event.type === 'external-message' + ? { addMessages: [{ participant: 'user', content: [{ type: 'text', text: 'Run probe' }] }], requestInference: true } + : {}; + }, + }; + const framework = await AgentFramework.create({ + storePath: join(dir, 'store.chronicle'), membrane: membrane.asMembrane(), + agents: [{ name: 'assistant', model: 'test', systemPrompt: 'Test', ...config }], + modules: [module], syncIntervalMs: 0, codeExecution: { enabled: true }, + }); + const traces: Array> = []; + framework.onTrace((event) => traces.push(event as unknown as Record)); + const lifecycle: ToolLifecycleParams[] = []; + const emitter = new ToolLifecycleEmitter({ + observers: () => [{ + id: 'observer', grant: new CapabilityGrant(new Set(['toolLifecycle.observe']), []), + toolObserveFilter: null, sendToolLifecycle: (event) => lifecycle.push(event), + }], + configFor: () => undefined, describe: () => ({ class: [] }), + }); + // Use the real emitter with an in-memory observer, without an MCPL process. + const internal = framework as unknown as { + toolLifecycleEmitter: ToolLifecycleEmitter; + dispatchScriptToolCall(agent: string, tool: string, input: Record): Promise; + }; + internal.toolLifecycleEmitter = emitter; + return { + framework, membrane, effects, traces, lifecycle, emitter, internal, + moduleCall: (call: ToolCall) => moduleContext.callTool(call), + async turn(calls: ToolCall[]) { + membrane.pushResponse(createMockResponse(calls.map((call) => ({ + type: 'tool_use', id: call.id, name: call.name, input: call.input as Record, + })), 'tool_use')); + membrane.pushResponse(createMockResponse([{ type: 'text', text: 'Done' }])); + framework.pushEvent({ type: 'external-message', source: 'probe', content: 'Run probe', metadata: {} }); + await framework.runUntilIdle(); + return membrane.lastStream!.receivedToolResults[0] as Array<{ toolUseId: string; content: string; isError?: boolean }>; + }, + async close() { await framework.stop(); rmSync(dir, { recursive: true, force: true }); }, + }; +} + +const generic: ToolCall = { id: 'generic-denied', name: 'probe--denied', input: {} }; +const builtin: ToolCall = { + id: 'builtin-denied', name: 'agent_settings', + input: { action: 'update', same_round_think_text_policy: 'private' }, +}; + +for (const allowedTools of [['probe--allowed'], []]) { + for (const calls of [[generic], [builtin]]) { + test(`normal model dispatch denies ${calls[0].name} with ${JSON.stringify(allowedTools)}`, async () => { + const h = await harness({ allowedTools }); + try { + const before = h.framework.getAgentRuntimeSettings('assistant'); + const results = await h.turn(calls); + assert.equal(h.effects.length, 0, 'unlisted generic tool must have zero effects'); + assert.deepEqual(h.framework.getAgentRuntimeSettings('assistant'), before, 'unlisted builtin must have zero effects'); + assert.deepEqual(results.map((r) => r.toolUseId), calls.map((c) => c.id)); + for (const result of results) { + assert.equal(result.isError, true); + assert.match(result.content, /not permitted.*allowedTools/); + } + assert.deepEqual(h.lifecycle, [], 'refused calls must emit no RFC-007 lifecycle events'); + assert.equal(h.emitter.openCount, 0); + for (const call of calls) { + const events = h.traces.filter((e) => e.callId === call.id && String(e.type).startsWith('tool:')); + assert.equal(events.length, 1); + assert.equal(events[0].type, 'tool:failed'); + assert.equal(events[0].tool, call.name); + } + const compiled = await h.framework.getAgent('assistant')!.compileContext(); + const blocks = compiled.messages.flatMap((m) => m.content); + for (const call of calls) { + assert.ok(blocks.some((b) => b.type === 'tool_use' && b.id === call.id)); + assert.ok(blocks.some((b) => b.type === 'tool_result' && b.toolUseId === call.id && b.isError)); + } + } finally { await h.close(); } + }); + } +} + +for (const allowedTools of [undefined, 'all', ['probe--denied', 'agent_settings']] as Array) { + test(`allowed model calls retain effects and paired results: ${JSON.stringify(allowedTools)}`, async () => { + const h = await harness({ allowedTools }); + try { + const results = await h.turn([generic, builtin]); + assert.equal(h.effects.length, 1); + assert.equal(h.framework.getAgentRuntimeSettings('assistant').sameRoundThinkTextPolicy, 'private'); + assert.deepEqual(results.map((r) => r.toolUseId).sort(), [generic.id, builtin.id].sort()); + assert.ok(results.every((r) => !r.isError)); + assert.deepEqual(h.lifecycle.map((e) => e.phase).sort(), ['completed', 'completed', 'started', 'started']); + assert.equal(h.emitter.openCount, 0); + } finally { await h.close(); } + }); +} + +test('explicit prose helper remains advertised and executable with an empty list', async () => { + const h = await harness({ allowedTools: [], proseRouting: 'explicit' }); + try { + assert.ok(h.framework.listToolClasses('assistant').some((t) => t.tool === 'prose_help')); + const [result] = await h.turn([{ id: 'help', name: 'prose_help', input: {} }]); + assert.equal(result.toolUseId, 'help'); + assert.ok(!result.isError); + assert.match(result.content, />>/); + } finally { await h.close(); } +}); + +test('shared script dispatch enforces the resident list and resolves its real event waiter', async () => { + const h = await harness({ allowedTools: ['probe--allowed'] }); + try { + const before = h.framework.getAgentRuntimeSettings('assistant'); + for (const call of [generic, builtin, { id: 'allowed', name: 'probe--allowed', input: {} }]) { + const pending = h.internal.dispatchScriptToolCall('assistant', call.name, call.input as Record); + await h.framework.runUntilIdle(); + const result = await pending; + assert.equal(result.success, call.name === 'probe--allowed'); + if (!result.success) { + assert.equal(result.isError, true); + assert.match(result.error!, /not permitted.*allowedTools/); + } + } + assert.deepEqual(h.effects.map((c) => c.name), ['allowed']); + assert.deepEqual(h.framework.getAgentRuntimeSettings('assistant'), before); + } finally { await h.close(); } +}); + +test('model-issued Python cannot forge unlisted inner generic or builtin calls', async () => { + const h = await harness({ allowedTools: ['code_execution', 'probe--allowed'] }); + try { + const before = h.framework.getAgentRuntimeSettings('assistant'); + const [result] = await h.turn([{ + id: 'script', name: 'code_execution', input: { code: [ + 'make = probe__allowed.__globals__["_make_tool_fn"]', + 'for name, args in [("probe--denied", {}), ("agent_settings", {"action": "update", "same_round_think_text_policy": "private"})]:', + ' try:', + ' print(await make(name, "forged")(args))', + ' except Exception as error:', + ' print(str(error))', + 'print(await probe__allowed())', + ].join('\n') }, + }]); + assert.equal(result.toolUseId, 'script'); + assert.ok(!result.isError); + const execution = JSON.parse(result.content) as { stdout: string; stderr: string; return_code: number }; + assert.equal(execution.return_code, 0, execution.stderr); + assert.equal((execution.stdout.match(/not permitted.*allowedTools/g) ?? []).length, 2); + assert.deepEqual(h.effects.map((c) => c.name), ['allowed']); + assert.deepEqual(h.framework.getAgentRuntimeSettings('assistant'), before); + } finally { await h.close(); } +}); + +test('unregistered programmatic ephemeral script callers retain their full surface', async () => { + const h = await harness({ allowedTools: [] }); + try { + const pending = h.framework.executeToolCall({ + id: 'ephemeral-script', name: 'code_execution', callerAgentName: 'unregistered-ephemeral', + input: { code: 'print(await probe__denied())' }, + }); + // The Python subprocess enqueues its inner call after startup. + const processing = setInterval(() => { void h.framework.runUntilIdle(); }, 10); + try { + const result = await pending; + assert.equal(result.success, true); + assert.equal((result.data as { return_code: number }).return_code, 0); + assert.equal(h.effects.length, 1); + } finally { clearInterval(processing); } + } finally { await h.close(); } +}); + +for (const call of [generic, { id: 'public-script', name: 'code_execution', input: { code: 'print("forbidden")' } }]) { + test(`registered public dispatch refuses ${call.name} before effects`, async () => { + const h = await harness({ allowedTools: [] }); + try { + const result = await h.framework.executeToolCall({ ...call, callerAgentName: 'assistant' }); + assert.equal(result.success, false); + assert.equal(result.isError, true); + assert.match(result.error!, /not permitted.*allowedTools/); + assert.equal(h.effects.length, 0); + assert.deepEqual(h.lifecycle, []); + assert.equal(h.traces.filter((e) => e.callId === call.id && e.type === 'tool:failed').length, 1); + // A refused code_execution must not even create an interpreter. + assert.equal((h.framework as unknown as { codeExecutionRunners: Map }).codeExecutionRunners.size, 0); + } finally { await h.close(); } + }); +} + +test('trusted module closure bypasses resident restrictions even with a registered caller identity', async () => { + const h = await harness({ allowedTools: [] }); + try { + const result = await h.moduleCall({ ...generic, callerAgentName: 'assistant' }); + assert.equal(result.success, true); + assert.deepEqual(h.effects.map((c) => c.name), ['denied']); + } finally { await h.close(); } +}); + +// Same channel capture contract as explicit-prose-routing.test.ts. +function captureRouting(framework: AgentFramework) { + const routed: string[] = []; + const registry: Record = { + handleChannelToolCall: async () => ({ success: true, data: 'ok' }), + routeSpeech: async (_agent: string, text: string) => { routed.push(text); return { delivered: true, channelId: 'world:commons' }; }, + resolveLocus: () => 'world:commons', getDefaultPublishChannel: () => null, + isChannelOpen: () => true, getDescriptor: () => undefined, getChannelTools: () => [], + }; + (framework as unknown as { channelRegistry: unknown }).channelRegistry = new Proxy(registry, { + get: (target, prop: string) => prop in target ? target[prop] : () => undefined, + }); + return routed; +} + +for (const fallback of [false, true]) { + for (const name of ['skip_reply', 'think', 'channel_publish']) { + for (const permitted of [false, true]) { + test(`${fallback ? 'fallback' : 'native'} prose disposition ${permitted ? 'honors permitted' : 'ignores denied'} ${name}`, async () => { + const h = await harness({ allowedTools: permitted ? [name] : [], sameRoundThinkTextPolicy: 'private' }); + try { + const routed = captureRouting(h.framework); + const call = { type: 'tool_use' as const, id: 'disposition', name, input: { reason: 'test', content: 'test' } }; + if (fallback) { + const streamYielding = h.membrane.streamYielding.bind(h.membrane); + h.membrane.streamYielding = (request, options) => { + const stream = streamYielding(request, options); + const iterate = stream[Symbol.asyncIterator].bind(stream); + stream[Symbol.asyncIterator] = async function* () { + for await (const event of { [Symbol.asyncIterator]: iterate }) { + if (event.type === 'tool-calls') { + yield { ...event, context: { ...event.context, roundContent: undefined } }; + } else if (event.type === 'complete') { + // Older membrane completion contains the cumulative tool round. + yield { ...event, response: { ...event.response, content: [call, ...event.response.content] } }; + } else yield event; + } + }; + return stream; + }; + } + h.membrane.pushResponse(createMockResponse([{ type: 'text', text: 'Public round prose' }, call], 'tool_use')); + h.membrane.pushResponse(createMockResponse([{ type: 'text', text: 'Public trailing prose' }])); + h.framework.pushEvent({ type: 'external-message', source: 'probe', content: 'Run probe', metadata: {} }); + await h.framework.runUntilIdle(); + const results = h.membrane.lastStream!.receivedToolResults[0] as Array<{ isError?: boolean; content: string }>; + if (!permitted) { + assert.equal(results[0].isError, true); + assert.match(results[0].content, /not permitted.*allowedTools/); + assert.ok(routed.includes('Public trailing prose'), 'denied tools cannot silence trailing public prose'); + if (!fallback) assert.ok(routed.includes('Public round prose'), 'denied think cannot privatize same-round public prose'); + } else if (name === 'think') { + assert.ok(routed.includes('Public trailing prose')); + if (!fallback) assert.ok(!routed.includes('Public round prose'), 'permitted private think retains same-round policy'); + } else { + assert.deepEqual(routed, [], 'permitted silencing calls retain prose disposition'); + } + } finally { await h.close(); } + }); + } + } +} From 3f7aafe6c304243e1525bc84a7e653b9a38c0ebf Mon Sep 17 00:00:00 2001 From: Petra-1012 Date: Wed, 7 Oct 2026 05:00:03 -0400 Subject: [PATCH 0022/1112] feat(mcpl): stamp each accepted inbound item with its source envelope MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every inbound item the host accepts now carries `metadata.inboundSource` (src/mcpl/inbound-source.ts), decided once at the framework's ingestion boundary: which conversation the item belongs to and when the host accepted it. Speech routing (shelf-355), the visible provenance header (shelf-356) and per-channel receipt clocks (shelf-354, room-220) all read this one value instead of re-deriving identity from the lane an item arrived on. Kinds: `channel` (MCPL server id, RFC-006 endpoint binding, canonical channel id — the registered composite id, also on the push lane — thread, message, producer eventId when supplied, reply edge from metadata.replyTo, the channel's registry label at receipt), `unscoped` (a push naming no channel: never a guessed channel), and `surface` (console/API input: module messages added in response to an `external-message` or `api:message` event that names no channel). Each carries `acceptedAt` (host acceptance, epoch ms) and, for MCPL items, the adapter's `sourceTimestamp` verbatim; neither is a version key. Acceptance time is stamped where the host actually admits an item: ChannelRegistry.handleIncoming and PushHandler for ordinary items, and pushCoalescer acceptance for RFC-006 work. A coalesced occurrence computes its envelope at acceptance (handleCoalescedIncoming/handleCoalescedPush) and stores it on the occurrence's event, so deferral, fan-out, replay and materialization deliver it unchanged (a materialized delivery only adds `materialized: true`; deferred work is marked `deferred: true`). A label is therefore the label when accepted, and a later rename never rewrites it. The field is written after adapter metadata is spread, so an adapter cannot supply or spoof it. It is not named `source` because adapter origins already carry a free-form `source` string (discord-mcpl sends `source: 'discord'`), which stays untouched. An optional InboundAcceptanceObserver (private framework field, installed by the receipt lane) hears each acceptance exactly once in-process: where an ordinary item's envelope is created in handleMcplChannelIncoming / handleMcplPushEvent, at a coalescer admission via a new CoalescerHost `accepted` hook (after the receipt is recorded, never for a deduplicated producer retry), and once per console/API event. Deliveries that reuse a frozen envelope are not acceptances. A throwing observer never affects delivery; it is reported on stderr and as an `inbound:observer-failed` trace. Interface agreed with Oscar-1005 and Hugo-1004 in room-220 (#41234, #41242, #41249, #41270, #41286, #41292). Tests (test/inbound-source.test.ts, over a real MCPL child fixture test/fixtures/speech-route-mcpl-server.mjs and the RFC-006 WebSocket fixture): incoming message stamp with label, reply edge, timestamps and an adapter spoof attempt overwritten; DM push with derived composite id and people-first label (origin.source preserved); unscoped push; console surface; label frozen across a rename; observer failure isolation and trace; coalesced acceptance observed once (retry not an acceptance, replacement carries its own frozen envelope); deferred work observed before render with the materialized delivery keeping that envelope; channel-scoped coalesced message. The trunk-channel-routing registry stubs gain getChannelLabel. npm test: 1303 pass / 0 fail (baseline 64c480b: 1298 pass / 0 fail). Co-Authored-By: Claude Opus 5.5 --- changelog.d/inbound-source-envelope.added.md | 12 + src/framework.ts | 211 ++++++++++++++- src/index.ts | 9 + src/mcpl/channel-registry.ts | 11 + src/mcpl/inbound-source.ts | 132 ++++++++++ src/mcpl/index.ts | 9 + src/mcpl/push-coalescer.ts | 9 + src/mcpl/push-handler.ts | 7 + src/types/events.ts | 4 + src/types/trace.ts | 10 + test/fixtures/speech-route-mcpl-server.mjs | 225 ++++++++++++++++ test/inbound-source.test.ts | 261 +++++++++++++++++++ test/trunk-channel-routing.test.ts | 5 + 13 files changed, 899 insertions(+), 6 deletions(-) create mode 100644 changelog.d/inbound-source-envelope.added.md create mode 100644 src/mcpl/inbound-source.ts create mode 100644 test/fixtures/speech-route-mcpl-server.mjs create mode 100644 test/inbound-source.test.ts diff --git a/changelog.d/inbound-source-envelope.added.md b/changelog.d/inbound-source-envelope.added.md new file mode 100644 index 000000000..30db418ea --- /dev/null +++ b/changelog.d/inbound-source-envelope.added.md @@ -0,0 +1,12 @@ +- **Inbound source envelope.** Every inbound item the host accepts now carries + `metadata.inboundSource`, stamped once where it was accepted: the + conversation it belongs to (`channel`: MCPL server, endpoint binding, + canonical channel id, thread, message, reply edge and the channel's label at + receipt; `unscoped` for a push that names no channel; `surface` for console + or API input from a module), the host acceptance time and the adapter's own + timestamp. It covers `channels/incoming` and `push/event` on both the + ordinary and the RFC-006 coalesced paths; a coalesced occurrence freezes its + envelope at acceptance, so deferral, fan-out and replay deliver it unchanged + and a later rename never rewrites it. Adapter-supplied metadata cannot set + the field (an adapter's own `origin.source` string is untouched). + `readInboundSource`, `conversationKey` and `INBOUND_SOURCE_KEY` are exported. diff --git a/src/framework.ts b/src/framework.ts index c527c488f..3934b1f9d 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -83,6 +83,7 @@ import { detectKnownToolWrapperProse } from './tool-wrapper-prose-guard.js'; import { InferenceRouter } from './mcpl/inference-router.js'; import { ChannelRegistry, type ChannelToolOrigin } from './mcpl/channel-registry.js'; import { ConversationRouter, CONVERSATION_ROUTING_DEPRECATION_NOTICE } from './mcpl/conversation-router.js'; +import { INBOUND_SOURCE_KEY, type InboundAcceptanceObserver, type InboundChannelSource, type InboundSource, type InboundUnscopedSource } from './mcpl/inbound-source.js'; import { safeSlice } from './safe-slice.js'; import type { WorkspaceModule } from './modules/workspace/index.js'; import { @@ -441,6 +442,11 @@ interface CoalescedChannelEvent { assemblingFor?: string; /** RFC-006 §3.2: deliver into this agent only (replacement / notice audience). */ deliverTo?: string; + /** Host acceptance time (epoch ms), stamped where the message was admitted. */ + acceptedAt?: number; + /** Framework-owned source envelope, frozen at a coalesced occurrence's + * acceptance so its later delivery cannot restamp it (inbound-source.ts). */ + inboundSource?: InboundSource; } type CoalescedDelivery = | { lane: 'channel'; event: CoalescedChannelEvent } @@ -1012,6 +1018,9 @@ export class AgentFramework { private processLoggingPersist: boolean; private processLoggingBroadcast: boolean; private activeStreams: Map> = new Map(); + /** Told once per inbound acceptance (mcpl/inbound-source.ts); unset unless + * a receipt consumer installs one. */ + private inboundAcceptanceObserver?: InboundAcceptanceObserver; /** Per-agent output locus FROZEN for the CURRENT logical turn. Resolved * eagerly in startAgentStream (home → addressed trigger → global default) @@ -6949,9 +6958,15 @@ export class AgentFramework { } } - // Apply responses + // Apply responses. Console/API input is conversation from a non-channel + // surface: one acceptance per event, whose envelope every message the + // modules add for it carries. + const surface = this.surfaceSourceFor(event); + if (surface && responses.some(({ response }) => (response.addMessages?.length ?? 0) > 0)) { + this.noteInboundAccepted(surface); + } for (const { moduleName, response } of responses) { - await this.applyProcessResponse(response, event, moduleName); + await this.applyProcessResponse(response, event, moduleName, surface); } // Handle tool calls specially @@ -6973,12 +6988,15 @@ export class AgentFramework { private async applyProcessResponse( response: EventResponse, event: ProcessEvent, - moduleName: string + moduleName: string, + /** The event's surface envelope, when it is console/API input. */ + surface?: InboundSource, ): Promise { // Add messages if (response.addMessages) { for (const msg of response.addMessages) { - const id = this.addMessage(msg.participant, msg.content, msg.metadata); + const metadata = surface ? { ...msg.metadata, [INBOUND_SOURCE_KEY]: surface } : msg.metadata; + const id = this.addMessage(msg.participant, msg.content, metadata); this.emitTrace({ type: 'message:added', messageId: id, source: event.type }); } } @@ -7065,6 +7083,8 @@ export class AgentFramework { eventId?: string; assemblingFor?: string; deliverTo?: string; + acceptedAt?: number; + inboundSource?: InboundSource; }): Promise { const metadata: Record = { ...event.metadata, @@ -7078,6 +7098,27 @@ export class AgentFramework { }; if (event.threadId) metadata.threadId = event.threadId; if (event.tags) metadata.tags = event.tags; + // The inbound source envelope (mcpl/inbound-source.ts): frozen at the + // occurrence's acceptance when the coalescer delivers, else stamped now + // from this event. Written after the adapter's metadata, so an adapter + // cannot supply it. + let source: InboundSource; + if (event.inboundSource) { + source = event.inboundSource; + } else { + source = this.channelSource({ + serverId: event.serverId, + channelId: event.channelId, + threadId: event.threadId, + messageId: event.messageId, + eventId: event.eventId, + replyTo: event.metadata?.replyTo, + acceptedAt: event.acceptedAt, + sourceTimestamp: event.timestamp, + }); + this.noteInboundAccepted(source); + } + metadata[INBOUND_SOURCE_KEY] = source; // Per-channel conversation routing: messages go to the channel's fork // agent (spawned from the template on first qualifying message), never @@ -7587,6 +7628,113 @@ export class AgentFramework { return createHash('sha256').update(JSON.stringify([serverId, config?.url ?? null, config?.command ?? null, config?.args ?? null])).digest('hex').slice(0, 16); } + /** + * Report one inbound acceptance to the observer, if one is installed + * (InboundAcceptanceObserver). Called where an envelope is first created — + * never for a delivery that reuses a frozen one. A failing observer is + * reported loudly (stderr + trace) and never affects delivery. + */ + private noteInboundAccepted(source: InboundSource): void { + if (!this.inboundAcceptanceObserver) return; + try { + this.inboundAcceptanceObserver.inboundAccepted(source); + } catch (err) { + const error = err instanceof Error ? err.message : String(err); + console.error(`[inbound] acceptance observer failed (${source.kind}): ${error}`); + this.emitTrace({ type: 'inbound:observer-failed', kind: source.kind, error }); + } + } + + /** + * The inbound source envelope (mcpl/inbound-source.ts) of an item that + * belongs to a registered channel. The one place the framework decides + * which conversation an inbound item came from; every lane calls it at the + * point the host accepted the item, or reuses the envelope frozen then. + * `acceptedAt` falls back to now only for an event no admission path + * stamped (a direct framework caller), which is the closest observation. + */ + private channelSource(fields: { + serverId: string; + channelId: string; + threadId?: unknown; + messageId?: unknown; + eventId?: unknown; + replyTo?: unknown; + labelFallback?: string; + acceptedAt?: number; + sourceTimestamp?: unknown; + deferred?: boolean; + }): InboundChannelSource { + const str = (v: unknown): string | undefined => (typeof v === 'string' && v.length > 0 ? v : undefined); + const label = this.channelRegistry?.getChannelLabel(fields.serverId, fields.channelId) ?? str(fields.labelFallback); + const threadId = str(fields.threadId); + const messageId = str(fields.messageId); + const eventId = str(fields.eventId); + const replyTo = str(fields.replyTo); + const sourceTimestamp = str(fields.sourceTimestamp); + return { + kind: 'channel', + serverId: fields.serverId, + binding: this.coalescingBinding(fields.serverId), + channelId: fields.channelId, + ...(threadId ? { threadId } : {}), + ...(messageId ? { messageId } : {}), + ...(eventId ? { eventId } : {}), + ...(label ? { label } : {}), + ...(replyTo ? { replyTo } : {}), + acceptedAt: fields.acceptedAt ?? Date.now(), + ...(sourceTimestamp ? { sourceTimestamp } : {}), + ...(fields.deferred ? { deferred: true as const } : {}), + }; + } + + /** Source envelope of a push event: its derived channel, else unscoped. */ + private pushSource(event: McplPushEvent, deferred = false): InboundSource { + const origin = (event.origin ?? {}) as Record; + const channel = this.derivePushEventChannel(event.origin); + if (channel) { + return this.channelSource({ + serverId: event.serverId, + channelId: channel.channelId, + threadId: origin.threadId, + messageId: origin.messageId, + eventId: event.eventId, + replyTo: origin.replyTo, + labelFallback: channel.label, + acceptedAt: event.acceptedAt, + sourceTimestamp: event.timestamp, + deferred, + }); + } + const unscoped: InboundUnscopedSource = { + kind: 'unscoped', + serverId: event.serverId, + binding: this.coalescingBinding(event.serverId), + ...(typeof event.eventId === 'string' && event.eventId ? { eventId: event.eventId } : {}), + acceptedAt: event.acceptedAt ?? Date.now(), + ...(typeof event.timestamp === 'string' && event.timestamp ? { sourceTimestamp: event.timestamp } : {}), + ...(deferred ? { deferred: true as const } : {}), + }; + return unscoped; + } + + /** + * Source envelope for conversational input from a non-channel surface: a + * module's messages in response to an `external-message` (console, TUI) or + * `api:message` event. An event whose metadata names a channel is not a + * surface conversation and gets none (never a guessed identity). + */ + private surfaceSourceFor(event: ProcessEvent): InboundSource | undefined { + if (event.type !== 'external-message' && event.type !== 'api:message') return undefined; + const metadata = (event as { metadata?: Record }).metadata; + if (typeof metadata?.channelId === 'string' && metadata.channelId) return undefined; + const declared = (event as { source?: unknown }).source; + const surface = typeof declared === 'string' && declared + ? declared + : event.type === 'api:message' ? 'api' : 'external'; + return { kind: 'surface', surface, acceptedAt: Date.now() }; + } + private initializePushCoalescer(): void { const coalescer = new PushCoalescer({ isUnread: (p) => this.isUnreadPlacement(p), @@ -7610,6 +7758,12 @@ export class AgentFramework { this.coalescingRecentReceipts.push(record); this.store.setStateJson(COALESCING_RECENT_ID, this.coalescingRecentReceipts); }, + // The acceptance of coalesced work (deferred included) is observed + // here, once; its later delivery reuses the frozen envelope. + accepted: (occ) => { + const source = occ.event.event.inboundSource; + if (source) this.noteInboundAccepted(source); + }, wasPublished: (subject, eventId) => { // Boot-time only: the occurrence's durable delivery identity // (subject + eventId in message metadata) in any agent's context. @@ -7710,6 +7864,18 @@ export class AgentFramework { if (typeof message.timestamp !== 'string') throw new CoalesceError('timestamp', 'timestamp is required'); validateCoalescedContent(message.content); const c = message.coalesce; + // The inbound source envelope, frozen at acceptance: deferral, fan-out + // and replay deliver it unchanged (mcpl/inbound-source.ts). + const inboundSource = this.channelSource({ + serverId, + channelId: message.channelId, + threadId: message.threadId, + messageId: message.messageId, + eventId: message.eventId, + replyTo: message.metadata?.replyTo, + acceptedAt: event.acceptedAt, + sourceTimestamp: message.timestamp, + }); const result = await this.pushCoalescer.accept({ serverId, binding: this.coalescingBinding(serverId), @@ -7723,7 +7889,7 @@ export class AgentFramework { tags: event.tags, content: message.content, identity: { messageId: message.messageId, author: message.author, threadId: message.threadId }, - event: { lane: 'channel', event: { ...event, eventId: message.eventId } }, + event: { lane: 'channel', event: { ...event, eventId: message.eventId, inboundSource } }, }); return { messageId: message.messageId, accepted: true, coalesce: result.coalesce }; }); @@ -7740,6 +7906,21 @@ export class AgentFramework { if (c.channelId) this.assertChannelScopedPush(serverId, c.channelId); const origin = params.origin ?? {}; const str = (v: unknown): string | undefined => (typeof v === 'string' && v ? v : undefined); + // The inbound source envelope, frozen at acceptance (see the channel + // lane). A channel-scoped push belongs to the channel its subject names. + const inboundSource: InboundSource = c.channelId + ? this.channelSource({ + serverId, + channelId: c.channelId, + threadId: origin.threadId, + messageId: origin.messageId, + eventId: params.eventId, + replyTo: origin.replyTo, + acceptedAt: event.acceptedAt, + sourceTimestamp: params.timestamp, + deferred: !!c.deferred, + }) + : this.pushSource(event, !!c.deferred); return this.pushCoalescer.accept({ serverId, binding: this.coalescingBinding(serverId), @@ -7761,7 +7942,7 @@ export class AgentFramework { ...(str(origin.authorId) ? { author: { id: str(origin.authorId)!, name: str(origin.authorName) ?? str(origin.authorId)! } } : {}), }, } : {}), - event: { lane: 'push', event }, + event: { lane: 'push', event: { ...event, inboundSource } }, }); }); } @@ -7835,6 +8016,12 @@ export class AgentFramework { ): Promise { const coalescingSubject = coalescingSubjectKey(occ.serverId, occ.binding, occ.scope, occ.key); const narrowing = occ.deliverTo ? { deliverTo: occ.deliverTo } : {}; + // The envelope frozen at acceptance travels unchanged; this delivery only + // adds the fact that it carries the rendered body. + const frozenSource = (source: InboundSource | undefined): { inboundSource?: InboundSource } => + source + ? { inboundSource: materialized && source.kind !== 'surface' ? { ...source, materialized: true } : source } + : {}; if (occ.event.lane === 'channel') { const event = { ...occ.event.event, coalescingSubject, ...narrowing, ...(assemblingFor ? { assemblingFor } : {}) }; const placement = await this.handleMcplChannelIncoming(event); @@ -7867,6 +8054,7 @@ export class AgentFramework { eventId: occ.eventId, ...narrowing, ...(assemblingFor ? { assemblingFor } : {}), + ...frozenSource(push.inboundSource), }; const placement = await this.handleMcplChannelIncoming(event); await this.dispatchProcessEventToModules(event as unknown as ProcessEvent); @@ -7882,6 +8070,7 @@ export class AgentFramework { // assembled; the wake that started it is not repeated. ...(assemblingFor ? { assemblingFor, triggerInference: false } : {}), ...(occ.deliverTo ? { targetAgents: [occ.deliverTo] } : {}), + ...frozenSource(occ.event.event.inboundSource), }; const placement = this.handleMcplPushEvent(event); await this.dispatchProcessEventToModules(event as unknown as ProcessEvent); @@ -8049,6 +8238,16 @@ export class AgentFramework { if (triggerChannel) { metadata.channelId = triggerChannel.channelId; } + // The inbound source envelope (mcpl/inbound-source.ts), after the + // adapter's origin fields so an adapter cannot supply it. + let source: InboundSource; + if (event.inboundSource) { + source = event.inboundSource; + } else { + source = this.pushSource(event); + this.noteInboundAccepted(source); + } + metadata[INBOUND_SOURCE_KEY] = source; const content = [...event.content]; if (triggerChannel) { diff --git a/src/index.ts b/src/index.ts index bd07d3547..5cc2c2461 100644 --- a/src/index.ts +++ b/src/index.ts @@ -83,6 +83,15 @@ export type { GateConfig, GateOptions, GatePolicy, GatePolicyMatch, GateBehavior // MCPL channel registry (exposed for modules that need channel-level operations) export { ChannelRegistry } from './mcpl/index.js'; +// Where each accepted inbound item came from (`metadata.inboundSource`) +export { INBOUND_SOURCE_KEY, readInboundSource, conversationKey } from './mcpl/index.js'; +export type { + InboundSource, + InboundChannelSource, + InboundUnscopedSource, + InboundSurfaceSource, +} from './mcpl/index.js'; + // MCPL server config (exposed for hosts that manage servers at runtime via // connectMcplServer / disconnectMcplServer / restartMcplServer) export type { McplServerConfig } from './mcpl/index.js'; diff --git a/src/mcpl/channel-registry.ts b/src/mcpl/channel-registry.ts index 9e31ef7e1..d999a6867 100644 --- a/src/mcpl/channel-registry.ts +++ b/src/mcpl/channel-registry.ts @@ -236,6 +236,9 @@ interface McplChannelIncomingEvent { tags?: string[]; triggerInference?: boolean; targetAgents?: string[]; + /** Host acceptance time (epoch ms), stamped where the message is admitted. + * The framework's inbound source envelope reads it; nothing re-stamps it. */ + acceptedAt?: number; } // ============================================================================ @@ -981,6 +984,7 @@ export class ChannelRegistry { metadata: message.metadata, ...(message.tags ? { tags: message.tags } : {}), triggerInference, + acceptedAt: Date.now(), }; if (coalesced) { @@ -1229,6 +1233,13 @@ export class ChannelRegistry { return this.findChannelEntry(channelId)?.descriptor; } + /** The registered label of `channelId` on exactly `serverId` (no + * cross-server first match), or undefined when it has none. */ + getChannelLabel(serverId: string, channelId: string): string | undefined { + const label = this.channels.get(`${serverId}:${channelId}`)?.descriptor.label; + return typeof label === 'string' && label.length > 0 ? label : undefined; + } + isChannelOpen(channelId: string): boolean { return this.findChannelEntry(channelId)?.open === true; } diff --git a/src/mcpl/inbound-source.ts b/src/mcpl/inbound-source.ts new file mode 100644 index 000000000..b7c2fea7d --- /dev/null +++ b/src/mcpl/inbound-source.ts @@ -0,0 +1,132 @@ +/** + * Where an inbound item came from, decided once at Agent Framework's + * ingestion boundary and stored on the message as `metadata.inboundSource`. + * + * Every later consumer reads this one value instead of re-deriving identity + * from the lane it happened to arrive on: speech routing compares + * conversations with it, the visible provenance header renders from it, and + * delivery bookkeeping keys on it. It is frozen when the host ACCEPTS the + * item. A coalesced occurrence (RFC-006) carries the value stamped at its + * acceptance through deferral, fan-out and replay, so a later rename, + * reconnect or re-render never rewrites where an old item says it came from. + * + * The field is framework-owned. It is written after any adapter-supplied + * metadata is spread, so an adapter cannot supply or spoof it (adapter + * `origin` objects already use a free-form `source` string, which is why this + * is not called `source`). + * + * Facts only, no policy: whether an item is a body, an edit or a deletion + * stays in its tags (`chat:edited`, `chat:deleted`); `deferred` and + * `materialized` record what RFC-006 did with it. Nothing here is a version + * key: `eventId` is present only when the producer supplied one. + */ + +/** An item that belongs to a registered channel. */ +export interface InboundChannelSource { + kind: 'channel'; + /** MCPL server (connection) id the item arrived through, as the host names it. */ + serverId: string; + /** RFC-006 endpoint binding of that connection at acceptance. A recipe that + * later points the same server id at a different endpoint gets a + * different binding, so old items never acquire the new endpoint's identity. */ + binding: string; + /** The registered canonical channel id (on the push lane: the composite id + * the host derived from `origin`, never the adapter's raw platform id). */ + channelId: string; + /** Thread within the channel, when the adapter supplied one. */ + threadId?: string; + /** The platform message this item is (or, for an edit/delete, refers to). */ + messageId?: string; + /** The producer's occurrence id, when it supplied one. */ + eventId?: string; + /** The channel's human label when the host accepted the item. */ + label?: string; + /** Message this one replies to, when the adapter supplied a reply edge. */ + replyTo?: string; + /** Host acceptance time (epoch ms). */ + acceptedAt: number; + /** The adapter's own timestamp for the item, verbatim. */ + sourceTimestamp?: string; + /** RFC-006: accepted as a deferred notification (body rendered later). */ + deferred?: true; + /** RFC-006: this delivery carries the materialized (rendered) body. */ + materialized?: true; +} + +/** An MCPL push that names no channel (heartbeats, timers, feature-set events). */ +export interface InboundUnscopedSource { + kind: 'unscoped'; + serverId: string; + binding: string; + eventId?: string; + acceptedAt: number; + sourceTimestamp?: string; + deferred?: true; + materialized?: true; +} + +/** Conversational input from a non-channel surface (console, TUI, API). */ +export interface InboundSurfaceSource { + kind: 'surface'; + /** The surface's name as its module reported it (e.g. `tui`, `api`). */ + surface: string; + acceptedAt: number; +} + +export type InboundSource = InboundChannelSource | InboundUnscopedSource | InboundSurfaceSource; + +/** + * Told once per inbound acceptance, with that acceptance's envelope: when an + * ordinary item reaches the framework, and when the RFC-006 coalescer admits + * an occurrence (deferred work included). Deliveries that reuse a frozen + * envelope (replay, fan-out, materialization, corrections) are not + * acceptances and are not reported. Observation only: the framework contains + * and reports a throw, which never affects delivery. + */ +export interface InboundAcceptanceObserver { + inboundAccepted(source: InboundSource): void; +} + +/** Metadata key the framework stamps. */ +export const INBOUND_SOURCE_KEY = 'inboundSource'; + +const str = (v: unknown): string | undefined => (typeof v === 'string' && v.length > 0 ? v : undefined); + +/** + * Read a stored item's source back from its metadata. Returns undefined for + * anything that is not a well-formed framework stamp (older messages, and any + * shape this version does not recognize) — never a guess. + */ +export function readInboundSource(metadata: unknown): InboundSource | undefined { + if (!metadata || typeof metadata !== 'object') return undefined; + const raw = (metadata as Record)[INBOUND_SOURCE_KEY]; + if (!raw || typeof raw !== 'object') return undefined; + const s = raw as Record; + if (typeof s.acceptedAt !== 'number') return undefined; + if (s.kind === 'channel') { + if (!str(s.serverId) || !str(s.binding) || !str(s.channelId)) return undefined; + return s as unknown as InboundChannelSource; + } + if (s.kind === 'unscoped') { + if (!str(s.serverId) || !str(s.binding)) return undefined; + return s as unknown as InboundUnscopedSource; + } + if (s.kind === 'surface') { + if (!str(s.surface)) return undefined; + return s as unknown as InboundSurfaceSource; + } + return undefined; +} + +/** + * The conversation an item belongs to, as a comparable key: server + channel + * + thread for a channel item, the surface for console input. Undefined for an + * unscoped push, which belongs to no conversation. + */ +export function conversationKey(source: InboundSource): string | undefined { + if (source.kind === 'channel') { + return `channel\u0000${source.serverId}\u0000${source.channelId}\u0000${source.threadId ?? ''}`; + } + if (source.kind === 'surface') return `surface\u0000${source.surface}`; + return undefined; +} diff --git a/src/mcpl/index.ts b/src/mcpl/index.ts index d82790c67..3b5fef3d5 100644 --- a/src/mcpl/index.ts +++ b/src/mcpl/index.ts @@ -157,6 +157,15 @@ export { InferenceRouter } from './inference-router.js'; // Channel registry (channel lifecycle, incoming messages, synthesized tools) export { ChannelRegistry } from './channel-registry.js'; +// Inbound source envelope: where each accepted item came from, stamped once +export { INBOUND_SOURCE_KEY, readInboundSource, conversationKey } from './inbound-source.js'; +export type { + InboundSource, + InboundChannelSource, + InboundUnscopedSource, + InboundSurfaceSource, +} from './inbound-source.js'; + // Per-channel conversation routing (fork-per-channel agents) /** @deprecated Per-channel conversation routing is deprecated and will be removed (anima-research/agent-framework#235). */ export { ConversationRouter, DEFAULT_CLOSURE_PROMPT } from './conversation-router.js'; diff --git a/src/mcpl/push-coalescer.ts b/src/mcpl/push-coalescer.ts index 9536962f7..1e1b1be6c 100644 --- a/src/mcpl/push-coalescer.ts +++ b/src/mcpl/push-coalescer.ts @@ -170,6 +170,10 @@ export interface CoalescerHost { /** Persist one receipt now, before the acceptance is acknowledged. Throws * on failure, which fails the acceptance (the producer retries). */ recordReceipt?(record: CoalescingReceiptRecord): void; + /** One call per ADMITTED occurrence (never for a producer retry the receipt + * deduplicates), after its receipt is recorded. Observation only: a throw + * is contained here and never fails the acceptance. */ + accepted?(occurrence: CoalescedOccurrence): void; /** Make every write so far DURABLE (fsync, group-committed by the host). * Awaited before an acceptance is acknowledged and before push/render is * issued: a kill after either boundary must find the receipt, the @@ -396,6 +400,11 @@ export class PushCoalescer { throw new CoalesceError('eventId', 'host could not persist the acceptance; retry', -32000); } this.receipts.set(receiptKey, entry); + try { + this.host.accepted?.(occurrence); + } catch (error) { + this.host.audit({ kind: 'accepted-observer-failed', subject, eventId: occurrence.eventId, error: String(error) }); + } this.prune(); this.persist(); // Written but not yet durable: the in-memory receipt deduplicates a retry diff --git a/src/mcpl/push-handler.ts b/src/mcpl/push-handler.ts index d85df767a..d6f9b39f6 100644 --- a/src/mcpl/push-handler.ts +++ b/src/mcpl/push-handler.ts @@ -19,6 +19,7 @@ import type { FeatureSetManager } from './feature-set-manager.js'; import { McplFeatureSetError } from './feature-set-manager.js'; import { expandCoreTags } from './tags.js'; import { validateCoalescedContent } from './push-coalescer.js'; +import type { InboundSource } from './inbound-source.js'; // ============================================================================ // McplPushEvent (the ProcessEvent shape pushed to the queue) @@ -47,6 +48,11 @@ export interface McplPushEvent { coalescingSubject?: string; /** RFC-006 assembly: materialized for this agent's turn (store directly). */ assemblingFor?: string; + /** Host acceptance time (epoch ms), stamped where the push is admitted. */ + acceptedAt?: number; + /** Framework-owned source envelope, frozen at a coalesced occurrence's + * acceptance so its later delivery cannot restamp it (inbound-source.ts). */ + inboundSource?: InboundSource; } // ============================================================================ @@ -286,6 +292,7 @@ export class PushHandler { timestamp: params.timestamp, inferenceId, triggerInference, + acceptedAt: Date.now(), }; if (coalesced) { // RFC-006: the coalescer decides whether this occurrence replaces an diff --git a/src/types/events.ts b/src/types/events.ts index b5e8afb8e..1c91b9592 100644 --- a/src/types/events.ts +++ b/src/types/events.ts @@ -231,6 +231,8 @@ export interface McplPushEvent { inferenceId: string; triggerInference?: boolean; targetAgents?: string[]; + /** Host acceptance time (epoch ms), stamped where the push is admitted. */ + acceptedAt?: number; } /** @@ -249,4 +251,6 @@ export interface McplChannelIncomingEvent { metadata?: Record; triggerInference?: boolean; targetAgents?: string[]; + /** Host acceptance time (epoch ms), stamped where the message is admitted. */ + acceptedAt?: number; } diff --git a/src/types/trace.ts b/src/types/trace.ts index bf7b2715d..c5c3064f5 100644 --- a/src/types/trace.ts +++ b/src/types/trace.ts @@ -485,6 +485,16 @@ export type TraceEvent = type: 'host:quiesced_boot'; reason?: string; since?: number; + }) + + // Inbound acceptance observation (mcpl/inbound-source.ts): an installed + // InboundAcceptanceObserver threw. Delivery is unaffected; the observer's + // own coverage is interrupted for that acceptance. + | (TraceEventBase & { + type: 'inbound:observer-failed'; + /** Envelope kind of the acceptance the observer failed on. */ + kind: 'channel' | 'unscoped' | 'surface'; + error: string; }); /** diff --git a/test/fixtures/speech-route-mcpl-server.mjs b/test/fixtures/speech-route-mcpl-server.mjs new file mode 100644 index 000000000..e4bffa711 --- /dev/null +++ b/test/fixtures/speech-route-mcpl-server.mjs @@ -0,0 +1,225 @@ +// MCPL fixture for speech routing, drafts and provenance tests. +// +// Registers the channels in CHANNELS (JSON array of {id,label}; two Discord +// guild channels by default) and, on command, sends inbound traffic over +// BOTH MCPL lanes. The TEST appends one JSON object per line to COMMAND_PATH: +// {"op":"incoming","channelId":…,"messageId":…,"mode":"ambient"|"addressed", +// "text":…, "threadId"?, "metadata"?, "eventId"?, "coalesce"?} +// {"op":"dm","eventId":…,"authorId":…,"authorName"?,"rawChannelId":…,"text":…} +// {"op":"push","eventId":…,"origin"?,"tags"?,"text":…,"featureSet"?} +// {"op":"rename","channelId":…,"label":…} (channels/changed) +// {"op":"publish-mode","mode":"delivered"|"not-delivered"|"no-receipt"|"error"|"hang"} +// {"op":"history","channelId":…,"history":[ChannelIncomingMessage…]} +// Host-side channels/publish and channels/open calls are recorded to +// STATUS_PATH as JSONL ({event:'publish', channelId, text, messageId}). +import { appendFileSync, existsSync, readFileSync } from 'node:fs'; +import { createInterface } from 'node:readline'; + +const statusPath = process.env.STATUS_PATH; +const commandPath = process.env.COMMAND_PATH; +const CHANNELS = process.env.CHANNELS + ? JSON.parse(process.env.CHANNELS) + : [ + { id: 'discord:g1:room', label: '#room (Guild One)' }, + { id: 'discord:g1:general', label: '#general (Guild One)' }, + ]; + +const log = (event, extra = {}) => { + if (!statusPath) return; + appendFileSync(statusPath, JSON.stringify({ event, ...extra }) + '\n'); +}; +const send = (message) => process.stdout.write(JSON.stringify(message) + '\n'); +const reply = (id, result) => send({ jsonrpc: '2.0', id, result }); +const replyError = (id, message) => send({ jsonrpc: '2.0', id, error: { code: -32000, message } }); + +let nextId = 500; +let processedCommands = 0; +let publishMode = 'delivered'; +let posted = 0; +const histories = new Map(); + +const textBlock = (text) => [{ type: 'text', text }]; + +function pollCommands() { + if (!commandPath || !existsSync(commandPath)) return; + const lines = readFileSync(commandPath, 'utf8').split('\n').filter(Boolean); + for (const line of lines.slice(processedCommands)) { + processedCommands++; + const c = JSON.parse(line); + if (c.op === 'incoming') { + log('incoming-sent', { channelId: c.channelId, messageId: c.messageId }); + send({ + jsonrpc: '2.0', + id: nextId++, + method: 'channels/incoming', + params: { + messages: [{ + channelId: c.channelId, + messageId: c.messageId, + ...(c.threadId ? { threadId: c.threadId } : {}), + ...(c.eventId ? { eventId: c.eventId } : {}), + ...(c.coalesce ? { coalesce: c.coalesce } : {}), + author: { id: c.authorId ?? 'U-other', name: c.authorName ?? 'someone' }, + timestamp: c.timestamp ?? new Date().toISOString(), + content: textBlock(c.text ?? ''), + ...(c.metadata ? { metadata: c.metadata } : {}), + tags: c.mode === 'addressed' + ? ['chat:mention', 'chat:addressed', 'chat:from-human'] + : ['chat:ambient', 'chat:from-human'], + }], + }, + }); + } else if (c.op === 'dm') { + log('dm-sent', { eventId: c.eventId }); + send({ + jsonrpc: '2.0', + id: nextId++, + method: 'push/event', + params: { + featureSet: 'chat', + eventId: c.eventId, + timestamp: c.timestamp ?? new Date().toISOString(), + origin: { + source: 'discord', + channelId: c.rawChannelId, + guildId: null, + isDM: true, + authorId: c.authorId, + authorName: c.authorName ?? 'antra', + messageId: c.messageId ?? c.eventId, + ...(c.origin ?? {}), + }, + tags: ['chat:dm', 'chat:addressed', 'chat:private'], + payload: { content: textBlock(c.text ?? '') }, + }, + }); + } else if (c.op === 'push') { + log('push-sent', { eventId: c.eventId }); + send({ + jsonrpc: '2.0', + id: nextId++, + method: 'push/event', + params: { + featureSet: c.featureSet ?? 'chat', + eventId: c.eventId, + timestamp: c.timestamp ?? new Date().toISOString(), + ...(c.origin ? { origin: c.origin } : {}), + ...(c.tags ? { tags: c.tags } : {}), + payload: { content: textBlock(c.text ?? '') }, + }, + }); + } else if (c.op === 'rename') { + const channel = CHANNELS.find((x) => x.id === c.channelId); + if (channel) channel.label = c.label; + send({ + jsonrpc: '2.0', + id: nextId++, + method: 'channels/changed', + params: { + updated: [{ id: c.channelId, type: 'discord', label: c.label, direction: 'bidirectional' }], + }, + }); + log('renamed', { channelId: c.channelId, label: c.label }); + } else if (c.op === 'publish-mode') { + publishMode = c.mode; + log('publish-mode', { mode: c.mode }); + } else if (c.op === 'history') { + histories.set(c.channelId, c.history); + } + } +} +const pollTimer = setInterval(pollCommands, 25); + +const rl = createInterface({ input: process.stdin }); +rl.on('close', () => { + clearInterval(pollTimer); + process.exit(0); +}); +rl.on('line', (line) => { + let msg; + try { + msg = JSON.parse(line); + } catch { + return; + } + if (msg.method === 'initialize') { + reply(msg.id, { + protocolVersion: '2024-11-05', + capabilities: { + experimental: { + mcpl: { + version: '0.5', + pushEvents: true, + channels: { register: true, lifecycle: true, incoming: true, publish: true }, + featureSets: { + chat: { description: 'chat connector', uses: ['pushEvents', 'channels.incoming', 'channels.publish'] }, + }, + }, + }, + }, + serverInfo: { name: 'speech-route-fixture', version: '0.0.0' }, + }); + return; + } + if (msg.method === 'featureSets/update') { + if (msg.id !== undefined && msg.id !== null) reply(msg.id, { accepted: true }); + return; + } + if (msg.method === 'notifications/initialized') { + send({ + jsonrpc: '2.0', + id: 400, + method: 'channels/register', + params: { + channels: CHANNELS.map((c) => ({ + id: c.id, type: 'discord', label: c.label, direction: 'bidirectional', initiallyOpen: true, + })), + }, + }); + return; + } + if (msg.method === 'channels/open') { + const channelId = msg.params?.channelId; + const channel = CHANNELS.find((x) => x.id === channelId) ?? { id: channelId, label: channelId }; + const history = msg.params?.history ? histories.get(channelId) : undefined; + log('channel-opened', { channelId, history: !!history }); + reply(msg.id, { + channel: { id: channel.id, type: 'discord', label: channel.label, direction: 'bidirectional' }, + ...(history ? { history } : {}), + }); + return; + } + if (msg.method === 'channels/close') { + reply(msg.id, { closed: true }); + return; + } + if (msg.method === 'channels/publish') { + const text = (msg.params?.content ?? []).filter((b) => b.type === 'text').map((b) => b.text).join('\n'); + const channelId = msg.params?.channelId; + if (publishMode === 'hang') { + log('publish-hung', { channelId, text }); + return; // never answered: the host's request times out + } + if (publishMode === 'error') { + log('publish-error', { channelId, text }); + if (msg.id !== undefined && msg.id !== null) replyError(msg.id, 'connector exploded'); + return; + } + if (publishMode === 'not-delivered') { + log('publish-refused', { channelId, text }); + if (msg.id !== undefined && msg.id !== null) reply(msg.id, { delivered: false }); + return; + } + const messageId = `posted-${++posted}`; + log('publish', { channelId, text, messageId }); + if (msg.id !== undefined && msg.id !== null) { + reply(msg.id, publishMode === 'no-receipt' ? {} : { delivered: true, messageId }); + } + return; + } + if (msg.method === 'tools/list') { + reply(msg.id, { tools: [] }); + return; + } + // Responses to our own requests (register, incoming, push) need no handling. +}); diff --git a/test/inbound-source.test.ts b/test/inbound-source.test.ts new file mode 100644 index 000000000..806b54a9c --- /dev/null +++ b/test/inbound-source.test.ts @@ -0,0 +1,261 @@ +/** + * The inbound source envelope (src/mcpl/inbound-source.ts), end to end over a + * real MCPL child: every accepted item is stamped once, at the framework's + * ingestion boundary, with the conversation it came from. Both MCPL lanes + * (channels/incoming and push/event), unscoped pushes, and console input + * from a module are covered; an adapter cannot supply or spoof the stamp, and + * an acceptance observer hears each acceptance exactly once. + */ +import { describe, it, beforeEach, afterEach } from 'node:test'; +import assert from 'node:assert/strict'; +import { tmpdir } from 'node:os'; +import { mkdtempSync, rmSync, readFileSync, existsSync, writeFileSync, appendFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { + AgentFramework, + INBOUND_SOURCE_KEY, + readInboundSource, + conversationKey, + type InboundSource, + type Module, + type ModuleContext, + type ProcessEvent, + type ProcessState, + type EventResponse, + type ToolDefinition, + type ToolCall, + type ToolResult, +} from '../src/index.js'; +import { MockMembrane } from './helpers/mock-membrane.js'; +import { fixture, eventually, TS } from './helpers/coalescing-fixture.js'; + +const FIXTURE = join(import.meta.dirname, 'fixtures/speech-route-mcpl-server.mjs'); +const ROOM = 'discord:g1:room'; +const RAW_DM = '1548000000000000001'; + +async function waitFor(cond: () => boolean, what: string, timeoutMs = 15_000): Promise { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + if (cond()) return; + await new Promise((r) => setTimeout(r, 20)); + } + throw new Error(`timed out waiting for: ${what}`); +} + +/** A console surface: the module that turns external-message into context. */ +class ConsoleModule implements Module { + readonly name = 'console'; + async start(_ctx: ModuleContext): Promise {} + async stop(): Promise {} + getTools(): ToolDefinition[] { return []; } + async handleToolCall(_call: ToolCall): Promise { return { success: true }; } + async onProcess(event: ProcessEvent, _state: ProcessState): Promise { + if (event.type !== 'external-message') return {}; + const text = String((event as { content?: unknown }).content); + return { addMessages: [{ participant: 'Operator', content: [{ type: 'text', text }] }] }; + } +} + +describe('inbound source envelope', () => { + let tempDir: string; + let commandPath: string; + let framework: AgentFramework; + let observed: InboundSource[]; + let n = 0; + + beforeEach(async () => { + tempDir = mkdtempSync(join(tmpdir(), 'inbound-source-')); + commandPath = join(tempDir, 'commands.jsonl'); + writeFileSync(commandPath, ''); + observed = []; + framework = await AgentFramework.create({ + storePath: join(tempDir, 'test.chronicle'), + membrane: new MockMembrane().asMembrane(), + agents: [{ name: 'scout', model: 'test-model', systemPrompt: 'You are scout.' }], + mcplServers: [{ + id: 'discord', + command: process.execPath, + args: [FIXTURE], + env: { STATUS_PATH: join(tempDir, 'status.jsonl'), COMMAND_PATH: commandPath }, + }], + modules: [new ConsoleModule()], + }); + (framework as unknown as { inboundAcceptanceObserver: unknown }).inboundAcceptanceObserver = { + inboundAccepted: (source: InboundSource) => observed.push(source), + }; + await framework.start(); + const registry = (framework as unknown as { channelRegistry: { listChannelsRaw(): unknown[] } }).channelRegistry; + await waitFor(() => registry.listChannelsRaw().length >= 2, 'channel registration'); + }); + + afterEach(async () => { + await framework.stop(); + rmSync(tempDir, { recursive: true, force: true }); + }); + + const command = (c: Record): void => appendFileSync(commandPath, JSON.stringify(c) + '\n'); + const stored = () => framework.getAgent('scout')!.getContextManager().getAllMessages(); + const storedWith = (pred: (m: Record) => boolean) => + stored().find((m) => pred((m.metadata ?? {}) as Record)); + + it('stamps a channels/incoming message with its registered channel, label, reply edge and acceptance time', async () => { + const before = Date.now(); + command({ + op: 'incoming', channelId: ROOM, messageId: 'm-1', mode: 'ambient', text: 'hello', + timestamp: '2026-10-07T01:02:03.000Z', + metadata: { + replyTo: 'm-0', + // An adapter cannot supply the framework's stamp. + inboundSource: { kind: 'channel', serverId: 'evil', binding: 'x', channelId: 'discord:g9:elsewhere', acceptedAt: 1 }, + }, + }); + await waitFor(() => !!storedWith((m) => m.messageId === 'm-1'), 'message stored'); + const message = storedWith((m) => m.messageId === 'm-1')!; + const source = readInboundSource(message.metadata); + assert.ok(source && source.kind === 'channel'); + assert.equal(source.serverId, 'discord'); + assert.match(source.binding, /^[0-9a-f]{16}$/); + assert.equal(source.channelId, ROOM); + assert.equal(source.messageId, 'm-1'); + assert.equal(source.label, '#room (Guild One)'); + assert.equal(source.replyTo, 'm-0'); + assert.equal(source.sourceTimestamp, '2026-10-07T01:02:03.000Z'); + assert.ok(source.acceptedAt >= before && source.acceptedAt <= Date.now()); + assert.deepEqual(observed, [source], 'observed exactly once, with the stored envelope'); + }); + + it('stamps a DM push with the derived composite channel and its people-first label', async () => { + command({ op: 'dm', eventId: `ev-${++n}`, authorId: '134', authorName: 'antra', rawChannelId: RAW_DM, text: 'psst' }); + await waitFor(() => !!storedWith((m) => readInboundSource(m)?.kind === 'channel'), 'dm stored'); + const source = readInboundSource(storedWith((m) => readInboundSource(m)?.kind === 'channel')!.metadata); + assert.ok(source && source.kind === 'channel'); + assert.equal(source.channelId, `discord:dm:${RAW_DM}`); + assert.equal(source.label, 'DM: antra'); + assert.equal(source.messageId, `ev-${n}`); + assert.equal(source.eventId, `ev-${n}`); + // The adapter's own free-form origin.source stays where it was. + assert.equal((storedWith((m) => m.eventId === `ev-${n}`)!.metadata as Record).source, 'discord'); + assert.equal(observed.length, 1); + }); + + it('stamps a push that names no channel as unscoped, never a guessed channel', async () => { + command({ op: 'push', eventId: 'tick-1', origin: { source: 'timer' }, text: 'tick' }); + await waitFor(() => !!storedWith((m) => m.eventId === 'tick-1'), 'push stored'); + const source = readInboundSource(storedWith((m) => m.eventId === 'tick-1')!.metadata); + assert.ok(source && source.kind === 'unscoped'); + assert.equal(source.serverId, 'discord'); + assert.equal(source.eventId, 'tick-1'); + assert.equal(conversationKey(source), undefined); + }); + + it('stamps console input from a module as a surface conversation', async () => { + framework.pushEvent({ type: 'external-message', source: 'tui', content: 'hi there', metadata: {} } as ProcessEvent); + await waitFor(() => stored().some((m) => m.participant === 'Operator'), 'console message stored'); + const message = stored().find((m) => m.participant === 'Operator')!; + const source = readInboundSource(message.metadata); + assert.deepEqual(source && { kind: source.kind, surface: (source as { surface: string }).surface }, { kind: 'surface', surface: 'tui' }); + assert.equal(conversationKey(source!), 'surface\u0000tui'); + assert.equal(observed.length, 1); + }); + + it('keeps a stored envelope when the channel is renamed later', async () => { + command({ op: 'incoming', channelId: ROOM, messageId: 'm-2', mode: 'ambient', text: 'before rename' }); + await waitFor(() => !!storedWith((m) => m.messageId === 'm-2'), 'first message stored'); + command({ op: 'rename', channelId: ROOM, label: '#lobby (Guild One)' }); + command({ op: 'incoming', channelId: ROOM, messageId: 'm-3', mode: 'ambient', text: 'after rename' }); + await waitFor(() => !!storedWith((m) => m.messageId === 'm-3'), 'second message stored'); + const first = readInboundSource(storedWith((m) => m.messageId === 'm-2')!.metadata); + const second = readInboundSource(storedWith((m) => m.messageId === 'm-3')!.metadata); + assert.equal(first?.kind === 'channel' && first.label, '#room (Guild One)'); + assert.equal(second?.kind === 'channel' && second.label, '#lobby (Guild One)'); + }); + + it('delivers normally when the acceptance observer throws, and reports the failure', async () => { + const traces: Array> = []; + framework.onTrace((e) => { if (e.type === 'inbound:observer-failed') traces.push(e as unknown as Record); }); + (framework as unknown as { inboundAcceptanceObserver: unknown }).inboundAcceptanceObserver = { + inboundAccepted: () => { throw new Error('ledger down'); }, + }; + command({ op: 'incoming', channelId: ROOM, messageId: 'm-4', mode: 'ambient', text: 'still here' }); + await waitFor(() => !!storedWith((m) => m.messageId === 'm-4'), 'message stored despite observer'); + assert.equal(traces.length, 1); + assert.equal(traces[0]!.kind, 'channel'); + assert.equal(traces[0]!.error, 'ledger down'); + }); +}); + +describe('readInboundSource', () => { + it('reads only well-formed framework stamps', () => { + const channel = { kind: 'channel', serverId: 's', binding: 'b', channelId: 'c', acceptedAt: 1 }; + assert.deepEqual(readInboundSource({ [INBOUND_SOURCE_KEY]: channel }), channel); + assert.equal(readInboundSource({ [INBOUND_SOURCE_KEY]: { ...channel, acceptedAt: 'soon' } }), undefined); + assert.equal(readInboundSource({ [INBOUND_SOURCE_KEY]: { ...channel, channelId: '' } }), undefined); + assert.equal(readInboundSource({ [INBOUND_SOURCE_KEY]: { kind: 'mystery', acceptedAt: 1 } }), undefined); + assert.equal(readInboundSource({ source: 'discord' }), undefined); + assert.equal(readInboundSource(undefined), undefined); + }); + + it('keys conversations by server, channel and thread', () => { + const base = { kind: 'channel' as const, serverId: 's', binding: 'b', channelId: 'c', acceptedAt: 1 }; + assert.notEqual(conversationKey(base), conversationKey({ ...base, serverId: 't' })); + assert.notEqual(conversationKey(base), conversationKey({ ...base, threadId: 'th' })); + assert.equal(conversationKey(base), conversationKey({ ...base, messageId: 'other', acceptedAt: 2 })); + }); +}); + +describe('inbound source envelope under RFC-006 coalescing', () => { + const install = (framework: AgentFramework): InboundSource[] => { + const observed: InboundSource[] = []; + (framework as unknown as { inboundAcceptanceObserver: unknown }).inboundAcceptanceObserver = { + inboundAccepted: (source: InboundSource) => observed.push(source), + }; + return observed; + }; + const sourceOf = (f: Awaited>, text: string): InboundSource | undefined => { + const message = f.framework.getAgent('agent')!.getContextManager().getAllMessages() + .find((m) => JSON.stringify(m.content).includes(text)); + return message ? readInboundSource(message.metadata) : undefined; + }; + + it('observes each acceptance once; a replacement carries its own frozen envelope and a retry is not an acceptance', async (t) => { + const f = await fixture(); t.after(f.close); + const observed = install(f.framework); + await f.send('push/event', f.params('1', 'edit_original', { initial: true })); + await f.send('push/event', f.params('2', 'edit_latest')); + await f.send('push/event', f.params('2', 'edit_latest')); + assert.deepEqual(observed.map((s) => (s as { eventId?: string }).eventId), ['1', '2']); + const stored = sourceOf(f, 'edit_latest'); + assert.ok(stored && stored.kind === 'unscoped'); + assert.deepEqual(stored, observed[1], 'the delivered copy carries the envelope frozen at its acceptance'); + }); + + it('observes deferred work at acceptance, before any render; its rendered delivery keeps that envelope and adds materialized', async (t) => { + const f = await fixture(); t.after(f.close); + const observed = install(f.framework); + await f.send('push/event', f.params('1', 'fallback_unavailable', { deferred: true })); + assert.equal(observed.length, 1, 'observed at acceptance, nothing rendered yet'); + assert.equal(f.renders.length, 0); + const accepted = observed[0]!; + assert.equal((accepted as { deferred?: boolean }).deferred, true); + await f.framework.runUntilIdle(); + assert.equal(f.renders.length, 1); + assert.equal(observed.length, 1, 'materialization is not a second acceptance'); + const stored = sourceOf(f, 'document_diff'); + assert.deepEqual(stored, { ...accepted, materialized: true }); + }); + + it('a channel-scoped coalesced message belongs to its channel, frozen at acceptance', async (t) => { + const f = await fixture(); t.after(f.close); + await f.register('chat'); + const observed = install(f.framework); + await f.send('channels/incoming', { messages: [f.channel('e1', 'first_words')] }); + await eventually(() => !!sourceOf(f, 'first_words'), 'coalesced message stored'); + const stored = sourceOf(f, 'first_words'); + assert.ok(stored && stored.kind === 'channel'); + assert.equal(stored.channelId, 'chat'); + assert.equal(stored.messageId, 'm'); + assert.equal(stored.eventId, 'e1'); + assert.equal(stored.sourceTimestamp, TS); + assert.deepEqual(observed, [stored]); + }); +}); diff --git a/test/trunk-channel-routing.test.ts b/test/trunk-channel-routing.test.ts index dc61fc56c..c9f79c75f 100644 --- a/test/trunk-channel-routing.test.ts +++ b/test/trunk-channel-routing.test.ts @@ -146,6 +146,7 @@ describe('Trunk channel routing (item-3 redux)', () => { i.channelRegistry = { ensureChannelRegistered: () => {}, isChannelOpen: () => false, + getChannelLabel: () => undefined, getDescriptor: () => ({ capabilities: { history: { maxMessages: 80 } } }), stopAll: () => {}, }; @@ -197,6 +198,7 @@ describe('Trunk channel routing (item-3 redux)', () => { i.channelRegistry = { ensureChannelRegistered: () => {}, isChannelOpen: () => false, + getChannelLabel: () => undefined, getDescriptor: () => ({ capabilities: { history: { maxMessages: 80 } } }), stopAll: () => {}, }; @@ -262,6 +264,7 @@ describe('Trunk channel routing (item-3 redux)', () => { i.channelRegistry = { ensureChannelRegistered: () => {}, isChannelOpen: () => false, + getChannelLabel: () => undefined, getDescriptor: () => ({ label: 'DM: _reim0n', capabilities: { history: { maxMessages: 80 } } }), proseTargetFor: (id: string, serverId?: string) => { asked.push(`${serverId}/${id}`); return '@_reim0n'; }, stopAll: () => {}, @@ -280,6 +283,7 @@ describe('Trunk channel routing (item-3 redux)', () => { i2.channelRegistry = { ensureChannelRegistered: () => {}, isChannelOpen: () => false, + getChannelLabel: () => undefined, getDescriptor: () => ({ label: 'DM: _reim0n', capabilities: { history: { maxMessages: 80 } } }), proseTargetFor: () => undefined, stopAll: () => {}, @@ -297,6 +301,7 @@ describe('Trunk channel routing (item-3 redux)', () => { i3.channelRegistry = { ensureChannelRegistered: () => {}, isChannelOpen: () => false, + getChannelLabel: () => undefined, getDescriptor: () => undefined, stopAll: () => {}, }; From eb2a588b1f28a81ea0f7e7fbe5a35edd0cbe9412 Mon Sep 17 00:00:00 2001 From: Petra-1012 Date: Wed, 7 Oct 2026 05:32:48 -0400 Subject: [PATCH 0023/1112] fix(mcpl): freeze ordinary inbound envelopes at admission; add lane facts Review findings on 3f7aafe from Hugo-1004 (room-220 #41587) and a request from Oscar-1005 (#41589). Freeze at admission. An ordinary (uncoalesced) channels/incoming message or push/event used to get only `acceptedAt` at admission; its envelope was built later, in handleMcplChannelIncoming/handleMcplPushEvent, from the registry's CURRENT label and endpoint binding, and the acceptance was observed there. An item accepted (and acknowledged) before a rename or rebind could therefore acquire the new label or binding while it waited in the queue, and a receipt clock read stale until the queue drained. ChannelRegistry.handleIncoming and PushHandler now call a framework-supplied `acceptInbound` at admission, before queueing and before the acknowledgement: the framework builds the envelope and observes the acceptance there, and the event carries the frozen envelope to its handler, as coalesced work already did. The handlers only fall back to building (and observing) an envelope for a direct caller that bypassed admission. Lane facts. Channel and unscoped envelopes now carry `lane` ('channels/incoming' | 'push/event') and `coalesced: true` for RFC-006 admission. What an eventId is worth depends on the admitting lane's contract (push/event dedups by eventId; coalesced admission guarantees stable retries and distinct versions; an uncoalesced channels/incoming eventId is adapter-supplied with no guarantee), and its mere presence cannot say which. Facts only: the receipt policy that uses them stays with the receipt lane. Strict reader. readInboundSource validates every field it returns: identity strings non-empty, acceptedAt finite, deferred/materialized/coalesced exactly true when present, a known lane, and refuses arrays. Tests: both ordinary lanes hold an admitted item between admission and processing (by intercepting the framework's queue), assert the observer ran at admission with the original label, rename the channel, release, and assert the stored envelope equals the admission envelope; lane/coalesced facts asserted on ordinary and coalesced items; the reader refuses each malformed field. Suite (machine load average ~40 during the run): all files except code-execution 1274/1275 with code-execution separately 36/36. The one failure is mcpl-coalescing's wall-clock bound (#196 vector 34c, 300 fsynced coalesced acceptances < 30 s): 40.0 s here, and 40.4 s for the same test on an untouched 64c480b build under the same load, so it is load, not this change (it passed at 24.1 s on 3f7aafe at lower load). The full-suite PyRunner "rejects concurrent execs" test hung twice under this load in the combined run and passes alone. Co-Authored-By: Claude Opus 5.5 --- changelog.d/inbound-source-envelope.added.md | 7 ++ src/framework.ts | 69 ++++++++++--- src/mcpl/channel-registry.ts | 21 +++- src/mcpl/inbound-source.ts | 48 ++++++--- src/mcpl/push-handler.ts | 9 ++ test/inbound-source.test.ts | 100 +++++++++++++++++-- 6 files changed, 220 insertions(+), 34 deletions(-) diff --git a/changelog.d/inbound-source-envelope.added.md b/changelog.d/inbound-source-envelope.added.md index 30db418ea..9afd3c00c 100644 --- a/changelog.d/inbound-source-envelope.added.md +++ b/changelog.d/inbound-source-envelope.added.md @@ -10,3 +10,10 @@ and a later rename never rewrites it. Adapter-supplied metadata cannot set the field (an adapter's own `origin.source` string is untouched). `readInboundSource`, `conversationKey` and `INBOUND_SOURCE_KEY` are exported. +- **Inbound source envelope: admission facts.** An ordinary `channels/incoming` + message or `push/event` now has its envelope frozen at admission, before it + is queued or acknowledged, so a rename or rebind while it waits cannot + change it. Every channel and unscoped envelope names the admission `lane` + (`channels/incoming` or `push/event`) and whether it came through RFC-006 + coalesced admission (`coalesced: true`): facts a consumer needs to know what + an `eventId` is worth. `readInboundSource` validates every field it returns. diff --git a/src/framework.ts b/src/framework.ts index 3934b1f9d..b6f771746 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -83,7 +83,7 @@ import { detectKnownToolWrapperProse } from './tool-wrapper-prose-guard.js'; import { InferenceRouter } from './mcpl/inference-router.js'; import { ChannelRegistry, type ChannelToolOrigin } from './mcpl/channel-registry.js'; import { ConversationRouter, CONVERSATION_ROUTING_DEPRECATION_NOTICE } from './mcpl/conversation-router.js'; -import { INBOUND_SOURCE_KEY, type InboundAcceptanceObserver, type InboundChannelSource, type InboundSource, type InboundUnscopedSource } from './mcpl/inbound-source.js'; +import { INBOUND_SOURCE_KEY, type InboundAcceptanceObserver, type InboundChannelSource, type InboundLane, type InboundSource, type InboundUnscopedSource } from './mcpl/inbound-source.js'; import { safeSlice } from './safe-slice.js'; import type { WorkspaceModule } from './modules/workspace/index.js'; import { @@ -7106,16 +7106,9 @@ export class AgentFramework { if (event.inboundSource) { source = event.inboundSource; } else { - source = this.channelSource({ - serverId: event.serverId, - channelId: event.channelId, - threadId: event.threadId, - messageId: event.messageId, - eventId: event.eventId, - replyTo: event.metadata?.replyTo, - acceptedAt: event.acceptedAt, - sourceTimestamp: event.timestamp, - }); + // Only a caller that bypassed admission (none in the framework's own + // lanes) arrives without a frozen envelope. + source = this.channelEventSource(event, 'channels/incoming'); this.noteInboundAccepted(source); } metadata[INBOUND_SOURCE_KEY] = source; @@ -7654,6 +7647,8 @@ export class AgentFramework { * stamped (a direct framework caller), which is the closest observation. */ private channelSource(fields: { + lane: InboundLane; + coalesced?: boolean; serverId: string; channelId: string; threadId?: unknown; @@ -7674,6 +7669,8 @@ export class AgentFramework { const sourceTimestamp = str(fields.sourceTimestamp); return { kind: 'channel', + lane: fields.lane, + ...(fields.coalesced ? { coalesced: true as const } : {}), serverId: fields.serverId, binding: this.coalescingBinding(fields.serverId), channelId: fields.channelId, @@ -7688,12 +7685,38 @@ export class AgentFramework { }; } + /** Source envelope of a channels/incoming event (lane facts as given). */ + private channelEventSource( + event: { + serverId: string; channelId: string; threadId?: string; messageId?: string; eventId?: string; + metadata?: Record; acceptedAt?: number; timestamp?: string; + }, + lane: InboundLane, + coalesced = false, + ): InboundChannelSource { + return this.channelSource({ + lane, + coalesced, + serverId: event.serverId, + channelId: event.channelId, + threadId: event.threadId, + messageId: event.messageId, + eventId: event.eventId, + replyTo: event.metadata?.replyTo, + acceptedAt: event.acceptedAt, + sourceTimestamp: event.timestamp, + }); + } + /** Source envelope of a push event: its derived channel, else unscoped. */ - private pushSource(event: McplPushEvent, deferred = false): InboundSource { + private pushSource(event: McplPushEvent, coalesced: { deferred: boolean } | undefined = undefined): InboundSource { const origin = (event.origin ?? {}) as Record; + const deferred = coalesced?.deferred ?? false; const channel = this.derivePushEventChannel(event.origin); if (channel) { return this.channelSource({ + lane: 'push/event', + coalesced: !!coalesced, serverId: event.serverId, channelId: channel.channelId, threadId: origin.threadId, @@ -7708,6 +7731,8 @@ export class AgentFramework { } const unscoped: InboundUnscopedSource = { kind: 'unscoped', + lane: 'push/event', + ...(coalesced ? { coalesced: true as const } : {}), serverId: event.serverId, binding: this.coalescingBinding(event.serverId), ...(typeof event.eventId === 'string' && event.eventId ? { eventId: event.eventId } : {}), @@ -7867,6 +7892,8 @@ export class AgentFramework { // The inbound source envelope, frozen at acceptance: deferral, fan-out // and replay deliver it unchanged (mcpl/inbound-source.ts). const inboundSource = this.channelSource({ + lane: 'channels/incoming', + coalesced: true, serverId, channelId: message.channelId, threadId: message.threadId, @@ -7910,6 +7937,8 @@ export class AgentFramework { // lane). A channel-scoped push belongs to the channel its subject names. const inboundSource: InboundSource = c.channelId ? this.channelSource({ + lane: 'push/event', + coalesced: true, serverId, channelId: c.channelId, threadId: origin.threadId, @@ -7920,7 +7949,7 @@ export class AgentFramework { sourceTimestamp: params.timestamp, deferred: !!c.deferred, }) - : this.pushSource(event, !!c.deferred); + : this.pushSource(event, { deferred: !!c.deferred }); return this.pushCoalescer.accept({ serverId, binding: this.coalescingBinding(serverId), @@ -13183,6 +13212,13 @@ export class AgentFramework { (event) => this.emitTrace(event as { type: TraceEvent['type']; [key: string]: unknown }), triggerFilter, (serverId, params, event) => this.handleCoalescedPush(serverId, params, event), + // An ordinary push: freeze its envelope and observe the acceptance at + // admission, before it is queued. + (event) => { + const source = this.pushSource(event); + this.noteInboundAccepted(source); + return source; + }, ); // Server-initiated inference router (Step 6) @@ -13214,6 +13250,13 @@ export class AgentFramework { store: this.store, handleCoalescedIncoming: (serverId, message, event) => this.handleCoalescedIncoming(serverId, message, event as CoalescedChannelEvent), + // An ordinary channels/incoming message: freeze its envelope and + // observe the acceptance at admission, before it is queued. + acceptInbound: (event) => { + const source = this.channelEventSource(event, 'channels/incoming'); + this.noteInboundAccepted(source); + return source; + }, sendTypingFn: (serverId, channelId, metadata, op) => { const server = this.mcplServerRegistry!.getServer(serverId); if (server) { diff --git a/src/mcpl/channel-registry.ts b/src/mcpl/channel-registry.ts index d999a6867..34912447b 100644 --- a/src/mcpl/channel-registry.ts +++ b/src/mcpl/channel-registry.ts @@ -39,6 +39,7 @@ import type { ToolDefinition, ToolResult, ProcessEvent } from '../types/index.js import { expandCoreTags } from './tags.js'; import { validateCoalescedContent } from './push-coalescer.js'; import { CapabilityGrant } from './capability-grant.js'; +import type { InboundSource } from './inbound-source.js'; // ============================================================================ // Typing indicator interval (Discord typing lasts ~10s, so 7s keeps it alive) @@ -236,9 +237,10 @@ interface McplChannelIncomingEvent { tags?: string[]; triggerInference?: boolean; targetAgents?: string[]; - /** Host acceptance time (epoch ms), stamped where the message is admitted. - * The framework's inbound source envelope reads it; nothing re-stamps it. */ + /** Host acceptance time (epoch ms), stamped where the message is admitted. */ acceptedAt?: number; + /** Source envelope frozen at admission (mcpl/inbound-source.ts). */ + inboundSource?: InboundSource; } // ============================================================================ @@ -457,6 +459,13 @@ const CHANNEL_TOOL_DEFINITIONS: ToolDefinition[] = [ // ============================================================================ interface ChannelRegistryOptions { + /** + * An ordinary (uncoalesced) `channels/incoming` message has just been + * admitted, before it is queued or acknowledged: return its source + * envelope, frozen now (mcpl/inbound-source.ts). The framework observes the + * acceptance here; coalesced work is stamped and observed by its own path. + */ + acceptInbound?: (event: McplChannelIncomingEvent) => InboundSource | undefined; /** * RFC-006: an admitted `channels/incoming` message carrying `coalesce`, with * the event the ordinary path would have queued. The handler decides @@ -660,6 +669,7 @@ export class ChannelRegistry { private legacyPolicies = new Map(); private migratedLegacyPolicies = new Set(); private handleCoalescedIncoming?: ChannelRegistryOptions['handleCoalescedIncoming']; + private acceptInbound?: ChannelRegistryOptions['acceptInbound']; constructor( serverRegistry: McplServerRegistry, @@ -676,6 +686,7 @@ export class ChannelRegistry { }, ) { this.handleCoalescedIncoming = options?.handleCoalescedIncoming; + this.acceptInbound = options?.acceptInbound; this.serverRegistry = serverRegistry; this.featureSetManager = featureSetManager; this.pushEventFn = pushEventFn; @@ -1007,6 +1018,12 @@ export class ChannelRegistry { continue; } + // The source envelope is frozen here, at admission, before the message + // is queued or acknowledged: a rename or rebind while it waits in the + // queue cannot change where it says it came from. + const inboundSource = this.acceptInbound?.(event); + if (inboundSource) event.inboundSource = inboundSource; + // Push to the processing queue // Cast through unknown because McplChannelIncomingEvent matches the // CustomEvent `${string}:${string}` type pattern but lacks an index signature. diff --git a/src/mcpl/inbound-source.ts b/src/mcpl/inbound-source.ts index b7c2fea7d..29c2e65e0 100644 --- a/src/mcpl/inbound-source.ts +++ b/src/mcpl/inbound-source.ts @@ -21,9 +21,20 @@ * key: `eventId` is present only when the producer supplied one. */ +/** The MCPL admission lane that accepted an item. Its contract decides what + * an `eventId` is worth: `push/event` deduplicates by eventId, and RFC-006 + * coalesced admission (`coalesced: true`) guarantees stable retries and + * distinct versions; a `channels/incoming` eventId outside coalescing is + * adapter-supplied with no such guarantee. */ +export type InboundLane = 'channels/incoming' | 'push/event'; + /** An item that belongs to a registered channel. */ export interface InboundChannelSource { kind: 'channel'; + /** The admission lane that accepted it (see InboundLane). */ + lane: InboundLane; + /** Accepted through RFC-006 coalesced admission. */ + coalesced?: true; /** MCPL server (connection) id the item arrived through, as the host names it. */ serverId: string; /** RFC-006 endpoint binding of that connection at acceptance. A recipe that @@ -56,6 +67,8 @@ export interface InboundChannelSource { /** An MCPL push that names no channel (heartbeats, timers, feature-set events). */ export interface InboundUnscopedSource { kind: 'unscoped'; + lane: 'push/event'; + coalesced?: true; serverId: string; binding: string; eventId?: string; @@ -90,30 +103,39 @@ export interface InboundAcceptanceObserver { /** Metadata key the framework stamps. */ export const INBOUND_SOURCE_KEY = 'inboundSource'; -const str = (v: unknown): string | undefined => (typeof v === 'string' && v.length > 0 ? v : undefined); +const isText = (v: unknown): v is string => typeof v === 'string' && v.length > 0; +const optionalText = (v: unknown): boolean => v === undefined || isText(v); +const optionalTrue = (v: unknown): boolean => v === undefined || v === true; /** - * Read a stored item's source back from its metadata. Returns undefined for - * anything that is not a well-formed framework stamp (older messages, and any - * shape this version does not recognize) — never a guess. + * Read a stored item's source back from its metadata, validating every field + * a consumer relies on: identity strings are non-empty strings, `acceptedAt` + * is a finite number, flags are exactly `true` when present, and the lane is + * one this version knows. Anything else (older messages, a shape this + * version does not recognize, a damaged import) is undefined — never a guess. */ export function readInboundSource(metadata: unknown): InboundSource | undefined { if (!metadata || typeof metadata !== 'object') return undefined; const raw = (metadata as Record)[INBOUND_SOURCE_KEY]; - if (!raw || typeof raw !== 'object') return undefined; + if (!raw || typeof raw !== 'object' || Array.isArray(raw)) return undefined; const s = raw as Record; - if (typeof s.acceptedAt !== 'number') return undefined; + if (typeof s.acceptedAt !== 'number' || !Number.isFinite(s.acceptedAt)) return undefined; + if (s.kind === 'surface') { + return isText(s.surface) ? (s as unknown as InboundSurfaceSource) : undefined; + } + const effectFlags = optionalTrue(s.coalesced) && optionalTrue(s.deferred) && optionalTrue(s.materialized); + if (!isText(s.serverId) || !isText(s.binding) || !effectFlags) return undefined; + if (!optionalText(s.eventId) || !optionalText(s.sourceTimestamp)) return undefined; if (s.kind === 'channel') { - if (!str(s.serverId) || !str(s.binding) || !str(s.channelId)) return undefined; + if (s.lane !== 'channels/incoming' && s.lane !== 'push/event') return undefined; + if (!isText(s.channelId)) return undefined; + for (const field of ['threadId', 'messageId', 'label', 'replyTo'] as const) { + if (!optionalText(s[field])) return undefined; + } return s as unknown as InboundChannelSource; } if (s.kind === 'unscoped') { - if (!str(s.serverId) || !str(s.binding)) return undefined; - return s as unknown as InboundUnscopedSource; - } - if (s.kind === 'surface') { - if (!str(s.surface)) return undefined; - return s as unknown as InboundSurfaceSource; + return s.lane === 'push/event' ? (s as unknown as InboundUnscopedSource) : undefined; } return undefined; } diff --git a/src/mcpl/push-handler.ts b/src/mcpl/push-handler.ts index d6f9b39f6..6a903ab00 100644 --- a/src/mcpl/push-handler.ts +++ b/src/mcpl/push-handler.ts @@ -183,6 +183,10 @@ export class PushHandler { emitTraceFn: (event: { type: string; [key: string]: unknown }) => void, shouldTriggerInference?: (content: string, metadata: Record) => boolean, private readonly handleCoalesced?: (serverId: string, params: PushEventParams, event: McplPushEvent) => Promise, + /** An ordinary (uncoalesced) push was just admitted, before it is queued + * or acknowledged: return its source envelope, frozen now + * (mcpl/inbound-source.ts). Coalesced work is stamped by its own path. */ + private readonly acceptInbound?: (event: McplPushEvent) => InboundSource | undefined, ) { this.featureSetManager = featureSetManager; this.pushEventFn = pushEventFn; @@ -310,6 +314,11 @@ export class PushHandler { } return; } + // The source envelope is frozen at admission, before queueing or the + // acknowledgement: nothing that changes while the push waits in the + // queue can rewrite where it came from. + const inboundSource = this.acceptInbound?.(pushEvent); + if (inboundSource) pushEvent.inboundSource = inboundSource; this.pushEventFn(pushEvent); // 7. Emit trace diff --git a/test/inbound-source.test.ts b/test/inbound-source.test.ts index 806b54a9c..5516268dc 100644 --- a/test/inbound-source.test.ts +++ b/test/inbound-source.test.ts @@ -114,6 +114,8 @@ describe('inbound source envelope', () => { const source = readInboundSource(message.metadata); assert.ok(source && source.kind === 'channel'); assert.equal(source.serverId, 'discord'); + assert.equal(source.lane, 'channels/incoming'); + assert.equal(source.coalesced, undefined); assert.match(source.binding, /^[0-9a-f]{16}$/); assert.equal(source.channelId, ROOM); assert.equal(source.messageId, 'm-1'); @@ -170,6 +172,62 @@ describe('inbound source envelope', () => { assert.equal(second?.kind === 'channel' && second.label, '#lobby (Guild One)'); }); + /** + * Hold MCPL events between admission and processing (the framework's + * queue), so a test can change the world while an accepted item waits. + */ + const holdQueue = () => { + const held: ProcessEvent[] = []; + const original = framework.pushEvent.bind(framework); + (framework as unknown as { pushEvent: (e: ProcessEvent) => void }).pushEvent = (event: ProcessEvent) => { + if (event.type === 'mcpl:channel-incoming' || event.type === 'mcpl:push-event') held.push(event); + else original(event); + }; + return { + held, + release: () => { + (framework as unknown as { pushEvent: (e: ProcessEvent) => void }).pushEvent = original; + for (const event of held.splice(0)) original(event); + }, + }; + }; + const registryLabel = (channelId: string) => + (framework as unknown as { channelRegistry: { getChannelLabel(s: string, c: string): string | undefined } }) + .channelRegistry.getChannelLabel('discord', channelId); + + it('freezes an ordinary channels/incoming envelope at admission: observed before queueing, and a rename while queued does not change it', async () => { + const queue = holdQueue(); + command({ op: 'incoming', channelId: ROOM, messageId: 'm-10', mode: 'ambient', text: 'queued words' }); + await waitFor(() => queue.held.length === 1, 'message admitted and held in the queue'); + assert.equal(observed.length, 1, 'observed at admission, before processing'); + assert.equal(observed[0]!.kind === 'channel' && observed[0]!.label, '#room (Guild One)'); + assert.equal(observed[0]!.kind === 'channel' && observed[0]!.lane, 'channels/incoming'); + command({ op: 'rename', channelId: ROOM, label: '#lobby (Guild One)' }); + await waitFor(() => registryLabel(ROOM) === '#lobby (Guild One)', 'rename applied'); + queue.release(); + await waitFor(() => !!storedWith((m) => m.messageId === 'm-10'), 'message stored'); + assert.deepEqual(readInboundSource(storedWith((m) => m.messageId === 'm-10')!.metadata), observed[0]); + assert.equal(observed.length, 1, 'processing is not a second acceptance'); + }); + + it('freezes an ordinary push envelope at admission: a rename while queued does not change it', async () => { + const queue = holdQueue(); + command({ + op: 'push', eventId: 'ev-room', text: 'pushed into the room', + origin: { source: 'discord', mcplChannelId: ROOM, messageId: 'pm-1', authorName: 'someone' }, + tags: ['chat:mention', 'chat:addressed'], + }); + await waitFor(() => queue.held.length === 1, 'push admitted and held in the queue'); + assert.equal(observed.length, 1); + assert.equal(observed[0]!.kind === 'channel' && observed[0]!.lane, 'push/event'); + assert.equal(observed[0]!.kind === 'channel' && observed[0]!.label, '#room (Guild One)'); + command({ op: 'rename', channelId: ROOM, label: '#lobby (Guild One)' }); + await waitFor(() => registryLabel(ROOM) === '#lobby (Guild One)', 'rename applied'); + queue.release(); + await waitFor(() => !!storedWith((m) => m.eventId === 'ev-room'), 'push stored'); + assert.deepEqual(readInboundSource(storedWith((m) => m.eventId === 'ev-room')!.metadata), observed[0]); + }); + it('delivers normally when the acceptance observer throws, and reports the failure', async () => { const traces: Array> = []; framework.onTrace((e) => { if (e.type === 'inbound:observer-failed') traces.push(e as unknown as Record); }); @@ -185,18 +243,44 @@ describe('inbound source envelope', () => { }); describe('readInboundSource', () => { + const channel = { kind: 'channel', lane: 'channels/incoming', serverId: 's', binding: 'b', channelId: 'c', acceptedAt: 1 }; + const read = (value: unknown) => readInboundSource({ [INBOUND_SOURCE_KEY]: value }); + it('reads only well-formed framework stamps', () => { - const channel = { kind: 'channel', serverId: 's', binding: 'b', channelId: 'c', acceptedAt: 1 }; - assert.deepEqual(readInboundSource({ [INBOUND_SOURCE_KEY]: channel }), channel); - assert.equal(readInboundSource({ [INBOUND_SOURCE_KEY]: { ...channel, acceptedAt: 'soon' } }), undefined); - assert.equal(readInboundSource({ [INBOUND_SOURCE_KEY]: { ...channel, channelId: '' } }), undefined); - assert.equal(readInboundSource({ [INBOUND_SOURCE_KEY]: { kind: 'mystery', acceptedAt: 1 } }), undefined); + assert.deepEqual(read(channel), channel); + assert.deepEqual(read({ ...channel, coalesced: true, deferred: true, materialized: true, eventId: 'e', label: '#c' }), + { ...channel, coalesced: true, deferred: true, materialized: true, eventId: 'e', label: '#c' }); + assert.deepEqual(read({ kind: 'unscoped', lane: 'push/event', serverId: 's', binding: 'b', acceptedAt: 1 }), + { kind: 'unscoped', lane: 'push/event', serverId: 's', binding: 'b', acceptedAt: 1 }); assert.equal(readInboundSource({ source: 'discord' }), undefined); assert.equal(readInboundSource(undefined), undefined); }); + it('refuses a stamp with any field its consumers rely on malformed', () => { + for (const bad of [ + { ...channel, acceptedAt: 'soon' }, + { ...channel, acceptedAt: Number.POSITIVE_INFINITY }, + { ...channel, acceptedAt: Number.NaN }, + { ...channel, channelId: '' }, + { ...channel, lane: 'carrier-pigeon' }, + { ...channel, lane: undefined }, + { ...channel, deferred: false }, + { ...channel, materialized: 'yes' }, + { ...channel, coalesced: 1 }, + { ...channel, messageId: 42 }, + { ...channel, label: '' }, + { ...channel, sourceTimestamp: {} }, + { kind: 'unscoped', lane: 'channels/incoming', serverId: 's', binding: 'b', acceptedAt: 1 }, + { kind: 'surface', surface: '', acceptedAt: 1 }, + { kind: 'mystery', acceptedAt: 1 }, + [channel], + ]) { + assert.equal(read(bad), undefined, JSON.stringify(bad)); + } + }); + it('keys conversations by server, channel and thread', () => { - const base = { kind: 'channel' as const, serverId: 's', binding: 'b', channelId: 'c', acceptedAt: 1 }; + const base = { kind: 'channel' as const, lane: 'channels/incoming' as const, serverId: 's', binding: 'b', channelId: 'c', acceptedAt: 1 }; assert.notEqual(conversationKey(base), conversationKey({ ...base, serverId: 't' })); assert.notEqual(conversationKey(base), conversationKey({ ...base, threadId: 'th' })); assert.equal(conversationKey(base), conversationKey({ ...base, messageId: 'other', acceptedAt: 2 })); @@ -226,6 +310,8 @@ describe('inbound source envelope under RFC-006 coalescing', () => { assert.deepEqual(observed.map((s) => (s as { eventId?: string }).eventId), ['1', '2']); const stored = sourceOf(f, 'edit_latest'); assert.ok(stored && stored.kind === 'unscoped'); + assert.equal(stored.lane, 'push/event'); + assert.equal(stored.coalesced, true); assert.deepEqual(stored, observed[1], 'the delivered copy carries the envelope frozen at its acceptance'); }); @@ -253,6 +339,8 @@ describe('inbound source envelope under RFC-006 coalescing', () => { const stored = sourceOf(f, 'first_words'); assert.ok(stored && stored.kind === 'channel'); assert.equal(stored.channelId, 'chat'); + assert.equal(stored.lane, 'channels/incoming'); + assert.equal(stored.coalesced, true); assert.equal(stored.messageId, 'm'); assert.equal(stored.eventId, 'e1'); assert.equal(stored.sourceTimestamp, TS); From 7144970b3370b7b6f020a1e29a80df5f8d1562e6 Mon Sep 17 00:00:00 2001 From: Petra-1012 Date: Wed, 7 Oct 2026 05:45:51 -0400 Subject: [PATCH 0024/1112] feat(mcpl): type MCPL request failures by what reached the server MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A failed request to an MCPL server used to reject with a plain Error, so the only way to tell "never written" from "written and unanswered" from "answered with an error" was to match message text. Callers that act in the world need that difference as a fact: a publish or a reaction that was never sent left the world as it was, while one that was handed to the transport and never answered may have landed. (A rollback reaction drain met exactly this: its pending reactions failed at shutdown, and only their message text said whether each had been written.) McplRequestError extends Error with `outcome`: - `not-sent`: sendRequest refused before writing (connection already closed); - `error-response`: a JSON-RPC error response; `code` and `data` kept verbatim (a connector can describe a partial multi-part post there); - `no-response`: the request was handed to the transport and no answer came (per-request timeout, a deliberate close while awaiting, or an unexpected disconnect). It does not prove the request reached the server — the transport can decline or drop a write — only that the host can no longer say it didn't. Messages are byte-identical to before, and every rejection is still an Error. Exported from the mcpl index and the package root. Tests (test/mcpl-request-outcome.test.ts, a real stdio server): an error response keeps code and data; an unanswered request and a process exit while awaiting are no-response; a request on a closed connection is not-sent; the class keeps Error identity. 12/12 with the existing request-timeout tests. Reviewed by Tessa-974 (room-203 #43780), whose wording correction for no-response this carries. Shared with room-225's lane 1 (Ada-1017). Co-Authored-By: Claude Opus 5.5 --- changelog.d/mcpl-request-outcome.added.md | 10 +++ src/index.ts | 2 +- src/mcpl/index.ts | 2 +- src/mcpl/server-connection.ts | 45 +++++++++-- test/mcpl-request-outcome.test.ts | 92 +++++++++++++++++++++++ 5 files changed, 144 insertions(+), 7 deletions(-) create mode 100644 changelog.d/mcpl-request-outcome.added.md create mode 100644 test/mcpl-request-outcome.test.ts diff --git a/changelog.d/mcpl-request-outcome.added.md b/changelog.d/mcpl-request-outcome.added.md new file mode 100644 index 000000000..241ec226e --- /dev/null +++ b/changelog.d/mcpl-request-outcome.added.md @@ -0,0 +1,10 @@ +- **`McplRequestError`: why an MCPL request produced no result.** A request + to an MCPL server that fails now rejects with `McplRequestError` (exported), + whose `outcome` says which of three things happened: `not-sent` (refused + before anything was written: the connection was already closed), + `error-response` (the server answered with a JSON-RPC error; `code` and the + error's `data` are kept), or `no-response` (the request was handed to the + transport and no answer came back: a timeout, or the connection closed + while awaiting; it may or may not have reached the server). Only + `not-sent` proves the server never saw the request. Messages are unchanged, + and it is still an `Error`, so existing handling keeps working. diff --git a/src/index.ts b/src/index.ts index bd07d3547..e6bf2a6b3 100644 --- a/src/index.ts +++ b/src/index.ts @@ -90,7 +90,7 @@ export type { McplServerConfig } from './mcpl/index.js'; // MCPL client connection (exposed so external tooling — server playtests, // health probes, protocol harnesses — can dial an MCPL server exactly the // way the framework host does, over stdio or WebSocket) -export { McplServerConnection } from './mcpl/index.js'; +export { McplServerConnection, McplRequestError } from './mcpl/index.js'; export type { McplHostCapabilities } from './mcpl/index.js'; // MCPL tool lifecycle (RFC-007) and tool classes (RFC-008): the types a host diff --git a/src/mcpl/index.ts b/src/mcpl/index.ts index d82790c67..9033077ec 100644 --- a/src/mcpl/index.ts +++ b/src/mcpl/index.ts @@ -106,7 +106,7 @@ export { } from './errors.js'; // Server connection and registry -export { McplServerConnection } from './server-connection.js'; +export { McplServerConnection, McplRequestError } from './server-connection.js'; export { McplServerRegistry, type McplCapabilityQuery } from './server-registry.js'; // Feature set management (permission layer) diff --git a/src/mcpl/server-connection.ts b/src/mcpl/server-connection.ts index f29fea22d..18f82d096 100644 --- a/src/mcpl/server-connection.ts +++ b/src/mcpl/server-connection.ts @@ -62,6 +62,34 @@ const DEFAULT_REQUEST_TIMEOUT_MS = 60_000; /** MCP protocol version used in the initialize handshake. */ const MCP_PROTOCOL_VERSION = '2024-11-05'; +/** + * Why a request to an MCPL server produced no result, as far as the host can + * know it. Messages are unchanged from plain errors; the `outcome` makes the + * difference between "never left the host" and "left, and no answer came + * back" a fact a caller can act on instead of a string to match: + * - `not-sent`: the request was refused before anything was written (the + * connection was already closed); + * - `error-response`: the server answered with a JSON-RPC error (it may have + * acted before failing; the error alone does not say); + * - `no-response`: the request was handed to the transport and no answer + * came back (timeout, or the connection closed while awaiting). It does + * not prove the request reached the server — the transport can decline or + * drop a write — only that the host can no longer say it didn't. + */ +export class McplRequestError extends Error { + constructor( + message: string, + readonly outcome: 'not-sent' | 'error-response' | 'no-response', + readonly code?: number, + /** The JSON-RPC error's `data`, verbatim, when the server supplied one + * (e.g. which parts of a multi-part send were posted). */ + readonly data?: unknown, + ) { + super(message); + this.name = 'McplRequestError'; + } +} + /** * Represents a pending JSON-RPC request awaiting a response. */ @@ -846,7 +874,7 @@ export class McplServerConnection extends EventEmitter { // Reject all pending requests for (const [id, pending] of this.pendingRequests) { if (pending.timer) clearTimeout(pending.timer); - pending.reject(new Error(`Connection to MCPL server "${this.id}" closed while awaiting response for ${pending.method} (id=${id})`)); + pending.reject(new McplRequestError(`Connection to MCPL server "${this.id}" closed while awaiting response for ${pending.method} (id=${id})`, 'no-response')); } this.pendingRequests.clear(); this.orphanedRequests.clear(); @@ -973,7 +1001,7 @@ export class McplServerConnection extends EventEmitter { options: RequestOptions = {}, ): Promise { if (this.closed) { - return Promise.reject(new Error(`Cannot send request: connection to "${this.id}" is closed`)); + return Promise.reject(new McplRequestError(`Cannot send request: connection to "${this.id}" is closed`, 'not-sent')); } const id = this.nextRequestId++; @@ -994,13 +1022,14 @@ export class McplServerConnection extends EventEmitter { this.orphanedRequests.set(id, { method, ...(method === 'push/render' ? { renderParams: { featureSet: params.featureSet, key: params.key, eventId: params.eventId } } : {}) }); if (this.orphanedRequests.size > 4096) this.orphanedRequests.delete(this.orphanedRequests.keys().next().value!); } - reject(new Error( + reject(new McplRequestError( `MCPL server "${this.id}" did not respond to ${method} (id=${id}) ` + `within ${timeoutMs}ms — the server may be hung. The response ` + `outcome is unknown: the request was abandoned locally but was not ` + `cancelled, so the tool may still have completed server-side; verify ` + `state before retrying; a blind retry of a stateful/side-effecting ` + `tool may duplicate it.`, + 'no-response', )); }, timeoutMs); // Don't hold the event loop open for the watchdog alone. @@ -1215,7 +1244,12 @@ export class McplServerConnection extends EventEmitter { if (response.error) { pending.reject( - new Error(`MCPL server "${this.id}" returned error for ${pending.method}: [${response.error.code}] ${response.error.message}`), + new McplRequestError( + `MCPL server "${this.id}" returned error for ${pending.method}: [${response.error.code}] ${response.error.message}`, + 'error-response', + typeof response.error.code === 'number' ? response.error.code : undefined, + (response.error as { data?: unknown }).data, + ), ); } else { pending.resolve(response.result); @@ -1267,8 +1301,9 @@ export class McplServerConnection extends EventEmitter { for (const [id, pending] of this.pendingRequests) { if (pending.timer) clearTimeout(pending.timer); pending.reject( - new Error( + new McplRequestError( `MCPL server "${this.id}" disconnected unexpectedly (code=${info.code ?? 'n/a'}, signal=${info.signal ?? 'n/a'}, reason=${info.reason ?? 'unknown'}) while awaiting ${pending.method} (id=${id})`, + 'no-response', ), ); } diff --git a/test/mcpl-request-outcome.test.ts b/test/mcpl-request-outcome.test.ts new file mode 100644 index 000000000..1b0f45647 --- /dev/null +++ b/test/mcpl-request-outcome.test.ts @@ -0,0 +1,92 @@ +/** + * McplRequestError: a request that produced no result says which of three + * things happened, as a fact rather than a message to parse. + * not-sent refused before anything was written (connection closed) + * error-response the server answered with a JSON-RPC error (code, data kept) + * no-response the request was handed to the transport and no answer + * came back (timeout, or the connection closed while + * awaiting): it may or may not have reached the server + * Callers that act in the world (publishing speech, placing reactions) need + * the difference: only `not-sent` proves the server never saw the request. + */ +import { test, afterEach } from 'node:test'; +import assert from 'node:assert/strict'; + +import { McplServerConnection, McplRequestError } from '../src/mcpl/server-connection.js'; +import type { McplHostCapabilities, McplServerConfig } from '../src/mcpl/types.js'; + +// Stdio server: answers initialize; tools/call 'fail' gets a JSON-RPC error +// with data, 'stuck' gets nothing, 'exit' kills the process mid-request. +const SERVER = ` +let buf = ''; +process.stdin.on('data', (c) => { + buf += c.toString('utf8'); + let i; + while ((i = buf.indexOf('\\n')) >= 0) { + const line = buf.slice(0, i); buf = buf.slice(i + 1); + if (!line.trim()) continue; + let m; try { m = JSON.parse(line); } catch { continue; } + const out = (o) => process.stdout.write(JSON.stringify({ jsonrpc: '2.0', id: m.id, ...o }) + '\\n'); + if (m.method === 'initialize') out({ result: { capabilities: {} } }); + else if (m.method === 'tools/call' && m.params.name === 'fail') { + out({ error: { code: -32001, message: 'POSTED: part 1 of 2', data: { posted: ['111'] } } }); + } else if (m.method === 'tools/call' && m.params.name === 'exit') { + process.exit(3); + } + // 'stuck': deliberately no response. + } +}); +setInterval(() => {}, 1 << 30); +`; + +const HOST_CAPS: McplHostCapabilities = { version: '0.4', pushEvents: true, contextHooks: { beforeInference: true }, featureSets: true }; +const config = (overrides?: Partial): McplServerConfig => + ({ id: 'outcome', command: process.execPath, args: ['-e', SERVER], ...overrides }); + +let connection: McplServerConnection | null = null; +afterEach(async () => { + await connection?.close(); + connection = null; +}); + +const rejectsWith = async (promise: Promise, outcome: McplRequestError['outcome']): Promise => { + let caught: unknown; + await assert.rejects(promise, (err: unknown) => { caught = err; return true; }); + assert.ok(caught instanceof McplRequestError, `expected McplRequestError, got ${String(caught)}`); + assert.equal((caught as McplRequestError).outcome, outcome); + return caught as McplRequestError; +}; + +test('an error response is error-response, with its code and data kept', async () => { + connection = await McplServerConnection.connect(config(), HOST_CAPS); + const err = await rejectsWith(connection.sendToolsCall('fail', {}), 'error-response'); + assert.equal(err.code, -32001); + assert.deepEqual(err.data, { posted: ['111'] }); + assert.match(err.message, /returned error for tools\/call: \[-32001\] POSTED: part 1 of 2/); +}); + +test('a request nobody answers is no-response, and the message is unchanged', async () => { + connection = await McplServerConnection.connect(config({ requestTimeoutMs: 200 }), HOST_CAPS); + const err = await rejectsWith(connection.sendToolsCall('stuck', {}), 'no-response'); + assert.match(err.message, /did not respond to tools\/call/); +}); + +test('a connection lost while awaiting is no-response', async () => { + connection = await McplServerConnection.connect(config({ requestTimeoutMs: 10_000 }), HOST_CAPS); + await rejectsWith(connection.sendToolsCall('exit', {}), 'no-response'); +}); + +test('a request on a closed connection is not-sent', async () => { + connection = await McplServerConnection.connect(config(), HOST_CAPS); + await connection.close(); + const err = await rejectsWith(connection.sendToolsCall('fail', {}), 'not-sent'); + assert.match(err.message, /Cannot send request: connection to "outcome" is closed/); + connection = null; +}); + +test('McplRequestError is an Error that keeps its message and name', () => { + const err = new McplRequestError('boom', 'not-sent'); + assert.ok(err instanceof Error); + assert.equal(err.name, 'McplRequestError'); + assert.equal(err.message, 'boom'); +}); From 2c591a7d9574a0ddfc3da57419ccae4138d4e45d Mon Sep 17 00:00:00 2001 From: Petra-1012 Date: Wed, 7 Oct 2026 06:01:35 -0400 Subject: [PATCH 0025/1112] feat(mcpl): one publish executor with structured delivery outcomes ChannelRegistry.publish(conversationId, text, {serverId?, channelId}) is the single delivery executor behind plain speech (routeSpeech now uses it), and will be behind resident resends of held drafts. It returns a PublishOutcome: - `delivered`: the connector returned `delivered: true` (with its messageId). A historical receipt, not a claim the message still exists. - `failed`: provably nothing posted. Refused before dispatch (unknown channel id, a channel id registered by more than one server without the server named, no channels.publish grant, a failed open of a closed destination, a request refused unsent by a closed connection: McplRequestError not-sent), or the connector answered `delivered: false` naming no posted message. - `unknown`: dispatched, and no valid confirmation came back: a JSON-RPC error response (which can follow a partial multi-part post, discord-mcpl #35), a timeout, a lost connection, `delivered: false` that names a posted message, or a missing/malformed receipt (#163's rule: a missing receipt is no longer counted as delivered). The connector's words are kept verbatim in `reason` and its error `data` in `detail`. Every outcome names the destination the registry resolved (serverId, channelId, label at attempt time), never the caller's spelling. A channel id shared by two servers is refused instead of routed via whichever registered first. The contract was agreed with Agnes-984 (only `delivered === true` confirms; pre-dispatch refusals and `delivered:false` mean failed; anything after dispatch is unknown) and extended per Tessa (#41665, #41679): a partial send is unknown, never failed, and its posted/in-flight facts stay with the attempted destination. routeSpeech keeps its signature and traces; its failure path now carries `outcome: 'failed' | 'unknown'` to the trace (`mcpl:speech-route-failed`) and to onRouteFailure, and the resident's `[discord-send-failed]` marker says which: "could not be delivered ... Nothing was posted" versus "was not confirmed ... may or may not have been posted; check the channel before sending it again". The marker metadata carries `outcome`. Tests: test/publish-outcome.test.ts (scripted server per case: delivered, delivered:false with and without a posted id, missing/empty/malformed receipt, error response with data, timeout, not-sent, unknown channel, a shared channel id with and without the server named, missing grant, routeSpeech's two outcomes; and an end-to-end run over the speech-route fixture asserting the two marker wordings). 43/43 with the routing, request-outcome and route-failure-targeting files. Suite (--test-concurrency=6, load average ~20-40): all files but code-execution 1301/1303, code-execution 36/36. The two failures are mcpl-coalescing timing tests that fail identically on an untouched 64c480b build in this environment: #196 vector 34c's wall-clock bound, and R17's 40 ms storage-outage backoff, which counted a fourth recovery wake; running the whole coalescing file on 64c480b reproduced the same R17 failure. Co-Authored-By: Claude Opus 5.5 --- changelog.d/publish-outcomes.fixed.md | 16 ++ src/framework.ts | 10 +- src/mcpl/channel-registry.ts | 299 ++++++++++++++++++-------- src/types/trace.ts | 3 + test/publish-outcome.test.ts | 213 ++++++++++++++++++ 5 files changed, 454 insertions(+), 87 deletions(-) create mode 100644 changelog.d/publish-outcomes.fixed.md create mode 100644 test/publish-outcome.test.ts diff --git a/changelog.d/publish-outcomes.fixed.md b/changelog.d/publish-outcomes.fixed.md new file mode 100644 index 000000000..62eb206a7 --- /dev/null +++ b/changelog.d/publish-outcomes.fixed.md @@ -0,0 +1,16 @@ +- **Speech delivery reports what a publish actually established.** Plain + speech now goes through one publish executor (`ChannelRegistry.publish`, + returning a `PublishOutcome`: `delivered`, `failed` or `unknown`, with the + destination the registry resolved). Only a connector's `delivered: true` + confirms a post; a missing or malformed receipt is no longer counted as + delivered (#163's rule). `failed` means nothing was posted: refused before + dispatch, or `delivered: false` naming no posted message. An error response, + a timeout or a lost connection after dispatch is `unknown`, keeping the + connector's own words (a multi-part send may already be partly posted). + The resident's `[discord-send-failed]` marker now says which: "could not be + delivered … Nothing was posted" for a failure, and "was not confirmed … may + or may not have been posted; check the channel before sending it again" + for an uncertain outcome. The marker's metadata and the + `mcpl:speech-route-failed` trace carry `outcome`. A channel id registered + by more than one MCPL server is refused rather than routed through + whichever server registered it first. diff --git a/src/framework.ts b/src/framework.ts index b6f771746..931af6bff 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -13286,7 +13286,7 @@ export class AgentFramework { // addMessage() alone does not request inference, so this never wakes // her (matching the `discord-send-failed-skip` gate intent: context // yes, wake no). - onRouteFailure: ({ channelId, reason, textLen }) => { + onRouteFailure: ({ channelId, reason, textLen, outcome }) => { try { // Render a human-readable channel name when we can — a bare // snowflake in the marker is unresolvable for the agent (the @@ -13305,9 +13305,13 @@ export class AgentFramework { 'user', [{ type: 'text', - text: `[discord-send-failed] Your previous reply (${textLen} chars) could not be delivered to ${where} (${reason}). It was saved to your archive but the human did not receive it.`, + text: outcome === 'unknown' + // A request went out and no valid receipt came back: it may + // have been posted (in part or whole). Never claim it wasn't. + ? `[discord-send-failed] Delivery of your previous reply (${textLen} chars) to ${where} was not confirmed (${reason}). It may or may not have been posted; check the channel before sending it again. It is saved in your archive.` + : `[discord-send-failed] Your previous reply (${textLen} chars) could not be delivered to ${where} (${reason}). Nothing was posted; it is saved in your archive.`, }], - { system: true, kind: 'discord-send-failed', channelId: channelId ?? '', reason }, + { system: true, kind: 'discord-send-failed', channelId: channelId ?? '', reason, ...(outcome ? { outcome } : {}) }, ); } catch (err) { console.error('onRouteFailure: failed to record send-failure marker:', err); diff --git a/src/mcpl/channel-registry.ts b/src/mcpl/channel-registry.ts index 34912447b..8333c860a 100644 --- a/src/mcpl/channel-registry.ts +++ b/src/mcpl/channel-registry.ts @@ -40,6 +40,7 @@ import { expandCoreTags } from './tags.js'; import { validateCoalescedContent } from './push-coalescer.js'; import { CapabilityGrant } from './capability-grant.js'; import type { InboundSource } from './inbound-source.js'; +import { McplRequestError } from './server-connection.js'; // ============================================================================ // Typing indicator interval (Discord typing lasts ~10s, so 7s keeps it alive) @@ -458,6 +459,41 @@ const CHANNEL_TOOL_DEFINITIONS: ToolDefinition[] = [ // Constructor Options // ============================================================================ +/** Where a publish was resolved to go — named by the registry, never by + * the caller's spelling. */ +export interface PublishDestination { + serverId: string; + channelId: string; + /** The channel's registered label when the attempt was made. */ + label?: string; +} + +/** + * What one channels/publish attempt established (ChannelRegistry.publish). + * - `delivered`: the connector returned `delivered: true`. A historical + * receipt (confirmed at `at`, to `destination`, message `messageId` when + * the connector named one), not a claim the message still exists. + * - `failed`: nothing was posted: the request never left the host + * (unresolvable or shared channel id, missing grant, failed open, closed + * connection), or the connector answered `delivered: false`. + * - `unknown`: the request was dispatched and no valid receipt came back (an + * error response, a timeout, a lost connection, or a missing or malformed + * receipt). It may or may not have been posted. + */ +export interface PublishOutcome { + status: 'delivered' | 'failed' | 'unknown'; + /** Absent only when no destination could be resolved. */ + destination?: PublishDestination; + messageId?: string; + /** Why it failed, or why its outcome is unknown — for a connector error, + * its message verbatim (which may say what was already posted). */ + reason?: string; + /** The connector error's structured `data`, verbatim, when it sent one. */ + detail?: unknown; + /** When the outcome was established (epoch ms). */ + at: number; +} + interface ChannelRegistryOptions { /** * An ordinary (uncoalesced) `channels/incoming` message has just been @@ -494,6 +530,9 @@ interface ChannelRegistryOptions { channelId: string | null; reason: string; textLen: number; + /** `failed`: nothing was posted; `unknown`: the request was dispatched + * and no valid receipt came back, so it may or may not have been. */ + outcome?: 'failed' | 'unknown'; }) => void; /** * Called when channels were opened WITHOUT the agent asking (subscription @@ -569,12 +608,7 @@ export class ChannelRegistry { op?: 'start' | 'stop', ) => void; private shouldTriggerInference?: (content: string, metadata: Record) => boolean; - private onRouteFailure?: (info: { - conversationId: string; - channelId: string | null; - reason: string; - textLen: number; - }) => void; + private onRouteFailure?: ChannelRegistryOptions['onRouteFailure']; private onChannelAutoOpened?: (info: { conversationId?: string; serverId: string; @@ -2968,6 +3002,161 @@ export class ChannelRegistry { return home ?? this.activeChannelResolver?.(conversationId) ?? this.defaultPublishChannel ?? null; } + /** + * Publish `text` to one registered channel and report what the attempt + * established (PublishOutcome). The single delivery executor behind plain + * speech (routeSpeech), resident resends of held drafts, and channel tools. + * + * Resolution is exact: `serverId` plus `channelId` when the caller has the + * server, otherwise the channel id must be registered by exactly one server + * (a shared id is refused, never routed through whichever server came + * first). It is not possible to send into a closed channel: a closed but + * registered destination is opened first (`openSource` names why), and a + * failed open sends nothing. + * + * Only the connector's `delivered: true` confirms a post. Anything that + * stops before the request is written is `failed` (nothing left the host), + * as is the connector's own `delivered: false`. Once the request has been + * dispatched, an error response, a timeout, a lost connection, or a missing + * or malformed receipt is `unknown`: the post may or may not exist, and the + * caller must not treat it as safe to retry blindly. + */ + async publish( + conversationId: string, + text: string, + target: { serverId?: string; channelId: string }, + openSource: 'opened-by-delivery' | 'opened-by-reply' = 'opened-by-delivery', + ): Promise { + const at = (): number => Date.now(); + const matches = [...this.channels.values()].filter( + (e) => e.descriptor.id === target.channelId && (!target.serverId || e.serverId === target.serverId), + ); + if (matches.length === 0) { + return { + status: 'failed', + reason: target.serverId + ? `no registered channel "${target.channelId}" on server "${target.serverId}"` + : `no registered channel "${target.channelId}"`, + at: at(), + }; + } + if (matches.length > 1) { + return { + status: 'failed', + reason: `channel id "${target.channelId}" is registered by more than one MCPL server; the destination must name its server`, + at: at(), + }; + } + const entry = matches[0]!; + const destination: PublishDestination = { + serverId: entry.serverId, + channelId: entry.descriptor.id, + ...(entry.descriptor.label ? { label: entry.descriptor.label } : {}), + }; + + if (!entry.open) { + try { + await this.openChannelNow(entry, openSource); + this.emitTraceFn({ + type: 'mcpl:channel-opened-by-send', + serverId: entry.serverId, + channelId: entry.descriptor.id, + }); + if (openSource === 'opened-by-delivery') { + try { + this.onChannelAutoOpened?.({ + conversationId, + serverId: entry.serverId, + source: 'opened-by-delivery', + channels: [{ channelId: entry.descriptor.id, label: entry.descriptor.label }], + }); + } catch (err) { + console.error('onChannelAutoOpened (delivery) failed:', err); + } + } + } catch (err) { + return { + status: 'failed', + destination, + reason: `channel is closed and open failed: ${(err as Error).message}`, + at: at(), + }; + } + } + + const server = this.serverRegistry.getServer(entry.serverId); + if (!server) { + return { status: 'failed', destination, reason: `server "${entry.serverId}" not found`, at: at() }; + } + // §14.1: channels/publish requires channels.publish in the grant. The + // host not sending is the enforcement. + if (!CapabilityGrant.of(server).has('channels.publish')) { + return { + status: 'failed', + destination, + reason: `channels.publish not in "${entry.serverId}"'s effective grant (§14.1)`, + at: at(), + }; + } + + let result: unknown; + try { + result = await server.sendChannelsPublish({ + conversationId, + channelId: entry.descriptor.id, + content: [{ type: 'text', text }], + }); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + // Only a request that never left the host proves nothing was posted. + // An error response may follow a partial post (a multi-part send whose + // later part failed), so it is unknown, with the connector's own words + // (and data) kept for the resident to judge. + const notSent = err instanceof McplRequestError && err.outcome === 'not-sent'; + const detail = err instanceof McplRequestError ? err.data : undefined; + return { + status: notSent ? 'failed' : 'unknown', + destination, + reason: notSent ? message : `${message} (delivery uncertain)`, + ...(detail !== undefined ? { detail } : {}), + at: at(), + }; + } + const receipt = (result ?? {}) as { delivered?: unknown; messageId?: unknown }; + const messageId = typeof receipt.messageId === 'string' && receipt.messageId ? receipt.messageId : undefined; + if (receipt.delivered === true) { + return { status: 'delivered', destination, ...(messageId ? { messageId } : {}), at: at() }; + } + if (receipt.delivered === false) { + // `delivered: false` proves nothing went out only when the receipt + // names no posted message; a named one means something did. + return messageId + ? { + status: 'unknown', + destination, + messageId, + reason: `server "${entry.serverId}" reported delivered:false but named posted message ${messageId} (delivery uncertain)`, + at: at(), + } + : { + status: 'failed', + destination, + reason: `server "${entry.serverId}" reported delivered:false`, + at: at(), + }; + } + // Final publish is a request, so success requires an explicit receipt; a + // missing or malformed one leaves delivery uncertain (anima-research/ + // agent-framework#163's rule). + return { + status: 'unknown', + destination, + ...(messageId ? { messageId } : {}), + reason: `server "${entry.serverId}" returned no valid delivery receipt (delivery uncertain)`, + at: at(), + }; + } + async routeSpeech( conversationId: string, text: string, @@ -2981,18 +3170,20 @@ export class ChannelRegistry { locusChannelId: string | null, ): Promise<{ delivered: boolean; channelId: string; messageId?: string } | null> { // Surface a routing failure: emit a trace AND notify the host (which drops - // a `[discord-send-failed]` marker into chronicle) so the agent learns her - // reply never reached the human, instead of it vanishing silently. - const fail = (channelId: string | null, reason: string): null => { - console.error(`[routeSpeech] ${conversationId}: ${reason} — speech NOT routed (${text.length} chars stay in chronicle)`); + // a `[discord-send-failed]` marker into chronicle) so the agent learns its + // reply did not (or may not have) reached anyone, instead of it vanishing + // silently. + const fail = (channelId: string | null, reason: string, outcome: 'failed' | 'unknown' = 'failed'): null => { + console.error(`[routeSpeech] ${conversationId}: ${reason} — speech ${outcome === 'unknown' ? 'delivery NOT confirmed' : 'NOT routed'} (${text.length} chars stay in chronicle)`); this.emitTraceFn({ type: 'mcpl:speech-route-failed', conversationId, channelId: channelId ?? '', reason, textLen: text.length, + outcome, }); - this.onRouteFailure?.({ conversationId, channelId, reason, textLen: text.length }); + this.onRouteFailure?.({ conversationId, channelId, reason, textLen: text.length, outcome }); return null; }; @@ -3002,95 +3193,35 @@ export class ChannelRegistry { if (locusChannelId === undefined) { return fail(null, 'caller passed no locus (routing bug: every speech path must snapshot the turn locus)'); } - const channelId = locusChannelId; - if (!channelId) { + if (!locusChannelId) { // The turn froze with no locus (no home, no triggering channel, no // global inbound ever seen) — the agent was told its prose stays in // the archive; honor that. return fail(null, 'turn has no locus (no home/trigger channel; nothing to deliver into)'); } - const entry = this.findChannelEntry(channelId); - if (!entry) { - return fail(channelId, `no registered channel for locus "${channelId}"`); + const outcome = await this.publish(conversationId, text, { channelId: locusChannelId }); + const channelId = outcome.destination?.channelId ?? locusChannelId; + if (outcome.status !== 'delivered') { + return fail(channelId, outcome.reason ?? 'delivery not confirmed', outcome.status); } - // INVARIANT: it is not possible to send into a closed channel. Speech - // routed into a closed-but-registered locus (the canonical case: a DM - // that woke the agent — DMs register closed) OPENS the channel first, - // so typing indicators, reaction machinery, and inbound forwarding come - // alive with the reply. If the open fails, the speech does NOT go out: - // a half-alive delivery is worse than a loud marker. - if (!entry.open) { - try { - await this.openChannelNow(entry, 'opened-by-delivery'); - console.error( - `[routeSpeech] ${conversationId}: locus ${channelId} was closed — opened by delivery (speech implies engagement)`, - ); - this.emitTraceFn({ - type: 'mcpl:channel-opened-by-send', - serverId: entry.serverId, - channelId, - }); - try { - this.onChannelAutoOpened?.({ - conversationId, - serverId: entry.serverId, - source: 'opened-by-delivery', - channels: [{ channelId, label: entry.descriptor.label }], - }); - } catch (err) { - console.error('onChannelAutoOpened (delivery) failed:', err); - } - } catch (err) { - return fail(channelId, `locus channel is closed and open failed: ${(err as Error).message}`); - } - } - - const server = this.serverRegistry.getServer(entry.serverId); - if (!server) { - return fail(channelId, `server "${entry.serverId}" not found`); - } - // §14.1: channels/publish requires channels.publish in the grant. The - // host not sending is the enforcement — a send to an ungranted server - // would have it act on authority it was never told it has. - if (!CapabilityGrant.of(server).has('channels.publish')) { - return fail(channelId, `channels.publish not in "${entry.serverId}"'s effective grant (§14.1)`); - } - - const publishParams: ChannelsPublishParams = { - conversationId, - channelId, - content: [{ type: 'text', text }], - }; - const result = await server.sendChannelsPublish(publishParams); - const delivered = (result as { delivered?: boolean } | undefined)?.delivered ?? true; - // Surface the posted message's id (ChannelsPublishResult.messageId) so - // trace consumers can act on the just-posted message — e.g. a TTS-relay - // tap editing it down to the words actually voiced on interruption. - // Previously this was silently dropped here. - const messageId = (result as { messageId?: string } | undefined)?.messageId; - - // The server accepted the publish RPC but reported the message was not - // actually delivered (e.g. missing Send Messages permission). Previously - // this returned `{ delivered: true }`, masking the failure. Surface it. - if (delivered === false) { - return fail(channelId, `server "${entry.serverId}" reported delivered:false for "${channelId}"`); - } - - console.error(`[routeSpeech] ${conversationId}: routed ${text.length} chars -> ${channelId} (server=${entry.serverId}, delivered=${delivered})`); + console.error(`[routeSpeech] ${conversationId}: routed ${text.length} chars -> ${channelId} (server=${outcome.destination!.serverId}, delivered=true)`); this.emitTraceFn({ type: 'mcpl:speech-routed', conversationId, - serverId: entry.serverId, + serverId: outcome.destination!.serverId, channelId, - delivered, + delivered: true, textLen: text.length, text, - ...(messageId !== undefined ? { messageId } : {}), + // Surface the posted message's id (ChannelsPublishResult.messageId) so + // trace consumers can act on the just-posted message — e.g. a TTS-relay + // tap editing it down to the words actually voiced on interruption. + ...(outcome.messageId !== undefined ? { messageId: outcome.messageId } : {}), }); - return { delivered, channelId, ...(messageId !== undefined ? { messageId } : {}) }; + return { delivered: true, channelId, ...(outcome.messageId !== undefined ? { messageId: outcome.messageId } : {}) }; } private async handleToolPublish(input: { channelId?: string; content?: string; text?: string }): Promise { diff --git a/src/types/trace.ts b/src/types/trace.ts index c5c3064f5..e14ab7088 100644 --- a/src/types/trace.ts +++ b/src/types/trace.ts @@ -369,6 +369,9 @@ export type TraceEvent = channelId: string; reason: string; textLen: number; + /** `failed`: nothing was posted; `unknown`: the request was dispatched + * and no valid receipt came back, so it may have been. */ + outcome?: 'failed' | 'unknown'; }) // Admin puppet: an operator executed a tool AS an agent and stored the diff --git a/test/publish-outcome.test.ts b/test/publish-outcome.test.ts new file mode 100644 index 000000000..b9bb16d3a --- /dev/null +++ b/test/publish-outcome.test.ts @@ -0,0 +1,213 @@ +/** + * ChannelRegistry.publish(): what one channels/publish attempt establishes. + * + * Only the connector's `delivered: true` confirms a post. `failed` is reserved + * for proof that nothing went out (refused before dispatch, or a connector + * `delivered: false` naming no posted message). Anything after dispatch that + * is not a valid confirmation is `unknown`: an error response (which may + * follow a partial multi-part post), a timeout, a lost connection, or a + * missing/malformed receipt. The attempted destination is always named by + * the registry, never by the caller's spelling. + */ +import { test, describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { tmpdir } from 'node:os'; +import { mkdtempSync, rmSync, writeFileSync, appendFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { CapabilityGrant, ALL_CAPABILITY_PATHS } from '../src/mcpl/capability-grant.js'; +import { ChannelRegistry } from '../src/mcpl/channel-registry.js'; +import { McplRequestError } from '../src/mcpl/server-connection.js'; +import type { McplServerRegistry } from '../src/mcpl/server-registry.js'; +import type { FeatureSetManager } from '../src/mcpl/feature-set-manager.js'; +import { AgentFramework } from '../src/index.js'; +import { MockMembrane, createMockResponse } from './helpers/mock-membrane.js'; + +type Publish = (params: { channelId: string }) => Promise; + +function registryWith(servers: Record, failures: unknown[] = []) { + const published: Array<{ serverId: string; channelId: string }> = []; + const mocks = new Map(); + for (const [id, s] of Object.entries(servers)) { + mocks.set(id, { + id, + grant: new CapabilityGrant(new Set(s.grant === false ? [] : ALL_CAPABILITY_PATHS), []), + sendChannelsPublish: async (params: { channelId: string }) => { + published.push({ serverId: id, channelId: params.channelId }); + return s.publish(params); + }, + sendChannelsOpen: async () => ({}), + }); + } + const registry = new ChannelRegistry( + { getServer: (id: string) => mocks.get(id) } as unknown as McplServerRegistry, + {} as FeatureSetManager, + () => {}, + () => {}, + { onRouteFailure: (info) => { failures.push(info); } }, + ); + const channels = (registry as unknown as { + channels: Map; + }).channels; + const seed = (serverId: string, id: string, label: string) => + channels.set(`${serverId}:${id}`, { serverId, descriptor: { id, type: 'discord', label }, open: true }); + return { registry, published, seed }; +} + +const ok: Publish = async () => ({ delivered: true, messageId: 'posted-1' }); + +describe('ChannelRegistry.publish outcomes', () => { + it('confirms only delivered:true, naming the destination from the registry', async () => { + const { registry, seed } = registryWith({ discord: { publish: ok } }); + seed('discord', 'discord:g1:room', '#room (Guild)'); + const outcome = await registry.publish('agent', 'hello', { channelId: 'discord:g1:room' }); + assert.equal(outcome.status, 'delivered'); + assert.equal(outcome.messageId, 'posted-1'); + assert.deepEqual(outcome.destination, { serverId: 'discord', channelId: 'discord:g1:room', label: '#room (Guild)' }); + assert.equal(typeof outcome.at, 'number'); + }); + + it('treats delivered:false with no posted message as failed', async () => { + const { registry, seed } = registryWith({ discord: { publish: async () => ({ delivered: false }) } }); + seed('discord', 'c', '#c'); + const outcome = await registry.publish('agent', 'x', { channelId: 'c' }); + assert.equal(outcome.status, 'failed'); + }); + + it('treats delivered:false that names a posted message as unknown', async () => { + const { registry, seed } = registryWith({ discord: { publish: async () => ({ delivered: false, messageId: 'part-1' }) } }); + seed('discord', 'c', '#c'); + const outcome = await registry.publish('agent', 'x', { channelId: 'c' }); + assert.equal(outcome.status, 'unknown'); + assert.equal(outcome.messageId, 'part-1'); + }); + + for (const [what, receipt] of [['a missing receipt', undefined], ['an empty receipt', {}], ['a malformed delivered field', { delivered: 'yes' }]] as const) { + it(`treats ${what} as unknown, never as delivered (#163's rule)`, async () => { + const { registry, seed } = registryWith({ discord: { publish: async () => receipt } }); + seed('discord', 'c', '#c'); + const outcome = await registry.publish('agent', 'x', { channelId: 'c' }); + assert.equal(outcome.status, 'unknown'); + assert.match(outcome.reason ?? '', /delivery uncertain/); + }); + } + + it('treats an error response as unknown, keeping the connector\'s words and data (a partial post may precede it)', async () => { + const partial = 'Send partially completed. POSTED: part 1 of 2 (message id 111). IN FLIGHT: part 2 of 2.'; + const { registry, seed } = registryWith({ + discord: { publish: async () => { throw new McplRequestError(`MCPL server "discord" returned error for channels/publish: [-32000] ${partial}`, 'error-response', -32000, { posted: ['111'] }); } }, + }); + seed('discord', 'c', '#c'); + const outcome = await registry.publish('agent', 'x', { channelId: 'c' }); + assert.equal(outcome.status, 'unknown'); + assert.ok(outcome.reason?.includes(partial), 'the connector\'s account is kept verbatim'); + assert.deepEqual(outcome.detail, { posted: ['111'] }); + assert.equal(outcome.destination?.channelId, 'c'); + }); + + it('treats a timeout or a lost connection as unknown', async () => { + const { registry, seed } = registryWith({ + discord: { publish: async () => { throw new McplRequestError('MCPL server "discord" did not respond to channels/publish (id=7) within 10ms', 'no-response'); } }, + }); + seed('discord', 'c', '#c'); + assert.equal((await registry.publish('agent', 'x', { channelId: 'c' })).status, 'unknown'); + }); + + it('treats a request refused before it was written as failed', async () => { + const { registry, seed } = registryWith({ + discord: { publish: async () => { throw new McplRequestError('Cannot send request: connection to "discord" is closed', 'not-sent'); } }, + }); + seed('discord', 'c', '#c'); + const outcome = await registry.publish('agent', 'x', { channelId: 'c' }); + assert.equal(outcome.status, 'failed'); + assert.match(outcome.reason ?? '', /connection to "discord" is closed/); + }); + + it('fails without a destination for an unknown channel', async () => { + const { registry, published } = registryWith({ discord: { publish: ok } }); + const outcome = await registry.publish('agent', 'x', { channelId: 'nowhere' }); + assert.equal(outcome.status, 'failed'); + assert.equal(outcome.destination, undefined); + assert.equal(published.length, 0); + }); + + it('refuses a channel id two servers share unless the server is named, and then uses exactly that server', async () => { + const { registry, published, seed } = registryWith({ alpha: { publish: ok }, beta: { publish: ok } }); + seed('alpha', 'discord:g1:room', '#room (Alpha)'); + seed('beta', 'discord:g1:room', '#room (Beta)'); + const shared = await registry.publish('agent', 'x', { channelId: 'discord:g1:room' }); + assert.equal(shared.status, 'failed'); + assert.match(shared.reason ?? '', /more than one MCPL server/); + assert.equal(published.length, 0, 'never routed through whichever server came first'); + const exact = await registry.publish('agent', 'x', { serverId: 'beta', channelId: 'discord:g1:room' }); + assert.equal(exact.status, 'delivered'); + assert.deepEqual(published, [{ serverId: 'beta', channelId: 'discord:g1:room' }]); + assert.equal(exact.destination?.label, '#room (Beta)'); + }); + + it('fails before dispatch without channels.publish in the grant', async () => { + const { registry, published, seed } = registryWith({ discord: { publish: ok, grant: false } }); + seed('discord', 'c', '#c'); + const outcome = await registry.publish('agent', 'x', { channelId: 'c' }); + assert.equal(outcome.status, 'failed'); + assert.equal(published.length, 0); + }); + + it('routeSpeech reports a failed and an uncertain delivery as different outcomes', async () => { + const failures: Array<{ outcome?: string }> = []; + const { registry, seed } = registryWith({ discord: { publish: async () => ({}) }, other: { publish: async () => ({ delivered: false }) } }, failures); + seed('discord', 'c', '#c'); + seed('other', 'd', '#d'); + assert.equal(await registry.routeSpeech('agent', 'x', 'c'), null); + assert.equal(await registry.routeSpeech('agent', 'x', 'd'), null); + assert.deepEqual(failures.map((f) => f.outcome), ['unknown', 'failed']); + }); +}); + +test('the resident\'s failure marker never claims an uncertain delivery did not happen', async () => { + const dir = mkdtempSync(join(tmpdir(), 'publish-outcome-')); + const commandPath = join(dir, 'commands.jsonl'); + writeFileSync(commandPath, ''); + const membrane = new MockMembrane(); + const framework = await AgentFramework.create({ + storePath: join(dir, 'store'), + membrane: membrane.asMembrane(), + agents: [{ name: 'scout', model: 'test-model', systemPrompt: 'You are scout.' }], + mcplServers: [{ + id: 'discord', + command: process.execPath, + args: [join(import.meta.dirname, 'fixtures/speech-route-mcpl-server.mjs')], + env: { STATUS_PATH: join(dir, 'status.jsonl'), COMMAND_PATH: commandPath }, + }], + modules: [], + }); + try { + await framework.start(); + const registry = (framework as unknown as { channelRegistry: { listChannelsRaw(): unknown[] } }).channelRegistry; + const until = async (cond: () => boolean, what: string) => { + const deadline = Date.now() + 15_000; + while (!cond()) { if (Date.now() > deadline) throw new Error(`timed out: ${what}`); await new Promise((r) => setTimeout(r, 20)); } + }; + await until(() => registry.listChannelsRaw().length >= 2, 'registration'); + const markers = () => framework.getAgent('scout')!.getContextManager().getAllMessages() + .filter((m) => (m.metadata as { kind?: string } | undefined)?.kind === 'discord-send-failed') + .map((m) => (m.content[0] as { text: string }).text); + const turn = async (mode: string, n: number) => { + appendFileSync(commandPath, JSON.stringify({ op: 'publish-mode', mode }) + '\n'); + membrane.pushResponse(createMockResponse([{ type: 'text', text: `reply ${n}` }])); + appendFileSync(commandPath, JSON.stringify({ op: 'incoming', channelId: 'discord:g1:room', messageId: `m-${n}`, mode: 'addressed', text: 'hello?' }) + '\n'); + await until(() => markers().length === n, `marker ${n}`); + await framework.runUntilIdle(); + }; + await turn('no-receipt', 1); + await turn('not-delivered', 2); + const [uncertain, failed] = markers(); + assert.match(uncertain!, /was not confirmed/); + assert.match(uncertain!, /may or may not have been posted/); + assert.doesNotMatch(uncertain!, /did not receive|Nothing was posted/); + assert.match(failed!, /could not be delivered/); + assert.match(failed!, /Nothing was posted/); + } finally { + await framework.stop(); + rmSync(dir, { recursive: true, force: true }); + } +}); From 7a9106685c595e121711e6584443647d9b38d226 Mon Sep 17 00:00:00 2001 From: Petra-1012 Date: Wed, 7 Oct 2026 07:58:31 -0400 Subject: [PATCH 0026/1112] feat(drafts): hold suppressed plain speech as resendable drafts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Shelf-351 (room-203). Suppressed plain speech used to survive only as a count in the turn receipt ("N segment(s) suppressed ... resend with a send tool"), and explicit mode's bounced text lived in one in-memory latest-wins clipboard: the suppression rule was right, the penalty was amnesia (a Connectome resident's report). Now the exact words are kept as drafts until the resident resends or dismisses them. What becomes a draft (src/prose-drafts.ts, reasons): - explicit-send: prose silenced by an explicit send in the same round or a sticky earlier one (channel_publish, send_message, reply_message, send_dm, say, whisper, and now a drafts resend), in locus and hybrid modes, mid-turn, trailing and text-only (a guard retry carrying the silence), native and XML tool modes. The turn tracks WHY it is silenced (silenceCause): an explicit send holds prose as drafts; skip_reply and silent turns keep it private. Hybrid prose is split into envelopes and a draft holds an envelope's body, not its routing syntax, with "written for >>>target" as a note. - no-destination: hybrid prose after a failed envelope (fail-closed until a new valid target), formerly counted as suppressed. - bounced: explicit/hybrid bounces. `{{unsent}}` keeps meaning the latest bounce while that draft is open; delivering through it journals an attempt on the draft; `>>skip_reply {{unsent}}` dismisses it; an unconfirmed latest bounce is never substituted silently. Re-bouncing `{{unsent}}` alone keeps the one draft. Never drafted (counted in the receipt): skip_reply, a same-round private think, silent turns, proseRouting=disabled narration. Storage: a RecordJournal (typed Chronicle records that do not follow branch switches), agreed in room-220 instead of the JSONL file first proposed to Tessa/Ada/June; the contract is the same. Entries held / attempt / outcome / dismissed / noticed, every one durable before its call returns; an attempt is journaled BEFORE its request leaves AF, so an attempt without an outcome reads as unconfirmed, never unsent. Undo, rollback, checkout and restarts never change drafts; nothing is ever sent from them on its own. A failed journal write reconciles (load) and reports the words it could not hold. Notices: a private notice naming new drafts at the next tool boundary when the live stream presents injections (native rounds; Membrane's supportsInjectedMessages when it ships, room-220 #41701/#41870); otherwise the turn-end [delivered] receipt names them ("held as drafts d-…"); drafts no notice reached (crash, aborted turn) are named at the next fresh turn's start. Notices name their resident: residents of one framework share a message slot (#197). The bounce notice now goes to the bouncing resident (forAgent) instead of the primary. The drafts tool (with MCPL channels): list | read | resend | dismiss, scoped to the caller; nullable unused fields (save_recent_image's pattern). resend publishes verbatim, in order, through ChannelRegistry.publish, stops at the first non-delivery, counts as an explicit send, marks the channel engaged, returns a delivered draft's historical receipt instead of resending, needs confirmDuplicate for an unconfirmed one, refuses dismissed drafts and a draft already in flight, and honours a conversation agent's home binding. Results are plain text blocks, so drafts read exactly as written. Registry: deliverSpeech (routeSpeech's PublishOutcome form; routeSpeech wraps it unchanged) and resolveDestination (exact resolution, shared ids refused). Text-only routing goes segment by segment when the content holds a membrane tool_attempt (Greta's 376, room-220 #43515). Tests: test/prose-drafts.test.ts, 18 (store: order/ids/isolation, reopen with an outcome-less attempt, branch independence, journal failure, latest-bounce; end to end over the speech-route fixture: three-segment reply held/named/injected/read/resent verbatim, XML mode, skip_reply and private think, disabled mode, resident scoping, failed then unknown then confirmed resend, June's rollback and restart cases and dismissal across a rollback, crash catch-up, explicit bounce across restart via {{unsent}}, resend silencing adjacent prose, parallel resend, refusals). Receipt expectations updated in present-while-acting and explicit-prose-routing; the explicit stub gains deliverSpeech/resolveDestination; the fixture answers tools/call at once. Suite (--test-concurrency=4, load ~20-30): 1315 pass, 1 fail (#196 vector 34c's wall-clock bound) and mcpl-coalescing.test.js cut off by the 900 s guard; that file alone afterwards: 66/66. code-execution alone: 36/36. Co-Authored-By: Claude Opus 5.5 --- changelog.d/prose-drafts.added.md | 27 + docs/explicit-prose-routing.md | 23 +- src/framework.ts | 971 ++++++++++++++++++--- src/mcpl/channel-registry.ts | 99 ++- src/prose-drafts.ts | 314 +++++++ src/types/trace.ts | 24 + test/explicit-prose-routing.test.ts | 18 +- test/fixtures/speech-route-mcpl-server.mjs | 5 + test/present-while-acting.test.ts | 13 +- test/prose-drafts.test.ts | 603 +++++++++++++ 10 files changed, 1950 insertions(+), 147 deletions(-) create mode 100644 changelog.d/prose-drafts.added.md create mode 100644 src/prose-drafts.ts create mode 100644 test/prose-drafts.test.ts diff --git a/changelog.d/prose-drafts.added.md b/changelog.d/prose-drafts.added.md new file mode 100644 index 000000000..a115c52fe --- /dev/null +++ b/changelog.d/prose-drafts.added.md @@ -0,0 +1,27 @@ +- **Held prose drafts: suppressed speech is kept, not destroyed.** Plain + speech that is not sent is now held as a private draft with its exact + words, instead of surviving only as a count: speech suppressed by an + explicit send in the same round (locus and hybrid modes, mid-turn, trailing + and text-only, native and XML tool modes), hybrid prose left without a + destination after a failed envelope, and explicit/hybrid bounces (which + replace the in-memory latest-wins clipboard; `{{unsent}}` keeps meaning the + latest bounce). Deliberate privacy is never drafted: `skip_reply`, a + same-round private `think`, silent turns and `proseRouting: disabled` + narration stay private and are counted in the receipt. +- Drafts live in typed Chronicle records (`RecordJournal`) that do not follow + branch switches: undo, rollback and checkout never change them, and they + survive restarts. Nothing is ever sent from them on its own. +- A private notice names held drafts at the next tool boundary when the live + stream presents mid-turn messages, and the turn-end `[delivered]` receipt + names every draft held in the turn ("held as drafts d-…"); a draft no notice + reached (a crash, an aborted turn) is named at the next turn's start. +- New `drafts` tool (whenever MCPL channels are configured): `list`, `read`, + `resend` and `dismiss`, scoped to the calling resident. `resend` publishes + drafts verbatim, in order, to one destination through the normal publish + path, journaling each attempt durably before the request leaves; it counts + as an explicit send. A delivered draft returns its historical receipt + instead of sending again; a draft whose last attempt's outcome is unknown + needs `confirmDuplicate: true`. Unused fields may be passed as null. +- `ChannelRegistry.deliverSpeech` (routeSpeech with its `PublishOutcome`) and + `ChannelRegistry.resolveDestination` are new; routeSpeech is unchanged. +- The bounce notice now goes to the bouncing resident rather than the primary. diff --git a/docs/explicit-prose-routing.md b/docs/explicit-prose-routing.md index 3f78c6ff4..1c428316d 100644 --- a/docs/explicit-prose-routing.md +++ b/docs/explicit-prose-routing.md @@ -30,15 +30,26 @@ A prose segment's FIRST line may carry a routing prefix: "keep going" ability tool turns have — wanted for robotics-like loops where an end-of-turn pause is undesirable. - `{{unsent}}` anywhere in the body is replaced at delivery with the - retained undelivered text (see below), enabling verbatim resend without retyping. + latest bounced text, held as a draft (see below), enabling verbatim resend + without retyping. ## The bounce (unprefixed prose) -Unprefixed prose with no sticky target is **never delivered**. It is retained -per-agent (latest-wins) and a system notice is appended telling the -agent how to resend: `>>#channel {{unsent}}`. The notice requests inference so -the resend can happen immediately — capped at 2 consecutive bounce-wakes per -agent (then notices append without waking, breaking any loop). +Unprefixed prose with no sticky target is **never delivered**. It is held as a +private draft (src/prose-drafts.ts), and a system notice naming the draft tells +the agent how to resend: `>>#channel {{unsent}}`, or the `drafts` tool by id. +The notice requests inference so the resend can happen immediately — capped at +2 consecutive bounce-wakes per agent (then notices append without waking, +breaking any loop). + +`{{unsent}}` means the latest bounce while that draft is open (latest-wins, as +the clipboard it replaced was), and `>>skip_reply {{unsent}}` dismisses it. +Earlier bounces stay in the agent's drafts, by id, until resent or dismissed; +drafts survive restarts and branch moves, and a delivery through `{{unsent}}` +is recorded on the draft like any resend. When the latest bounce's last +delivery attempt may already have been posted (an unknown outcome), +`{{unsent}}` is not substituted: the agent is told, and can resend it with the +`drafts` tool's `confirmDuplicate`. ## What explicit mode retires (for that agent) diff --git a/src/framework.ts b/src/framework.ts index 931af6bff..4fdab2234 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -81,7 +81,8 @@ import { parseProsePrefix, parseHybridProsePrefix } from './mcpl/prose-grammar.j import { ProseStreamRouter } from './mcpl/prose-stream-router.js'; import { detectKnownToolWrapperProse } from './tool-wrapper-prose-guard.js'; import { InferenceRouter } from './mcpl/inference-router.js'; -import { ChannelRegistry, type ChannelToolOrigin } from './mcpl/channel-registry.js'; +import { ChannelRegistry, type ChannelToolOrigin, type PublishDestination, type PublishOutcome } from './mcpl/channel-registry.js'; +import { ProseDraftStore, draftState, type Draft, type DraftReason, type DraftSource } from './prose-drafts.js'; import { ConversationRouter, CONVERSATION_ROUTING_DEPRECATION_NOTICE } from './mcpl/conversation-router.js'; import { INBOUND_SOURCE_KEY, type InboundAcceptanceObserver, type InboundChannelSource, type InboundLane, type InboundSource, type InboundUnscopedSource } from './mcpl/inbound-source.js'; import { safeSlice } from './safe-slice.js'; @@ -175,6 +176,38 @@ const isSilencingTool = (name: string): boolean => { return SILENCING_TOOLS.has(bare) || WORLD_PUBLICATION_TOOLS.has(bare); }; +/** True when a model-issued call silences the round's auto-routed prose: a + * silencing tool by name, or the resident's `drafts` resend (an explicit + * send, though inspecting drafts is not). */ +const isSilencingCall = (call: { name: string; input?: unknown }): boolean => { + if (isSilencingTool(call.name)) return true; + return bareToolName(call.name) === 'drafts' + && (call.input as { action?: unknown } | null | undefined)?.action === 'resend'; +}; + +/** Hybrid prose envelopes: every line starting with `>>>` opens a new one. + * Leading whitespace before a `>>>` is dropped; blank envelopes are skipped. */ +const splitHybridEnvelopes = (rawText: string): string[] => { + const envelopes: string[] = []; + let current: string[] = []; + for (const line of rawText.split('\n')) { + if (line.trimStart().startsWith('>>>') && current.length > 0) { + envelopes.push(current.join('\n')); + current = []; + } + current.push(line); + } + if (current.length > 0) envelopes.push(current.join('\n')); + return envelopes.filter((e) => e.trim()).map((e) => e.replace(/^\s+(?=>>>)/, '')); +}; + +/** Why a round's calls silence its prose: `skip_reply` deliberately keeps it + * private; an explicit send holds it back as resendable drafts. */ +const silenceCauseOf = (calls: Array<{ name: string; input?: unknown }>): 'send' | 'private' | null => { + if (calls.some((c) => bareToolName(c.name) === 'skip_reply')) return 'private'; + return calls.some(isSilencingCall) ? 'send' : null; +}; + /** * True when an injected message is real conversational input — something the * agent might actually be replying to — rather than ambient machinery. @@ -313,9 +346,11 @@ const PROSE_ROUTING_HELP = ' Append " !" after the destination (e.g. ">>#ops !") to start your next turn\n' + ' immediately when this one ends, instead of pausing until the next event.\n' + ' >>skip_reply — text stays in your context only, like the skip_reply tool.\n' + - 'Text without a destination is not delivered: it is retained, and a notice will\n' + - 'prompt you to resend — reply e.g. ">>#channel {{unsent}}" to deliver the retained\n' + - 'text unchanged. Send tools (send_message, send_dm, …) are unaffected.'; + 'Text without a destination is not delivered: it is held as a draft, and a notice\n' + + 'names it — reply e.g. ">>#channel {{unsent}}" to deliver your latest held text\n' + + 'unchanged, or use the drafts tool to list, resend or dismiss any draft by id.\n' + + '">>skip_reply {{unsent}}" sets the latest one aside. Send tools (send_message,\n' + + 'send_dm, …) are unaffected.'; const PROSE_HELP_TOOL: import('./types/index.js').ToolDefinition = { name: 'prose_help', @@ -1079,14 +1114,28 @@ export class AgentFramework { * first turn's locus is announced once to re-establish the baseline. */ private lastAnnouncedLocus: Map = new Map(); + // ---- Held prose drafts (src/prose-drafts.ts) --------------------------- + /** Every resident's held drafts: suppressed, bounced or ambiguous plain + * speech, kept privately until resent or dismissed. */ + private proseDrafts: ProseDraftStore; + /** Drafts held during the current logical turn, for its `[delivered]` + * receipt. Cleared each fresh turn; restarts keep it. */ + private turnDrafts: Map = new Map(); + /** Plain-speech segments this turn kept private by the resident's own + * choice (skip_reply in the same round) — counted, never drafted. */ + private turnProsePrivate: Map = new Map(); + /** Segments that should have been held as drafts but could not be + * journaled (the words remain in the resident's history only). */ + private turnDraftFailures: Map = new Map(); + // ---- Explicit prose routing (docs/explicit-prose-routing.md) ---------- - /** Latest bounced (undelivered) prose per agent — the `{{unsent}}` source. */ - private proseClipboards: Map = new Map(); /** Sticky per-TURN delivery target set by the turn's first `>>` prefix. * Cleared at every non-restart turn start; survives context restarts. */ private proseTargetPins: Map = new Map(); - /** Hybrid router is fail-closed after a malformed/unresolved envelope until a new valid target. */ - private proseHybridSuppressed: Set = new Set(); + /** Hybrid router is fail-closed after a `>>>skip_reply` envelope (the + * rest is private) or a malformed/unresolved/undelivered one (the rest is + * held as drafts, having no destination) until a new valid target. */ + private proseHybridSuppressed: Map = new Map(); /** Agents whose current turn requested `!` continuation — re-woken when the * turn completes instead of pausing until the next external event. */ private proseContinuations: Set = new Set(); @@ -1393,8 +1442,8 @@ export class AgentFramework { // Session-level token usage tracking (always-on) private usageTracker: UsageTracker; - /** Explicit-send suppression carried into a tool-result-guard retry. */ - private guardRetryTurnSilenced = new Map(); + /** Explicit-send or private suppression carried into a tool-result-guard retry. */ + private guardRetryTurnSilenced = new Map(); /** Presentation-only wall-clock zone; persistence remains UTC/epoch. */ private readonly timeZone: string; @@ -1417,6 +1466,9 @@ export class AgentFramework { this.operatorLog = operatorLog; this.store = store; this.ownsStore = ownsStore; + // Held prose drafts live in typed Chronicle records that do not follow + // branch switches (RecordJournal): replayed here, never re-sent. + this.proseDrafts = new ProseDraftStore(store); this.membrane = membrane; this.inferencePolicy = inferencePolicy; this.errorPolicy = errorPolicy; @@ -2450,6 +2502,9 @@ export class AgentFramework { // Copy: getChannelTools() returns the registry's shared definitions // array; pushing onto it would append another tune_out on every call. const channelTools = [...(this.channelRegistry?.getChannelTools() ?? [])]; + if (this.channelRegistry) { + channelTools.push(AgentFramework.DRAFTS_TOOL); + } if (this.tuneOutCoordinator) { channelTools.push(AgentFramework.TUNE_OUT_TOOL); } @@ -4595,6 +4650,51 @@ export class AgentFramework { inputSchema: { type: 'object' }, }; + /** Synthesized drafts tool — present whenever MCPL channels are + * configured. A resident's held drafts (src/prose-drafts.ts): plain speech + * that was not sent, kept privately until resent or dismissed. Nullable + * fields, cast like save_recent_image's: a caller whose provider presents + * every property as required can still say "not in use". */ + private static readonly DRAFTS_TOOL = { + name: 'drafts', + description: + 'Your held drafts: plain speech of yours that was NOT sent — held back because an explicit send in the ' + + 'same round holds plain speech back, because a routing prefix bounced, or because it had no destination. ' + + 'Drafts are private to you and stay until you resend or dismiss them; nothing is ever sent from here on ' + + 'its own. list: your open drafts, newest first. read: drafts in full, with every delivery attempt. ' + + 'resend: publish drafts verbatim, in the order given, to one destination ("#channel", "@person" or a ' + + 'channel id) — an explicit send, like send_message; it stops at the first draft not confirmed delivered. ' + + 'A draft already delivered returns its receipt instead of sending again. If an earlier attempt may ' + + 'already have been posted (its outcome is unknown), resend needs confirmDuplicate: true. dismiss: set ' + + 'drafts aside (your history keeps the words). A field you are not using may be omitted or passed as null.', + inputSchema: { + type: 'object', + properties: { + action: { type: 'string', enum: ['list', 'read', 'resend', 'dismiss'], description: 'What to do.' }, + draftIds: { + type: ['array', 'null'], + items: { type: 'string' }, + description: 'read, resend, dismiss: draft ids such as "d-k7x3q", in order. null for list.', + }, + destination: { + type: ['string', 'null'], + description: 'resend: where to publish — "#channel", "@person" or a channel id. null otherwise.', + }, + confirmDuplicate: { + type: ['boolean', 'null'], + description: + 'resend: true to send a draft again although an earlier attempt may already have been posted. ' + + 'null or false otherwise.', + }, + offset: { + type: ['integer', 'null'], + description: 'list: how many of the newest drafts to skip. null, like omitting it, means 0.', + }, + }, + required: ['action'], + }, + } as unknown as import('./types/index.js').ToolDefinition; + /** Synthesized save_image tool — present when a workspace module is * registered. Lets the agent persist an image it has already seen in its * own context (Discord attachments arrive inlined as base64 and are @@ -8680,12 +8780,468 @@ export class AgentFramework { } } - /** Record prose segments suppressed by explicit-send silencing. */ + /** Record prose segments suppressed without drafts (proseRouting=disabled). */ private recordProseSuppression(agentName: string, count: number): void { if (count <= 0) return; this.turnProseSuppressed.set(agentName, (this.turnProseSuppressed.get(agentName) ?? 0) + count); } + /** + * Silenced plain speech: an explicit send holds it as resendable drafts; a + * deliberate silence (skip_reply, a silent turn, a same-round private + * think) keeps it private — counted for the receipt, never drafted. + */ + private holdSilencedProse( + agent: Agent, + segments: string[], + cause: 'send' | 'private' | null, + privateThink: boolean, + opts: { round?: number; notice: 'now' | 'later' }, + ): void { + if (segments.length === 0) return; + if (cause === 'private' || privateThink) { + this.turnProsePrivate.set(agent.name, (this.turnProsePrivate.get(agent.name) ?? 0) + segments.length); + return; + } + if (agent.proseRouting !== 'hybrid') { + this.holdProseDrafts(agent, segments, 'explicit-send', opts); + return; + } + // Hybrid prose may carry `>>>` envelopes: a draft holds the words, not + // the routing syntax (a resend must not publish the prefix), and the + // destination the resident wrote stays with it as a note. A + // `>>>skip_reply` envelope was private by the resident's own choice. + let privateCount = 0; + const parts: Array<{ text: string; note?: string }> = []; + for (const segment of segments) { + for (const envelope of splitHybridEnvelopes(segment)) { + const parsed = parseHybridProsePrefix(envelope); + if (parsed.kind === 'private') privateCount++; + else if (parsed.kind === 'target') parts.push({ text: parsed.body, note: `written for >>>${parsed.target}` }); + else parts.push({ text: envelope }); + } + } + if (privateCount > 0) { + this.turnProsePrivate.set(agent.name, (this.turnProsePrivate.get(agent.name) ?? 0) + privateCount); + } + this.holdProseDrafts(agent, parts, 'explicit-send', opts); + } + + /** + * Hold plain speech as private drafts (src/prose-drafts.ts) instead of + * letting it vanish: journaled durably and recorded for this turn's + * receipt. `notice: 'now'` also queues a private notice naming them, which + * the resident hears at its next tool boundary — only when the live stream + * presents mid-turn injections. Otherwise the turn-end receipt names them + * (or, after a crash, the next turn's catch-up). Drafts the journal cannot + * hold are reported in the receipt: their words remain in history only. + */ + private holdProseDrafts( + agent: Agent, + texts: Array, + reason: DraftReason, + opts: { round?: number; note?: string; notice: 'now' | 'later' }, + ): Draft[] { + const segments = texts + .map((t) => (typeof t === 'string' ? { text: t, note: opts.note } : { text: t.text, note: t.note ?? opts.note })) + .filter((t) => t.text.trim().length > 0); + if (segments.length === 0) return []; + const turn = this.logicalTurnToolCalls.get(agent)?.turnToken ?? 0; + let branch = 'unknown'; + try { + branch = this.store.currentBranch().name; + } catch { + // provenance only + } + const { held, notHeld } = this.proseDrafts.hold( + agent.name, + segments.map(({ text, note }, segment) => ({ + text, + ...(note ? { note } : {}), + source: { branch, turn, round: opts.round ?? 0, segment } satisfies DraftSource, + })), + reason, + ); + if (notHeld) { + console.error(`[drafts] ${agent.name}: ${notHeld.count} segment(s) could not be held as drafts: ${notHeld.error}`); + const prior = this.turnDraftFailures.get(agent.name); + this.turnDraftFailures.set(agent.name, { count: (prior?.count ?? 0) + notHeld.count, error: notHeld.error }); + } + if (held.length === 0) return held; + const list = this.turnDrafts.get(agent.name) ?? []; + list.push(...held); + this.turnDrafts.set(agent.name, list); + console.error(`[drafts] ${agent.name}: held ${held.length} segment(s) as ${held.map((d) => d.id).join(', ')} (${reason})`); + this.emitTrace({ + type: 'prose:drafts-held', + agentName: agent.name, + reason, + draftIds: held.map((d) => d.id), + textLen: held.reduce((n, d) => n + d.text.length, 0), + }); + if (opts.notice === 'now') { + try { + this.addMessage( + 'user', + [{ type: 'text', text: this.draftsHeldNotice(agent.name, held, reason) }], + { system: true, kind: 'prose-drafts', draftIds: held.map((d) => d.id) } as MessageMetadata, + { forAgent: agent.name }, + ); + } catch (err) { + console.error('[drafts] held-draft notice failed:', err); + } + } + return held; + } + + private static readonly DRAFT_REASON_TEXT: Record = { + 'explicit-send': 'an explicit send in the same round holds plain speech back', + 'no-destination': 'it had no destination — after a routing envelope failed, plain speech waits for a new valid target', + ambiguous: 'more than one conversation was waiting for you, so unaddressed speech was held rather than guessed', + bounced: 'its routing prefix bounced', + }; + + private static readonly DRAFT_REASON_SHORT: Record = { + 'explicit-send': 'explicit send in the same round', + 'no-destination': 'no destination', + ambiguous: 'competing conversations', + bounced: 'routing prefix bounced', + }; + + private draftsHeldNotice(agentName: string, held: Draft[], reason: DraftReason): string { + const ids = held.map((d) => d.id); + const many = held.length > 1; + const idList = ids.map((id) => `"${id}"`).join(', '); + // Named: residents that share one message slot (#197) share this window. + return ( + `[drafts] ${agentName}: not sent — ${many ? `${held.length} plain-speech segments` : 'a plain-speech segment'} ` + + `held as draft${many ? 's' : ''} ${ids.join(', ')} — ${AgentFramework.DRAFT_REASON_TEXT[reason]}. ` + + `Drafts are private to you and stay until you act: drafts(action: "resend", draftIds: [${idList}], ` + + `destination: "#channel") delivers ${many ? 'them' : 'it'} unchanged; drafts(action: "dismiss", ` + + `draftIds: [${idList}]) sets ${many ? 'them' : 'it'} aside.` + ); + } + + /** Compact local time for draft listings. */ + private draftTime(t: number): string { + return formatZonedDateTime(t, this.timeZone).replace(/:\d\d\.\d{3}/, ''); + } + + private static draftPreview(text: string, max = 80): string { + const flat = text.replace(/\s+/g, ' ').trim(); + return JSON.stringify(flat.length > max ? `${flat.slice(0, max)}…` : flat); + } + + private static destinationText(d: PublishDestination): string { + const label = d.label && d.label !== d.channelId ? `${d.label.startsWith('#') || d.label.startsWith('DM') ? d.label : `#${d.label}`} ` : ''; + return `${label}(${d.channelId})`; + } + + /** One line per draft: id, state, when, why, size, preview. */ + private draftLine(draft: Draft): string { + const state = draftState(draft); + const last = draft.attempts.at(-1); + let stateText: string = state; + if (state === 'unconfirmed' && last) { + stateText = `UNCONFIRMED — an attempt to ${AgentFramework.destinationText(last.destination)} at ` + + `${this.draftTime(last.at)} may have been posted`; + } else if (state === 'held' && last) { + stateText = `held (last attempt failed: nothing was posted)`; + } + return `${draft.id} · ${stateText} · held ${this.draftTime(draft.heldAt)} · ` + + `${AgentFramework.DRAFT_REASON_SHORT[draft.reason]} · ${draft.text.length} chars · ` + + AgentFramework.draftPreview(draft.text); + } + + /** A delivered draft's historical receipt. */ + private draftReceipt(draft: Draft): string { + const delivered = draft.attempts.find((a) => a.outcome?.status === 'delivered'); + if (!delivered) return `${draft.id}: not delivered`; + return `${draft.id}: delivered — confirmed at ${this.draftTime(delivered.outcome!.at)} to ` + + `${AgentFramework.destinationText(delivered.destination)}` + + (delivered.outcome!.messageId ? `, message ${delivered.outcome!.messageId}` : '') + + ' (a record of that delivery, not a check that the message is still there)'; + } + + /** + * Turn start (or after a crash): name any open drafts no notice has named + * yet, privately, before the turn's compile. + */ + private noticeUnnoticedDrafts(agent: Agent): void { + const pending = this.proseDrafts.unnoticed(agent.name); + if (pending.length === 0) return; + const shown = pending.slice(0, 8).map((d) => + `${d.id} (held ${this.draftTime(d.heldAt)}, ${AgentFramework.DRAFT_REASON_SHORT[d.reason]}, ` + + `${d.text.length} chars: ${AgentFramework.draftPreview(d.text, 60)})`); + const more = pending.length > shown.length ? ` · and ${pending.length - shown.length} more` : ''; + const text = + `[drafts] ${agent.name}: ${pending.length === 1 ? 'a held draft' : `${pending.length} held drafts`} of yours ` + + `${pending.length === 1 ? 'has' : 'have'} not been named to you yet: ${shown.join(' · ')}${more}. ` + + 'They were not sent. drafts(action: "list") shows your open drafts; resend or dismiss them by id.'; + try { + const mid = agent.getContextManager().addMessage( + 'user', + [{ type: 'text', text }], + { system: true, kind: 'prose-drafts', draftIds: pending.map((d) => d.id) } as MessageMetadata, + ); + this.emitTrace({ type: 'message:added', messageId: mid, source: 'prose-drafts' }); + this.proseDrafts.markNoticed(agent.name, pending.map((d) => d.id)); + } catch (err) { + console.error('[drafts] catch-up notice failed:', err); + } + } + + /** Turn end: every draft held this turn has now been named (receipt or notice). */ + private settleTurnDrafts(agent: Agent): void { + const held = this.turnDrafts.get(agent.name); + this.turnDrafts.delete(agent.name); + this.turnDraftFailures.delete(agent.name); + this.turnProsePrivate.delete(agent.name); + if (!held?.length) return; + try { + this.proseDrafts.markNoticed(agent.name, held.map((d) => d.id)); + } catch (err) { + // Unmarked drafts are named again by the next turn's catch-up. + console.error('[drafts] could not record notice:', err); + } + } + + /** Page size for drafts(action: "list"). */ + private static readonly DRAFTS_PAGE = 10; + /** Drafts with a delivery attempt in flight, `${agent}\u0000${id}`: a + * parallel resend (or `{{unsent}}`) of the same draft is refused. */ + private draftsInFlight = new Set(); + + /** + * The resident's `drafts` tool: list, read, resend and dismiss its own held + * drafts. Private to the calling resident. `resend` is an explicit send: + * each draft is published verbatim, in order, through the normal publish + * executor, its attempt journaled durably before the request leaves. + */ + private async handleDraftsTool(agentName: string, rawInput: unknown): Promise { + const input = (rawInput && typeof rawInput === 'object' && !Array.isArray(rawInput) ? rawInput : {}) as Record; + const refuse = (error: string): ToolResult => ({ success: false, error, isError: true }); + // Plain text, not a JSON-quoted string: draft text must read exactly as written. + const ok = (text: string): ToolResult => ({ success: true, data: [{ type: 'text', text }] }); + // null and omission mean the same thing: the field is not in use. + const present = (v: unknown): boolean => v !== undefined && v !== null; + const action = input.action; + if (action !== 'list' && action !== 'read' && action !== 'resend' && action !== 'dismiss') { + return refuse('action must be one of "list", "read", "resend" or "dismiss".'); + } + const unused = (field: string): ToolResult => + refuse(`${field} is not used by action "${action}" — pass null or omit it.`); + + let ids: string[] = []; + if (present(input.draftIds)) { + if (action === 'list') return unused('draftIds'); + if (!Array.isArray(input.draftIds) || input.draftIds.some((id) => typeof id !== 'string' || !id.trim())) { + return refuse('draftIds must be a list of draft ids such as "d-k7x3q", or null.'); + } + ids = (input.draftIds as string[]).map((id) => id.trim()); + if (new Set(ids).size !== ids.length) return refuse('draftIds names a draft more than once.'); + if (ids.length > 20) return refuse('draftIds names more than 20 drafts; act on them in smaller groups.'); + } + if (present(input.destination) && action !== 'resend') return unused('destination'); + if (present(input.confirmDuplicate) && action !== 'resend') return unused('confirmDuplicate'); + if (present(input.offset) && action !== 'list') return unused('offset'); + + if (action === 'list') { + let offset = 0; + if (present(input.offset)) { + const raw = input.offset; + const n = typeof raw === 'number' ? raw : typeof raw === 'string' && /^\d+$/.test(raw.trim()) ? Number(raw) : NaN; + if (!Number.isInteger(n) || n < 0) return refuse('offset must be a whole number of drafts to skip (0 or more), or null.'); + offset = n; + } + const open = this.proseDrafts.open(agentName); + if (open.length === 0) return ok('You have no open drafts.'); + const page = open.slice(offset, offset + AgentFramework.DRAFTS_PAGE); + const header = `${open.length} open draft${open.length === 1 ? '' : 's'}, newest first` + + (offset > 0 || page.length < open.length ? ` (showing ${page.length} from #${offset + 1})` : '') + ':'; + const more = offset + page.length < open.length + ? `\nMore: drafts(action: "list", offset: ${offset + page.length}).` + : ''; + return ok( + `${header}\n${page.map((d) => this.draftLine(d)).join('\n')}${more}\n` + + 'drafts(action: "read", draftIds: [...]) shows full text and every attempt.', + ); + } + + if (ids.length === 0) return refuse(`action "${action}" needs draftIds: the drafts to ${action}.`); + const drafts: Draft[] = []; + const missing: string[] = []; + for (const id of ids) { + const draft = this.proseDrafts.get(agentName, id); + if (draft) drafts.push(draft); + else missing.push(id); + } + if (missing.length > 0) { + return refuse(`No draft of yours with id ${missing.join(', ')}. drafts(action: "list") shows your open drafts.`); + } + + if (action === 'read') { + const blocks = drafts.map((d) => { + const lines = [ + `${d.id} — ${draftState(d)}; held ${this.draftTime(d.heldAt)} (${AgentFramework.DRAFT_REASON_TEXT[d.reason]})` + + (d.note ? `; ${d.note}` : ''), + ...d.attempts.map((a) => { + const o = a.outcome; + const status = !o + ? 'outcome never recorded — it may have been posted' + : o.status === 'delivered' + ? `delivered${o.messageId ? `, message ${o.messageId}` : ''}` + : o.status === 'unknown' + ? `UNKNOWN — may have been posted: ${o.reason ?? 'no valid receipt'}` + : `failed, nothing posted: ${o.reason ?? 'refused'}`; + return ` attempt ${this.draftTime(a.at)} → ${AgentFramework.destinationText(a.destination)}: ${status}` + + (a.confirmedDuplicate ? ' (resent knowing it might duplicate)' : ''); + }), + `Text (${d.text.length} chars, exactly as written):`, + d.text, + ]; + return lines.join('\n'); + }); + return ok(blocks.join('\n\n')); + } + + if (action === 'dismiss') { + const lines: string[] = []; + for (const d of drafts) { + const state = draftState(d); + if (state === 'delivered') { + lines.push(`${d.id}: already delivered — nothing to dismiss. ${this.draftReceipt(d)}`); + continue; + } + if (state === 'dismissed') { + lines.push(`${d.id}: already dismissed.`); + continue; + } + try { + this.proseDrafts.dismiss(agentName, d.id); + } catch (err) { + return refuse( + `${lines.length > 0 ? `${lines.join('\n')}\n` : ''}Could not record dismissing ${d.id}: ` + + `${err instanceof Error ? err.message : String(err)}. It is still open.`, + ); + } + lines.push(`${d.id}: dismissed (your history keeps the words).`); + } + this.emitTrace({ type: 'prose:drafts-dismissed', agentName, draftIds: drafts.map((d) => d.id) }); + return ok(lines.join('\n')); + } + + // resend + const registry = this.channelRegistry; + if (!registry) return refuse('No channels are configured, so there is nowhere to resend to.'); + const home = this.conversationAgentHomes.get(agentName); + let spec = typeof input.destination === 'string' ? input.destination.trim() : ''; + if (present(input.destination) && typeof input.destination !== 'string') { + return refuse('destination must be "#channel", "@person" or a channel id.'); + } + if (!spec && home) spec = home; + if (!spec) return refuse('resend needs a destination: "#channel", "@person" or a channel id.'); + let confirmDuplicate = false; + if (present(input.confirmDuplicate)) { + if (typeof input.confirmDuplicate !== 'boolean') return refuse('confirmDuplicate must be true, false or null.'); + confirmDuplicate = input.confirmDuplicate; + } + const resolved = registry.resolveProseTarget(spec); + if ('error' in resolved) { + const cand = resolved.candidates?.length ? ` It could mean: ${resolved.candidates.join(', ')}.` : ''; + return refuse(`Destination "${spec}" did not resolve: ${resolved.error}.${cand} Nothing was sent.`); + } + const target = registry.resolveDestination({ channelId: resolved.channelId }); + if ('error' in target) return refuse(`Destination "${spec}": ${target.error}. Nothing was sent.`); + const destination = target.destination; + if (home && destination.channelId !== home) { + return refuse(`This conversation is bound to channel ${home}; resending to ${destination.channelId} is not allowed. Nothing was sent.`); + } + + // Refuse up front, before anything is published, rather than half-way. + for (const d of drafts) { + if (this.draftsInFlight.has(`${agentName}\u0000${d.id}`)) { + return refuse(`${d.id} is being sent right now; wait for that result before sending it again. Nothing was sent.`); + } + const state = draftState(d); + if (state === 'dismissed') { + return refuse(`${d.id} was dismissed, so it can't be resent. Nothing was sent.`); + } + if (state === 'unconfirmed' && !confirmDuplicate) { + const last = d.attempts.at(-1)!; + return refuse( + `${d.id}'s last attempt (to ${AgentFramework.destinationText(last.destination)} at ${this.draftTime(last.at)}) ` + + `may already have been posted: ${last.outcome?.reason ?? 'its outcome was never recorded'}. ` + + 'Check that channel; to send it again anyway, resend with confirmDuplicate: true. Nothing was sent.', + ); + } + } + + const lines: string[] = []; + let stopped = false; + for (const d of drafts) { + if (stopped) { + lines.push(`${d.id}: not attempted (an earlier draft was not confirmed delivered).`); + continue; + } + if (draftState(d) === 'delivered') { + lines.push(`${this.draftReceipt(d)} — not sent again.`); + continue; + } + let attemptId: string; + try { + attemptId = this.proseDrafts.beginAttempt(agentName, d.id, destination, 'resend', draftState(d) === 'unconfirmed'); + } catch (err) { + lines.push(`${d.id}: not sent — the attempt could not be recorded durably (${err instanceof Error ? err.message : String(err)}).`); + stopped = true; + continue; + } + const flight = `${agentName}\u0000${d.id}`; + this.draftsInFlight.add(flight); + let outcome: PublishOutcome; + try { + outcome = await registry.publish(agentName, d.text, { serverId: destination.serverId, channelId: destination.channelId }); + } finally { + this.draftsInFlight.delete(flight); + } + let recorded = true; + try { + this.proseDrafts.recordOutcome(agentName, d.id, attemptId, outcome); + } catch (err) { + recorded = false; + console.error(`[drafts] ${agentName}: outcome of ${d.id} not recorded:`, err); + } + this.emitTrace({ + type: 'prose:draft-resent', + agentName, + draftId: d.id, + status: outcome.status, + serverId: outcome.destination?.serverId ?? destination.serverId, + channelId: outcome.destination?.channelId ?? destination.channelId, + ...(outcome.messageId ? { messageId: outcome.messageId } : {}), + }); + const where = AgentFramework.destinationText(outcome.destination ?? destination); + const unrecorded = recorded ? '' : ' (this outcome could not be recorded; the draft will read as unconfirmed)'; + if (outcome.status === 'delivered') { + lines.push(`${d.id}: delivered to ${where}${outcome.messageId ? `, message ${outcome.messageId}` : ''}${unrecorded}.`); + let engaged = this.turnEngagedChannels.get(agentName); + if (!engaged) { + engaged = new Set(); + this.turnEngagedChannels.set(agentName, engaged); + } + engaged.add(destination.channelId); + continue; + } + stopped = true; + lines.push(outcome.status === 'unknown' + ? `${d.id}: delivery to ${where} NOT confirmed — it may or may not have been posted: ${outcome.reason ?? 'no valid receipt'}. ` + + `Check the channel before sending it again (that needs confirmDuplicate: true)${unrecorded}.` + : `${d.id}: not sent to ${where} — ${outcome.reason ?? 'refused'}. Nothing was posted; the draft stays held${unrecorded}.`); + } + const allDelivered = !stopped; + return allDelivered ? ok(lines.join('\n')) : refuse(lines.join('\n')); + } + /** Record a successful plain-prose delivery for this turn's receipt. */ private recordProseDelivery( agentName: string, @@ -8716,7 +9272,10 @@ export class AgentFramework { private appendProseDeliveryReceipt(agent: Agent): void { const list = this.turnProseDeliveries.get(agent.name); const suppressed = this.turnProseSuppressed.get(agent.name) ?? 0; - if ((!list || list.length === 0) && suppressed === 0) return; + const drafts = this.turnDrafts.get(agent.name) ?? []; + const kept = this.turnProsePrivate.get(agent.name) ?? 0; + const notHeld = this.turnDraftFailures.get(agent.name); + if ((!list || list.length === 0) && suppressed === 0 && drafts.length === 0 && kept === 0 && !notHeld) return; this.turnProseDeliveries.delete(agent.name); this.turnProseSuppressed.delete(agent.name); const seen = new Set(); @@ -8731,12 +9290,27 @@ export class AgentFramework { : id, ); } - const suppressedNote = - suppressed > 0 - ? agent.proseRouting === 'disabled' - ? `${suppressed} plain-speech segment(s) suppressed (proseRouting=disabled — publish only with an explicit send tool)` - : `${suppressed} plain-speech segment(s) suppressed (explicit send in the same round — resend with a send tool if it was meant to be heard)` - : ''; + const notes: string[] = []; + if (drafts.length > 0) { + const ids = drafts.slice(0, 8).map((d) => d.id).join(', ') + (drafts.length > 8 ? `, and ${drafts.length - 8} more` : ''); + notes.push( + `${drafts.length} plain-speech segment(s) held as draft${drafts.length === 1 ? '' : 's'} ${ids} ` + + '(not sent — drafts can resend them unchanged, or dismiss them)', + ); + } + if (notHeld) { + notes.push( + `${notHeld.count} plain-speech segment(s) suppressed and NOT held as drafts (${notHeld.error}) — ` + + 'the words remain only in your history', + ); + } + if (kept > 0) notes.push(`${kept} plain-speech segment(s) kept private (skip_reply)`); + if (suppressed > 0) { + notes.push(agent.proseRouting === 'disabled' + ? `${suppressed} plain-speech segment(s) suppressed (proseRouting=disabled — publish only with an explicit send tool)` + : `${suppressed} plain-speech segment(s) suppressed`); + } + const suppressedNote = notes.join(' · '); const text = shown.length > 0 ? `[delivered] plain speech → ${shown.join(' · ')}${suppressedNote ? ` · ${suppressedNote}` : ''}` @@ -8854,7 +9428,7 @@ export class AgentFramework { * nothing is ever sent to a destination the model did not name (directly * or via the turn's sticky target). */ - private async deliverProse(agent: Agent, rawText: string): Promise { + private async deliverProse(agent: Agent, rawText: string, hold: { round?: number; notice: 'now' | 'later' } = { notice: 'later' }): Promise { // Multi-envelope: a single prose block may address SEVERAL destinations — // every line beginning with `>>` opens a new envelope, routed // independently (first live use: Tilde 2026-07-24, one block carrying a @@ -8878,38 +9452,94 @@ export class AgentFramework { for (const envelope of envelopes) { if (!envelope.trim()) continue; - await this.deliverProseEnvelope(agent, envelope.replace(/^\s+(?=>>)/, '')); + await this.deliverProseEnvelope(agent, envelope.replace(/^\s+(?=>>)/, ''), hold); } } + /** + * `{{unsent}}` in an envelope body: the latest bounce draft's words. An + * unconfirmed one (its last attempt may have been posted) is never + * substituted silently — the resident gets the facts and the explicit + * route (drafts resend with confirmDuplicate) instead. + */ + private takeUnsent(agent: Agent, prefix: '>>' | '>>>'): { draft?: Draft; refused?: true } { + const draft = this.proseDrafts.latestBounce(agent.name); + if (!draft || draftState(draft) !== 'unconfirmed') return { draft }; + const last = draft.attempts.at(-1)!; + const text = + `[prose-routing] {{unsent}} is draft ${draft.id}, and its last delivery attempt (to ` + + `${AgentFramework.destinationText(last.destination)} at ${this.draftTime(last.at)}) may already have been ` + + `posted: ${last.outcome?.reason ?? 'its outcome was never recorded'}. Nothing was sent. Check that channel; ` + + `to send it again anyway use drafts(action: "resend", draftIds: ["${draft.id}"], destination: "#channel", confirmDuplicate: true).`; + try { + this.addMessage('user', [{ type: 'text', text }], { system: true, kind: 'prose-bounce' }, { forAgent: agent.name }); + } catch (err) { + console.error('[drafts] unconfirmed-unsent notice failed:', err); + } + return { draft, refused: true }; + } + + /** + * Deliver an envelope that substituted `{{unsent}}`: the substituted + * draft's attempt is journaled durably before the request leaves, and its + * outcome after, exactly as a drafts resend would. + */ + private async deliverWithUnsent( + agent: Agent, + body: string, + channelId: string, + unsent: Draft | undefined, + ): Promise { + const registry = this.channelRegistry!; + let attemptId: string | undefined; + if (unsent) { + const target = registry.resolveDestination({ channelId }); + if (!('error' in target)) { + try { + attemptId = this.proseDrafts.beginAttempt(agent.name, unsent.id, target.destination, 'unsent-token', false); + } catch (err) { + const reason = `the attempt for draft ${unsent.id} could not be recorded durably (${err instanceof Error ? err.message : String(err)})`; + console.error(`[drafts] ${agent.name}: ${reason} — not sending`); + return { status: 'failed', reason, at: Date.now() }; + } + } + } + const flight = unsent ? `${agent.name}\u0000${unsent.id}` : undefined; + if (flight) this.draftsInFlight.add(flight); + let outcome: PublishOutcome; + try { + outcome = await registry.deliverSpeech(agent.name, body, channelId); + } finally { + if (flight) this.draftsInFlight.delete(flight); + } + if (unsent && attemptId) { + try { + this.proseDrafts.recordOutcome(agent.name, unsent.id, attemptId, outcome); + } catch (err) { + console.error(`[drafts] ${agent.name}: outcome of ${unsent.id} not recorded:`, err); + } + } + return outcome; + } + /** Locus-preserving explicit publication. Source remains byte-identical in Chronicle. */ private async deliverHybridProse( agent: Agent, rawText: string, locus: string | null, allowLocus: boolean, + hold: { round?: number; notice: 'now' | 'later' } = { notice: 'later' }, ): Promise { - const lines = rawText.split('\n'); - const envelopes: string[] = []; - let current: string[] = []; - for (const line of lines) { - if (line.trimStart().startsWith('>>>') && current.length > 0) { - envelopes.push(current.join('\n')); - current = []; - } - current.push(line); - } - if (current.length > 0) envelopes.push(current.join('\n')); - - for (const envelope of envelopes) { - if (!envelope.trim()) continue; - const normalized = envelope.replace(/^\s+(?=>>>)/, ''); + for (const envelope of splitHybridEnvelopes(rawText)) { + const normalized = envelope; const attempted = normalized.startsWith('>>>'); const parsed = parseHybridProsePrefix(normalized); if (parsed.continueTurn) this.proseContinuations.add(agent.name); if (parsed.kind === 'private') { this.proseTargetPins.delete(agent.name); - this.proseHybridSuppressed.add(agent.name); + this.proseHybridSuppressed.set(agent.name, 'private'); + // `>>>skip_reply {{unsent}}`: the resident sets the bounced draft aside. + if (parsed.body.includes('{{unsent}}')) this.dismissLatestBounce(agent); console.error(`[prose] ${agent.name}: >>>skip_reply — ${parsed.body.length} chars kept in context (not sent)`); continue; } @@ -8917,13 +9547,19 @@ export class AgentFramework { const resolved = this.channelRegistry!.resolveProseTarget(parsed.target!); if ('error' in resolved) { this.proseTargetPins.delete(agent.name); - this.proseHybridSuppressed.add(agent.name); - this.bounceProse(agent, parsed.body, resolved.error, resolved.candidates, '>>>'); + this.proseHybridSuppressed.set(agent.name, 'failed'); + this.bounceProse(agent, parsed.body, resolved.error, resolved.candidates, '>>>', hold); continue; } let body = parsed.body; const usedClipboard = body.includes('{{unsent}}'); - if (usedClipboard) body = body.replaceAll('{{unsent}}', this.proseClipboards.get(agent.name) ?? ''); + const unsent = usedClipboard ? this.takeUnsent(agent, '>>>') : {}; + if (unsent.refused) { + this.proseTargetPins.delete(agent.name); + this.proseHybridSuppressed.set(agent.name, 'failed'); + continue; + } + if (usedClipboard) body = body.replaceAll('{{unsent}}', unsent.draft?.text ?? ''); this.proseTargetPins.set(agent.name, resolved.channelId); this.proseHybridSuppressed.delete(agent.name); if (!body.trim()) { @@ -8931,36 +9567,45 @@ export class AgentFramework { continue; } try { - const outcome = await this.channelRegistry!.routeSpeech(agent.name, body, resolved.channelId); - this.recordProseDelivery(agent.name, outcome); - if (outcome?.delivered) { + const outcome = await this.deliverWithUnsent(agent, body, resolved.channelId, unsent.draft); + this.recordProseDelivery(agent.name, outcome.status === 'delivered' ? { delivered: true, channelId: outcome.destination!.channelId } : null); + if (outcome.status === 'delivered') { this.proseBounceStreaks.delete(agent.name); - if (usedClipboard) this.proseClipboards.delete(agent.name); } else { this.proseTargetPins.delete(agent.name); - this.proseHybridSuppressed.add(agent.name); - this.bounceProse(agent, parsed.body, `delivery to ${resolved.channelId} was not confirmed`, undefined, '>>>'); + this.proseHybridSuppressed.set(agent.name, 'failed'); + // An uncertain delivery is reported by the failure marker; it is + // not bounced, so its words are not re-held as a fresh draft that + // invites a duplicate. + if (outcome.status === 'failed') { + this.bounceProse(agent, parsed.body, `delivery to ${resolved.channelId} failed: ${outcome.reason ?? 'refused'}`, undefined, '>>>', hold); + } } } catch (err) { this.proseTargetPins.delete(agent.name); - this.proseHybridSuppressed.add(agent.name); - this.bounceProse(agent, parsed.body, `delivery to ${resolved.channelId} failed: ${err instanceof Error ? err.message : String(err)}`, undefined, '>>>'); + this.proseHybridSuppressed.set(agent.name, 'failed'); + this.bounceProse(agent, parsed.body, `delivery to ${resolved.channelId} failed: ${err instanceof Error ? err.message : String(err)}`, undefined, '>>>', hold); } continue; } if (attempted) { this.proseTargetPins.delete(agent.name); - this.proseHybridSuppressed.add(agent.name); - this.bounceProse(agent, normalized, 'malformed >>> routing envelope: destination is missing', undefined, '>>>'); + this.proseHybridSuppressed.set(agent.name, 'failed'); + this.bounceProse(agent, normalized, 'malformed >>> routing envelope: destination is missing', undefined, '>>>', hold); + continue; + } + const suppressedBy = this.proseHybridSuppressed.get(agent.name); + if (suppressedBy === 'private') { + this.turnProsePrivate.set(agent.name, (this.turnProsePrivate.get(agent.name) ?? 0) + 1); continue; } - if (this.proseHybridSuppressed.has(agent.name)) { - this.recordProseSuppression(agent.name, 1); + if (suppressedBy === 'failed') { + this.holdProseDrafts(agent, [envelope], 'no-destination', hold); continue; } const sticky = this.proseTargetPins.get(agent.name); if (!allowLocus && !sticky) { - this.recordProseSuppression(agent.name, 1); + this.holdProseDrafts(agent, [envelope], 'no-destination', hold); continue; } const target = sticky ?? locus; @@ -8973,16 +9618,28 @@ export class AgentFramework { } } - private async deliverProseEnvelope(agent: Agent, rawText: string): Promise { + /** `>>skip_reply {{unsent}}` and its hybrid form: set the latest bounce draft aside. */ + private dismissLatestBounce(agent: Agent): void { + const draft = this.proseDrafts.latestBounce(agent.name); + if (!draft) return; + try { + this.proseDrafts.dismiss(agent.name, draft.id); + this.emitTrace({ type: 'prose:drafts-dismissed', agentName: agent.name, draftIds: [draft.id] }); + } catch (err) { + console.error(`[drafts] ${agent.name}: could not dismiss ${draft.id}:`, err); + } + } + + private async deliverProseEnvelope(agent: Agent, rawText: string, hold: { round?: number; notice: 'now' | 'later' }): Promise { const name = agent.name; const parsed = parseProsePrefix(rawText); if (parsed.continueTurn) this.proseContinuations.add(name); if (parsed.kind === 'private') { // The text already lives in the assistant message (window/chronicle); - // "delivery" is deliberately a no-op. Consumes the retained text if the - // segment embedded it. - if (parsed.body.includes('{{unsent}}')) this.proseClipboards.delete(name); + // "delivery" is deliberately a no-op. `>>skip_reply {{unsent}}` sets + // the bounced draft aside. + if (parsed.body.includes('{{unsent}}')) this.dismissLatestBounce(agent); console.error(`[prose] ${name}: >>skip_reply — ${parsed.body.length} chars kept in context (not sent)`); return; } @@ -8992,7 +9649,7 @@ export class AgentFramework { if (parsed.kind === 'target') { const res = this.channelRegistry!.resolveProseTarget(parsed.target!); if ('error' in res) { - this.bounceProse(agent, parsed.body, res.error, res.candidates); + this.bounceProse(agent, parsed.body, res.error, res.candidates, '>>', hold); return; } targetChannel = res.channelId; @@ -9002,7 +9659,7 @@ export class AgentFramework { } else { const sticky = this.proseTargetPins.get(name); if (!sticky) { - this.bounceProse(agent, rawText, 'no destination prefix and no destination set yet this turn'); + this.bounceProse(agent, rawText, 'no destination prefix and no destination set yet this turn', undefined, '>>', hold); return; } targetChannel = sticky; @@ -9010,8 +9667,10 @@ export class AgentFramework { } const usedClipboard = body.includes('{{unsent}}'); + const unsent = usedClipboard ? this.takeUnsent(agent, '>>') : {}; + if (unsent.refused) return; if (usedClipboard) { - body = body.replaceAll('{{unsent}}', this.proseClipboards.get(name) ?? ''); + body = body.replaceAll('{{unsent}}', unsent.draft?.text ?? ''); } if (!body.trim()) { console.error(`[prose] ${name}: empty body after prefix/substitution — nothing to send`); @@ -9019,11 +9678,8 @@ export class AgentFramework { } try { - const result = await this.channelRegistry!.routeSpeech(name, body, targetChannel); - if (result?.delivered) { - this.proseBounceStreaks.delete(name); - if (usedClipboard) this.proseClipboards.delete(name); - } + const outcome = await this.deliverWithUnsent(agent, body, targetChannel, unsent.draft); + if (outcome.status === 'delivered') this.proseBounceStreaks.delete(name); } catch (err) { console.error(`[prose] ${name}: delivery to ${targetChannel} failed:`, err); } @@ -9032,17 +9688,41 @@ export class AgentFramework { /** Cap on consecutive bounce-triggered wakes (notices still append after). */ private static readonly PROSE_BOUNCE_WAKE_CAP = 2; - private bounceProse(agent: Agent, text: string, reason: string, candidates?: string[], prefix: '>>' | '>>>' = '>>'): void { + /** + * A routing envelope that could not be delivered: its words are held as a + * draft (the `{{unsent}}` source) and the resident is told, privately. + * Bouncing `{{unsent}}` itself again keeps the one draft rather than + * minting a copy; a body that adds words around it holds the expanded text. + */ + private bounceProse( + agent: Agent, + text: string, + reason: string, + candidates?: string[], + prefix: '>>' | '>>>' = '>>', + hold: { round?: number; notice: 'now' | 'later' } = { notice: 'later' }, + ): void { const name = agent.name; - this.proseClipboards.set(name, text); + const previous = this.proseDrafts.latestBounce(name); + let draft: Draft | undefined; + if (previous && text.trim() === '{{unsent}}') { + draft = previous; + } else { + const expanded = text.includes('{{unsent}}') ? text.replaceAll('{{unsent}}', previous?.text ?? '') : text; + // The bounce notice below names the draft, so no separate held notice. + draft = this.holdProseDrafts(agent, [expanded], 'bounced', { round: hold.round, note: reason, notice: 'later' })[0]; + } const streak = (this.proseBounceStreaks.get(name) ?? 0) + 1; this.proseBounceStreaks.set(name, streak); const cand = candidates?.length ? ` Known channels it could mean: ${candidates.join(', ')}.` : ''; + const kept = draft + ? `It is held as draft ${draft.id}; nothing is lost. To deliver it unchanged, reply with a destination plus the ` + + `token {{unsent}}, e.g. "${prefix}#channel {{unsent}}" or "${prefix}@person {{unsent}}", or use ` + + `drafts(action: "resend", draftIds: ["${draft.id}"], destination: "#channel"). ` + + `"${prefix}skip_reply {{unsent}}" sets it aside.` + : 'It could not be held as a draft; the words remain in your history.'; const notice = - `[prose-routing] Your text (${text.length} chars) was not delivered — ${reason}.${cand} ` + - 'The text is retained; nothing is lost. To deliver it unchanged, reply with a ' + - `destination plus the token {{unsent}}, e.g. "${prefix}#channel {{unsent}}" or "${prefix}@person {{unsent}}". ` + - `"${prefix}skip_reply {{unsent}}" keeps it in context only.` + + `[prose-routing] Your text (${text.length} chars) was not delivered — ${reason}.${cand} ${kept}` + (prefix === '>>' ? ' The prose_help tool shows the full syntax.' : ''); try { // Through framework.addMessage, NOT the context manager directly: while @@ -9051,11 +9731,12 @@ export class AgentFramework { // stream (hear-while-acting) — the model sees the error and can correct // the prefix within the same turn. A direct append here would land the // notice before the turn's assistant blocks (cache bust, invisible - // until the next compile). + // until the next compile). Addressed to the bouncing resident alone. const id = this.addMessage( 'user', [{ type: 'text', text: notice }], - { system: true, kind: 'prose-bounce' }, + { system: true, kind: 'prose-bounce', ...(draft ? { draftIds: [draft.id] } : {}) } as MessageMetadata, + { forAgent: name }, ); if (id) this.emitTrace({ type: 'message:added', messageId: id, source: 'prose-bounce' }); else this.emitTrace({ type: 'message:added', messageId: 'deferred', source: 'prose-bounce' }); @@ -9334,6 +10015,9 @@ export class AgentFramework { if (!continuingTurn) { if (attempt === 0) this.maybePrimeProseMode(agent); + // Name any held drafts no notice reached (a crash, an aborted turn), + // privately, before this turn's compile. + if (attempt === 0) this.noticeUnnoticedDrafts(agent); const previousLogicalToolState = this.logicalTurnToolCalls.get(agent); if (attempt === 0) { // A true new turn gets a fresh generation and count. @@ -9348,6 +10032,9 @@ export class AgentFramework { this.turnEngagedChannels.delete(agent.name); this.turnProseDeliveries.delete(agent.name); this.turnProseSuppressed.delete(agent.name); + this.turnDrafts.delete(agent.name); + this.turnProsePrivate.delete(agent.name); + this.turnDraftFailures.delete(agent.name); this.proseHybridSuppressed.delete(agent.name); if (turnProseRouting === 'hybrid') this.proseTargetPins.delete(agent.name); if (turnProseRouting === 'explicit' || turnProseRouting === 'disabled') { @@ -9649,10 +10336,16 @@ export class AgentFramework { // Sticky explicit-send suppression: prose after send_message stays quiet // to prevent a redundant "sent it" postscript. Fresh injected input // clears it, because the following prose is a reply to a new message. - let turnSilenced = trigger?.suppressProse === true - || (trigger?.reason === 'tool_result_guard_retry' - && this.guardRetryTurnSilenced.get(agent.name) === true); + // Why it is silenced decides what happens to the held words: an explicit + // send ('send') holds them as resendable drafts; a deliberate silence + // ('private': skip_reply, a silent control-plane turn) keeps them private. + let silenceCause: 'send' | 'private' | null = trigger?.suppressProse === true + ? 'private' + : (trigger?.reason === 'tool_result_guard_retry' ? this.guardRetryTurnSilenced.get(agent.name) : undefined) ?? null; + let turnSilenced = silenceCause !== null; this.guardRetryTurnSilenced.delete(agent.name); + /** Provider rounds of this physical stream, for draft provenance. */ + let draftRound = 0; // Live routing is only trusted when the membrane provides verbatim // round-scoped blocks (roundContent, native tool mode, membrane ≥0.5.64). @@ -9762,6 +10455,7 @@ export class AgentFramework { // earlier explicit delivery has completed its conversational job. // Routing is NOT touched: the turn locus stays frozen. turnSilenced = trigger?.suppressProse === true; + silenceCause = turnSilenced ? 'private' : null; }; try { @@ -9942,16 +10636,26 @@ export class AgentFramework { // from the round it occurs (see SILENCING_TOOLS). Only rounds // with verbatim roundContent are live-routed (see liveProseRouting // note above — the fallback preamble is cumulative in XML mode). + draftRound++; if (this.channelRegistry) { const roundToolNames = event.calls.map((c) => c.name); const hasSameRoundPrivateThink = roundToolNames.includes('think') && requestSnapshot.sameRoundThinkTextPolicy === 'private'; - if (roundToolNames.some(isSilencingTool)) { + const roundCause = silenceCauseOf(event.calls); + if (roundCause) { turnSilenced = true; + silenceCause = roundCause === 'private' || silenceCause === 'private' ? 'private' : 'send'; } if (roundContent && roundContent.length > 0) { liveProseRouting = true; + // Whether this round's tool-result batch (and anything queued + // with it) will be presented to the model before it speaks + // again: the producer says so when it can; otherwise native + // rounds (verbatim roundContent) carry injections. + const presentsInjections = + (event.context as { supportsInjectedMessages?: boolean }).supportsInjectedMessages ?? true; + const holdOpts = { round: draftRound, notice: presentsInjections ? 'now' as const : 'later' as const }; const roundSegments = splitProseSegments(assistantBlocks); if (roundSegments.length > 0) { if (turnProseRouting === 'disabled') { @@ -9961,12 +10665,12 @@ export class AgentFramework { this.recordProseSuppression(agent.name, roundSegments.length); } else if (turnProseRouting === 'hybrid') { if (turnSilenced) { - this.recordProseSuppression(agent.name, roundSegments.length); + this.holdSilencedProse(agent, roundSegments, silenceCause, hasSameRoundPrivateThink, holdOpts); } else if (!hasSameRoundPrivateThink) { const locus = resolveTurnLocus(); for (const seg of roundSegments) { turnSpeechChain = turnSpeechChain - .then(() => this.deliverHybridProse(agent, seg, locus, true)) + .then(() => this.deliverHybridProse(agent, seg, locus, true, holdOpts)) .catch((err) => console.error('mid-turn hybrid prose delivery failed:', err)); } } @@ -9980,17 +10684,17 @@ export class AgentFramework { ); for (const seg of roundSegments) { turnSpeechChain = turnSpeechChain - .then(() => this.deliverProse(agent, seg)) + .then(() => this.deliverProse(agent, seg, holdOpts)) .catch((err) => console.error('mid-turn prose delivery failed:', err)); } } } else if (turnSilenced) { console.error( - `[routing] ${agent.name}: mid-turn round [${roundToolNames.join(', ')}] -> prose NOT routed (turn silenced)`, + `[routing] ${agent.name}: mid-turn round [${roundToolNames.join(', ')}] -> prose NOT routed (turn silenced: ${silenceCause})`, ); - // Visible in the turn-end receipt — silencing must never - // be a silent black hole (n=8: the flying-scene reply). - this.recordProseSuppression(agent.name, roundSegments.length); + // Never a silent black hole (n=8: the flying-scene reply): + // held as drafts the resident can resend, and named. + this.holdSilencedProse(agent, roundSegments, silenceCause, hasSameRoundPrivateThink, holdOpts); } else if (hasSameRoundPrivateThink) { console.error( `[routing] ${agent.name}: mid-turn round [${roundToolNames.join(', ')}] -> prose NOT routed (same_round_think_text_policy=private)`, @@ -10077,7 +10781,7 @@ export class AgentFramework { // Carry same-turn explicit-send suppression into the retry: // a text-only recovery after a successful send must not post // a postscript to a message already delivered. - if (turnSilenced) this.guardRetryTurnSilenced.set(agent.name, true); + if (turnSilenced) this.guardRetryTurnSilenced.set(agent.name, silenceCause ?? 'send'); // Restart inference inside this logical turn. No tool is // executed again and no settle/checkpoint/locus reset occurs. await this.startAgentStream(agent, { @@ -10426,31 +11130,44 @@ export class AgentFramework { .map((b) => (b as ContentBlock & { type: 'text' }).text) .join('\n') .trim(); - if (speechText) { + // A text-only turn is one message — unless an all-refused XML + // round (membrane `tool_attempt`, answered in-band with a + // `tool_notice`) separates its prose: nothing was called, but + // the words before the attempt and after its notice are two + // messages, routed as segments like a tool turn's. + const speechSegments = response.content.some((b) => (b.type as string) === 'tool_attempt') + ? splitProseSegments(response.content) + : speechText ? [speechText] : []; + const textOnlyHold = { round: draftRound + 1, notice: 'later' as const }; + if (speechSegments.length > 0) { if (turnProseRouting === 'disabled') { console.error(`[routing] ${agent.name}: text-only prose NOT routed (proseRouting=disabled)`); - this.recordProseSuppression(agent.name, 1); + this.recordProseSuppression(agent.name, speechSegments.length); } else if (turnSilenced && turnProseRouting !== 'explicit') { // Only reachable on a guard recovery that carried the // same-turn send suppression, or a suppressProse trigger // (a fresh text-only turn starts unsilenced). Explicit // mode is exempt, as mid-turn. - console.error(`[routing] ${agent.name}: text-only prose NOT routed (turn silenced)`); - this.recordProseSuppression(agent.name, 1); + console.error(`[routing] ${agent.name}: text-only prose NOT routed (turn silenced: ${silenceCause})`); + this.holdSilencedProse(agent, speechSegments, silenceCause, false, textOnlyHold); } else if (turnProseRouting === 'hybrid') { const locus = resolveTurnLocus(); console.error(`[prose] ${agent.name}: text-only turn -> hybrid prose gateway`); - try { - await this.deliverHybridProse(agent, speechText, locus, true); - } catch (err) { - console.error('text-only hybrid prose delivery failed:', err); + for (const segment of speechSegments) { + try { + await this.deliverHybridProse(agent, segment, locus, true, textOnlyHold); + } catch (err) { + console.error('text-only hybrid prose delivery failed:', err); + } } } else if (turnProseRouting === 'explicit') { console.error(`[prose] ${agent.name}: text-only turn -> prose gateway`); - try { - await this.deliverProse(agent, speechText); - } catch (err) { - console.error('text-only prose delivery failed:', err); + for (const segment of speechSegments) { + try { + await this.deliverProse(agent, segment, textOnlyHold); + } catch (err) { + console.error('text-only prose delivery failed:', err); + } } } else { // Route to the TURN-FROZEN locus, like every other speech @@ -10463,11 +11180,13 @@ export class AgentFramework { console.error( `[routing] ${agent.name}: text-only turn -> routing speech -> ${locus ?? '(none)'}`, ); - try { - const outcome = await this.channelRegistry.routeSpeech(agent.name, speechText, locus); - this.recordProseDelivery(agent.name, outcome); - } catch (err) { - console.error('speech routing failed:', err); + for (const segment of speechSegments) { + try { + const outcome = await this.channelRegistry.routeSpeech(agent.name, segment, locus); + this.recordProseDelivery(agent.name, outcome); + } catch (err) { + console.error('speech routing failed:', err); + } } } } @@ -10486,13 +11205,18 @@ export class AgentFramework { // injected message can reset. The legacy/fallback path has no // reliable round boundaries, so retain its historical turn-wide // scan to avoid double-posting. - const toolNames = response.content + const toolCalls = response.content .filter((b) => b.type === 'tool_use') - .map((b) => (b as unknown as { name?: string }).name) - .filter((n): n is string => typeof n === 'string'); + .map((b) => b as unknown as { name?: string; input?: unknown }) + .filter((c): c is { name: string; input?: unknown } => typeof c.name === 'string'); + const toolNames = toolCalls.map((c) => c.name); + const fallbackCause = liveProseRouting ? null : silenceCauseOf(toolCalls); const silenced = liveProseRouting ? turnSilenced - : turnSilenced || toolNames.some(isSilencingTool); + : turnSilenced || fallbackCause !== null; + const trailingCause: 'send' | 'private' | null = + silenceCause === 'private' || fallbackCause === 'private' ? 'private' : (silenceCause ?? fallbackCause); + const trailingHold = { round: draftRound + 1, notice: 'later' as const }; const segments = splitProseSegments(liveProseRouting ? terminalContent : response.content); @@ -10510,12 +11234,12 @@ export class AgentFramework { } } else if (turnProseRouting === 'hybrid') { if (silenced && segments.length > 0) { - this.recordProseSuppression(agent.name, segments.length); + this.holdSilencedProse(agent, segments, trailingCause, false, trailingHold); } else if (segments.length > 0) { const locus = resolveTurnLocus(); for (const seg of segments) { try { - await this.deliverHybridProse(agent, seg, locus, true); + await this.deliverHybridProse(agent, seg, locus, true, trailingHold); } catch (err) { console.error('trailing hybrid prose delivery failed:', err); } @@ -10528,7 +11252,7 @@ export class AgentFramework { ); for (const seg of segments) { try { - await this.deliverProse(agent, seg); + await this.deliverProse(agent, seg, trailingHold); } catch (err) { console.error('trailing prose delivery failed:', err); } @@ -10537,10 +11261,10 @@ export class AgentFramework { } else if (silenced || segments.length === 0) { console.error( `[routing] ${agent.name}: tool-call turn [${toolNames.join(', ') || 'none'}] -> trailing prose NOT routed ` + - `(${silenced ? 'silencing tool / explicit send' : 'no trailing prose'})`, + `(${silenced ? `turn silenced: ${trailingCause}` : 'no trailing prose'})`, ); if (silenced && segments.length > 0) { - this.recordProseSuppression(agent.name, segments.length); + this.holdSilencedProse(agent, segments, trailingCause, false, trailingHold); } } else { // Reuse the locus pinned at the turn's first live-routed @@ -10577,6 +11301,10 @@ export class AgentFramework { if (!trigger?.suppressProse && turnProseRouting !== 'explicit') { this.appendProseDeliveryReceipt(agent); } + // The turn's drafts have now been named (receipt, held notice or + // bounce notice); a crash before this leaves them for the next + // turn's catch-up. + this.settleTurnDrafts(agent); // Explicit-prose `!` continuation: a prose segment this turn asked // to keep going (`>>#x !` / `>>private !`) — start another turn @@ -10747,6 +11475,10 @@ export class AgentFramework { await turnSpeechChain; this.appendProseDeliveryReceipt(agent); } + if (cancelKind === 'turn_ended') { + await turnSpeechChain; + this.settleTurnDrafts(agent); + } return; } } @@ -12454,6 +13186,25 @@ export class AgentFramework { return; } + // drafts: the resident's own held drafts (list/read/resend/dismiss). + // Answered by the framework; resend publishes through the registry's + // publish executor, so the result arrives asynchronously. + if (enrichedCall.name === 'drafts' && this.channelRegistry) { + const startedAt = Date.now(); + this.emitTrace({ type: 'tool:started', module: 'framework', tool: 'drafts', callId: enrichedCall.id, input: enrichedCall.input }); + void this.handleDraftsTool(agentName, enrichedCall.input) + .catch((err): ToolResult => ({ + success: false, + error: `drafts failed: ${err instanceof Error ? err.message : String(err)}`, + isError: true, + })) + .then((result) => { + this.emitTrace({ type: 'tool:completed', module: 'framework', tool: 'drafts', callId: enrichedCall.id, durationMs: Date.now() - startedAt }); + this.pushEvent({ type: 'tool-result', callId: enrichedCall.id, agentName, moduleName: 'framework', result }); + }); + return; + } + // prose_help: on-demand routing reference for explicit-prose agents. // Answered inline — the grammar is a tool RESULT (model-requested), never // ambient context (see PROSE_ROUTING_HELP / teach-by-bounce). diff --git a/src/mcpl/channel-registry.ts b/src/mcpl/channel-registry.ts index 8333c860a..6aa477bf2 100644 --- a/src/mcpl/channel-registry.ts +++ b/src/mcpl/channel-registry.ts @@ -3021,33 +3021,57 @@ export class ChannelRegistry { * or malformed receipt is `unknown`: the post may or may not exist, and the * caller must not treat it as safe to retry blindly. */ - async publish( - conversationId: string, - text: string, + /** + * Resolve a publish target to exactly one registered channel: `serverId` + * plus `channelId` when the caller has the server, otherwise a channel id + * registered by exactly one server. A shared id is refused, never resolved + * to whichever server registered it first. + */ + resolveDestination( target: { serverId?: string; channelId: string }, - openSource: 'opened-by-delivery' | 'opened-by-reply' = 'opened-by-delivery', - ): Promise { - const at = (): number => Date.now(); + ): { destination: PublishDestination } | { error: string } { + const entry = this.findExactEntry(target); + if ('error' in entry) return entry; + return { + destination: { + serverId: entry.entry.serverId, + channelId: entry.entry.descriptor.id, + ...(entry.entry.descriptor.label ? { label: entry.entry.descriptor.label } : {}), + }, + }; + } + + private findExactEntry( + target: { serverId?: string; channelId: string }, + ): { entry: ChannelEntry } | { error: string } { const matches = [...this.channels.values()].filter( (e) => e.descriptor.id === target.channelId && (!target.serverId || e.serverId === target.serverId), ); if (matches.length === 0) { return { - status: 'failed', - reason: target.serverId + error: target.serverId ? `no registered channel "${target.channelId}" on server "${target.serverId}"` : `no registered channel "${target.channelId}"`, - at: at(), }; } if (matches.length > 1) { return { - status: 'failed', - reason: `channel id "${target.channelId}" is registered by more than one MCPL server; the destination must name its server`, - at: at(), + error: `channel id "${target.channelId}" is registered by more than one MCPL server; the destination must name its server`, }; } - const entry = matches[0]!; + return { entry: matches[0]! }; + } + + async publish( + conversationId: string, + text: string, + target: { serverId?: string; channelId: string }, + openSource: 'opened-by-delivery' | 'opened-by-reply' = 'opened-by-delivery', + ): Promise { + const at = (): number => Date.now(); + const found = this.findExactEntry(target); + if ('error' in found) return { status: 'failed', reason: found.error, at: at() }; + const entry = found.entry; const destination: PublishDestination = { serverId: entry.serverId, channelId: entry.descriptor.id, @@ -3157,7 +3181,13 @@ export class ChannelRegistry { }; } - async routeSpeech( + /** + * Plain speech for one turn segment: publish `text` to the turn's frozen + * locus and report the PublishOutcome. A failure (or an uncertain outcome) + * is surfaced as a trace and through onRouteFailure, so the agent learns + * its words did not (or may not have) reached anyone. + */ + async deliverSpeech( conversationId: string, text: string, /** The turn's FROZEN locus, snapshotted once by the caller (resolveLocus @@ -3168,42 +3198,41 @@ export class ChannelRegistry { * PR #32, and the 2026-07-22 Sol DM misroute). Explicit `null` means * "this turn is pinned to no locus": fail loudly rather than guess. */ locusChannelId: string | null, - ): Promise<{ delivered: boolean; channelId: string; messageId?: string } | null> { - // Surface a routing failure: emit a trace AND notify the host (which drops - // a `[discord-send-failed]` marker into chronicle) so the agent learns its - // reply did not (or may not have) reached anyone, instead of it vanishing - // silently. - const fail = (channelId: string | null, reason: string, outcome: 'failed' | 'unknown' = 'failed'): null => { - console.error(`[routeSpeech] ${conversationId}: ${reason} — speech ${outcome === 'unknown' ? 'delivery NOT confirmed' : 'NOT routed'} (${text.length} chars stay in chronicle)`); + ): Promise { + const fail = (channelId: string | null, reason: string, outcome: PublishOutcome): PublishOutcome => { + const status = outcome.status === 'unknown' ? 'unknown' : 'failed'; + console.error(`[routeSpeech] ${conversationId}: ${reason} — speech ${status === 'unknown' ? 'delivery NOT confirmed' : 'NOT routed'} (${text.length} chars stay in chronicle)`); this.emitTraceFn({ type: 'mcpl:speech-route-failed', conversationId, channelId: channelId ?? '', reason, textLen: text.length, - outcome, + outcome: status, }); - this.onRouteFailure?.({ conversationId, channelId, reason, textLen: text.length, outcome }); - return null; + this.onRouteFailure?.({ conversationId, channelId, reason, textLen: text.length, outcome: status }); + return outcome; }; // Runtime backstop for JS callers the compiler can't see: an omitted // locus is a routing bug at the call site, never something to paper over // with a live re-resolution. if (locusChannelId === undefined) { - return fail(null, 'caller passed no locus (routing bug: every speech path must snapshot the turn locus)'); + const reason = 'caller passed no locus (routing bug: every speech path must snapshot the turn locus)'; + return fail(null, reason, { status: 'failed', reason, at: Date.now() }); } if (!locusChannelId) { // The turn froze with no locus (no home, no triggering channel, no // global inbound ever seen) — the agent was told its prose stays in // the archive; honor that. - return fail(null, 'turn has no locus (no home/trigger channel; nothing to deliver into)'); + const reason = 'turn has no locus (no home/trigger channel; nothing to deliver into)'; + return fail(null, reason, { status: 'failed', reason, at: Date.now() }); } const outcome = await this.publish(conversationId, text, { channelId: locusChannelId }); const channelId = outcome.destination?.channelId ?? locusChannelId; if (outcome.status !== 'delivered') { - return fail(channelId, outcome.reason ?? 'delivery not confirmed', outcome.status); + return fail(channelId, outcome.reason ?? 'delivery not confirmed', outcome); } console.error(`[routeSpeech] ${conversationId}: routed ${text.length} chars -> ${channelId} (server=${outcome.destination!.serverId}, delivered=true)`); @@ -3220,8 +3249,22 @@ export class ChannelRegistry { // tap editing it down to the words actually voiced on interruption. ...(outcome.messageId !== undefined ? { messageId: outcome.messageId } : {}), }); + return outcome; + } - return { delivered: true, channelId, ...(outcome.messageId !== undefined ? { messageId: outcome.messageId } : {}) }; + /** deliverSpeech, reduced to the confirmed delivery (null when there was none). */ + async routeSpeech( + conversationId: string, + text: string, + locusChannelId: string | null, + ): Promise<{ delivered: boolean; channelId: string; messageId?: string } | null> { + const outcome = await this.deliverSpeech(conversationId, text, locusChannelId); + if (outcome.status !== 'delivered') return null; + return { + delivered: true, + channelId: outcome.destination!.channelId, + ...(outcome.messageId !== undefined ? { messageId: outcome.messageId } : {}), + }; } private async handleToolPublish(input: { channelId?: string; content?: string; text?: string }): Promise { diff --git a/src/prose-drafts.ts b/src/prose-drafts.ts new file mode 100644 index 000000000..57e18706f --- /dev/null +++ b/src/prose-drafts.ts @@ -0,0 +1,314 @@ +/** + * Held prose drafts: a resident's plain speech that was not sent. + * + * Plain speech is held back for reasons that are right in themselves: an + * explicit send in the same round suppresses the round's prose (so a "sent + * it" postscript is not posted twice), a turn may have no destination, two + * conversations may compete for an unaddressed reply, and a malformed or + * unresolvable routing prefix bounces. Before drafts, the held words were + * kept only as a count (or, in explicit mode, as one in-memory latest-wins + * clipboard): the law was right, and the penalty was amnesia. A draft keeps + * the exact words, privately, until the resident resends or dismisses them. + * + * Durability is outside the conversation. Drafts live in a RecordJournal of + * typed Chronicle records, which do not follow branch switches: undo, + * rollback, suppression and branch moves never touch them. A draft written + * in a turn that is later undone stays usable (the words were genuinely + * written and held back), and a delivered draft stays delivered. Its + * originating branch is recorded as provenance only. + * + * A delivery attempt is journaled durably BEFORE its request leaves the + * host, and its outcome after: an attempt with no recorded outcome (a crash + * mid-send) reads as unknown, never as unsent. A delivered draft carries a + * historical receipt — confirmed at a time, to a destination, with the + * connector's message id — not a claim that the message still exists. + * Nothing here publishes anything: every send is the resident's explicit + * resend, and nothing replays at boot or on a branch move. + */ + +import { randomInt } from 'node:crypto'; +import type { JsStore } from '@animalabs/chronicle'; +import { RecordJournal } from './record-journal.js'; +import type { PublishDestination, PublishOutcome } from './mcpl/channel-registry.js'; + +/** Chronicle record type of the drafts journal. */ +export const PROSE_DRAFT_RECORD_TYPE = 'framework/prose-draft'; + +/** Why words were held instead of sent. */ +export type DraftReason = + /** An explicit send in the same round suppressed the round's plain speech. */ + | 'explicit-send' + /** The speech had no destination (no route this turn, or a failed envelope). */ + | 'no-destination' + /** Two conversations competed for an unaddressed reply. */ + | 'ambiguous' + /** An explicit/hybrid routing prefix was missing, malformed or unresolvable. */ + | 'bounced'; + +/** Where a draft's words were written. Provenance only: nothing routes by it. */ +export interface DraftSource { + /** Chronicle branch current when the words were held. */ + branch: string; + /** The resident's logical turn (process-local counter) and the round in it. */ + turn: number; + round: number; + /** Position of the segment within its round's speech. */ + segment: number; +} + +export interface DraftAttemptOutcome { + status: PublishOutcome['status']; + at: number; + messageId?: string; + reason?: string; + detail?: unknown; +} + +export interface DraftAttempt { + attemptId: string; + at: number; + destination: PublishDestination; + /** How the words went out: the drafts tool, or explicit mode's `{{unsent}}`. */ + via: 'resend' | 'unsent-token'; + /** The resident resent knowing an earlier attempt may have been posted. */ + confirmedDuplicate?: true; + /** Absent until recorded; an attempt without one reads as unknown. */ + outcome?: DraftAttemptOutcome; +} + +export interface Draft { + id: string; + agent: string; + /** The held words, byte for byte. */ + text: string; + reason: DraftReason; + /** Short context for the resident: the bounce reason, or the competing conversations. */ + note?: string; + heldAt: number; + source: DraftSource; + attempts: DraftAttempt[]; + dismissedAt?: number; + /** When an in-band notice named this draft. */ + noticedAt?: number; +} + +/** + * - `held`: never attempted, or the last attempt definitely failed — resend freely; + * - `unconfirmed`: the last attempt's outcome is unknown (or was never + * recorded): it may have been posted, so a resend needs explicit confirmation; + * - `delivered`: an attempt was confirmed — its receipt is historical; + * - `dismissed`: the resident set it aside. + */ +export type DraftState = 'held' | 'unconfirmed' | 'delivered' | 'dismissed'; + +export function draftState(draft: Draft): DraftState { + if (draft.attempts.some((a) => a.outcome?.status === 'delivered')) return 'delivered'; + if (draft.dismissedAt !== undefined) return 'dismissed'; + const last = draft.attempts.at(-1); + if (last && (!last.outcome || last.outcome.status === 'unknown')) return 'unconfirmed'; + return 'held'; +} + +type DraftEntry = + | { op: 'held'; draft: Omit } + | { op: 'attempt'; agent: string; id: string; attemptId: string; at: number; destination: PublishDestination; via: DraftAttempt['via']; confirmedDuplicate?: true } + | { op: 'outcome'; agent: string; id: string; attemptId: string; outcome: DraftAttemptOutcome } + | { op: 'dismissed'; agent: string; id: string; at: number } + | { op: 'noticed'; agent: string; ids: string[]; at: number }; + +/** Unambiguous lowercase alphabet for short ids (no i/l/o/0/1). */ +const ID_ALPHABET = 'abcdefghjkmnpqrstuvwxyz23456789'; + +export class ProseDraftStore { + private readonly journal: RecordJournal; + /** agent → draft id → draft, in journal (hold) order. */ + private drafts = new Map>(); + + constructor(store: JsStore) { + this.journal = new RecordJournal(store, { type: PROSE_DRAFT_RECORD_TYPE }); + this.reload(); + } + + /** Rebuild the projection from the journal (at open, and to reconcile). */ + reload(): void { + const { entries } = this.journal.load(); + this.drafts = new Map(); + for (const { entry } of entries) this.reduce(entry); + } + + private reduce(entry: DraftEntry): void { + if (entry.op === 'held') { + this.agentDrafts(entry.draft.agent).set(entry.draft.id, { ...entry.draft, attempts: [] }); + return; + } + if (entry.op === 'noticed') { + for (const id of entry.ids) { + const draft = this.drafts.get(entry.agent)?.get(id); + if (draft && draft.noticedAt === undefined) draft.noticedAt = entry.at; + } + return; + } + const draft = this.drafts.get(entry.agent)?.get(entry.id); + if (!draft) return; + if (entry.op === 'dismissed') { + draft.dismissedAt ??= entry.at; + } else if (entry.op === 'attempt') { + draft.attempts.push({ + attemptId: entry.attemptId, + at: entry.at, + destination: entry.destination, + via: entry.via, + ...(entry.confirmedDuplicate ? { confirmedDuplicate: true as const } : {}), + }); + } else if (entry.op === 'outcome') { + const attempt = draft.attempts.find((a) => a.attemptId === entry.attemptId); + if (attempt) attempt.outcome = entry.outcome; + } + } + + /** + * Journal and apply one entry. A write that fails after reaching the store + * leaves the journal unreconciled: rebuild from it before rethrowing, so + * the projection never disagrees with what the journal holds. + */ + private write(entry: DraftEntry, durable: boolean): void { + try { + this.journal.append(entry, { durable }); + } catch (err) { + if (this.journal.needsReconcile) this.reload(); + throw err; + } + this.reduce(entry); + } + + private agentDrafts(agent: string): Map { + let map = this.drafts.get(agent); + if (!map) { + map = new Map(); + this.drafts.set(agent, map); + } + return map; + } + + private newId(agent: string): string { + const existing = this.agentDrafts(agent); + for (;;) { + let id = 'd-'; + for (let i = 0; i < 5; i++) id += ID_ALPHABET[randomInt(ID_ALPHABET.length)]; + if (!existing.has(id)) return id; + } + } + + /** + * Hold words as drafts, in order, each journaled durably before this + * returns. If the journal refuses a write (after reconciling), the drafts + * already written stand and the rest are reported as not held: their words + * remain in the resident's history only. + */ + hold( + agent: string, + segments: Array<{ text: string; source: DraftSource; note?: string }>, + reason: DraftReason, + note?: string, + ): { held: Draft[]; notHeld?: { count: number; error: string } } { + const held: Draft[] = []; + for (let i = 0; i < segments.length; i++) { + const segment = segments[i]!; + const segmentNote = segment.note ?? note; + const draft = { + id: this.newId(agent), + agent, + text: segment.text, + reason, + ...(segmentNote ? { note: segmentNote } : {}), + heldAt: Date.now(), + source: segment.source, + }; + try { + this.write({ op: 'held', draft }, true); + } catch (err) { + return { held, notHeld: { count: segments.length - i, error: err instanceof Error ? err.message : String(err) } }; + } + held.push(this.get(agent, draft.id)!); + } + return { held }; + } + + get(agent: string, id: string): Draft | undefined { + return this.drafts.get(agent)?.get(id); + } + + /** Held and unconfirmed drafts, newest first (journal order, which is + * exact even for segments held in the same millisecond). */ + open(agent: string): Draft[] { + return [...(this.drafts.get(agent)?.values() ?? [])] + .filter((d) => { const s = draftState(d); return s === 'held' || s === 'unconfirmed'; }) + .reverse(); + } + + /** Open drafts no in-band notice has named yet (a crash before the notice). */ + unnoticed(agent: string): Draft[] { + return this.open(agent).filter((d) => d.noticedAt === undefined).reverse(); + } + + markNoticed(agent: string, ids: string[]): void { + if (ids.length === 0) return; + this.write({ op: 'noticed', agent, ids, at: Date.now() }, true); + } + + dismiss(agent: string, id: string): void { + this.write({ op: 'dismissed', agent, id, at: Date.now() }, true); + } + + /** + * Journal a delivery attempt durably, BEFORE the request leaves the host. + * Throws (having reconciled) when it cannot be made durable: the caller + * must then not send. + */ + beginAttempt( + agent: string, + id: string, + destination: PublishDestination, + via: DraftAttempt['via'], + confirmedDuplicate: boolean, + ): string { + const attemptId = `a-${Date.now().toString(36)}-${randomInt(36 ** 4).toString(36)}`; + this.write({ + op: 'attempt', agent, id, attemptId, at: Date.now(), destination, via, + ...(confirmedDuplicate ? { confirmedDuplicate: true as const } : {}), + }, true); + return attemptId; + } + + /** Journal what an attempt established. */ + recordOutcome(agent: string, id: string, attemptId: string, outcome: PublishOutcome): void { + this.write({ + op: 'outcome', agent, id, attemptId, + outcome: { + status: outcome.status, + at: outcome.at, + ...(outcome.messageId ? { messageId: outcome.messageId } : {}), + ...(outcome.reason ? { reason: outcome.reason } : {}), + ...(outcome.detail !== undefined ? { detail: outcome.detail } : {}), + }, + }, true); + } + + /** + * Explicit and hybrid mode's `{{unsent}}`: the most recent bounce, while + * that draft is still open (held or unconfirmed — the caller must not + * resend an unconfirmed one without the resident's confirmation). + * Latest-wins, like the clipboard it replaces: once the latest bounce is + * delivered or dismissed there is nothing to substitute, even if older + * bounces are still held (they stay in the collection, by id). + */ + latestBounce(agent: string): Draft | undefined { + let latest: Draft | undefined; + for (const draft of this.drafts.get(agent)?.values() ?? []) { + if (draft.reason === 'bounced') latest = draft; + } + if (!latest) return undefined; + const state = draftState(latest); + return state === 'held' || state === 'unconfirmed' ? latest : undefined; + } +} diff --git a/src/types/trace.ts b/src/types/trace.ts index e14ab7088..a7465b550 100644 --- a/src/types/trace.ts +++ b/src/types/trace.ts @@ -498,6 +498,30 @@ export type TraceEvent = /** Envelope kind of the acceptance the observer failed on. */ kind: 'channel' | 'unscoped' | 'surface'; error: string; + }) + + // Held prose drafts (src/prose-drafts.ts): plain speech kept privately + // instead of sent; a resident's explicit resend; a deliberate dismissal. + | (TraceEventBase & { + type: 'prose:drafts-held'; + agentName: string; + reason: 'explicit-send' | 'no-destination' | 'ambiguous' | 'bounced'; + draftIds: string[]; + textLen: number; + }) + | (TraceEventBase & { + type: 'prose:draft-resent'; + agentName: string; + draftId: string; + status: 'delivered' | 'failed' | 'unknown'; + serverId: string; + channelId: string; + messageId?: string; + }) + | (TraceEventBase & { + type: 'prose:drafts-dismissed'; + agentName: string; + draftIds: string[]; }); /** diff --git a/test/explicit-prose-routing.test.ts b/test/explicit-prose-routing.test.ts index 98a000067..588f08774 100644 --- a/test/explicit-prose-routing.test.ts +++ b/test/explicit-prose-routing.test.ts @@ -85,6 +85,14 @@ function stubRegistry(framework: AgentFramework, plan: Array<'delivered' | 'fals if (behavior === 'throw') throw new Error('synthetic publication failure'); return { delivered: behavior === 'delivered', channelId: locus ?? '' }; }, + // The registry's outcome-returning form of routeSpeech, over the same plan. + deliverSpeech: async (agent: string, text: string, locus?: string | null) => { + const result = await (explicit.routeSpeech as (a: string, t: string, l?: string | null) => Promise<{ delivered: boolean; channelId: string }>)(agent, text, locus); + return result.delivered + ? { status: 'delivered', destination: { serverId: 'stub', channelId: result.channelId }, at: Date.now() } + : { status: 'failed', destination: { serverId: 'stub', channelId: result.channelId }, reason: 'stub reported delivered:false', at: Date.now() }; + }, + resolveDestination: ({ channelId }: { channelId: string }) => ({ destination: { serverId: 'stub', channelId } }), resolveLocus: () => 'world:commons', getDefaultPublishChannel: () => null, isChannelOpen: () => true, @@ -443,7 +451,15 @@ describe('explicit prose routing', () => { const cm = framework.getAgent('assistant')!.getContextManager(); const texts = cm.getAllMessages().flatMap(m => m.content).filter(b => b.type === 'text').map(b => (b as { text: string }).text); assert.ok(texts.includes(authored)); - assert.ok(texts.some(t => t.startsWith('[delivered] nothing') && t.includes('suppressed'))); + assert.ok(texts.some(t => t.startsWith('[delivered] nothing') && /2 plain-speech segment\(s\) held as drafts d-[a-z2-9]{5}, d-[a-z2-9]{5}/.test(t))); + // The held words are the envelope's body, not its routing syntax; the + // destination the resident wrote stays with the draft as a note. + const drafts = (framework as unknown as { proseDrafts: { open(a: string): Array<{ text: string; note?: string; reason: string }> } }) + .proseDrafts.open('assistant').reverse(); + assert.deepEqual(drafts.map(d => [d.text, d.note, d.reason]), [ + ['wrong-locus duplicate', 'written for >>>#cafe', 'explicit-send'], + ['tool completed', undefined, 'explicit-send'], + ]); await framework.stop(); }); diff --git a/test/fixtures/speech-route-mcpl-server.mjs b/test/fixtures/speech-route-mcpl-server.mjs index e4bffa711..8cebbf70c 100644 --- a/test/fixtures/speech-route-mcpl-server.mjs +++ b/test/fixtures/speech-route-mcpl-server.mjs @@ -221,5 +221,10 @@ rl.on('line', (line) => { reply(msg.id, { tools: [] }); return; } + if (msg.method === 'tools/call') { + // No tools: answer at once (a rollback's awareness markers call one). + replyError(msg.id, `no tool "${msg.params?.name}" on this fixture`); + return; + } // Responses to our own requests (register, incoming, push) need no handling. }); diff --git a/test/present-while-acting.test.ts b/test/present-while-acting.test.ts index c5c3157bc..f2dfaafef 100644 --- a/test/present-while-acting.test.ts +++ b/test/present-while-acting.test.ts @@ -624,8 +624,13 @@ describe('present while acting', () => { .filter((b) => b.type === 'text') .map((b) => b.text ?? '') .filter((t) => t.startsWith('[delivered]')); + // The suppressed words are held as drafts, named in the receipt. + const drafts = (framework as unknown as { proseDrafts: { open(a: string): Array<{ id: string; text: string }> } }) + .proseDrafts.open('assistant').reverse(); + assert.deepEqual(drafts.map((d) => d.text), ['the reply that silencing will eat', 'trailing prose, also suppressed (sticky)']); assert.deepEqual(receipts, [ - '[delivered] nothing — 2 plain-speech segment(s) suppressed (explicit send in the same round — resend with a send tool if it was meant to be heard)', + `[delivered] nothing — 2 plain-speech segment(s) held as drafts ${drafts.map((d) => d.id).join(', ')} ` + + '(not sent — drafts can resend them unchanged, or dismiss them)', ]); await framework.stop(); @@ -665,8 +670,12 @@ describe('present while acting', () => { .filter((b) => b.type === 'text') .map((b) => b.text ?? '') .filter((t) => t.startsWith('[delivered]')); + const drafts = (framework as unknown as { proseDrafts: { open(a: string): Array<{ id: string; text: string }> } }) + .proseDrafts.open('assistant').reverse(); + assert.deepEqual(drafts.map((d) => d.text), ['adjacent prose that must NOT auto-publish', 'trailing prose, also suppressed (sticky)']); assert.deepEqual(receipts, [ - '[delivered] nothing — 2 plain-speech segment(s) suppressed (explicit send in the same round — resend with a send tool if it was meant to be heard)', + `[delivered] nothing — 2 plain-speech segment(s) held as drafts ${drafts.map((d) => d.id).join(', ')} ` + + '(not sent — drafts can resend them unchanged, or dismiss them)', ], `${verb}: suppression is visible in the receipt`); await framework.stop(); diff --git a/test/prose-drafts.test.ts b/test/prose-drafts.test.ts new file mode 100644 index 000000000..f4c0e1e3d --- /dev/null +++ b/test/prose-drafts.test.ts @@ -0,0 +1,603 @@ +/** + * Held prose drafts (shelf-351): plain speech that is not sent is kept as a + * private, explicitly resendable draft instead of vanishing. + * + * Unit tests drive ProseDraftStore over a real Chronicle store; end-to-end + * tests run real turns (MockMembrane) against a real MCPL child (the + * speech-route fixture), which records every channels/publish it receives. + */ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { tmpdir } from 'node:os'; +import { appendFileSync, existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { JsStore } from '@animalabs/chronicle'; +import type { ContentBlock, NormalizedRequest, StreamEvent, YieldingStream } from '@animalabs/membrane'; +import { AgentFramework } from '../src/index.js'; +import { ProseDraftStore, draftState, PROSE_DRAFT_RECORD_TYPE } from '../src/prose-drafts.js'; +import { MockMembrane, createMockResponse } from './helpers/mock-membrane.js'; + +const ROOM = 'discord:g1:room'; +const GENERAL = 'discord:g1:general'; +const source = { branch: 'main', turn: 1, round: 1, segment: 0 }; +const dest = (channelId = ROOM) => ({ serverId: 'discord', channelId, label: '#room (Guild One)' }); + +describe('ProseDraftStore', () => { + const open = () => { + const dir = mkdtempSync(join(tmpdir(), 'prose-drafts-unit-')); + const store = JsStore.openOrCreate({ path: join(dir, 'store') }); + return { dir, store, drafts: new ProseDraftStore(store) }; + }; + + it('holds exact words in order, with stable ids, per resident', () => { + const { dir, store, drafts } = open(); + try { + const { held } = drafts.hold('a', [{ text: ' one\n', source }, { text: 'two ', source: { ...source, segment: 1 } }], 'explicit-send'); + assert.equal(held.length, 2); + assert.match(held[0]!.id, /^d-[a-z2-9]{5}$/); + assert.notEqual(held[0]!.id, held[1]!.id); + assert.deepEqual(drafts.open('a').map((d) => d.text), ['two ', ' one\n'], 'newest first, bytes exact'); + assert.deepEqual(drafts.open('b'), [], 'another resident sees none of them'); + assert.equal(drafts.get('b', held[0]!.id), undefined); + assert.equal(draftState(held[0]!), 'held'); + } finally { + store.close(); + rmSync(dir, { recursive: true, force: true }); + } + }); + + it('replays the journal at reopen: an attempt with no recorded outcome reads as unconfirmed', () => { + const { dir, store, drafts } = open(); + const path = join(dir, 'store'); + const { held } = drafts.hold('a', [{ text: 'words', source }, { text: 'more', source }], 'explicit-send'); + drafts.beginAttempt('a', held[0]!.id, dest(), 'resend', false); // the process "dies" here + const second = drafts.beginAttempt('a', held[1]!.id, dest(), 'resend', false); + drafts.recordOutcome('a', held[1]!.id, second, { status: 'delivered', messageId: 'm-9', at: 5, destination: dest() }); + store.close(); + const reopened = JsStore.openOrCreate({ path }); + try { + const again = new ProseDraftStore(reopened); + assert.equal(draftState(again.get('a', held[0]!.id)!), 'unconfirmed'); + assert.equal(draftState(again.get('a', held[1]!.id)!), 'delivered'); + assert.equal(again.get('a', held[1]!.id)!.attempts[0]!.outcome!.messageId, 'm-9'); + } finally { + reopened.close(); + rmSync(dir, { recursive: true, force: true }); + } + }); + + it('is not moved by branch switches: undo, rollback and checkout leave every draft as it was', () => { + const { dir, store, drafts } = open(); + try { + const main = store.currentBranch().name; + const { held } = drafts.hold('a', [{ text: 'kept', source }, { text: 'set aside', source }], 'explicit-send'); + const attempt = drafts.beginAttempt('a', held[0]!.id, dest(), 'resend', false); + drafts.recordOutcome('a', held[0]!.id, attempt, { status: 'delivered', at: 7, destination: dest() }); + drafts.dismiss('a', held[1]!.id); + store.createBranch('elsewhere', main); + store.switchBranch('elsewhere'); + drafts.reload(); + assert.equal(draftState(drafts.get('a', held[0]!.id)!), 'delivered', 'a delivered draft stays delivered'); + assert.equal(draftState(drafts.get('a', held[1]!.id)!), 'dismissed', 'a dismissed draft stays dismissed'); + const later = drafts.hold('a', [{ text: 'held on another branch', source }], 'explicit-send').held[0]!; + store.switchBranch(main); + drafts.reload(); + assert.equal(draftState(drafts.get('a', later.id)!), 'held', 'a draft is usable from any branch'); + assert.equal(store.getRecordIdsByType(PROSE_DRAFT_RECORD_TYPE).length, 6); + } finally { + store.close(); + rmSync(dir, { recursive: true, force: true }); + } + }); + + it('reports words it could not hold, and stays consistent with its journal', () => { + const { dir, store, drafts } = open(); + try { + const real = store.sync.bind(store); + let fail = true; + (store as unknown as { sync: () => void }).sync = () => { if (fail) throw new Error('EIO'); real(); }; + const result = drafts.hold('a', [{ text: 'x', source }, { text: 'y', source }], 'explicit-send'); + assert.equal(result.held.length, 0); + assert.deepEqual(result.notHeld, { count: 2, error: 'EIO' }); + // The first record reached the store before its barrier failed: the + // journal was reconciled, so the projection shows what it holds. + assert.deepEqual(drafts.open('a').map((d) => d.text), ['x']); + fail = false; + assert.equal(drafts.hold('a', [{ text: 'z', source }], 'explicit-send').held.length, 1); + } finally { + store.close(); + rmSync(dir, { recursive: true, force: true }); + } + }); + + it('{{unsent}} is the latest bounce while it is open, latest-wins', () => { + const { dir, store, drafts } = open(); + try { + const first = drafts.hold('a', [{ text: 'first', source }], 'bounced').held[0]!; + const second = drafts.hold('a', [{ text: 'second', source }], 'bounced').held[0]!; + drafts.hold('a', [{ text: 'not a bounce', source }], 'explicit-send'); + assert.equal(drafts.latestBounce('a')?.id, second.id); + drafts.dismiss('a', second.id); + assert.equal(drafts.latestBounce('a'), undefined, 'an older bounce is not substituted'); + assert.equal(draftState(drafts.get('a', first.id)!), 'held', 'it stays in the collection by id'); + } finally { + store.close(); + rmSync(dir, { recursive: true, force: true }); + } + }); +}); + +// --------------------------------------------------------------------------- +// End to end: real turns against a real MCPL child +// --------------------------------------------------------------------------- + +type Publish = { event: string; channelId: string; text: string; messageId?: string }; + +/** A stream in XML tool mode: tool-calls carry no verbatim roundContent. */ +class XmlModeStream implements YieldingStream { + constructor(private readonly inner: YieldingStream) {} + provideToolResults(...args: Parameters): void { + this.inner.provideToolResults(...args); + } + cancel(): void { this.inner.cancel(); } + get isWaitingForTools(): boolean { return this.inner.isWaitingForTools; } + get pendingToolCallIds(): string[] { return this.inner.pendingToolCallIds; } + get toolDepth(): number { return this.inner.toolDepth; } + async *[Symbol.asyncIterator](): AsyncIterator { + for await (const event of this.inner) { + if (event.type !== 'tool-calls') { + yield event; + continue; + } + const context = { ...(event as unknown as { context: Record }).context }; + delete context.roundContent; + yield { ...event, context } as unknown as StreamEvent; + } + } +} + +/** A membrane in XML tool mode (the inner mock's streams, wrapped). */ +function xmlMode(inner: MockMembrane): MockMembrane { + const streamYielding = inner.streamYielding.bind(inner); + (inner as unknown as { streamYielding: (r: NormalizedRequest, o?: unknown) => YieldingStream }).streamYielding = + (request, options) => new XmlModeStream(streamYielding(request, options)); + return inner; +} + +async function harness(opts: { + agents?: Array<{ name: string; proseRouting?: 'locus' | 'explicit' | 'hybrid' | 'disabled'; sameRoundThinkTextPolicy?: 'public' | 'private' }>; + xml?: boolean; +} = {}) { + const dir = mkdtempSync(join(tmpdir(), 'prose-drafts-e2e-')); + const commandPath = join(dir, 'commands.jsonl'); + const statusPath = join(dir, 'status.jsonl'); + writeFileSync(commandPath, ''); + const membrane = opts.xml ? xmlMode(new MockMembrane()) : new MockMembrane(); + // A wake the test did not script (an explicit-mode bounce wake, say) gets + // an empty completion instead of a stream with no events. + const streamYielding = membrane.streamYielding.bind(membrane); + (membrane as unknown as { streamYielding: typeof streamYielding }).streamYielding = (request, options) => { + const internals = membrane as unknown as { responses: unknown[]; responseIndex: number }; + if (internals.responseIndex >= internals.responses.length) membrane.pushResponse(createMockResponse([])); + return streamYielding(request, options); + }; + const agents = (opts.agents ?? [{ name: 'scout' }]).map((a) => ({ + name: a.name, model: 'test-model', systemPrompt: `You are ${a.name}.`, + ...(a.proseRouting ? { proseRouting: a.proseRouting } : {}), + ...(a.sameRoundThinkTextPolicy ? { sameRoundThinkTextPolicy: a.sameRoundThinkTextPolicy } : {}), + })); + const create = async () => { + const framework = await AgentFramework.create({ + storePath: join(dir, 'store'), + membrane: membrane.asMembrane(), + agents, + mcplServers: [{ + id: 'discord', + command: process.execPath, + args: [join(import.meta.dirname, 'fixtures/speech-route-mcpl-server.mjs')], + env: { STATUS_PATH: statusPath, COMMAND_PATH: commandPath }, + }], + modules: [], + }); + await framework.start(); + const registry = (framework as unknown as { channelRegistry: { listChannelsRaw(): unknown[] } }).channelRegistry; + await until(() => registry.listChannelsRaw().length >= 2, 'registration'); + return framework; + }; + const until = async (cond: () => boolean, what: string) => { + const deadline = Date.now() + 15_000; + while (!cond()) { + if (Date.now() > deadline) throw new Error(`timed out: ${what}`); + await new Promise((r) => setTimeout(r, 20)); + } + }; + let framework = await create(); + let incoming = 0; + const h = { + dir, + membrane, + get framework() { return framework; }, + until, + command: (c: Record) => appendFileSync(commandPath, JSON.stringify(c) + '\n'), + publishes: (): Publish[] => existsSync(statusPath) + ? readFileSync(statusPath, 'utf8').split('\n').filter(Boolean).map((l) => JSON.parse(l) as Publish) + .filter((e) => e.event === 'publish' || e.event === 'publish-refused' || e.event === 'publish-error') + : [], + /** One addressed message in #room starts a turn (or, with `request`, a + * direct inference request for just that resident); resolves when it settles. */ + turn: async (responses: Array>, agentName = 'scout', via: 'incoming' | 'request' = 'incoming') => { + for (const r of responses) membrane.pushResponse(r); + const before = membrane.calls.length; + if (via === 'incoming') { + h.command({ op: 'incoming', channelId: ROOM, messageId: `in-${++incoming}`, mode: 'addressed', text: `message ${incoming}` }); + } else { + (framework as unknown as { pendingRequests: unknown[] }).pendingRequests.push({ + agentName, reason: 'test', source: 'test', timestamp: Date.now(), + }); + } + await until(() => membrane.calls.length > before, 'turn start'); + await framework.runUntilIdle(); + await until(() => !(framework as unknown as { activeTurnTokens: Map }).activeTurnTokens.has(agentName), 'turn end'); + await framework.runUntilIdle(); + }, + drafts: (agent = 'scout') => (framework as unknown as { proseDrafts: ProseDraftStore }).proseDrafts.open(agent), + store: () => (framework as unknown as { proseDrafts: ProseDraftStore }).proseDrafts, + texts: (agent = 'scout') => framework.getAgent(agent)!.getContextManager().getAllMessages() + .flatMap((m) => m.content).filter((b) => b.type === 'text').map((b) => (b as { text: string }).text), + messages: (agent = 'scout') => framework.getAgent(agent)!.getContextManager().getAllMessages(), + /** Text of every tool result the last stream received, in call order + * within each round (results arrive in completion order). */ + toolResults: () => (membrane.lastStream?.receivedToolResults ?? []).flatMap((round, i) => { + const order = (membrane.lastStream as unknown as { responses: Array<{ toolCalls: Array<{ id: string }> }> }) + .responses[i]?.toolCalls.map((c) => c.id) ?? []; + const byId = (r: unknown) => order.indexOf((r as { toolUseId?: string }).toolUseId ?? ''); + return [...round].sort((a, b) => byId(a) - byId(b)).map((r) => { + const content = (r as { content?: unknown }).content; + if (typeof content === 'string') return content; + if (Array.isArray(content)) return content.map((b) => (b as { text?: string }).text ?? '').join(''); + return JSON.stringify(r); + }); + }), + restart: async () => { + await framework.stop(); + // A fresh fixture child reads the command file from the top: start it + // on an empty one so earlier inbound traffic is not replayed. + writeFileSync(commandPath, ''); + framework = await create(); + }, + close: async () => { + await framework.stop(); + rmSync(dir, { recursive: true, force: true }); + }, + }; + return h; +} + +const text = (t: string) => ({ type: 'text', text: t }) as ContentBlock; +const call = (id: string, name: string, input: Record) => + ({ type: 'tool_use', id, name, input }) as ContentBlock; +const drafts = (id: string, input: Record) => + call(id, 'drafts', { action: null, draftIds: null, destination: null, confirmDuplicate: null, offset: null, ...input }); +const explicitSend = (id: string) => call(id, 'channel_publish', { channelId: GENERAL, content: 'an explicit note elsewhere' }); + +describe('held prose drafts, end to end', () => { + it('a suppressed three-segment reply is held, named, inspected and delivered without retyping', async () => { + const h = await harness(); + try { + const segments = ['First — the actual reply.', 'Second, with two spaces\nand a newline.', 'Third: trailing words.']; + await h.turn([ + createMockResponse([text(segments[0]!), explicitSend('s1')], 'tool_use'), + createMockResponse([text(segments[1]!), call('l1', 'channel_list', {})], 'tool_use'), + createMockResponse([text(segments[2]!)]), + ]); + assert.deepEqual(h.publishes().map((p) => p.text), ['an explicit note elsewhere'], 'only the explicit send went out'); + const held = h.drafts().reverse(); + assert.deepEqual(held.map((d) => d.text), segments, 'exact words, in order'); + assert.ok(held.every((d) => d.reason === 'explicit-send')); + const ids = held.map((d) => d.id); + + // Safe in-band ordering: each mid-turn notice is stored after the tool + // result it rode with, and was injected into the live stream there. + const msgs = h.messages(); + const noticeIdx = msgs.findIndex((m) => (m.metadata as { kind?: string } | undefined)?.kind === 'prose-drafts'); + assert.ok(noticeIdx > 0, 'a held-draft notice was stored'); + assert.ok(msgs[noticeIdx - 1]!.content.some((b) => b.type === 'tool_result'), 'right after the tool result'); + const injected = (h.membrane.lastStream?.receivedToolResultOptions ?? []) + .flatMap((o) => o?.injectedMessages ?? []).map((m) => JSON.stringify(m.content)); + assert.ok(injected.some((t) => t.includes(ids[0]!)), 'the resident heard about the first draft mid-turn'); + const receipt = h.texts().find((t) => t.startsWith('[delivered]'))!; + assert.equal(receipt, `[delivered] nothing — 3 plain-speech segment(s) held as drafts ${ids.join(', ')} (not sent — drafts can resend them unchanged, or dismiss them)`); + + // Next turn: list (every field present, nulls), read, then resend all three. + await h.turn([ + createMockResponse([drafts('d1', { action: 'list' })], 'tool_use'), + createMockResponse([drafts('d2', { action: 'read', draftIds: [ids[1]] })], 'tool_use'), + createMockResponse([drafts('d3', { action: 'resend', draftIds: ids, destination: ROOM })], 'tool_use'), + createMockResponse([]), + ]); + const [list, read, resend] = h.toolResults(); + assert.match(list!, /3 open drafts, newest first/); + assert.ok(read!.endsWith(`exactly as written):\n${segments[1]}`), `read shows the text in full: ${read}`); + assert.match(resend!, new RegExp(`${ids[0]}: delivered to #room \\(Guild One\\) \\(${ROOM}\\), message posted-2`)); + const sent = h.publishes().slice(1); + assert.deepEqual(sent.map((p) => [p.channelId, p.text]), segments.map((s) => [ROOM, s]), 'verbatim, in order'); + assert.ok(h.store().open('scout').length === 0, 'nothing left open'); + } finally { + await h.close(); + } + }); + + it('XML tool mode: the whole turn\'s prose is held at turn end and named in the receipt, never mid-turn', async () => { + const h = await harness({ xml: true }); + try { + // XML mode completes with the turn's cumulative content (its tool + // calls already ran: the completion carries none of its own). + await h.turn([ + createMockResponse([text('Narration before the send.'), explicitSend('s1')], 'tool_use'), + { ...createMockResponse([text('Narration before the send.'), explicitSend('s1'), text('And after it.')]), toolCalls: [] }, + ]); + const held = h.drafts().reverse(); + assert.deepEqual(held.map((d) => d.text), ['Narration before the send.', 'And after it.']); + assert.equal(h.messages().filter((m) => (m.metadata as { kind?: string } | undefined)?.kind === 'prose-drafts').length, 0, + 'no mid-turn notice: XML rounds present no injections'); + assert.ok(h.texts().some((t) => t.startsWith('[delivered] nothing — 2 plain-speech segment(s) held as drafts'))); + } finally { + await h.close(); + } + }); + + it('deliberate privacy is never drafted: skip_reply and a same-round private think', async () => { + const h = await harness({ agents: [{ name: 'scout', sameRoundThinkTextPolicy: 'private' }] }); + try { + await h.turn([ + createMockResponse([text('Thinking aloud, privately.'), call('k1', 'skip_reply', {})], 'tool_use'), + createMockResponse([]), + ]); + assert.deepEqual(h.drafts(), []); + assert.ok(h.texts().some((t) => t === '[delivered] nothing — 1 plain-speech segment(s) kept private (skip_reply)')); + await h.turn([ + createMockResponse([text('Prose beside a private think.'), call('t1', 'think', { thought: 'hmm' }), explicitSend('s1')], 'tool_use'), + createMockResponse([]), + ]); + assert.deepEqual(h.drafts(), [], 'a same-round private think keeps its prose private, even beside a send'); + assert.deepEqual(h.publishes().map((p) => p.text), ['an explicit note elsewhere']); + } finally { + await h.close(); + } + }); + + it('disabled mode stays draft-free: its narration is private by configuration', async () => { + const h = await harness({ agents: [{ name: 'scout', proseRouting: 'disabled' }] }); + try { + await h.turn([ + createMockResponse([text('Narration that never publishes.'), explicitSend('s1')], 'tool_use'), + createMockResponse([text('More of it.')]), + ]); + assert.deepEqual(h.drafts(), []); + assert.ok(h.texts().some((t) => t.startsWith('[delivered] nothing — 2 plain-speech segment(s) suppressed (proseRouting=disabled'))); + } finally { + await h.close(); + } + }); + + it('drafts belong to their resident: another resident can neither list nor resend them', async () => { + const h = await harness({ agents: [{ name: 'scout' }, { name: 'other' }] }); + try { + // Direct requests: a channel message would wake both residents. + await h.turn([ + createMockResponse([text('scout words'), explicitSend('s1')], 'tool_use'), + createMockResponse([]), + ], 'scout', 'request'); + const [mine] = h.drafts('scout'); + assert.ok(mine); + await h.turn([ + createMockResponse([drafts('o1', { action: 'list' })], 'tool_use'), + createMockResponse([drafts('o2', { action: 'resend', draftIds: [mine.id], destination: ROOM })], 'tool_use'), + createMockResponse([]), + ], 'other', 'request'); + const [list, resend] = h.toolResults(); + assert.match(list!, /You have no open drafts/); + assert.match(resend!, new RegExp(`No draft of yours with id ${mine.id}`)); + assert.equal(h.publishes().filter((p) => p.text === 'scout words').length, 0); + // Residents of one framework share a message slot (#197), so the + // notice is visible there too; it names whose drafts it means. + assert.ok(h.texts('scout').some((t) => t.startsWith(`[drafts] scout: not sent — a plain-speech segment held as draft ${mine.id}`))); + } finally { + await h.close(); + } + }); + + it('failed and uncertain resends: a refusal stays resendable; an unknown outcome needs confirmDuplicate', async () => { + const h = await harness(); + try { + await h.turn([ + createMockResponse([text('fragile words'), explicitSend('s1')], 'tool_use'), + createMockResponse([]), + ]); + const [draft] = h.drafts(); + const resend = (id: string, confirm?: boolean) => + drafts(id, { action: 'resend', draftIds: [draft!.id], destination: ROOM, ...(confirm ? { confirmDuplicate: true } : {}) }); + + h.command({ op: 'publish-mode', mode: 'not-delivered' }); + await h.turn([createMockResponse([resend('r1')], 'tool_use'), createMockResponse([])]); + assert.match(h.toolResults()[0]!, /not sent to .* Nothing was posted; the draft stays held/); + assert.equal(draftState(h.store().get('scout', draft!.id)!), 'held'); + + h.command({ op: 'publish-mode', mode: 'no-receipt' }); + await h.turn([createMockResponse([resend('r2')], 'tool_use'), createMockResponse([])]); + assert.match(h.toolResults()[0]!, /NOT confirmed — it may or may not have been posted/); + assert.equal(draftState(h.store().get('scout', draft!.id)!), 'unconfirmed'); + + h.command({ op: 'publish-mode', mode: 'delivered' }); + await h.turn([createMockResponse([resend('r3')], 'tool_use'), createMockResponse([])]); + assert.match(h.toolResults()[0]!, /may already have been posted.*confirmDuplicate: true\. Nothing was sent/); + const before = h.publishes().length; + await h.turn([createMockResponse([resend('r4', true)], 'tool_use'), createMockResponse([])]); + assert.match(h.toolResults()[0]!, /delivered to/); + assert.equal(h.publishes().length, before + 1); + const attempts = h.store().get('scout', draft!.id)!.attempts; + assert.deepEqual(attempts.map((a) => [a.outcome?.status, a.confirmedDuplicate ?? false]), [ + ['failed', false], ['unknown', false], ['delivered', true], + ]); + } finally { + await h.close(); + } + }); + + it('June\'s cases: a rollback to before a confirmed resend returns the receipt, no new publish; a dismissal survives it', async () => { + const h = await harness(); + try { + await h.turn([ + createMockResponse([text('to deliver'), explicitSend('s1')], 'tool_use'), + createMockResponse([text('to dismiss')]), + ]); + const [deliver, dismiss] = h.drafts().reverse(); + const anchor = h.messages().at(-1)!.id; + await h.turn([ + createMockResponse([drafts('r1', { action: 'resend', draftIds: [deliver!.id], destination: ROOM })], 'tool_use'), + createMockResponse([drafts('x1', { action: 'dismiss', draftIds: [dismiss!.id] })], 'tool_use'), + createMockResponse([]), + ]); + const published = h.publishes().length; + await h.framework.rollbackToMessage('scout', { messageId: anchor }); + assert.ok(!h.texts().some((t) => t.includes('delivered to')), 'the resend turn is gone from the branch'); + await h.turn([ + createMockResponse([drafts('r2', { action: 'resend', draftIds: [deliver!.id], destination: ROOM })], 'tool_use'), + createMockResponse([drafts('r3', { action: 'resend', draftIds: [dismiss!.id], destination: ROOM })], 'tool_use'), + createMockResponse([]), + ]); + const [receipt, refused] = h.toolResults(); + assert.match(receipt!, new RegExp(`${deliver!.id}: delivered — confirmed at .* to #room \\(Guild One\\) \\(${ROOM}\\), message posted-\\d+ .*not sent again`)); + assert.match(refused!, new RegExp(`${dismiss!.id} was dismissed`)); + assert.equal(h.publishes().length, published, 'no new publish'); + } finally { + await h.close(); + } + }); + + it('June\'s case: after a restart, an attempt that never recorded an outcome stays unknown and needs confirmDuplicate', async () => { + const h = await harness(); + try { + await h.turn([ + createMockResponse([text('half-sent words'), explicitSend('s1')], 'tool_use'), + createMockResponse([]), + ]); + const [draft] = h.drafts(); + // The process "died" between journaling the attempt and its outcome. + h.store().beginAttempt('scout', draft!.id, dest(), 'resend', false); + await h.restart(); + assert.equal(draftState(h.store().get('scout', draft!.id)!), 'unconfirmed'); + const before = h.publishes().length; + await h.turn([ + createMockResponse([drafts('r1', { action: 'resend', draftIds: [draft!.id], destination: ROOM })], 'tool_use'), + createMockResponse([]), + ]); + assert.match(h.toolResults()[0]!, /its outcome was never recorded.*confirmDuplicate: true\. Nothing was sent/); + assert.equal(h.publishes().length, before, 'nothing replayed at boot or sent without confirmation'); + } finally { + await h.close(); + } + }); + + it('a draft held in a turn that crashed is named by the next turn\'s catch-up notice', async () => { + const h = await harness(); + try { + // Held directly, as if the turn that held it died before its receipt. + h.store().hold('scout', [{ text: 'orphaned words', source }], 'explicit-send'); + await h.turn([createMockResponse([])]); + const notice = h.texts().find((t) => t.startsWith('[drafts]'))!; + assert.match(notice, /^\[drafts\] scout: a held draft of yours has not been named to you yet: d-[a-z2-9]{5} .*"orphaned words"/); + await h.turn([createMockResponse([])]); + assert.equal(h.texts().filter((t) => t.startsWith('[drafts]')).length, 1, 'named once'); + } finally { + await h.close(); + } + }); + + it('explicit mode: a bounce is a draft that survives a restart, and {{unsent}} delivers and settles it', async () => { + const h = await harness({ agents: [{ name: 'scout', proseRouting: 'explicit' }] }); + try { + await h.turn([createMockResponse([text('words with nowhere to go')])]); + const [bounced] = h.drafts(); + assert.equal(bounced?.reason, 'bounced'); + assert.ok(h.texts().some((t) => t.includes(`It is held as draft ${bounced!.id}`))); + await h.restart(); + await h.turn([createMockResponse([text(`>>${ROOM} {{unsent}}`)])]); + assert.deepEqual(h.publishes().map((p) => p.text), ['words with nowhere to go']); + const after = h.store().get('scout', bounced!.id)!; + assert.equal(draftState(after), 'delivered'); + assert.equal(after.attempts[0]!.via, 'unsent-token'); + } finally { + await h.close(); + } + }); + + it('resend is an explicit send: prose beside it is held, not auto-routed', async () => { + const h = await harness(); + try { + await h.turn([ + createMockResponse([text('earlier words'), explicitSend('s1')], 'tool_use'), + createMockResponse([]), + ]); + const [draft] = h.drafts(); + await h.turn([ + createMockResponse([text('a postscript'), drafts('r1', { action: 'resend', draftIds: [draft!.id], destination: ROOM })], 'tool_use'), + createMockResponse([]), + ]); + assert.deepEqual(h.publishes().map((p) => p.text), ['an explicit note elsewhere', 'earlier words']); + assert.deepEqual(h.drafts().map((d) => d.text), ['a postscript']); + } finally { + await h.close(); + } + }); + + it('two resends of one draft in the same round: one is sent, the other is refused while it is in flight', async () => { + const h = await harness(); + try { + await h.turn([ + createMockResponse([text('once only'), explicitSend('s1')], 'tool_use'), + createMockResponse([]), + ]); + const [draft] = h.drafts(); + const resend = (id: string) => + drafts(id, { action: 'resend', draftIds: [draft!.id], destination: ROOM, confirmDuplicate: true }); + await h.turn([createMockResponse([resend('a'), resend('b')], 'tool_use'), createMockResponse([])]); + const [first, second] = h.toolResults(); + assert.match(first!, /delivered to/); + assert.match(second!, /is being sent right now; wait for that result/); + assert.equal(h.publishes().filter((p) => p.text === 'once only').length, 1); + } finally { + await h.close(); + } + }); + + it('refuses unusable calls before sending anything', async () => { + const h = await harness(); + try { + await h.turn([ + createMockResponse([text('words'), explicitSend('s1')], 'tool_use'), + createMockResponse([]), + ]); + const [draft] = h.drafts(); + await h.turn([ + createMockResponse([ + drafts('a', { action: 'resend', draftIds: [draft!.id] }), + drafts('b', { action: 'resend', draftIds: [draft!.id], destination: '#nowhere' }), + drafts('c', { action: 'list', draftIds: [draft!.id] }), + drafts('d', { action: 'read', draftIds: ['d-zzzzz'] }), + drafts('e', { action: 'shout' }), + ], 'tool_use'), + createMockResponse([]), + ]); + const [noDest, badDest, unused, unknown, badAction] = h.toolResults(); + assert.match(noDest!, /resend needs a destination/); + assert.match(badDest!, /Destination "#nowhere" did not resolve.*Nothing was sent/); + assert.match(unused!, /draftIds is not used by action "list"/); + assert.match(unknown!, /No draft of yours with id d-zzzzz/); + assert.match(badAction!, /action must be one of/); + assert.deepEqual(h.publishes().map((p) => p.text), ['an explicit note elsewhere']); + } finally { + await h.close(); + } + }); +}); From 15ec398ac9ca454b21b67ea1ea533b96d8862fe9 Mon Sep 17 00:00:00 2001 From: Petra-1012 Date: Wed, 7 Oct 2026 08:12:34 -0400 Subject: [PATCH 0027/1112] fix(mcpl): refuse invalid publish selectors; a contradictory receipt is unknown Tessa-974's review of the publish executor (room-203 #43780, probe /tmp/tessa-974-publish-boundary-probe.mjs): 1. An explicitly empty serverId acted like omission and dispatched to the sole server registering the channel id. A supplied selector is now checked, never read as omission: resolveDestination/publish refuse an empty or non-string serverId and an empty or missing channelId (failed, nothing dispatched). Omission remains the only "not in use" form for this API. 2. `{delivered:false, messageId:123}` became `failed` once the malformed id was discarded, which does not support "nothing was posted". `delivered: false` is now `failed` only when the receipt names no message at all (messageId absent or null); a message id beside it, valid or malformed, is a contradictory receipt and reads `unknown` (a valid id is kept). 3. The publish() doc called it the executor behind channel tools, which is prospective: channel_publish (and publishForAgent, which shares its handler) move onto it with shelf-356. The doc says so, and it now sits on publish() itself (351 had left it above resolveDestination). Tests (publish-outcome.test.ts): delivered:false beside a numeric, empty and object message id is unknown with "contradictory receipt"; a null id is failed; empty/null/non-string serverId and empty/missing channelId are refused with nothing dispatched, while the exact selector still delivers. 43/43 with channel-registry-routing; 62/62 across prose-drafts, explicit-prose-routing, present-while-acting and mcpl-request-outcome. Co-Authored-By: Claude Opus 5.5 --- changelog.d/publish-outcomes.fixed.md | 11 ++-- src/mcpl/channel-registry.ts | 89 +++++++++++++++++---------- test/publish-outcome.test.ts | 36 +++++++++++ 3 files changed, 98 insertions(+), 38 deletions(-) diff --git a/changelog.d/publish-outcomes.fixed.md b/changelog.d/publish-outcomes.fixed.md index 62eb206a7..659f0742b 100644 --- a/changelog.d/publish-outcomes.fixed.md +++ b/changelog.d/publish-outcomes.fixed.md @@ -4,13 +4,16 @@ destination the registry resolved). Only a connector's `delivered: true` confirms a post; a missing or malformed receipt is no longer counted as delivered (#163's rule). `failed` means nothing was posted: refused before - dispatch, or `delivered: false` naming no posted message. An error response, - a timeout or a lost connection after dispatch is `unknown`, keeping the - connector's own words (a multi-part send may already be partly posted). + dispatch, or `delivered: false` naming no message. An error response, a + timeout or a lost connection after dispatch is `unknown`, keeping the + connector's own words (a multi-part send may already be partly posted), and + so is a contradictory receipt (`delivered: false` beside a message id, + valid or not). The resident's `[discord-send-failed]` marker now says which: "could not be delivered … Nothing was posted" for a failure, and "was not confirmed … may or may not have been posted; check the channel before sending it again" for an uncertain outcome. The marker's metadata and the `mcpl:speech-route-failed` trace carry `outcome`. A channel id registered by more than one MCPL server is refused rather than routed through - whichever server registered it first. + whichever server registered it first, and a supplied but empty or + non-string server selector is refused rather than read as omitted. diff --git a/src/mcpl/channel-registry.ts b/src/mcpl/channel-registry.ts index 6aa477bf2..a17ffd3a2 100644 --- a/src/mcpl/channel-registry.ts +++ b/src/mcpl/channel-registry.ts @@ -3002,25 +3002,6 @@ export class ChannelRegistry { return home ?? this.activeChannelResolver?.(conversationId) ?? this.defaultPublishChannel ?? null; } - /** - * Publish `text` to one registered channel and report what the attempt - * established (PublishOutcome). The single delivery executor behind plain - * speech (routeSpeech), resident resends of held drafts, and channel tools. - * - * Resolution is exact: `serverId` plus `channelId` when the caller has the - * server, otherwise the channel id must be registered by exactly one server - * (a shared id is refused, never routed through whichever server came - * first). It is not possible to send into a closed channel: a closed but - * registered destination is opened first (`openSource` names why), and a - * failed open sends nothing. - * - * Only the connector's `delivered: true` confirms a post. Anything that - * stops before the request is written is `failed` (nothing left the host), - * as is the connector's own `delivered: false`. Once the request has been - * dispatched, an error response, a timeout, a lost connection, or a missing - * or malformed receipt is `unknown`: the post may or may not exist, and the - * caller must not treat it as safe to retry blindly. - */ /** * Resolve a publish target to exactly one registered channel: `serverId` * plus `channelId` when the caller has the server, otherwise a channel id @@ -3044,6 +3025,14 @@ export class ChannelRegistry { private findExactEntry( target: { serverId?: string; channelId: string }, ): { entry: ChannelEntry } | { error: string } { + // A supplied selector is checked, never read as omission: an empty or + // non-string serverId would otherwise widen to "any server". + if (typeof target.channelId !== 'string' || !target.channelId) { + return { error: 'the destination names no channel id' }; + } + if (target.serverId !== undefined && (typeof target.serverId !== 'string' || !target.serverId)) { + return { error: 'serverId must name a server (omit it to resolve the channel id alone)' }; + } const matches = [...this.channels.values()].filter( (e) => e.descriptor.id === target.channelId && (!target.serverId || e.serverId === target.serverId), ); @@ -3062,6 +3051,33 @@ export class ChannelRegistry { return { entry: matches[0]! }; } + /** + * Publish `text` to one registered channel and report what the attempt + * established (PublishOutcome). The delivery executor behind plain speech + * (deliverSpeech / routeSpeech) and resident resends of held drafts; + * channel_publish (and publishForAgent, which shares its handler) moves + * onto it with shelf-356 — until then that tool reads its own receipt. + * + * Resolution is exact (resolveDestination): `serverId` plus `channelId` + * when the caller has the server, otherwise the channel id must be + * registered by exactly one server (a shared id is refused, never routed + * through whichever server came first). An omitted `serverId` means "not + * in use"; a supplied one must be a non-empty server id, and an empty or + * non-string selector is refused rather than read as omission. It is not + * possible to send into a closed channel: a closed but registered + * destination is opened first (`openSource` names why), and a failed open + * sends nothing. + * + * Only the connector's `delivered: true` confirms a post. `failed` means + * nothing was posted: the attempt stopped before the request was written, + * or the connector answered `delivered: false` and named no message (no + * `messageId`, or null). Everything else is `unknown`: once the request + * has been handed to the transport, an error response, a timeout, a lost + * connection, or a missing, malformed or contradictory receipt (such as + * `delivered: false` beside a message id, valid or not) means the post may + * or may not exist, and the caller must not treat it as safe to retry + * blindly. + */ async publish( conversationId: string, text: string, @@ -3153,21 +3169,26 @@ export class ChannelRegistry { } if (receipt.delivered === false) { // `delivered: false` proves nothing went out only when the receipt - // names no posted message; a named one means something did. - return messageId - ? { - status: 'unknown', - destination, - messageId, - reason: `server "${entry.serverId}" reported delivered:false but named posted message ${messageId} (delivery uncertain)`, - at: at(), - } - : { - status: 'failed', - destination, - reason: `server "${entry.serverId}" reported delivered:false`, - at: at(), - }; + // names no message at all. A message id beside it — valid or + // malformed — contradicts it: something may have been posted. + if (receipt.messageId === undefined || receipt.messageId === null) { + return { + status: 'failed', + destination, + reason: `server "${entry.serverId}" reported delivered:false`, + at: at(), + }; + } + return { + status: 'unknown', + destination, + ...(messageId ? { messageId } : {}), + reason: messageId + ? `server "${entry.serverId}" reported delivered:false but named posted message ${messageId} (delivery uncertain)` + : `server "${entry.serverId}" reported delivered:false beside a malformed message id ` + + `${JSON.stringify(receipt.messageId)} (contradictory receipt; delivery uncertain)`, + at: at(), + }; } // Final publish is a request, so success requires an explicit receipt; a // missing or malformed one leaves delivery uncertain (anima-research/ diff --git a/test/publish-outcome.test.ts b/test/publish-outcome.test.ts index b9bb16d3a..d9d2d79d3 100644 --- a/test/publish-outcome.test.ts +++ b/test/publish-outcome.test.ts @@ -73,6 +73,42 @@ describe('ChannelRegistry.publish outcomes', () => { assert.equal(outcome.status, 'failed'); }); + for (const [what, messageId] of [['a numeric', 123], ['an empty', ''], ['an object', { id: 'x' }]] as const) { + it(`treats delivered:false beside ${what} message id as unknown: a contradictory receipt proves nothing`, async () => { + const { registry, seed } = registryWith({ discord: { publish: async () => ({ delivered: false, messageId }) } }); + seed('discord', 'c', '#c'); + const outcome = await registry.publish('agent', 'x', { channelId: 'c' }); + assert.equal(outcome.status, 'unknown'); + assert.match(outcome.reason ?? '', /contradictory receipt/); + assert.equal(outcome.messageId, undefined); + }); + } + + it('treats delivered:false with a null message id as failed (nothing named)', async () => { + const { registry, seed } = registryWith({ discord: { publish: async () => ({ delivered: false, messageId: null }) } }); + seed('discord', 'c', '#c'); + assert.equal((await registry.publish('agent', 'x', { channelId: 'c' })).status, 'failed'); + }); + + it('refuses a supplied but empty or non-string selector instead of reading it as omitted', async () => { + const { registry, published, seed } = registryWith({ discord: { publish: ok } }); + seed('discord', 'c', '#c'); + for (const target of [ + { serverId: '', channelId: 'c' }, + { serverId: null as unknown as string, channelId: 'c' }, + { serverId: 7 as unknown as string, channelId: 'c' }, + { channelId: '' }, + { channelId: undefined as unknown as string }, + ]) { + const outcome = await registry.publish('agent', 'x', target); + assert.equal(outcome.status, 'failed', JSON.stringify(target)); + assert.equal(outcome.destination, undefined); + assert.ok('error' in registry.resolveDestination(target), JSON.stringify(target)); + } + assert.equal(published.length, 0, 'nothing dispatched to the sole matching server'); + assert.equal((await registry.publish('agent', 'x', { serverId: 'discord', channelId: 'c' })).status, 'delivered', 'the exact selector still works'); + }); + it('treats delivered:false that names a posted message as unknown', async () => { const { registry, seed } = registryWith({ discord: { publish: async () => ({ delivered: false, messageId: 'part-1' }) } }); seed('discord', 'c', '#c'); From 2cb501af62033293e0e375b181a7f69061f5f0b2 Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Wed, 7 Oct 2026 07:42:15 -0400 Subject: [PATCH 0028/1112] feat(awareness): make Discord awareness marks an explicit, one-shot choice that never gates MCPL MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Room-225's awareness-marks contract (Connectome incident: a web-UI rollback queued a 💤 reaction on each of 918 removed messages, mostly other people's ambient messages; the serial drain held the store and every MCPL data plane). Seven points, carried to nissa by Tessa (#41730) and designed with June-1016: 1. Marks are opt-in, default none. `marks: 'none' | { scope: 'addressed' | 'all', refs? }` on rollbackToMessage/suppressMessages; `marks` on host/command undo and hide (invalid values refused, never guessed); `--marks` on agent-framework-recover. `addressed` = MCPL tag chat:addressed. previewSurgeryMarks() shows per-scope counts/channels/refs; passing a scope's refs back freezes the choice (selection = refs ∩ removed, later arrivals removed locally but never marked; receipt counts unmarked and notRemoved). 2. One-shot. The ledger is rewritten as an append-only fsynced JSONL journal (recovery/discord-awareness-journal.jsonl): batch / activated / held / released / cancelled / retract / requested / op-cancelled / dispatching (write-ahead, with a dispatchId shared by ops riding one request) / outcome records. Branch switches, undo/redo and restarts derive nothing; reconcileForBranch and the API server's branch-switch syncs are removed. Startup: unanswered attempts become `unknown`; a prepared batch whose target is exactly the active branch is crash-completed; anything else prepared is held for explicit release. Suppression bodies still resume at startup (body integrity), but a held/cancelled batch's marks stay held. 3. Delivery is background and never gates: the DiscordAwarenessBarrier generation machinery, installMcplDataPlaneGate/release/complete/fail and the handlers' barrier awaits are gone; startup/connect/reconnect open planes via readyMcplPlanes (quiesce-aware) after the §5.3 grant, and kick delivery. A disconnected route keeps work queued; delivery failure is an ops alert, never a connection recycle. A journal that cannot be read still aborts startup (it carries the suppression resume record). 4. Outcomes come from McplRequestError.outcome (Petra's 023b439): not-sent stays requested; no-response is `unknown` forever (a later attempt's confirmation never resolves it); Discord errors are failed (permanent when the message/channel is gone). An unknown op is retried only while it is the newest live op for its key; one request per key on the wire at a time (including a cancelled op still awaiting its outcome). 5. Operator controls (framework methods, host/command `marks` verb, CLI --awareness): cancel stops all further sends/retries of a batch (prepared batches included; never removes); retract queues removal across batches by (route, channel, message, emoji), supersedes adds never sent, skips keys where nothing ever left the host, and discloses unresolved add requests; release queues a held batch. All operator-logged. 6. Multi-record appends are transactions (txn ids + commit record), so a torn append applies nothing; a torn final line is truncated (or a complete but unterminated record terminated) before the next append. 7. The v2 JSON ledger is imported once (deterministic ids, sha256 import-complete record): applied -> confirmed, permanent-failure -> failed, pending work held; the file is renamed .migrated-v2 (retried if needed). Receipts: SurgeryMarkerFacts (scope, unmarked, notRemoved) on every markers result; marks chosen without any journal path are `not-scheduled`. Offline recovery collects only addressing metadata + tags (never content). Tests: discord-awareness-outbox (18: selection, write-ahead, startup recovery, cancel/retract/release, retry-vs-unknown, txn/torn tail, v2 import, drain classification, suppression resume, host/command undo/marks/ hide), surgery-marks-receipt (10, real store + fake Discord MCPL server: default local, addressed, frozen refs, non-gating, not-scheduled/unresolved, cancel/retract with a timed-out add, no-journal), discord-awareness-delivery (5: startup not gated, mandatory deadline with requestTimeoutMs 0, route down then up, corrupt journal, branch switch inert), offline-recovery-branch (6). mcpl-awareness-barrier and its fixture are removed (behaviour gone). --- src/api/server.ts | 8 +- src/framework.ts | 1042 ++++++------ src/index.ts | 21 +- src/mcpl/server-connection.ts | 14 +- src/operator-log.ts | 60 +- src/recovery/discord-awareness-outbox.ts | 1434 +++++++++++++---- src/recovery/offline-branch.ts | 100 +- src/recovery/recover-cli.ts | 80 +- test/discord-awareness-delivery.test.ts | 207 +++ test/discord-awareness-outbox.test.ts | 763 ++++++--- .../awareness-barrier-mcpl-server.mjs | 276 ---- test/mcpl-awareness-barrier.test.ts | 798 --------- test/mcpl-reconnect-reregistration.test.ts | 7 +- test/offline-recovery-branch.test.ts | 65 +- test/operator-surgery.test.ts | 5 +- test/surgery-marks-receipt.test.ts | 337 ++-- 16 files changed, 2887 insertions(+), 2330 deletions(-) create mode 100644 test/discord-awareness-delivery.test.ts delete mode 100644 test/fixtures/awareness-barrier-mcpl-server.mjs delete mode 100644 test/mcpl-awareness-barrier.test.ts diff --git a/src/api/server.ts b/src/api/server.ts index 58664705b..51b7f0d9d 100644 --- a/src/api/server.ts +++ b/src/api/server.ts @@ -541,10 +541,8 @@ export class ApiServer { store.switchBranch(params.name); this.currentBranch = params.name; - - // Reversible Discord awareness markers are part of the branch projection. - // Do not acknowledge the switch until add/remove operations were attempted. - await this.framework.syncDiscordAwarenessMarkers(); + // Awareness marks are one-shot: a branch switch never adds or removes + // them (retract or release explicitly instead). // Materialize config files from the new branch const ws = this.framework.getModule('workspace'); @@ -598,7 +596,6 @@ export class ApiServer { if (result.undone) { this.currentBranch = result.toBranch!; - await this.framework.syncDiscordAwarenessMarkers(); this.broadcast('branch:switched', { from: result.fromBranch, to: result.toBranch, @@ -618,7 +615,6 @@ export class ApiServer { if (result.redone) { this.currentBranch = result.toBranch!; - await this.framework.syncDiscordAwarenessMarkers(); this.broadcast('branch:switched', { from: result.fromBranch, to: result.toBranch, diff --git a/src/framework.ts b/src/framework.ts index 9b80c3985..8ea76cbc8 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -387,7 +387,7 @@ import { isToolAllowed } from './mcpl/tool-policy.js'; import { EventGate, formatShadowWarning } from './gate/event-gate.js'; import { UsageTracker, type PersistedUsageState } from './usage/usage-tracker.js'; import type { SessionUsageSnapshot, UsageUpdatedEvent } from './usage/types.js'; -import type { McplServerConnection } from './mcpl/server-connection.js'; +import { McplRequestError, type McplServerConnection } from './mcpl/server-connection.js'; import type { McplServerConfig, McplHostCapabilities, @@ -406,14 +406,23 @@ import { DiscordAwarenessOutbox, defaultDiscordAwarenessOutboxPath, extractDiscordAwarenessRefs, + selectDiscordAwarenessRefs, type DiscordAwarenessBatch, + type DiscordAwarenessBatchView, + type DiscordAwarenessCancelReceipt, + type DiscordAwarenessMarks, + type DiscordAwarenessReleaseReceipt, + type DiscordAwarenessRetractReceipt, + type DiscordAwarenessScope, } from './recovery/discord-awareness-outbox.js'; import { OperatorActionError, OperatorLog, capIds, defaultOperatorLogPath, + type SurgeryMarkerFacts, type SurgeryMarkerReceipt, + type SurgeryMarksPreview, type OperatorLogEntry, type OperatorLogInput, type OperatorRequester, @@ -507,12 +516,6 @@ type DiscordAwarenessDrainOutcome = | { status: 'delivered'; delivered: number; failed: number } | { status: 'unavailable'; accounted: number }; -interface DiscordAwarenessBarrier { - generation: number; - requiresBarrier: boolean; - promise: Promise; -} - function normalizeDiscordAwarenessDeadline(value: number | undefined): number { if (value === undefined || !Number.isFinite(value) || value <= 0) { return DEFAULT_DISCORD_AWARENESS_DEADLINE_MS; @@ -719,8 +722,19 @@ function bodyGroupRun( }; } +/** The batch scope a publication choice records (`none` for local surgery). */ +function marksScope(marks: DiscordAwarenessMarks | undefined): 'none' | DiscordAwarenessScope { + return !marks || marks === 'none' ? 'none' : marks.scope; +} + +/** A publication choice, summarized for the operator log (refs counted, not listed). */ +function describeMarksChoice(marks: DiscordAwarenessMarks | undefined): Record | 'none' { + if (!marks || marks === 'none') return 'none'; + return { scope: marks.scope, ...(marks.refs ? { authorizedRefs: marks.refs.length } : {}) }; +} + function describeMarkers(m: SurgeryMarkerReceipt): string { - if (m.status === 'queued') return `queued:${m.queued}`; + if (m.status === 'queued') return `queued:${m.queued}(${m.scope}; unmarked ${m.unmarked})`; if (m.status === 'not-scheduled') return 'not-scheduled(batch retired)'; if (m.status === 'unresolved') return `unresolved(batch ${m.batchId} may still be delivered)`; return 'none'; @@ -883,6 +897,14 @@ interface HostCommandParams { * (regardless of participant) are no longer on the active branch. * Mutually exclusive with `turns`. */ messages?: number; + /** Message-granular undo and `hide`: awareness marks on the removed + * Discord messages, `none` (the default), `addressed` or `all`. Any other + * value is refused. */ + marks?: string; + /** For the `marks` command: `list`, `cancel`, `retract` or `release`. */ + action?: string; + /** For the `marks` command: the batch to act on (`all` for retract). */ + batchId?: string; /** For the `hide` command: Discord message id of the (first) message to * remove. With `toMessageId`, removes the inclusive range between them. */ fromMessageId?: string; @@ -1385,8 +1407,6 @@ export class AgentFramework { /** Serialize per-server drains so reconnect and an online undo cannot race. */ private discordAwarenessDrains: Map> = new Map(); /** Framework-global inference gate; older generations cannot release it. */ - private discordAwarenessBarrier: DiscordAwarenessBarrier | null = null; - private discordAwarenessBarrierGeneration = 0; // EventGate (null when FrameworkConfig.gate is omitted) private eventGate: EventGate | null = null; @@ -1565,24 +1585,24 @@ export class AgentFramework { ); framework.providerHoldHook = config.providerHold; - // If an offline recovery process crashed after switching Chronicle but - // before committing its prepared marker batch, the active branch is the - // commit record. Promote it now; no quarantined content is read. + // Startup reconciliation of the awareness journal, before any delivery. + // It derives nothing from branch ancestry: a surgery that crashed after + // switching to exactly the active branch completes its activation; any + // other prepared batch is held for an operator; an attempt the previous + // process left on the wire is recorded unknown. No content is read. if (discordAwarenessOutbox) { try { - const activated = discordAwarenessOutbox.activatePreparedForBranch( - store.currentBranch().name, - store.listBranches(), - ); - if (activated > 0) { + const recovered = discordAwarenessOutbox.recoverAtStartup(store.currentBranch().name); + if (recovered.activated.length > 0 || recovered.held.length > 0 || recovered.unknownAttempts > 0) { console.error( - `[discord-awareness] recovered ${activated} prepared batch(es) for active branch ${store.currentBranch().name}`, + `[discord-awareness] startup: activated ${recovered.activated.length} interrupted batch(es) ` + + `for ${store.currentBranch().name}, held ${recovered.held.length}, ` + + `recorded ${recovered.unknownAttempts} unanswered attempt(s) as unknown`, ); } } catch (error) { // The branch may be safe, but reporting the framework ready while its - // durable awareness projection is unreadable creates a half-ready - // host whose data plane can never be released safely. + // durable awareness journal is unreadable creates a half-ready host. throw new DiscordAwarenessAccountingError('startup reconciliation', error); } } @@ -1751,12 +1771,12 @@ export class AgentFramework { } // Restore persisted quiesce mode (issue #122) BEFORE initializeMcpl: the - // flag must be set before any data-plane barrier completion can run, or - // the startup funnel would open the data planes on a host that shut down - // mid-maintenance. Staged connections boot with both planes closed, so a - // quiesced boot needs no re-pause — completeMcplDataPlaneGate consults - // the flag and holds data planes (control planes come up normally). The - // gate already exists at this point, so the suppression is wired here too. + // flag must be set before startup opens any plane, or it would open the + // data planes on a host that shut down mid-maintenance. Staged + // connections boot with both planes closed, so a quiesced boot needs no + // re-pause — readyMcplPlanes consults the flag and holds data planes + // (control planes come up normally). The gate already exists at this + // point, so the suppression is wired here too. // // Both records live OUTSIDE branch history (recovery/ files next to the // store): a historical rollback must not be able to erase the marker of @@ -3265,26 +3285,10 @@ export class AgentFramework { await this.flushDeferredWrites('resume'); this.persistHostMode(null); } finally { - // Reopen MCPL data planes through the existing barrier funnel — NOT a - // bespoke ready() loop. The funnel inherits completeMcplDataPlaneGate's - // nested-install guard (a flushed tools-list-changed can install a - // newer barrier mid-flush), drains any awareness work accumulated - // during the window before opening, and replaces a stale failed - // barrier by identity. In `finally` so that nothing above can leave - // the host un-quiesced with every data plane still paused. - if (this.mcplServerRegistry) { - const barrier = this.installMcplDataPlaneGate(); - this.releaseMcplDataPlaneGate(barrier); - try { - await barrier.promise; - this.completeMcplDataPlaneGate(barrier); - } catch (error) { - // The host IS resumed — don't rethrow. failMcplDataPlaneGate - // recycles the connections, and their reconnect flows re-run the - // funnel with quiesced=false, self-healing the data planes. - await this.failMcplDataPlaneGate(barrier, 'quiesce resume', error); - } - } + // Reopen MCPL data planes (quiesced is already false). In `finally` + // so that nothing above can leave the host un-quiesced with every data + // plane still paused. + this.readyMcplPlanes(); this.emitTrace({ type: 'host:resume', @@ -5077,12 +5081,21 @@ export class AgentFramework { /** Discord addresses removed by message-granular undo. The durable outbox * owns eventual delivery; this is also returned for immediate surfaces. */ removedRefs?: Array<{ serverId: string; channelId: string; messageId: string }>; - /** For message-granular undo: whether awareness marks were scheduled - * (not whether Discord has accepted them). */ + /** For message-granular undo and `hide`: whether awareness marks were + * scheduled (not whether Discord has accepted them). */ markers?: SurgeryMarkerReceipt; + /** For the `marks` command: the journal's batches (`list`) or the + * action's receipt. */ + awareness?: + | DiscordAwarenessBatchView[] + | DiscordAwarenessCancelReceipt + | DiscordAwarenessRetractReceipt + | DiscordAwarenessReleaseReceipt; hidden?: number; /** For `hide`: the Discord (channelId, messageId) of each removed message - * that carried one — so the surface can mark them with a reaction. */ + * that carried one. Marking is the framework's (see `marks`/`markers`); + * a surface that also reacts itself should do so only when its operator + * chose marks and the framework returned no `markers`. */ hiddenRefs?: Array<{ channelId: string; messageId: string }>; lastVisible?: { participant?: string; role?: string; preview?: string } | null; /** For `unstick`: acknowledges the forced-rewind loop has started; the @@ -5104,13 +5117,48 @@ export class AgentFramework { params.command !== 'quiesce' && params.command !== 'resume' && params.command !== 'maintain' && - params.command !== 'host-status' + params.command !== 'host-status' && + params.command !== 'marks' ) { return { ok: false, error: `Unknown host command: ${String(params.command)}` }; } + if (params.marks !== undefined && params.marks !== 'none' && params.marks !== 'addressed' && params.marks !== 'all') { + // A publication choice is never guessed: a mistyped scope is refused + // rather than silently treated as none (or as anything else). + return { ok: false, error: `marks must be none, addressed or all (got ${JSON.stringify(params.marks)})` }; + } // Host-scoped verbs (issue #122) — no agent resolution. const requester = params.requesterName ?? params.requesterId ?? `mcpl:${serverId}`; + if (params.command === 'marks') { + // The awareness journal's operator controls, for surfaces that act + // through host/command (the web UI calls the framework methods). + const by = { requester: hostCommandRequester(serverId, params) }; + const batchId = params.batchId; + try { + switch (params.action) { + case 'list': + return { ok: true, awareness: this.listDiscordAwareness() }; + case 'cancel': + if (!batchId) return { ok: false, error: 'marks cancel needs batchId' }; + return { ok: true, awareness: this.cancelDiscordAwareness(batchId, by) }; + case 'retract': + if (!batchId) return { ok: false, error: 'marks retract needs batchId (or all)' }; + return { ok: true, awareness: this.retractDiscordAwareness(batchId, by) }; + case 'release': + if (!batchId) return { ok: false, error: 'marks release needs batchId' }; + return { ok: true, awareness: this.releaseDiscordAwareness(batchId, by) }; + default: + return { ok: false, error: 'marks needs action list, cancel, retract or release' }; + } + } catch (error) { + return { + ok: false, + error: error instanceof Error ? error.message : String(error), + ...(error instanceof OperatorActionError ? { code: error.code } : {}), + }; + } + } if (params.command === 'quiesce') { // quiesce() clamps timeoutMs to [1s, 10m] itself — one clamp for every ingress. console.error(`[host-command] quiesce by=${requester} (server=${serverId})`); @@ -5204,6 +5252,9 @@ export class AgentFramework { if (agent.state.status !== 'idle') { return { ok: false, error: `Cannot hide while agent is ${agent.state.status}` }; } + const hideMarks: DiscordAwarenessMarks = params.marks === 'addressed' || params.marks === 'all' + ? { scope: params.marks } + : 'none'; if (!params.fromMessageId) { return { ok: false, error: 'hide: fromMessageId is required' }; } @@ -5245,6 +5296,7 @@ export class AgentFramework { const refs = refsIn(lo, hi); const rangeIds = all.slice(lo, hi + 1).map((m) => String(m.id)); cm.removeMessages(all[lo].id, all[hi].id); + const markers = this.scheduleHideMarks(agentName, serverId, all.slice(lo, hi + 1), hideMarks); console.error( `[host-command] hide agent=${agentName} range removed=${hi - lo + 1} ` + `(${params.fromMessageId}..${params.toMessageId}) by=${params.requesterName ?? params.requesterId ?? 'unknown'} (server=${serverId})`, @@ -5254,24 +5306,31 @@ export class AgentFramework { kind: 'hide', agent: agentName, requester: hostCommandRequester(serverId, params), - params: { fromMessageId: params.fromMessageId, toMessageId: params.toMessageId }, + params: { + fromMessageId: params.fromMessageId, + toMessageId: params.toMessageId, + marks: describeMarksChoice(hideMarks), + }, result: { branch: this.store.currentBranch().name, hidden: hi - lo + 1, removedIds: loggedRange.ids, ...(loggedRange.truncated ? { removedIdsTruncated: true } : {}), + markers, }, }); return { ok: true, hidden: hi - lo + 1, hiddenRefs: refs, + markers, lastVisible: await this.lastVisiblePreview(agentName), }; } const refs = refsIn(fromIdx, fromIdx); const hiddenId = String(all[fromIdx].id); cm.removeMessage(all[fromIdx].id); + const markers = this.scheduleHideMarks(agentName, serverId, [all[fromIdx]], hideMarks); console.error( `[host-command] hide agent=${agentName} removed=1 (${params.fromMessageId}) ` + `by=${params.requesterName ?? params.requesterId ?? 'unknown'} (server=${serverId})`, @@ -5280,13 +5339,14 @@ export class AgentFramework { kind: 'hide', agent: agentName, requester: hostCommandRequester(serverId, params), - params: { fromMessageId: params.fromMessageId }, - result: { branch: this.store.currentBranch().name, hidden: 1, removedIds: [hiddenId] }, + params: { fromMessageId: params.fromMessageId, marks: describeMarksChoice(hideMarks) }, + result: { branch: this.store.currentBranch().name, hidden: 1, removedIds: [hiddenId], markers }, }); return { ok: true, hidden: 1, hiddenRefs: refs, + markers, lastVisible: await this.lastVisiblePreview(agentName), }; } catch (error) { @@ -5318,6 +5378,11 @@ export class AgentFramework { messageId: String(target.id), branchName: `undo-msgs/${agentName}/${Date.now()}`, requester: hostCommandRequester(serverId, params), + // An immediate act: the choice and its application are the same + // moment, so the scope is evaluated now (no frozen refs needed). + marks: params.marks === 'addressed' || params.marks === 'all' + ? { scope: params.marks } + : 'none', }); return { ok: true, @@ -5360,8 +5425,8 @@ export class AgentFramework { return { ok: true, undone: 0, requested, lastVisible: null }; } - await this.syncDiscordAwarenessMarkers(); - + // Turn undo is a branch move: awareness marks are one-shot and never + // follow branch state, so there is nothing to reconcile here. return { ok: true, undone, requested, lastVisible: await this.lastVisiblePreview(agentName) }; } @@ -5952,14 +6017,17 @@ export class AgentFramework { * Roll the active branch back so `messageId` becomes its tail: fork the * chronicle at that message (origin-sequence time-travel branch) and switch * to the fork. Everything after the message stays on the source branch. - * Discord messages that left the live context get awareness markers via the - * durable outbox, exactly as message-granular `undo` does. + * + * The rollback is local to the resident. Marking removed Discord messages + * with an awareness reaction is a separate, explicit publication choice + * (`marks`, default `none`); `previewSurgeryMarks` shows what each scope + * would cover, and passing its `refs` binds the choice to exactly that set. * * Returns once the switch has landed: the body is applied, with a * marker-scheduling receipt (`markers`). It does not wait for Discord to - * accept the reactions: delivery is started, behind the MCPL data-plane - * gate, and continues without the caller or the store reservation waiting - * on it; its per-message outcomes stay in the outbox ledger. + * accept the reactions: delivery is started and continues without the + * caller, the store reservation or MCPL traffic waiting on it; its + * per-request outcomes stay in the awareness journal. * * Throws `OperatorActionError` (`agent-busy`, `unknown-message`, …) — the * agent must be idle; nothing is queued. @@ -5972,6 +6040,8 @@ export class AgentFramework { note?: string; /** Branch name for the fork (default `rollback//`). */ branchName?: string; + /** Publication choice for awareness marks; default `none`. */ + marks?: DiscordAwarenessMarks; }, ): Promise<{ agentName: string; @@ -5992,29 +6062,12 @@ export class AgentFramework { agent: agentName, ...(opts.requester ? { requester: opts.requester } : {}), ...(opts.note ? { note: opts.note } : {}), - params: { messageId: opts.messageId }, + params: { messageId: opts.messageId, marks: describeMarksChoice(opts.marks) }, }; try { - const agent = this.agents.get(agentName); - if (!agent) throw new OperatorActionError('unknown-agent', `Unknown agent: ${agentName}`); - const cm = agent.getContextManager(); - const total = cm.getMessageCount(); - const requestedIndex = locateMessageIndex(cm, opts.messageId); - if (requestedIndex < 0) { - throw new OperatorActionError('unknown-message', `Message ${opts.messageId} is not on the active branch`); - } - // Never bisect a body group: chronicle refuses that for removals - // (byte-faithful reassembly needs the whole run) and branchAt has no - // such guard of its own. Snap to the run's last shard. - const run = bodyGroupRun(cm, requestedIndex); - const targetIndex = run.to; - const tailMessageId = run.toId; - const messagesRemoved = total - targetIndex - 1; - if (messagesRemoved <= 0) { - throw new OperatorActionError('invalid', `Message ${opts.messageId} is already the tail of the active branch`); - } - const discarded = cm.getMessageWindow(targetIndex + 1, messagesRemoved, { resolveBlobs: false }).messages; + const { cm, tailMessageId, messagesRemoved, discarded } = this.planRollback(agentName, opts.messageId); const removedRefs = extractDiscordAwarenessRefs(discarded); + const selection = selectDiscordAwarenessRefs(discarded, opts.marks ?? 'none'); const sourceBranch = this.store.currentBranch().name; const targetBranch = opts.branchName ?? `rollback/${agentName}/${Date.now()}`; if (targetBranch === sourceBranch) { @@ -6038,7 +6091,10 @@ export class AgentFramework { agentName, sourceBranch, targetBranch, - refs: removedRefs, + refs: selection.refs, + scope: marksScope(opts.marks), + unmarked: selection.unmarked, + notRemoved: selection.notRemoved, emoji: this.discordAwarenessEmoji, }) ?? null; @@ -6061,10 +6117,13 @@ export class AgentFramework { // The body change has landed. Nothing after this point turns the // rollback into a failure: marks are an outbox outcome, reported // apart from the body (`markers`). - markers = this.activateSurgeryMarkers(markerBatch, 'rollback', agentName); - // Installs the MCPL data-plane gate synchronously, before the store is - // released, and starts delivery without awaiting it (the first - // reaction request may already be on the wire when this returns). + markers = this.activateSurgeryMarkers(markerBatch, 'rollback', agentName, { + scope: marksScope(opts.marks) as SurgeryMarkerReceipt['scope'], + unmarked: selection.unmarked, + notRemoved: selection.notRemoved, + }, selection.refs.length); + // Starts delivery without awaiting it (the first reaction request may + // already be on the wire when this returns); it never gates traffic. this.deliverDiscordAwarenessInBackground('rollback', agentName); this.materializeConfigMountAfterBranchSwitch(); } finally { @@ -6127,6 +6186,8 @@ export class AgentFramework { note?: string; /** Branch name for the fork (default `suppress//`). */ branchName?: string; + /** Publication choice for awareness marks; default `none`. */ + marks?: DiscordAwarenessMarks; }, ): Promise<{ agentName: string; @@ -6145,43 +6206,13 @@ export class AgentFramework { agent: agentName, ...(opts.requester ? { requester: opts.requester } : {}), ...(opts.note ? { note: opts.note } : {}), - params: { messageIds: requestedIds }, + params: { messageIds: requestedIds, marks: describeMarksChoice(opts.marks) }, }; try { - const agent = this.agents.get(agentName); - if (!agent) throw new OperatorActionError('unknown-agent', `Unknown agent: ${agentName}`); - if (requestedIds.length === 0) throw new OperatorActionError('invalid', 'No message ids given'); - const cm = agent.getContextManager(); - const total = cm.getMessageCount(); - - const located = locateMessageIndices(cm, new Set(requestedIds)); - const missing = requestedIds.filter((id) => !located.has(id)); - if (missing.length > 0) { - throw new OperatorActionError( - 'unknown-message', - `Not on the active branch: ${missing.slice(0, 5).join(', ')}${missing.length > 5 ? ` (+${missing.length - 5})` : ''}`, - ); - } - - // Build removal intervals (index-ordered, newest first). A shard expands - // to its whole body group — chronicle refuses to bisect one — and a - // group interval is always removed as a RANGE: `removeMessage` refuses - // any sharded message even when the group is down to one shard. - const intervals = new Map>(); - for (const id of requestedIds) { - const run = bodyGroupRun(cm, located.get(id)!.index); - intervals.set(`${run.from}-${run.to}`, run); - } - const ordered = [...intervals.values()].sort((a, b) => b.from - a.from); - const messagesRemoved = ordered.reduce((n, iv) => n + (iv.to - iv.from + 1), 0); - if (messagesRemoved >= total) { - throw new OperatorActionError('invalid', `Cannot suppress all ${total} message(s) — at least one must remain`); - } - const targeted = ordered.flatMap((iv) => - cm.getMessageWindow(iv.from, iv.to - iv.from + 1, { resolveBlobs: false }).messages, - ); + const { cm, ordered, messagesRemoved, targeted } = this.planSuppression(agentName, requestedIds); const removedIds = targeted.map((m) => String(m.id)); const removedRefs = extractDiscordAwarenessRefs(targeted); + const selection = selectDiscordAwarenessRefs(targeted, opts.marks ?? 'none'); const sourceBranch = this.store.currentBranch().name; const targetBranch = opts.branchName ?? `suppress/${agentName}/${Date.now()}`; @@ -6200,7 +6231,10 @@ export class AgentFramework { agentName, sourceBranch, targetBranch, - refs: removedRefs, + refs: selection.refs, + scope: marksScope(opts.marks), + unmarked: selection.unmarked, + notRemoved: selection.notRemoved, emoji: this.discordAwarenessEmoji, // Seeing targetBranch active does not prove the removals finished; // only this operation activates the batch, after the last redaction. @@ -6208,10 +6242,9 @@ export class AgentFramework { suppressionIntervals: ordered.map((iv) => ({ fromId: iv.fromId, toId: iv.toId })), }) ?? null; // Anything short of activation retires the batch: a prepared explicit - // batch re-arms through preparedSuppressionsForBranch the moment its - // target branch (or a descendant) becomes active — e.g. an operator - // opening the failed fork to look — and an un-completable resume - // there aborts framework start. + // batch is resumed at startup whenever its exact target branch is + // active — e.g. after an operator opens the failed fork to look — and + // an un-completable resume there aborts framework start. const retireBatch = (): void => { if (markerBatch) this.discordAwarenessOutbox!.discard(markerBatch.id); }; @@ -6253,10 +6286,13 @@ export class AgentFramework { // stands and the receipt says the marks were not scheduled. (No // interval needs resuming any more, so retiring the batch there // loses nothing.) - markers = this.activateSurgeryMarkers(markerBatch, 'suppress', agentName); - // Installs the MCPL data-plane gate synchronously, before the store is - // released, and starts delivery without awaiting it (the first - // reaction request may already be on the wire when this returns). + markers = this.activateSurgeryMarkers(markerBatch, 'suppress', agentName, { + scope: marksScope(opts.marks) as SurgeryMarkerReceipt['scope'], + unmarked: selection.unmarked, + notRemoved: selection.notRemoved, + }, selection.refs.length); + // Starts delivery without awaiting it (the first reaction request may + // already be on the wire when this returns); it never gates traffic. this.deliverDiscordAwarenessInBackground('suppress', agentName); this.materializeConfigMountAfterBranchSwitch(); } finally { @@ -6296,6 +6332,177 @@ export class AgentFramework { } } + /** + * The messages a rollback to `messageId` would remove, validated, without + * changing anything. Never bisects a body group: chronicle refuses that + * for removals and branchAt has no such guard, so the target snaps to the + * group's last shard. + */ + private planRollback(agentName: string, messageId: string) { + const agent = this.agents.get(agentName); + if (!agent) throw new OperatorActionError('unknown-agent', `Unknown agent: ${agentName}`); + const cm = agent.getContextManager(); + const total = cm.getMessageCount(); + const requestedIndex = locateMessageIndex(cm, messageId); + if (requestedIndex < 0) { + throw new OperatorActionError('unknown-message', `Message ${messageId} is not on the active branch`); + } + const run = bodyGroupRun(cm, requestedIndex); + const targetIndex = run.to; + const tailMessageId = run.toId; + const messagesRemoved = total - targetIndex - 1; + if (messagesRemoved <= 0) { + throw new OperatorActionError('invalid', `Message ${messageId} is already the tail of the active branch`); + } + const discarded = cm.getMessageWindow(targetIndex + 1, messagesRemoved, { resolveBlobs: false }).messages; + return { cm, tailMessageId, messagesRemoved, discarded }; + } + + /** + * The removal intervals a suppression of `requestedIds` would apply + * (index-ordered, newest first), validated, without changing anything. A + * shard expands to its whole body group (chronicle refuses to bisect one), + * and a group interval is always removed as a range. + */ + private planSuppression(agentName: string, requestedIds: string[]) { + const agent = this.agents.get(agentName); + if (!agent) throw new OperatorActionError('unknown-agent', `Unknown agent: ${agentName}`); + if (requestedIds.length === 0) throw new OperatorActionError('invalid', 'No message ids given'); + const cm = agent.getContextManager(); + const total = cm.getMessageCount(); + const located = locateMessageIndices(cm, new Set(requestedIds)); + const missing = requestedIds.filter((id) => !located.has(id)); + if (missing.length > 0) { + throw new OperatorActionError( + 'unknown-message', + `Not on the active branch: ${missing.slice(0, 5).join(', ')}${missing.length > 5 ? ` (+${missing.length - 5})` : ''}`, + ); + } + const intervals = new Map>(); + for (const id of requestedIds) { + const run = bodyGroupRun(cm, located.get(id)!.index); + intervals.set(`${run.from}-${run.to}`, run); + } + const ordered = [...intervals.values()].sort((a, b) => b.from - a.from); + const messagesRemoved = ordered.reduce((n, iv) => n + (iv.to - iv.from + 1), 0); + if (messagesRemoved >= total) { + throw new OperatorActionError('invalid', `Cannot suppress all ${total} message(s) — at least one must remain`); + } + const targeted = ordered.flatMap((iv) => + cm.getMessageWindow(iv.from, iv.to - iv.from + 1, { resolveBlobs: false }).messages, + ); + return { cm, ordered, messagesRemoved, targeted }; + } + + /** + * What a rollback or suppression would remove, and which Discord messages + * each publication scope would mark, without changing anything or reading + * message content. A surface shows this before an operator chooses + * `marks`; passing a scope's `refs` back as `marks.refs` binds the choice + * to exactly that set, so a later application never widens it. + */ + previewSurgeryMarks( + agentName: string, + target: { rollbackTo: string } | { suppress: string[] }, + ): SurgeryMarksPreview { + const messages = 'rollbackTo' in target + ? this.planRollback(agentName, target.rollbackTo).discarded + : this.planSuppression(agentName, [...new Set(target.suppress.map(String))]).targeted; + const scopeView = (scope: DiscordAwarenessScope): SurgeryMarksPreview['scopes'][DiscordAwarenessScope] => { + const { refs } = selectDiscordAwarenessRefs(messages, { scope }); + const channels = new Map(); + for (const ref of refs) channels.set(ref.channelId, (channels.get(ref.channelId) ?? 0) + 1); + return { + count: refs.length, + channels: [...channels].map(([channelId, count]) => ({ channelId, count })), + refs, + }; + }; + return { + messagesRemoved: messages.length, + addressable: extractDiscordAwarenessRefs(messages).length, + emoji: this.discordAwarenessEmoji, + scopes: { addressed: scopeView('addressed'), all: scopeView('all') }, + }; + } + + /** The awareness journal's batches, for operator surfaces. */ + listDiscordAwareness(): DiscordAwarenessBatchView[] { + return this.discordAwarenessOutbox?.view() ?? []; + } + + /** + * Stop a batch's awareness marks that have not been sent. Never removes a + * reaction: requests already on the wire, or whose outcome is unknown, are + * reported and may still land; retract explicitly to remove them. + */ + cancelDiscordAwareness( + batchId: string, + opts: { requester?: OperatorRequester; note?: string } = {}, + ): DiscordAwarenessCancelReceipt { + return this.awarenessOperatorAction('awareness-cancel', { batchId }, opts, () => + this.requireDiscordAwarenessOutbox().cancel(batchId, describeRequester(opts.requester))); + } + + /** + * Remove this bot's awareness mark, through each ref's configured MCPL + * route, from a batch's refs, or from every ref any batch sent an add for + * (`all`). The receipt says how many removals were queued and how many + * earlier add attempts are unresolved, any of which may land afterwards. + */ + retractDiscordAwareness( + target: string | 'all', + opts: { requester?: OperatorRequester; note?: string } = {}, + ): DiscordAwarenessRetractReceipt { + const receipt = this.awarenessOperatorAction('awareness-retract', { target }, opts, () => + this.requireDiscordAwarenessOutbox().retract(target, describeRequester(opts.requester))); + this.deliverDiscordAwarenessInBackground('retract', 'operator'); + return receipt; + } + + /** Release a held batch's recorded operations, explicitly. */ + releaseDiscordAwareness( + batchId: string, + opts: { requester?: OperatorRequester; note?: string } = {}, + ): DiscordAwarenessReleaseReceipt { + const receipt = this.awarenessOperatorAction('awareness-release', { batchId }, opts, () => + this.requireDiscordAwarenessOutbox().release(batchId, describeRequester(opts.requester))); + this.deliverDiscordAwarenessInBackground('release', 'operator'); + return receipt; + } + + private requireDiscordAwarenessOutbox(): DiscordAwarenessOutbox { + if (!this.discordAwarenessOutbox) { + throw new OperatorActionError('invalid', 'This framework has no Discord awareness journal (no storePath)'); + } + return this.discordAwarenessOutbox; + } + + private awarenessOperatorAction( + kind: string, + params: Record, + opts: { requester?: OperatorRequester; note?: string }, + act: () => T, + ): T { + const logBase: OperatorLogInput = { + kind, + agent: '*', + ...(opts.requester ? { requester: opts.requester } : {}), + ...(opts.note ? { note: opts.note } : {}), + params, + }; + try { + const result = act(); + this.recordOperatorAction({ ...logBase, result: { ...result } as Record }); + console.error(`[discord-awareness] ${kind} ${JSON.stringify(params)} by=${describeRequester(opts.requester)}: ${JSON.stringify(result)}`); + return result; + } catch (error) { + this.recordOperatorAction({ ...logBase, error: error instanceof Error ? error.message : String(error) }); + if (error instanceof OperatorActionError) throw error; + throw new OperatorActionError('invalid', error instanceof Error ? error.message : String(error), { cause: error }); + } + } + /** * Record a surgery's prepared marker batch as active once its body change * has landed, and say what happened. Never throws: the body is already @@ -6311,10 +6518,27 @@ export class AgentFramework { */ private activateSurgeryMarkers( batch: DiscordAwarenessBatch | null, - verb: 'rollback' | 'suppress', + verb: 'rollback' | 'suppress' | 'hide', agentName: string, + facts: SurgeryMarkerFacts, + selected: number, ): SurgeryMarkerReceipt { - if (!batch) return { status: 'none', queued: 0 }; + if (!batch) { + if (selected > 0 && !this.discordAwarenessOutbox) { + // Chosen and in scope, but there is nowhere to record them: say so + // rather than report that none were wanted. + console.error( + `[discord-awareness] ${verb} agent=${agentName}: ${selected} mark(s) chosen, but no awareness journal is configured`, + ); + return { + ...facts, + status: 'not-scheduled', + queued: 0, + error: 'no awareness journal is configured (no storePath or discordAwarenessOutboxPath)', + }; + } + return { ...facts, status: 'none', queued: 0 }; + } try { this.discordAwarenessOutbox!.activate(batch.id); } catch (error) { @@ -6331,7 +6555,7 @@ export class AgentFramework { `could not be activated (${detail}); ` + (retired ? 'batch retired' : `batch left prepared${retireDetail ? ` (retire failed: ${retireDetail})` : ''}`), ); - if (batch.refs.length === 0) return { status: 'none', queued: 0 }; + if (batch.refs.length === 0) return { ...facts, status: 'none', queued: 0 }; this.opsAlert( retired ? 'discord-awareness-not-scheduled' : 'discord-awareness-unresolved', agentName, @@ -6343,33 +6567,83 @@ export class AgentFramework { { data: { batchId: batch.id, retired } }, ); return retired - ? { status: 'not-scheduled', queued: 0, error: detail } - : { status: 'unresolved', queued: 0, batchId: batch.id, error: detail }; + ? { ...facts, status: 'not-scheduled', queued: 0, error: detail } + : { ...facts, status: 'unresolved', queued: 0, batchId: batch.id, error: detail }; } - if (batch.refs.length === 0) return { status: 'none', queued: 0 }; - return { status: 'queued', queued: batch.refs.length, batchId: batch.id }; + if (batch.refs.length === 0) return { ...facts, status: 'none', queued: 0 }; + return { ...facts, status: 'queued', queued: batch.refs.length, batchId: batch.id }; } /** - * Start awareness delivery without the caller awaiting it. - * `syncDiscordAwarenessMarkers` installs the MCPL data-plane gate - * synchronously, before its first await, so calling this while a surgery - * still holds the store keeps MCPL-borne wakes behind the gate exactly as - * before. The drain starts in the same synchronous run (the first reaction - * request can be written before this returns) and continues independently; - * only the surgery's response and its store reservation stop waiting on - * remote reactions. A failure is accounted by the gate itself (connections - * recycled); here it is logged and raised as an ops alert. + * Marks for an in-place `hide`, scheduled after its redaction committed. + * The batch targets the current branch and is explicit (activated here); + * a crash between its two journal records leaves it prepared for exactly + * this branch, which startup completes. Old records without a serverId are + * routed through the server that issued the command. */ - private deliverDiscordAwarenessInBackground(verb: 'rollback' | 'suppress', agentName: string): void { + private scheduleHideMarks( + agentName: string, + serverId: string, + removed: Array<{ metadata?: Record }>, + marks: DiscordAwarenessMarks, + ): SurgeryMarkerReceipt { + const carriers = removed.map((message) => ({ + metadata: { ...(message.metadata ?? {}), serverId: message.metadata?.serverId ?? serverId }, + })); + const selection = selectDiscordAwarenessRefs(carriers, marks); + const facts: SurgeryMarkerFacts = { + scope: marksScope(marks) as SurgeryMarkerReceipt['scope'], + unmarked: selection.unmarked, + notRemoved: selection.notRemoved, + }; + let batch: DiscordAwarenessBatch | null; + try { + const branch = this.store.currentBranch().name; + batch = this.discordAwarenessOutbox?.prepare({ + agentName, + sourceBranch: branch, + targetBranch: branch, + refs: selection.refs, + scope: marksScope(marks), + unmarked: selection.unmarked, + notRemoved: selection.notRemoved, + emoji: this.discordAwarenessEmoji, + activationPolicy: 'explicit', + }) ?? null; + } catch (error) { + // The redaction stands; nothing was recorded, so nothing will be sent. + const detail = error instanceof Error ? error.message : String(error); + this.opsAlert( + 'discord-awareness-not-scheduled', + agentName, + `hide applied, but its ${selection.refs.length} awareness mark(s) were not scheduled: ${detail}`, + ); + return { ...facts, status: 'not-scheduled', queued: 0, error: detail }; + } + const receipt = this.activateSurgeryMarkers(batch, 'hide', agentName, facts, selection.refs.length); + if (receipt.status === 'queued') this.deliverDiscordAwarenessInBackground('hide', agentName); + return receipt; + } + + /** + * Start awareness delivery without the caller awaiting it. The drain starts + * in the same synchronous run (the first request can be written before + * this returns) and continues independently. Delivery never gates MCPL + * traffic or turns; a failure is logged and raised as an ops alert, and + * the queued work stays in the journal for the next pass. + */ + private deliverDiscordAwarenessInBackground( + occasion: 'rollback' | 'suppress' | 'hide' | 'retract' | 'release' | 'startup' | 'connect' | 'list-change' | 'reconnect', + agentName: string, + ): void { if (!this.discordAwarenessOutbox) return; this.syncDiscordAwarenessMarkers().catch((error) => { const detail = error instanceof Error ? error.message : String(error); - console.error(`[discord-awareness] delivery after ${verb} agent=${agentName} failed: ${detail}`); + console.error(`[discord-awareness] delivery after ${occasion} failed: ${detail}`); this.opsAlert( 'discord-awareness-delivery', agentName, - `Awareness mark delivery after ${verb} failed: ${detail}`, + `Awareness mark delivery after ${occasion} failed: ${detail}`, ); }); } @@ -13251,16 +13525,11 @@ export class AgentFramework { for (const config of serverConfigs) { try { - // Stage every startup connection with both planes closed. This removes - // server-order dependence: no early heartbeat/shell push can become - // model-visible before a later Discord connection has joined the same - // awareness generation. + // Stage every startup connection with both planes closed, and open + // them together once all are staged, so startup does not depend on + // server order. await this.connectMcplServerInternal(config, true); } catch (error) { - if (error instanceof DiscordAwarenessAccountingError) { - await this.mcplServerRegistry.closeAll(); - throw error; - } // Fail-open: log and continue with remaining servers const err = error instanceof Error ? error : new Error(String(error)); console.error(`Failed to connect MCPL server "${config.id}":`, err.message); @@ -13278,48 +13547,34 @@ export class AgentFramework { } } - // Start the durable drain only after all startup connections are staged. - // The awareness tools/call is written before control-plane registration is - // flushed, preserving the server-side queue -> registration -> call order. - const startupBarrier = this.installMcplDataPlaneGate(); - this.releaseMcplDataPlaneGate(startupBarrier); - try { - await startupBarrier.promise; - this.completeMcplDataPlaneGate(startupBarrier); - } catch (error) { - await this.mcplServerRegistry.closeAll(); - throw error; - } + // Every startup connection is staged with its grant established: open + // them together (a quiesced boot keeps data planes held). Awareness marks + // never gate this: queued marks are delivered in the background. + this.readyMcplPlanes(); + this.deliverDiscordAwarenessInBackground('startup', 'host'); // Discover tools from all connected servers await this.refreshMcplTools(); } /** - * Reconcile the durable ledger with Chronicle, then deliver every server's - * work, joining any drain still running. + * Deliver every server's queued awareness operations now and wait for this + * pass. Delivery never gates MCPL traffic or turns: callers that do not + * need to wait use the background form. A server that is not connected is + * skipped (its work stays queued until it connects). */ async syncDiscordAwarenessMarkers(_onlyServerId?: string): Promise { if (!this.discordAwarenessOutbox) return; - // A targeted retry may discover pending work for other Discord servers - // during reconciliation. Safety is global, so one generation accounts for - // all pending operations before releasing any MCPL data plane. - const barrier = this.installMcplDataPlaneGate(); - this.releaseMcplDataPlaneGate(barrier); - try { - await barrier.promise; - this.completeMcplDataPlaneGate(barrier); - } catch (error) { - await this.failMcplDataPlaneGate(barrier, 'explicit synchronization', error); - throw error; - } + const serverIds = [...new Set( + this.readDiscordAwarenessDispatches().map((dispatch) => dispatch.key.serverId), + )]; + await Promise.all(serverIds.map((serverId) => this.drainDiscordAwarenessOutbox(serverId))); } private async resumePreparedDiscordSuppressions(): Promise { if (!this.discordAwarenessOutbox) return; const batches = this.discordAwarenessOutbox.preparedSuppressionsForBranch( this.store.currentBranch().name, - this.store.listBranches(), ); for (const batch of batches) { const agent = this.agents.get(batch.agentName); @@ -13342,15 +13597,28 @@ export class AgentFramework { if (interval.fromId === interval.toId) cm.removeMessage(from.id); else cm.removeMessages(from.id, to.id); } - this.discordAwarenessOutbox.activate(batch.id); - console.error(`[discord-awareness] resumed suppression batch ${batch.id}`); + if (batch.status === 'held' || batch.cancelled) { + // Body integrity only: its marks were held at an earlier startup (and + // are released explicitly, never by a branch becoming active), or an + // operator cancelled them. + this.discordAwarenessOutbox.recordSuppressionComplete(batch.id); + console.error(`[discord-awareness] completed held suppression ${batch.id} (marks stay held)`); + } else { + this.discordAwarenessOutbox.activate(batch.id); + console.error(`[discord-awareness] resumed suppression batch ${batch.id}`); + } } } /** - * Deliver every due ref independently. Permanent Discord failures are kept - * in the ledger for audit but do not block later operations; retryable - * failures remain pending for the next reconnect/list-change attempt. + * Deliver one server's due awareness operations, serially, in request + * order per reaction key. Every dispatch is written ahead to the journal + * before the request can leave, and its outcome is recorded from what the + * connection can actually know (McplRequestError.outcome): a request that + * was never written stays queued, one written and never answered is + * `unknown` for good, and a Discord answer is definitive (permanent when + * Discord says the message or channel is gone or inaccessible). A server + * that is not connected is skipped: its work simply stays queued. */ private drainDiscordAwarenessOutbox( serverId: string, @@ -13362,84 +13630,99 @@ export class AgentFramework { if (existing) return existing; const drain = (async () => { + const outbox = this.discordAwarenessOutbox!; const connection = this.mcplServerRegistry?.getServer(serverId); if (!connection?.isConnected) { - const operations = this.readDiscordAwarenessPending(serverId); - for (const operation of operations) { - this.writeDiscordAwarenessFailure( - operation.batchId, - operation.ref, - operation.action, - 'Awareness delivery not attempted: MCPL connection unavailable', - false, - ); - } - if (operations.length > 0) { + const waiting = this.readDiscordAwarenessDispatches(serverId).length; + if (waiting > 0) { console.error( - `[discord-awareness] ${serverId}: connection unavailable; ` + - `durably deferred=${operations.length}`, + `[discord-awareness] ${serverId}: connection unavailable; ${waiting} operation(s) stay queued`, ); } - return { status: 'unavailable' as const, accounted: operations.length }; + return { status: 'unavailable' as const, accounted: waiting }; } let delivered = 0; let failed = 0; const attempted = new Set(); + let announced = false; while (true) { - const operations = this.readDiscordAwarenessPending(serverId).filter((operation) => { - const key = `${operation.batchId}\0${operation.ref.channelId}\0${operation.ref.messageId}\0${operation.action}`; + const dispatches = this.readDiscordAwarenessDispatches(serverId).filter((dispatch) => { + const key = `${dispatch.opIds.join(',')}\0${dispatch.action}`; if (attempted.has(key)) return false; attempted.add(key); return true; }); - if (operations.length === 0) break; - for (const operation of operations) { - const ref = operation.ref; + if (dispatches.length === 0) break; + if (!announced) { + announced = true; + console.error( + `[discord-awareness] ${serverId}: delivering ${dispatches.length} queued mark operation(s)`, + ); + } + for (const dispatch of dispatches) { + const ref = dispatch.key; const channelId = ref.channelId.startsWith('discord:') ? ref.channelId.split(':').at(-1)! : ref.channelId; - let deliveryError: string | undefined; + const tool = dispatch.action === 'add' ? 'add_reaction' : 'remove_reaction'; + let attempts: Array<{ opId: string; attempt: number }>; + try { + attempts = outbox.recordDispatching(dispatch); + } catch (error) { + throw new DiscordAwarenessAccountingError('dispatch write-ahead', error); + } + let outcome: 'confirmed' | 'failed' | 'not-sent' | 'unknown'; + let detail: string | undefined; try { - const tool = operation.action === 'add' ? 'add_reaction' : 'remove_reaction'; const result = await connection.sendToolsCallWithDeadline(tool, { channelId, messageId: ref.messageId, - emoji: operation.emoji, + emoji: ref.emoji, }, this.discordAwarenessDeadlineMs); if (result.isError) { - deliveryError = result.content + outcome = 'failed'; + detail = result.content .map((content) => content.text ?? '') .filter(Boolean) .join('; ') || `Discord ${tool} returned an error`; + } else { + outcome = 'confirmed'; } } catch (error) { - deliveryError = error instanceof Error ? error.message : String(error); + detail = error instanceof Error ? error.message : String(error); + if (error instanceof McplRequestError) { + outcome = error.outcome === 'not-sent' + ? 'not-sent' + : error.outcome === 'error-response' ? 'failed' : 'unknown'; + } else { + // Not classified by the connection: it may have been written. + outcome = 'unknown'; + } } - - if (deliveryError !== undefined) { - const permanent = isPermanentDiscordReactionFailure(deliveryError); - this.writeDiscordAwarenessFailure( - operation.batchId, - ref, - operation.action, - deliveryError, - permanent, - ); + const permanent = outcome === 'failed' && isPermanentDiscordReactionFailure(detail ?? ''); + try { + outbox.recordOutcome(attempts, outcome, { + ...(permanent ? { permanent: true } : {}), + ...(detail !== undefined ? { error: detail } : {}), + }); + } catch (error) { + throw new DiscordAwarenessAccountingError('outcome write', error); + } + if (outcome === 'confirmed') { + delivered++; + } else { failed++; console.error( - `[discord-awareness] ${operation.action} failed for ${ref.channelId}/${ref.messageId}` + - ` (${permanent ? 'permanent' : 'retryable'}): ${deliveryError}`, + `[discord-awareness] ${dispatch.action} ${ref.channelId}/${ref.messageId}: ${outcome}` + + `${outcome === 'failed' ? (permanent ? ' (permanent)' : ' (retryable)') : ''}: ${detail ?? ''}`, ); - } else { - this.writeDiscordAwarenessSuccess(operation.batchId, ref, operation.action); - delivered++; } } } if (delivered > 0 || failed > 0) { console.error( - `[discord-awareness] ${serverId}: delivered=${delivered} failed=${failed}`, + `[discord-awareness] ${serverId}: confirmed=${delivered} not-confirmed=${failed}`, ); } return { status: 'delivered' as const, delivered, failed }; @@ -13451,206 +13734,29 @@ export class AgentFramework { return drain; } - private readDiscordAwarenessPending(serverId?: string) { + private readDiscordAwarenessDispatches(serverId?: string) { try { - return this.discordAwarenessOutbox!.pending(serverId); + return this.discordAwarenessOutbox!.pendingDispatches(serverId); } catch (error) { throw new DiscordAwarenessAccountingError( - `ledger read${serverId ? ` for ${serverId}` : ''}`, + `journal read${serverId ? ` for ${serverId}` : ''}`, error, ); } } - private writeDiscordAwarenessSuccess( - batchId: string, - ref: import('./recovery/discord-awareness-outbox.js').DiscordAwarenessRef, - action: import('./recovery/discord-awareness-outbox.js').DiscordAwarenessAction, - ): void { - try { - this.discordAwarenessOutbox!.recordSuccess(batchId, ref, action); - } catch (error) { - throw new DiscordAwarenessAccountingError('recordSuccess ledger write', error); - } - } - - private writeDiscordAwarenessFailure( - batchId: string, - ref: import('./recovery/discord-awareness-outbox.js').DiscordAwarenessRef, - action: import('./recovery/discord-awareness-outbox.js').DiscordAwarenessAction, - errorMessage: string, - permanent: boolean, - ): void { - try { - this.discordAwarenessOutbox!.recordFailure( - batchId, - ref, - action, - errorMessage, - permanent, - ); - } catch (error) { - throw new DiscordAwarenessAccountingError('recordFailure ledger write', error); - } - } - - private beginDiscordAwarenessBarrier(): { - requiresBarrier: boolean; - promise: Promise; - } { - if (!this.discordAwarenessOutbox) { - return { - requiresBarrier: false, - promise: Promise.resolve({ status: 'delivered', delivered: 0, failed: 0 }), - }; - } - try { - this.discordAwarenessOutbox.reconcileForBranch( - this.store.currentBranch().name, - this.store.listBranches(), - ); - } catch (error) { - throw new DiscordAwarenessAccountingError('branch reconciliation', error); - } - const pending = this.readDiscordAwarenessPending(); - // A drain that is still running holds operations it read before this - // reconciliation, and it sends them whatever the ledger now says. A branch - // move can hide them from `pending`: restoring a rollback's source makes - // its marks undesired, and an add whose reply has not arrived is not yet - // present, so the ledger shows nothing to do while the drain goes on to - // send its saved adds and then the removals they need. So a generation - // accounts for every running drain as well as for every pending - // operation, and joins those drains instead of releasing the data planes - // beside them. - const serverIds = [...new Set([ - ...pending.map((operation) => operation.ref.serverId), - ...this.discordAwarenessDrains.keys(), - ])]; - if (serverIds.length === 0) { - return { - requiresBarrier: false, - promise: Promise.resolve({ status: 'delivered', delivered: 0, failed: 0 }), - }; - } - const promise = Promise.all( - serverIds.map((serverId) => this.drainDiscordAwarenessOutbox(serverId)), - ).then((outcomes): DiscordAwarenessDrainOutcome => { - if (outcomes.every((outcome) => outcome.status === 'unavailable')) { - return { - status: 'unavailable', - accounted: outcomes.reduce( - (count, outcome) => count + (outcome.status === 'unavailable' ? outcome.accounted : 0), - 0, - ), - }; - } - return { - status: 'delivered', - delivered: outcomes.reduce( - (count, outcome) => count + (outcome.status === 'delivered' ? outcome.delivered : 0), - 0, - ), - failed: outcomes.reduce( - (count, outcome) => count + (outcome.status === 'delivered' ? outcome.failed : 0), - 0, - ), - }; - }); - return { requiresBarrier: true, promise }; - } - /** - * Install one framework-global Discord-awareness generation before releasing - * any inference-bearing event. With pending work or a drain still running, - * every MCPL data plane is paused while all control planes remain live. - * With neither, ready() runs synchronously so request responders preserve - * their historical same-stack behavior. + * Open MCPL connections' planes (every connection when none is given). A + * quiesced host (issue #122) opens only control planes, so data planes stay + * held for the whole maintenance window; resume() calls this again once + * the flag is cleared. Control planes always come up, since host/command + * (and with it resume) rides the control plane. */ - private installMcplDataPlaneGate(): DiscordAwarenessBarrier { - for (const connection of this.mcplServerRegistry?.getAllServers() ?? []) { - connection.pauseDataPlane(); - } - const generation = ++this.discordAwarenessBarrierGeneration; - - let begun: ReturnType; - try { - begun = this.beginDiscordAwarenessBarrier(); - } catch (error) { - begun = { - requiresBarrier: true, - promise: Promise.reject(error), - }; + private readyMcplPlanes(connections?: McplServerConnection[]): void { + for (const connection of connections ?? this.mcplServerRegistry?.getAllServers() ?? []) { + if (this.quiesced) connection.readyControlPlane(); + else connection.ready(); } - const barrier: DiscordAwarenessBarrier = { generation, ...begun }; - this.discordAwarenessBarrier = barrier; - return barrier; - } - - private releaseMcplDataPlaneGate( - barrier: DiscordAwarenessBarrier, - ): void { - if (this.discordAwarenessBarrier !== barrier) return; - if (barrier.requiresBarrier) { - for (const connection of this.mcplServerRegistry?.getAllServers() ?? []) { - connection.readyControlPlane(); - } - return; - } - this.completeMcplDataPlaneGate(barrier); - } - - private completeMcplDataPlaneGate( - barrier: DiscordAwarenessBarrier, - ): boolean { - if (this.discordAwarenessBarrier !== barrier) return false; - this.discordAwarenessBarrier = null; - for (const connection of this.mcplServerRegistry?.getAllServers() ?? []) { - // ready() can synchronously flush a nested list-change notification that - // installs a newer global generation. Never let this older completion - // release any remaining server behind that newer gate. (The quiesce - // check sits AFTER this guard: readyControlPlane can also synchronously - // flush a control event that installs a newer barrier.) - if (this.discordAwarenessBarrier !== null) return false; - // Host quiesce (issue #122): this is the single point every barrier - // completion and reconnect flow funnels through, so holding here keeps - // data planes paused across awareness drains and reconnects for the - // whole window. Control planes come up normally (host/command rides the - // control plane, so resume stays deliverable); resume() performs the - // real ready() flush through this same funnel once its feasibility gate - // passes and the flag is cleared. - if (this.quiesced) { - connection.readyControlPlane(); - continue; - } - connection.ready(); - } - return this.discordAwarenessBarrier === null; - } - - private async failMcplDataPlaneGate( - barrier: DiscordAwarenessBarrier, - context: string, - error: unknown, - ): Promise { - if (this.discordAwarenessBarrier !== barrier) return; - const err = error instanceof Error ? error : new Error(String(error)); - console.error(`[discord-awareness] ${context} failed globally:`, err.message); - const connections = this.mcplServerRegistry?.getAllServers() ?? []; - for (const connection of connections) { - this.emitTrace({ - type: 'mcpl:server-error', - serverId: connection.id, - error: `Discord awareness accounting unhealthy: ${err.message}`, - }); - } - await Promise.all(connections.map(async (connection) => { - await connection.reconnectAfterFailure().catch((closeError) => { - console.error( - `[discord-awareness] could not recycle unhealthy connection ${connection.id}:`, - closeError instanceof Error ? closeError.message : closeError, - ); - }); - })); } /** @@ -13661,7 +13767,7 @@ export class AgentFramework { */ private async connectMcplServerInternal( config: import('./mcpl/types.js').McplServerConfig, - deferAwareness = false, + staged = false, ): Promise { if (!this.mcplServerRegistry || !this.mcplHostCapabilities) { throw new Error('MCPL subsystem is not initialized'); @@ -13684,49 +13790,22 @@ export class AgentFramework { const connection = await this.mcplServerRegistry.addServer(config, this.mcplHostCapabilities); - // Wire listeners before either startup staging or the runtime global gate - // releases control traffic needed for registration and marker service. + // Wire listeners before any plane opens. this.wireMcplEvents(connection); - if (deferAwareness) { - // Staged startup: both planes are closed, so the §5.3 policy - // round-trip cannot let any buffered traffic slip — establish the - // grant now, before staging ever releases. - await this.registerMcplServerFeatures(config, connection); - this.emitTrace({ type: 'module:added', moduleName: `mcpl:${config.id}` }); - return; - } - - const awarenessBarrier = this.installMcplDataPlaneGate(); - // NO early release: both planes stay closed through accounting AND the - // §5.3 policy round-trip below (PR #79 review blocker 3 — the early - // control flush released channels-register/changed before any grant - // existed). The awareness drain rides tools/call RESPONSES, which are - // never event-buffered, so it needs no plane release to make progress. - try { - await awarenessBarrier.promise; - } catch (error) { - // Startup cannot fail open on a broken awareness ledger. Disable the - // reconnecting stub/connection before propagating the distinct error to - // initializeMcpl, which tears down any other servers and aborts create(). - // Ordering matters: awareness accounting settles BEFORE the §5.3 - // policy round-trip below, so an accounting failure aborts with the - // grant still empty and every buffered event still behind the gate — - // a failed startup never reports a released data plane. (The reverse - // order let a push arrive during the policy await and get flushed by - // teardown.) - await connection.close().catch(() => {}); - throw error; - } - - // §5.3 initial policy handshake — after accounting, before the gate - // completes: the grant is established (or knowingly left empty) before - // ANY buffered traffic (control or data) flows. complete() then - // ready()s both planes through the admission gate. + // §5.3 initial policy handshake with both planes still closed (PR #79 + // review blocker 3: an early control flush released channels-register/ + // changed before any grant existed). The grant is established (or + // knowingly left empty) before ANY buffered traffic flows. await this.registerMcplServerFeatures(config, connection); - this.completeMcplDataPlaneGate(awarenessBarrier); - this.emitTrace({ type: 'module:added', moduleName: `mcpl:${config.id}` }); + // Staged startup opens every connection together (initializeMcpl). + if (staged) return; + + this.readyMcplPlanes([connection]); + // A connection can make queued awareness work deliverable; it never + // waits on it. + this.deliverDiscordAwarenessInBackground('connect', 'host'); } /** Per-connection §17.8 fetch limiter: at most one in-flight fetch, at @@ -14229,8 +14308,6 @@ export class AgentFramework { params: PushEventParams, responder?: { respond: (result: unknown) => void; respondError: (code: number, message: string) => void }, ) => { - const barrier = this.discordAwarenessBarrier; - if (barrier) await barrier.promise; await this.pushHandler?.handlePushEvent(connection.id, params, responder as never); }); @@ -14245,8 +14322,6 @@ export class AgentFramework { return; } if (this.inferenceRouter && responder) { - const barrier = this.discordAwarenessBarrier; - if (barrier) await barrier.promise; await this.inferenceRouter.handleInferenceRequest(connection.id, params, { respond: responder.respond, respondError: responder.respondError, @@ -14332,8 +14407,6 @@ export class AgentFramework { params: ChannelsIncomingParams, responder?: { respond: (result: unknown) => void }, ) => { - const barrier = this.discordAwarenessBarrier; - if (barrier) await barrier.promise; await this.channelRegistry?.handleIncoming(connection.id, params, responder as never); }); @@ -14344,8 +14417,6 @@ export class AgentFramework { ) => { if (!responder) return; try { - const barrier = this.discordAwarenessBarrier; - if (barrier) await barrier.promise; const result = await this.handleHostCommand(connection.id, params ?? {}); responder.respond(result); } catch (error) { @@ -14356,19 +14427,10 @@ export class AgentFramework { // Handle dynamic tool list changes (notifications/tools/list_changed) connection.on('tools-list-changed', () => { - // Pause before starting any async refresh. This listener runs - // synchronously in the transport line callback, so a following inbound - // data event cannot pass before the new barrier exists. - const awarenessBarrier = this.installMcplDataPlaneGate(); - this.releaseMcplDataPlaneGate(awarenessBarrier); this.handleToolsListChanged(connection.id); - void awarenessBarrier.promise.then(() => { - this.completeMcplDataPlaneGate(awarenessBarrier); - }).catch((error) => this.failMcplDataPlaneGate( - awarenessBarrier, - 'tools-list reconciliation', - error, - )); + // A server re-listing its tools is a natural moment to retry queued + // awareness work for it; delivery never holds its traffic. + this.deliverDiscordAwarenessInBackground('list-change', 'host'); }); // Re-establish full server registration on reconnect. The 'close' handler @@ -14379,14 +14441,13 @@ export class AgentFramework { // preserved across the transient close are resumed by idempotent // registration. Then refresh tools (server may have different tools). connection.on('reconnect', (info?: { attempts?: number }) => { - // Install the barrier synchronously so any inbound event emitted after - // reconnect observes it before doing work that could wake an agent. The - // connection paused its data plane before wiring the fresh transport. - const awarenessBarrier = this.installMcplDataPlaneGate(); - // Async: the §5.3 policy Request must be answered (re-establishing the - // grant) before the data-plane barrier releases — otherwise the first + // The connection paused its data plane before wiring the fresh + // transport. Let control traffic through for registration, then + // re-establish the grant: the §5.3 policy Request must be answered + // before this connection's data plane opens, otherwise the first // post-reconnect events land while the grant is still empty and are // rejected fail-closed instead of delivered. + connection.readyControlPlane(); void (async () => { try { const config = this.mcplServerConfigs.get(connection.id); @@ -14399,14 +14460,8 @@ export class AgentFramework { error instanceof Error ? error.message : error, ); } - this.releaseMcplDataPlaneGate(awarenessBarrier); + this.readyMcplPlanes([connection]); this.handleToolsListChanged(connection.id); - })(); - void awarenessBarrier.promise.then(() => { - if ( - awarenessBarrier.requiresBarrier - && !this.completeMcplDataPlaneGate(awarenessBarrier) - ) return; this.emitTrace({ type: 'mcpl:server-reconnected', serverId: connection.id, @@ -14415,11 +14470,10 @@ export class AgentFramework { // Mirror the module:removed emitted on 'close' so module-lifecycle // consumers see the server come back, not just vanish. this.emitTrace({ type: 'module:added', moduleName: `mcpl:${connection.id}` }); - }).catch((error) => this.failMcplDataPlaneGate( - awarenessBarrier, - 'reconnect reconciliation', - error, - )); + // Work queued while the route was down can be sent now; it never + // holds this connection's traffic. + this.deliverDiscordAwarenessInBackground('reconnect', 'host'); + })(); }); // Surface connect/reconnect failures. Before these traces existed the diff --git a/src/index.ts b/src/index.ts index 9f7efcbc5..fe873b1ce 100644 --- a/src/index.ts +++ b/src/index.ts @@ -126,18 +126,31 @@ export { PassthroughStrategy, AutobiographicalStrategy, KnowledgeStrategy } from export type { KnowledgeConfig, PhaseType } from '@animalabs/context-manager'; export type { Membrane, NormalizedMessage, NormalizedRequest, ContentBlock } from '@animalabs/membrane'; -// Offline outage recovery and branch-independent Discord awareness markers +// Offline outage recovery and the Discord awareness-mark journal export { DiscordAwarenessOutbox, DEFAULT_DISCORD_AWARENESS_EMOJI, defaultDiscordAwarenessOutboxPath, extractDiscordAwarenessRefs, + selectDiscordAwarenessRefs, } from './recovery/discord-awareness-outbox.js'; export type { + DiscordAwarenessAction, + DiscordAwarenessAttempt, DiscordAwarenessBatch, - DiscordAwarenessEntry, - DiscordAwarenessOperation, + DiscordAwarenessBatchRecord, + DiscordAwarenessBatchView, + DiscordAwarenessCancelReceipt, + DiscordAwarenessDispatch, + DiscordAwarenessMarks, + DiscordAwarenessOp, + DiscordAwarenessOpStatus, + DiscordAwarenessOutcome, DiscordAwarenessRef, + DiscordAwarenessReleaseAction, + DiscordAwarenessReleaseReceipt, + DiscordAwarenessRetractReceipt, + DiscordAwarenessScope, DiscordSuppressionInterval, } from './recovery/discord-awareness-outbox.js'; // Live operator surgery (rollback / suppress) and its durable action log @@ -146,7 +159,9 @@ export type { OperatorLogEntry, OperatorLogInput, OperatorRequester, + SurgeryMarkerFacts, SurgeryMarkerReceipt, + SurgeryMarksPreview, } from './operator-log.js'; export { createOfflineRecoveryBranch } from './recovery/offline-branch.js'; export type { diff --git a/src/mcpl/server-connection.ts b/src/mcpl/server-connection.ts index 18f82d096..273f7b61a 100644 --- a/src/mcpl/server-connection.ts +++ b/src/mcpl/server-connection.ts @@ -354,8 +354,8 @@ export class McplServerConnection extends EventEmitter { /** * Close the data plane synchronously while leaving an already-released - * control plane live. Used before reconnect adoption and list-change - * reconciliation so no new inference event can beat barrier installation. + * control plane live. Used before reconnect adoption, so no data event can + * arrive before the server's grant is re-established, and by quiesce. */ pauseDataPlane(): void { this.dataPlaneReady = false; @@ -363,8 +363,8 @@ export class McplServerConnection extends EventEmitter { /** * Override emit to buffer server→host events until the corresponding plane - * is ready. Lifecycle events always pass through so reconnect can install a - * new data-plane gate before the fresh transport is exposed. + * is ready. Lifecycle events always pass through so the reconnect listener + * runs before the fresh transport's buffered traffic is released. */ override emit(event: string | symbol, ...args: unknown[]): boolean { const name = typeof event === 'string' ? event : ''; @@ -954,9 +954,9 @@ export class McplServerConnection extends EventEmitter { ); // Close the data plane before the new transport can emit. The reconnect - // lifecycle event bypasses buffering; its framework listener installs the - // awareness barrier synchronously, then releases control traffic needed - // to establish the server while data remains held. + // lifecycle event bypasses buffering; its framework listener releases + // the control traffic needed to re-establish the server's grant while + // data stays held until the grant is in place. this.pauseDataPlane(); this.resetPolicyForTransportBoundary(); this.transport = transport; diff --git a/src/operator-log.ts b/src/operator-log.ts index 726ee93ed..173192c26 100644 --- a/src/operator-log.ts +++ b/src/operator-log.ts @@ -118,32 +118,62 @@ export function capIds(ids: string[]): { ids: string[]; count: number; truncated : { ids, count: ids.length }; } +/** + * Facts every surgery marker receipt carries, whatever happened: the + * publication scope chosen (`none` unless the operator chose marks), how many + * removed addressable Discord messages the surgery left unmarked, and how + * many authorized refs it did not remove (a delayed application whose anchor + * no longer covered them; their marks are never widened to anything else). + */ +export interface SurgeryMarkerFacts { + scope: 'none' | 'addressed' | 'all'; + unmarked: number; + notRemoved: number; +} + /** * What a live surgery did about Discord awareness marks, reported apart from * the body change itself. A surgery that returns has applied its body change; * this says only whether marks were scheduled: * - * - `none` — no marks were scheduled: the change removed no addressable - * Discord message, or the awareness outbox is disabled (no - * `discordAwarenessOutboxPath` and no `storePath`). It is not proof that - * no addressable Discord message was removed: the surgery's `removedRefs` - * (`discordRefs` in the operator log) say whether any was. - * - `queued` — the outbox batch is durably active and its delivery has been - * handed to the background drain. It is not a delivery claim: per-message - * outcomes stay in the outbox ledger. - * - `not-scheduled` — the body change landed, but recording the batch as - * active failed, so it was retired from the ledger: none of its marks will - * be delivered. - * - `unresolved` — the ledger could record neither the activation nor the + * - `none` — no marks were requested (the default), or none of the removed + * messages fell within the chosen scope and authorized refs. It says + * nothing about whether Discord messages were removed: `unmarked` counts + * the addressable ones left unmarked, and the surgery's `removedRefs` + * (`discordRefs` in the operator log) list every one. + * - `queued` — one add request per mark is durably recorded and delivery has + * started. It is not a delivery claim: per-request outcomes stay in the + * awareness journal. + * - `not-scheduled` — marks were chosen and in scope, but none will be + * delivered: either this framework has no awareness journal (no storePath + * or discordAwarenessOutboxPath), or the body change landed and recording + * the batch as active failed, so it was retired from the journal. `error` + * says which. + * - `unresolved` — the journal could record neither the activation nor the * retirement. `batchId` names a retained batch whose scheduling outcome is * unresolved: any later reconciliation or restart that can read it may - * promote and deliver it, and nothing here promises otherwise. + * activate and deliver it, and nothing here promises otherwise. */ -export type SurgeryMarkerReceipt = +export type SurgeryMarkerReceipt = SurgeryMarkerFacts & ( | { status: 'none'; queued: 0 } | { status: 'queued'; queued: number; batchId: string } | { status: 'not-scheduled'; queued: 0; error: string } - | { status: 'unresolved'; queued: 0; batchId: string; error: string }; + | { status: 'unresolved'; queued: 0; batchId: string; error: string } +); + +/** What a surgery would remove and which messages each marks scope covers. */ +export interface SurgeryMarksPreview { + messagesRemoved: number; + /** Removed messages that carry a Discord address at all. */ + addressable: number; + emoji: string; + scopes: Record<'addressed' | 'all', { + count: number; + channels: Array<{ channelId: string; count: number }>; + /** Pass back as `marks.refs` to bind the choice to exactly these. */ + refs: Array<{ serverId: string; channelId: string; messageId: string }>; + }>; +} /** Thrown by live surgery methods when a request cannot be honored. `code` * lets surfaces distinguish "agent busy — quiesce first" from bad input. */ diff --git a/src/recovery/discord-awareness-outbox.ts b/src/recovery/discord-awareness-outbox.ts index 5ea0b4109..cde3dfc3a 100644 --- a/src/recovery/discord-awareness-outbox.ts +++ b/src/recovery/discord-awareness-outbox.ts @@ -1,478 +1,1266 @@ import { + closeSync, + existsSync, + fsyncSync, mkdirSync, + openSync, readFileSync, renameSync, - writeFileSync, + truncateSync, + writeSync, } from 'node:fs'; -import { dirname, join } from 'node:path'; -import { randomUUID } from 'node:crypto'; +import { basename, dirname, join } from 'node:path'; +import { createHash, randomUUID } from 'node:crypto'; + +/** + * Discord awareness marks: a reaction (💤 by default) that an operator may + * choose to place on Discord messages a surgery removed from a resident's + * context, so the people who wrote them can see the resident no longer has + * them in view. + * + * Marks are an explicit publication choice, separate from the surgery itself + * (`DiscordAwarenessMarks`, default `none`). Once requested, they are + * one-shot: branch moves and restarts never derive new operations from branch + * state; an operator retracts, cancels or releases explicitly. + * + * The ledger is an append-only journal of what was asked and what was + * answered, never a model of what is on Discord. Each operation is recorded + * as `requested`, then `dispatching` (written before the request can leave), + * then an outcome. A request that was dispatched and never answered stays + * `unknown` for good: a later retry's confirmation does not resolve it, and a + * later opposite operation discloses it. + * + * Operations are keyed by (MCPL server id, channel, message, emoji). The + * server id is the configured route to a bot, not the bot's identity: if the + * route is re-pointed at another bot, retained operations act as that bot, + * and nothing here can detect it. + */ export const DEFAULT_DISCORD_AWARENESS_EMOJI = '💤'; +/** The journal file under a store, next to the other recovery state. */ export function defaultDiscordAwarenessOutboxPath(storePath: string): string { - return join(storePath, 'recovery', 'discord-awareness-outbox.json'); + return join(storePath, 'recovery', 'discord-awareness-journal.jsonl'); } +/** The pre-journal (version 1 and 2) ledger file name, imported once. */ +const LEGACY_LEDGER_NAME = 'discord-awareness-outbox.json'; + export interface DiscordAwarenessRef { serverId: string; channelId: string; messageId: string; } +/** Which removed messages a publication choice covers. */ +export type DiscordAwarenessScope = 'addressed' | 'all'; + +/** + * An operator's publication choice for one surgery. `none`, the default, + * keeps the surgery local. With `refs`, the choice authorizes exactly those + * refs (as a preview showed them): the surgery marks only the ones it + * actually removes, and never widens the set. Without `refs`, the scope is + * evaluated when the surgery applies; that form is for an immediate act whose + * choice and application are the same moment. + */ +export type DiscordAwarenessMarks = + | 'none' + | { scope: DiscordAwarenessScope; refs?: DiscordAwarenessRef[] }; + export type DiscordAwarenessAction = 'add' | 'remove'; -export type DiscordAwarenessDeliveryStatus = 'pending' | 'applied' | 'permanent-failure'; -export interface DiscordAwarenessEntry extends DiscordAwarenessRef { - desired: boolean; - markerPresent: boolean; - deliveryStatus: DiscordAwarenessDeliveryStatus; - attempts: number; - lastAction?: DiscordAwarenessAction; - lastAttemptAt?: number; - lastError?: string; +export interface DiscordSuppressionInterval { + fromId: string; + toId: string; } -export interface DiscordAwarenessBatch { +export interface DiscordAwarenessReleaseAction extends DiscordAwarenessRef { + action: DiscordAwarenessAction; + /** Earlier attempts recorded for this ref before it was held (legacy import). */ + priorAttempts?: number; + /** The last error recorded for this ref before it was held, verbatim. */ + priorError?: string; +} + +/** One surgery's request, as recorded when the surgery prepared it. */ +export interface DiscordAwarenessBatchRecord { id: string; - status: 'prepared' | 'active'; agentName: string; sourceBranch: string; targetBranch: string; emoji: string; createdAt: number; - refs: DiscordAwarenessEntry[]; + /** `none` for a suppression journal with no marks; `legacy` for imports. */ + scope: 'none' | DiscordAwarenessScope | 'legacy'; + /** The refs this batch requests marks on (authorized ∩ removed). */ + refs: DiscordAwarenessRef[]; + /** Removed addressable messages this batch leaves unmarked. */ + unmarked?: number; + /** Authorized refs the surgery did not remove. */ + notRemoved?: number; /** - * `target-branch` may be promoted when its target branch (or a descendant) - * is active. `explicit` requires activate() after a second mutation. + * `target-branch`: startup activates the batch only if its target is + * exactly the active branch (the surgery crashed after switching). + * `explicit`: only the surgery (or startup's resume of its intervals) does. */ activationPolicy?: 'target-branch' | 'explicit'; /** Idempotent interval operations used to resume an interrupted suppression. */ suppressionIntervals?: DiscordSuppressionInterval[]; } -export interface DiscordSuppressionInterval { - fromId: string; - toId: string; +export type DiscordAwarenessBatchStatus = 'prepared' | 'active' | 'held' | 'discarded'; + +export interface DiscordAwarenessBatch extends DiscordAwarenessBatchRecord { + status: DiscordAwarenessBatchStatus; + /** A held suppression whose body was completed at a later startup. */ + suppressionComplete?: boolean; + held?: { reason: string; at: number; releaseActions: DiscordAwarenessReleaseAction[] }; + cancelled?: { at: number; by?: string }; + released?: { at: number; by?: string }; } -export interface DiscordAwarenessOperation { - batchId: string; - emoji: string; +export type DiscordAwarenessOutcome = 'confirmed' | 'failed' | 'not-sent' | 'unknown'; + +export interface DiscordAwarenessAttempt { + attempt: number; + /** One id per request on the wire; operations sharing a dispatch share it. */ + dispatchId?: string; + dispatchedAt: number; + /** Absent while the request is on the wire in this process. */ + outcome?: DiscordAwarenessOutcome; + /** For `failed`: Discord's refusal is final for this request. */ + permanent?: boolean; + outcomeAt?: number; + error?: string; +} + +export interface DiscordAwarenessOpCause { + kind: 'batch' | 'retract' | 'release' | 'import'; + batchId?: string; + requestId?: string; +} + +export type DiscordAwarenessOpStatus = + | 'requested' + | 'dispatching' + | 'confirmed' + | 'failed' + | 'unknown' + | 'cancelled'; + +export interface DiscordAwarenessOp { + opId: string; + key: DiscordAwarenessRef & { emoji: string }; + action: DiscordAwarenessAction; + cause: DiscordAwarenessOpCause; + requestedAt: number; + attempts: DiscordAwarenessAttempt[]; + cancelled?: { at: number; reason: 'batch-cancelled' | 'superseded' }; +} + +/** One dispatch: the head operation for a reaction key, plus any consecutive + * pending operations with the same action that share it. */ +export interface DiscordAwarenessDispatch { + key: DiscordAwarenessRef & { emoji: string }; action: DiscordAwarenessAction; - ref: DiscordAwarenessEntry; + opIds: string[]; } -interface DiscordAwarenessOutboxDocument { - version: 2; - batches: DiscordAwarenessBatch[]; +/** Receipt for cancel(). Nothing is ever removed from Discord by a cancel. */ +export interface DiscordAwarenessCancelReceipt { + batchId: string; + /** Requests that will now never be sent (for a batch not yet activated, + * every mark it would have requested). */ + cancelled: number; + /** Held release actions dropped (a held batch was cancelled). */ + heldDropped: number; + /** Operations on the wire right now: they may still land. */ + inFlight: number; + /** Operations whose outcome is unknown: they may have landed. */ + unknown: number; + /** Operations Discord already confirmed (cancel does not undo them). */ + confirmed: number; +} + +/** Receipt for retract(). */ +export interface DiscordAwarenessRetractReceipt { + requestId: string; + /** Removals queued (one per reaction key). */ + removalsQueued: number; + /** Add requests for those keys that had not been sent, now superseded. */ + addsSuperseded: number; + /** Keys whose earlier add attempts are unresolved (on the wire or unknown): + * such an add may land after this removal. */ + keysWithUnresolvedAdds: number; + /** Unresolved earlier add requests on those keys (one per request sent, + * however many batches' operations it carried). */ + unresolvedAddAttempts: number; } -interface BranchDescriptor { +export interface DiscordAwarenessReleaseReceipt { + batchId: string; + addsQueued: number; + removalsQueued: number; +} + +/** Per-batch view for operators: what was asked and what is known. */ +export interface DiscordAwarenessBatchView { id: string; - name: string; - parentId?: string; + status: DiscordAwarenessBatchStatus; + scope: DiscordAwarenessBatchRecord['scope']; + agentName: string; + sourceBranch: string; + targetBranch: string; + emoji: string; + createdAt: number; + refs: number; + unmarked?: number; + notRemoved?: number; + held?: { reason: string; at: number; releaseActions: number }; + cancelled?: { at: number; by?: string }; + released?: { at: number; by?: string }; + /** Counts over the add operations this batch requested, by status. */ + adds: Record; + /** Removals requested (by any retract) on this batch's refs, by status. */ + removals: Record; + /** Attempts on this batch's refs whose outcome is unknown or on the wire. */ + unresolvedAttempts: number; } +/** Extract Discord addressing metadata without reading message content. */ export interface DiscordMessageMetadataCarrier { metadata?: Record; } -/** Extract Discord addressing metadata without reading message content. */ export function extractDiscordAwarenessRefs( messages: DiscordMessageMetadataCarrier[], forcedServerId?: string, ): DiscordAwarenessRef[] { const refs = new Map(); + for (const message of messages) { + const ref = discordRef(message, forcedServerId); + if (ref) refs.set(refKey(ref), ref); + } + return [...refs.values()]; +} +/** + * Select the refs a publication choice covers among removed messages. + * `addressed` means the message carried the MCPL `chat:addressed` tag (a + * mention, a reply to the bot, or a DM, as the server classified it); the + * same signal the framework uses for loci and wakes. With `marks.refs`, only + * those refs are eligible as well. + */ +export function selectDiscordAwarenessRefs( + messages: DiscordMessageMetadataCarrier[], + marks: DiscordAwarenessMarks, + forcedServerId?: string, +): { refs: DiscordAwarenessRef[]; addressable: number; unmarked: number; notRemoved: number } { + const addressable = new Map(); for (const message of messages) { - const metadata = message.metadata ?? {}; - const channelId = typeof metadata.channelId === 'string' ? metadata.channelId : ''; - const messageId = typeof metadata.messageId === 'string' ? metadata.messageId : ''; - const metadataServerId = typeof metadata.serverId === 'string' ? metadata.serverId : ''; - const serverId = forcedServerId ?? metadataServerId; - if (!channelId || !messageId || !serverId) continue; + const ref = discordRef(message, forcedServerId); + if (!ref) continue; + const key = refKey(ref); + const addressed = isAddressed(message); + const existing = addressable.get(key); + addressable.set(key, { ref, addressed: addressed || existing?.addressed === true }); + } + if (marks === 'none') { + return { refs: [], addressable: addressable.size, unmarked: addressable.size, notRemoved: 0 }; + } + const authorized = marks.refs ? new Set(marks.refs.map(refKey)) : null; + const refs: DiscordAwarenessRef[] = []; + for (const [key, candidate] of addressable) { + if (marks.scope === 'addressed' && !candidate.addressed) continue; + if (authorized && !authorized.has(key)) continue; + refs.push(candidate.ref); + } + const notRemoved = authorized + ? [...authorized].filter((key) => !addressable.has(key)).length + : 0; + return { refs, addressable: addressable.size, unmarked: addressable.size - refs.length, notRemoved }; +} - const isDiscord = serverId.toLowerCase().includes('discord') || channelId.startsWith('discord:'); - if (!isDiscord) continue; +function discordRef(message: DiscordMessageMetadataCarrier, forcedServerId?: string): DiscordAwarenessRef | null { + const metadata = message.metadata ?? {}; + const channelId = typeof metadata.channelId === 'string' ? metadata.channelId : ''; + const messageId = typeof metadata.messageId === 'string' ? metadata.messageId : ''; + const metadataServerId = typeof metadata.serverId === 'string' ? metadata.serverId : ''; + const serverId = forcedServerId ?? metadataServerId; + if (!channelId || !messageId || !serverId) return null; + const isDiscord = serverId.toLowerCase().includes('discord') || channelId.startsWith('discord:'); + if (!isDiscord) return null; + return { serverId, channelId, messageId }; +} - const ref = { serverId, channelId, messageId }; - refs.set(refKey(ref), ref); - } +function isAddressed(message: DiscordMessageMetadataCarrier): boolean { + const tags = message.metadata?.tags; + return Array.isArray(tags) && tags.includes('chat:addressed'); +} - return [...refs.values()]; +// --------------------------------------------------------------------------- +// Journal records +// --------------------------------------------------------------------------- + +type JournalRecord = + | { t: 'batch'; at: number; batch: DiscordAwarenessBatchRecord } + | { t: 'activated'; at: number; batchId: string } + | { t: 'discarded'; at: number; batchId: string } + | { t: 'held'; at: number; batchId: string; reason: string; releaseActions: DiscordAwarenessReleaseAction[] } + | { t: 'released'; at: number; batchId: string; by?: string } + | { t: 'cancelled'; at: number; batchId: string; by?: string } + | { + t: 'requested'; + at: number; + opId: string; + key: DiscordAwarenessRef & { emoji: string }; + action: DiscordAwarenessAction; + cause: DiscordAwarenessOpCause; + } + | { t: 'op-cancelled'; at: number; opId: string; reason: 'batch-cancelled' | 'superseded' } + | { t: 'dispatching'; at: number; opId: string; attempt: number; dispatchId?: string } + | { + t: 'outcome'; + at: number; + opId: string; + attempt: number; + outcome: DiscordAwarenessOutcome; + permanent?: boolean; + error?: string; + } + | { t: 'retract'; at: number; requestId: string; target: string; by?: string } + | { t: 'suppression-complete'; at: number; batchId: string } + | { t: 'import-complete'; at: number; source: 'v2'; sha256: string } + | { t: 'commit'; at: number; txn: string }; + +/** + * Every multi-record append is one transaction: its records carry a `txn` + * id and only take effect when the closing `commit` record is present. A + * torn append (disk full, crash mid-write) therefore applies none of its + * records, never a prefix, so an activation can't half-queue its marks and a + * retract can't supersede adds without queuing its removals. + */ +type StoredRecord = JournalRecord & { txn?: string }; + +interface JournalState { + batches: Map; + /** Operations in request order. */ + ops: Map; + importedSources: Set; } /** - * A branch-independent, durable operation ledger for Discord awareness marks. - * - * Completed entries are retained. Reconciliation derives whether their emoji - * should currently exist from Chronicle branch ancestry, queuing `remove` - * after switching away from a recovery branch and `add` when returning to it. + * The durable awareness-mark ledger: an append-only journal (one JSON record + * per line, each append fsynced) and the operations derived from it. */ export class DiscordAwarenessOutbox { - constructor(readonly path: string) {} + readonly path: string; + private readonly legacyPath: string; + /** Operations dispatched by this process and still awaiting an outcome. */ + private readonly inFlight = new Set(); + + /** + * @param path the journal file. A path ending in `.json` names a pre-journal + * ledger (as an older `discordAwarenessOutboxPath` would): the journal then + * lives beside it as `.journal.jsonl` and the file itself is imported. + */ + constructor(path: string) { + if (path.endsWith('.json')) { + this.legacyPath = path; + this.path = `${path.slice(0, -'.json'.length)}.journal.jsonl`; + } else { + this.path = path; + this.legacyPath = join(dirname(path), LEGACY_LEDGER_NAME); + } + } + // -- surgery side ----------------------------------------------------------- + + /** + * Record a surgery's request before its branch switch. Returns null when it + * carries neither marks nor a suppression journal. A prepared batch never + * delivers: activate() (or startup's crash completion) does. + */ prepare(input: { agentName: string; sourceBranch: string; targetBranch: string; refs: DiscordAwarenessRef[]; + scope: DiscordAwarenessBatchRecord['scope']; emoji?: string; activationPolicy?: 'target-branch' | 'explicit'; suppressionIntervals?: DiscordSuppressionInterval[]; + unmarked?: number; + notRemoved?: number; }): DiscordAwarenessBatch | null { const refs = dedupeRefs(input.refs); if (refs.length === 0 && !input.suppressionIntervals?.length) return null; - - const document = this.read(); - const batch: DiscordAwarenessBatch = { + const batch: DiscordAwarenessBatchRecord = { id: randomUUID(), - status: 'prepared', agentName: input.agentName, sourceBranch: input.sourceBranch, targetBranch: input.targetBranch, emoji: input.emoji ?? DEFAULT_DISCORD_AWARENESS_EMOJI, createdAt: Date.now(), - refs: refs.map((ref) => ({ - ...ref, - desired: false, - markerPresent: false, - deliveryStatus: 'applied', - attempts: 0, - })), + scope: input.scope, + refs, + ...(input.unmarked ? { unmarked: input.unmarked } : {}), + ...(input.notRemoved ? { notRemoved: input.notRemoved } : {}), activationPolicy: input.activationPolicy ?? 'target-branch', - suppressionIntervals: input.suppressionIntervals?.map((interval) => ({ ...interval })), + ...(input.suppressionIntervals?.length + ? { suppressionIntervals: input.suppressionIntervals.map((interval) => ({ ...interval })) } + : {}), }; - document.batches.push(batch); - this.write(document); - return structuredClone(batch); + this.append([{ t: 'batch', at: batch.createdAt, batch }]); + return { ...structuredClone(batch), status: 'prepared' }; } - /** Activate after Chronicle has safely switched to the target branch. */ - activate(batchId: string): void { - const document = this.read(); - const batch = document.batches.find((candidate) => candidate.id === batchId); - if (!batch) throw new Error(`Discord awareness outbox batch not found: ${batchId}`); - batch.status = 'active'; - for (const ref of batch.refs) setDesired(ref, true); - this.write(document); + /** + * Activate a prepared batch after its body change landed: one `add` request + * per ref, durable before this returns. Returns how many were queued. + */ + activate(batchId: string): number { + const state = this.load(); + const batch = state.batches.get(batchId); + if (!batch) throw new Error(`Discord awareness batch not found: ${batchId}`); + if (batch.status === 'active') return 0; + if (batch.status !== 'prepared') { + throw new Error(`Discord awareness batch ${batchId} is ${batch.status}, not prepared`); + } + if (batch.cancelled) { + // An operator cancelled it while its surgery was still running. + throw new Error(`Discord awareness batch ${batchId} was cancelled before activation`); + } + const at = Date.now(); + const records: JournalRecord[] = [{ t: 'activated', at, batchId }]; + records.push(...this.requestRecords(state, batch.refs.map((ref) => ({ ...ref, action: 'add' as const })), batch.emoji, { + kind: 'batch', + batchId, + }, at)); + this.append(records); + return batch.refs.length; } /** - * Retire a batch that was prepared but whose operation did not complete - * (fork failed, a removal threw and the source branch was restored). A - * prepared batch is armed: it re-fires at boot or on branch switch through - * `activatePreparedForBranch` / `preparedSuppressionsForBranch`, and an - * un-completable suppression there aborts framework start. Only prepared - * batches can be discarded — an active batch owns delivered markers and - * must be reconciled, not dropped. Returns false when nothing was removed. + * Retire a batch whose surgery did not complete. Only a prepared batch can + * be discarded; returns false when nothing was retired. */ discard(batchId: string): boolean { - const document = this.read(); - const index = document.batches.findIndex((candidate) => candidate.id === batchId); - if (index < 0) return false; - if (document.batches[index].status !== 'prepared') return false; - document.batches.splice(index, 1); - this.write(document); + const batch = this.load().batches.get(batchId); + if (!batch || batch.status !== 'prepared') return false; + this.append([{ t: 'discarded', at: Date.now(), batchId }]); return true; } /** - * Reconcile every retained ledger entry against the active branch. A marker - * is desired when any active batch for the same ref+emoji has a target branch - * in the active branch's ancestry. + * Suppressions whose body may still need resuming on the active branch: + * explicit batches with intervals whose target is exactly that branch, + * prepared (crash completion: resume, then activate) or held at an earlier + * startup (resume the body only; marks stay held). Branch ancestry is never + * consulted. + */ + preparedSuppressionsForBranch(branchName: string): DiscordAwarenessBatch[] { + return [...this.load().batches.values()] + .filter((batch) => (batch.status === 'prepared' || batch.status === 'held') + && !batch.suppressionComplete + && batch.activationPolicy === 'explicit' + && !!batch.suppressionIntervals?.length + && batch.targetBranch === branchName) + .map((batch) => structuredClone(batch)); + } + + /** Record that a held suppression's body was completed (its marks stay held). */ + recordSuppressionComplete(batchId: string): void { + this.append([{ t: 'suppression-complete', at: Date.now(), batchId }]); + } + + /** + * Startup reconciliation, run once before any delivery. It derives nothing + * from branch ancestry: + * - an attempt left `dispatching` by a previous process is recorded + * `unknown` (it may have reached Discord); + * - a prepared batch whose target is exactly the active branch completes + * its crash window (activated), except a suppression, whose resume + * completes its body first; + * - any other prepared batch is held for an operator; + * - a batch an operator cancelled while it was prepared stays as it is. */ - reconcileForBranch( - branchName: string, - branches: BranchDescriptor[] = [], - ): { activated: number; queued: number } { - const document = this.read(); - let activated = 0; - - for (const batch of document.batches) { - if ( - batch.status === 'prepared' - && (batch.activationPolicy ?? 'target-branch') === 'target-branch' - && isTargetActive(branchName, batch.targetBranch, branches) - ) { - batch.status = 'active'; - activated++; + recoverAtStartup(activeBranchName: string): { activated: string[]; held: string[]; unknownAttempts: number } { + const state = this.load(); + const at = Date.now(); + const records: JournalRecord[] = []; + let unknownAttempts = 0; + for (const op of state.ops.values()) { + for (const attempt of op.attempts) { + if (attempt.outcome !== undefined) continue; + records.push({ + t: 'outcome', + at, + opId: op.opId, + attempt: attempt.attempt, + outcome: 'unknown', + error: 'the previous process ended before an outcome was recorded', + }); + unknownAttempts++; } } + const activated: string[] = []; + const held: string[] = []; + for (const batch of state.batches.values()) { + if (batch.status !== 'prepared' || batch.cancelled) continue; + const suppression = (batch.activationPolicy ?? 'target-branch') === 'explicit' + && !!batch.suppressionIntervals?.length; + if (batch.targetBranch === activeBranchName) { + if (suppression) continue; // resumePreparedDiscordSuppressions completes it - const desiredKeys = new Set(); - for (const batch of document.batches) { - if (batch.status !== 'active') continue; - if (!isTargetActive(branchName, batch.targetBranch, branches)) continue; - for (const ref of batch.refs) desiredKeys.add(markerKey(batch.emoji, ref)); + records.push({ t: 'activated', at, batchId: batch.id }); + records.push(...this.requestRecords( + state, + batch.refs.map((ref) => ({ ...ref, action: 'add' as const })), + batch.emoji, + { kind: 'batch', batchId: batch.id }, + at, + )); + activated.push(batch.id); + } else { + records.push({ + t: 'held', + at, + batchId: batch.id, + reason: suppression + ? `its suppression on ${batch.targetBranch} was interrupted; the body resumes when that branch is active at startup, the marks only on release` + : `its surgery was interrupted before the switch to ${batch.targetBranch} was confirmed`, + releaseActions: batch.refs.map((ref) => ({ ...ref, action: 'add' })), + }); + held.push(batch.id); + } } + if (records.length > 0) this.append(records); + return { activated, held, unknownAttempts }; + } + + // -- operator controls ------------------------------------------------------ - let queued = 0; - for (const batch of document.batches) { - if (batch.status !== 'active') continue; - for (const ref of batch.refs) { - setDesired(ref, desiredKeys.has(markerKey(batch.emoji, ref))); - if (ref.deliveryStatus === 'pending') queued++; + /** + * Stop all future dispatch and retry of a batch's requests: unsent ones, + * ones on the wire (their outcome is still recorded when it comes), and + * ones whose outcome is unknown. A held batch's release actions are + * dropped. Nothing is ever removed from Discord by a cancel: the receipt + * reports what may already have landed, and retract removes it. + */ + cancel(batchId: string, by?: string): DiscordAwarenessCancelReceipt { + const state = this.load(); + const batch = state.batches.get(batchId); + if (!batch) throw new Error(`Discord awareness batch not found: ${batchId}`); + const receipt: DiscordAwarenessCancelReceipt = { + batchId, + // A prepared batch (its surgery still running, or its bookkeeping + // unresolved) has no requests yet: every one of its marks is stopped. + cancelled: batch.status === 'prepared' && !batch.cancelled ? batch.refs.length : 0, + heldDropped: batch.held?.releaseActions.length ?? 0, + inFlight: 0, + unknown: 0, + confirmed: 0, + }; + if (batch.cancelled) return receipt; + const at = Date.now(); + const records: JournalRecord[] = [{ t: 'cancelled', at, batchId, ...(by ? { by } : {}) }]; + for (const op of state.ops.values()) { + if (op.cause.batchId !== batchId) continue; + const status = this.opStatus(op); + if (status === 'confirmed') { + receipt.confirmed++; + continue; } + if (status === 'requested') receipt.cancelled++; + else if (status === 'dispatching') receipt.inFlight++; + else if (status === 'unknown') receipt.unknown++; + else continue; // failed or already cancelled + records.push({ t: 'op-cancelled', at, opId: op.opId, reason: 'batch-cancelled' }); } + this.append(records); + return receipt; + } - this.write(document); - return { activated, queued }; + /** + * Remove this bot's mark (through the configured route) from a batch's refs, + * or from every ref any batch ever sent an add for. Acts across batches: a + * reaction is one per (route, message, emoji), whichever batches asked for + * it. Keys where no add attempt ever left the host are skipped. Add + * requests not yet sent for those keys are superseded. The removal is + * always sent, whatever earlier outcomes say. + */ + retract(target: string | 'all', by?: string): DiscordAwarenessRetractReceipt { + const state = this.load(); + let candidateKeys: Array; + if (target === 'all') { + candidateKeys = uniqueKeys([...state.ops.values()].filter((op) => op.action === 'add').map((op) => op.key)); + } else { + const batch = state.batches.get(target); + if (!batch) throw new Error(`Discord awareness batch not found: ${target}`); + candidateKeys = batch.refs.map((ref) => ({ ...ref, emoji: batch.emoji })); + } + const requestId = randomUUID(); + const at = Date.now(); + const receipt: DiscordAwarenessRetractReceipt = { + requestId, + removalsQueued: 0, + addsSuperseded: 0, + keysWithUnresolvedAdds: 0, + unresolvedAddAttempts: 0, + }; + const toRemove: Array = []; + const records: JournalRecord[] = [{ t: 'retract', at, requestId, target, ...(by ? { by } : {}) }]; + for (const key of candidateKeys) { + const adds = this.opsForKey(state, key).filter((op) => op.action === 'add'); + const left = adds.some((op) => op.attempts.some((attempt) => attempt.outcome !== 'not-sent')); + if (!left) { + // Nothing was ever sent for this key: stop its unsent adds, and + // there is no reaction of ours to remove. + for (const op of adds) { + if (this.opStatus(op) !== 'requested') continue; + records.push({ t: 'op-cancelled', at, opId: op.opId, reason: 'superseded' }); + receipt.addsSuperseded++; + } + continue; + } + const unresolved = unresolvedDispatches(adds, (op, attempt) => this.attemptUnresolved(op, attempt)); + if (unresolved > 0) { + receipt.keysWithUnresolvedAdds++; + receipt.unresolvedAddAttempts += unresolved; + } + toRemove.push({ ...key, action: 'remove' }); + } + const requested = this.requestRecords(state, toRemove, undefined, { kind: 'retract', requestId }, at); + receipt.addsSuperseded += requested.filter((record) => record.t === 'op-cancelled').length; + receipt.removalsQueued = requested.filter((record) => record.t === 'requested').length; + records.push(...requested); + this.append(records); + return receipt; } - /** Backward-compatible startup helper. */ - activatePreparedForBranch(branchName: string, branches: BranchDescriptor[] = []): number { - return this.reconcileForBranch(branchName, branches).activated; + /** Release a held batch: queue its recorded release actions, explicitly. */ + release(batchId: string, by?: string): DiscordAwarenessReleaseReceipt { + const state = this.load(); + const batch = state.batches.get(batchId); + if (!batch) throw new Error(`Discord awareness batch not found: ${batchId}`); + if (batch.status !== 'held' || !batch.held) { + throw new Error(`Discord awareness batch ${batchId} is ${batch.status}, not held`); + } + const at = Date.now(); + const actions = batch.held.releaseActions; + const records: JournalRecord[] = [{ t: 'released', at, batchId, ...(by ? { by } : {}) }]; + records.push(...this.requestRecords(state, actions.map((action) => ({ ...action })), batch.emoji, { + kind: 'release', + batchId, + }, at)); + this.append(records); + return { + batchId, + addsQueued: actions.filter((action) => action.action === 'add').length, + removalsQueued: actions.filter((action) => action.action === 'remove').length, + }; } - pending(serverId?: string): DiscordAwarenessOperation[] { - const operations: DiscordAwarenessOperation[] = []; - for (const batch of this.read().batches) { - if (batch.status !== 'active') continue; - for (const ref of batch.refs) { - if (ref.deliveryStatus !== 'pending') continue; - if (serverId !== undefined && ref.serverId !== serverId) continue; - operations.push({ - batchId: batch.id, - emoji: batch.emoji, - action: ref.desired ? 'add' : 'remove', - ref: { ...ref }, - }); + // -- delivery --------------------------------------------------------------- + + /** + * The next dispatch for each reaction key that has work and nothing on the + * wire. A key's operations go out in request order; consecutive requested + * operations with the same action share one dispatch. An `unknown` + * operation is retried only while it is still the newest live operation + * for its key. + */ + pendingDispatches(serverId?: string): DiscordAwarenessDispatch[] { + const state = this.load(); + const byKey = new Map(); + for (const op of state.ops.values()) { + if (serverId !== undefined && op.key.serverId !== serverId) continue; + const key = markKey(op.key); + const list = byKey.get(key); + if (list) list.push(op); + else byKey.set(key, [op]); + } + const dispatches: Array = []; + for (const ops of byKey.values()) { + // One request per reaction key on the wire at a time, including a + // cancelled one still awaiting its outcome: opposite actions must not + // overtake each other. + if (ops.some((op) => op.attempts.at(-1)?.outcome === undefined && op.attempts.length > 0 + && this.inFlight.has(op.opId))) continue; + const live = ops.filter((op) => !op.cancelled); + let head: DiscordAwarenessOp | undefined; + for (let index = 0; index < live.length; index++) { + const op = live[index]; + const status = this.opStatus(op); + if (status === 'requested') { head = op; break; } + if (status === 'unknown' && index === live.length - 1) { head = op; break; } } + if (!head) continue; + const opIds = [head.opId]; + const start = live.indexOf(head); + for (let index = start + 1; index < live.length; index++) { + const next = live[index]; + if (next.action !== head.action || this.opStatus(next) !== 'requested') break; + opIds.push(next.opId); + } + dispatches.push({ key: head.key, action: head.action, opIds, order: head.requestedAt }); } - return operations; - } - - recordSuccess(batchId: string, ref: DiscordAwarenessRef, action: DiscordAwarenessAction): void { - const document = this.read(); - const entry = findEntry(document, batchId, ref); - if (!entry) return; - entry.markerPresent = action === 'add'; - entry.deliveryStatus = entry.markerPresent === entry.desired ? 'applied' : 'pending'; - entry.attempts++; - entry.lastAction = action; - entry.lastAttemptAt = Date.now(); - delete entry.lastError; - this.write(document); - } - - recordFailure( - batchId: string, - ref: DiscordAwarenessRef, - action: DiscordAwarenessAction, - error: string, - permanent: boolean, - ): void { - const document = this.read(); - const entry = findEntry(document, batchId, ref); - if (!entry) return; - entry.deliveryStatus = permanent ? 'permanent-failure' : 'pending'; - entry.attempts++; - entry.lastAction = action; - entry.lastAttemptAt = Date.now(); - entry.lastError = error; - this.write(document); + return dispatches + .sort((a, b) => a.order - b.order) + .map(({ order: _order, ...dispatch }) => dispatch); + } + + /** Write-ahead: record that a dispatch is about to leave the host. */ + recordDispatching(dispatch: DiscordAwarenessDispatch): Array<{ opId: string; attempt: number }> { + const state = this.load(); + const at = Date.now(); + const dispatchId = randomUUID(); + const attempts = dispatch.opIds.map((opId) => { + const op = state.ops.get(opId); + if (!op) throw new Error(`Discord awareness operation not found: ${opId}`); + return { opId, attempt: op.attempts.length + 1 }; + }); + this.append(attempts.map(({ opId, attempt }) => ({ t: 'dispatching' as const, at, opId, attempt, dispatchId }))); + for (const { opId } of attempts) this.inFlight.add(opId); + return attempts; } - /** Compatibility alias for older callers that only acknowledged adds. */ - acknowledge(batchId: string, ref: DiscordAwarenessRef): void { - this.recordSuccess(batchId, ref, 'add'); + /** Record the outcome of a dispatch's attempts. */ + recordOutcome( + attempts: Array<{ opId: string; attempt: number }>, + outcome: DiscordAwarenessOutcome, + detail: { permanent?: boolean; error?: string } = {}, + ): void { + const at = Date.now(); + try { + this.append(attempts.map(({ opId, attempt }) => ({ + t: 'outcome' as const, + at, + opId, + attempt, + outcome, + ...(outcome === 'failed' && detail.permanent ? { permanent: true } : {}), + ...(detail.error !== undefined ? { error: detail.error } : {}), + }))); + } finally { + for (const { opId } of attempts) this.inFlight.delete(opId); + } } + // -- views ------------------------------------------------------------------ + batches(): DiscordAwarenessBatch[] { - return structuredClone(this.read().batches); + return [...this.load().batches.values()].map((batch) => structuredClone(batch)); } - preparedSuppressionsForBranch( - branchName: string, - branches: BranchDescriptor[] = [], - ): DiscordAwarenessBatch[] { - return structuredClone(this.read().batches.filter((batch) => - batch.status === 'prepared' - && batch.activationPolicy === 'explicit' - && !!batch.suppressionIntervals?.length - && isTargetActive(branchName, batch.targetBranch, branches))); + operations(): DiscordAwarenessOp[] { + const state = this.load(); + return [...state.ops.values()].map((op) => structuredClone(op)); } - private read(): DiscordAwarenessOutboxDocument { - let parsed: unknown; + /** Status of an operation as this process can know it. */ + operationStatus(op: DiscordAwarenessOp): DiscordAwarenessOpStatus { + return this.opStatus(op); + } + + view(): DiscordAwarenessBatchView[] { + const state = this.load(); + const views: DiscordAwarenessBatchView[] = []; + for (const batch of state.batches.values()) { + if (batch.status === 'discarded') continue; + const keys = new Set(batch.refs.map((ref) => markKey({ ...ref, emoji: batch.emoji }))); + const adds = emptyCounts(); + const removals = emptyCounts(); + const onKeys: DiscordAwarenessOp[] = []; + for (const op of state.ops.values()) { + if (!keys.has(markKey(op.key))) continue; + onKeys.push(op); + if (op.action === 'add' && op.cause.batchId === batch.id) adds[this.opStatus(op)]++; + if (op.action === 'remove') removals[this.opStatus(op)]++; + } + const unresolvedAttempts = unresolvedDispatches(onKeys, (op, attempt) => this.attemptUnresolved(op, attempt)); + views.push({ + id: batch.id, + status: batch.status, + scope: batch.scope, + agentName: batch.agentName, + sourceBranch: batch.sourceBranch, + targetBranch: batch.targetBranch, + emoji: batch.emoji, + createdAt: batch.createdAt, + refs: batch.refs.length, + ...(batch.unmarked !== undefined ? { unmarked: batch.unmarked } : {}), + ...(batch.notRemoved !== undefined ? { notRemoved: batch.notRemoved } : {}), + ...(batch.held + ? { held: { reason: batch.held.reason, at: batch.held.at, releaseActions: batch.held.releaseActions.length } } + : {}), + ...(batch.cancelled ? { cancelled: batch.cancelled } : {}), + ...(batch.released ? { released: batch.released } : {}), + adds, + removals, + unresolvedAttempts, + }); + } + return views; + } + + // -- internals -------------------------------------------------------------- + + private opStatus(op: DiscordAwarenessOp): DiscordAwarenessOpStatus { + const last = op.attempts.at(-1); + // A confirmation is a fact about Discord, even for a request cancelled + // while it was on the wire. + if (last?.outcome === 'confirmed') return 'confirmed'; + if (op.cancelled) return 'cancelled'; + if (!last) return 'requested'; + if (last.outcome === undefined) return this.inFlight.has(op.opId) ? 'dispatching' : 'unknown'; + if (last.outcome === 'unknown') return 'unknown'; + if (last.outcome === 'not-sent') return 'requested'; + return last.permanent ? 'failed' : 'requested'; + } + + /** + * An attempt that may have reached Discord without an answer: `unknown`, + * or still without an outcome (on the wire in this process, or left by a + * previous process before recoverAtStartup recorded it unknown). + */ + private attemptUnresolved(_op: DiscordAwarenessOp, attempt: DiscordAwarenessAttempt): boolean { + return attempt.outcome === 'unknown' || attempt.outcome === undefined; + } + + private opsForKey(state: JournalState, key: DiscordAwarenessRef & { emoji: string }): DiscordAwarenessOp[] { + const wanted = markKey(key); + return [...state.ops.values()].filter((op) => markKey(op.key) === wanted); + } + + /** + * Records for new requests. A request supersedes requests for the same + * key with the opposite action that were never sent (the newest explicit + * intent wins without churning the reaction); operations already sent are + * never touched. + */ + private requestRecords( + state: JournalState, + requests: Array, + emoji: string | undefined, + cause: DiscordAwarenessOpCause, + at: number, + ): JournalRecord[] { + const records: JournalRecord[] = []; + const superseded = new Set(); + for (const request of requests) { + const key = { + serverId: request.serverId, + channelId: request.channelId, + messageId: request.messageId, + emoji: request.emoji ?? emoji ?? DEFAULT_DISCORD_AWARENESS_EMOJI, + }; + for (const op of this.opsForKey(state, key)) { + if (op.action === request.action || superseded.has(op.opId)) continue; + if (this.opStatus(op) !== 'requested' || op.attempts.some((attempt) => attempt.outcome !== 'not-sent')) continue; + records.push({ t: 'op-cancelled', at, opId: op.opId, reason: 'superseded' }); + superseded.add(op.opId); + } + records.push({ t: 'requested', at, opId: randomUUID(), key, action: request.action, cause }); + } + return records; + } + + private load(): JournalState { + const { records } = this.readJournal(); + const state = fold(records); + const legacy = this.pendingLegacyImport(state); + if (legacy?.records.length) return fold([...records, ...legacy.records]); + return state; + } + + /** + * The journal's records, and how to repair an unterminated final line + * before the next append: a torn fragment (never applied) is truncated + * away; a complete record that only lost its newline is terminated. + * Without the repair, the next append would share that line and turn a + * harmless torn tail into corruption. + */ + private readJournal(): { records: StoredRecord[]; repair?: { truncateTo: number } | { terminate: true } } { + let raw: Buffer; try { - parsed = JSON.parse(readFileSync(this.path, 'utf8')); + raw = readFileSync(this.path); } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'ENOENT') { - return { version: 2, batches: [] }; - } + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return { records: [] }; throw new Error( - `Could not read Discord awareness outbox ${this.path}: ${error instanceof Error ? error.message : String(error)}`, + `Could not read Discord awareness journal ${this.path}: ${error instanceof Error ? error.message : String(error)}`, ); } + const terminated = raw.length === 0 || raw[raw.length - 1] === 0x0a; + const lastNewline = raw.lastIndexOf(0x0a); + const body = terminated ? raw : raw.subarray(0, lastNewline + 1); + const lines = body.toString('utf8').split('\n'); + const records: StoredRecord[] = []; + for (let index = 0; index < lines.length; index++) { + const line = lines[index]; + if (!line.trim()) continue; + try { + records.push(JSON.parse(line) as StoredRecord); + } catch { + throw new Error(`Corrupt Discord awareness journal ${this.path} at line ${index + 1}`); + } + } + if (terminated) return { records }; + const tail = raw.subarray(lastNewline + 1).toString('utf8'); + try { + records.push(JSON.parse(tail) as StoredRecord); + return { records, repair: { terminate: true } }; + } catch { + // A torn append: none of it ever applied. + return { records, repair: { truncateTo: lastNewline + 1 } }; + } + } - if (isDocumentV2(parsed)) return parsed; - if (isDocumentV1(parsed)) return migrateV1(parsed); - throw new Error(`Invalid Discord awareness outbox document: ${this.path}`); + /** + * A pre-journal ledger beside the journal: null when there is none; + * otherwise its import records, empty when its hash is already imported + * (a crash between the import and the rename), so the file is only renamed. + */ + private pendingLegacyImport(state: JournalState): { records: JournalRecord[]; sha256: string } | null { + let raw: string; + try { + raw = readFileSync(this.legacyPath, 'utf8'); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return null; + throw new Error( + `Could not read Discord awareness ledger ${this.legacyPath}: ${error instanceof Error ? error.message : String(error)}`, + ); + } + const sha256 = createHash('sha256').update(raw).digest('hex'); + if (state.importedSources.has(sha256)) return { records: [], sha256 }; + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch { + throw new Error(`Invalid Discord awareness ledger ${this.legacyPath}`); + } + return { records: importLegacy(parsed, this.legacyPath, sha256), sha256 }; } - private write(document: DiscordAwarenessOutboxDocument): void { + private append(records: JournalRecord[]): void { + if (records.length === 0) return; + const journal = this.readJournal(); + const state = fold(journal.records); + const legacy = this.pendingLegacyImport(state); + const all: StoredRecord[] = legacy ? [...legacy.records, ...records] : records; + let lines: StoredRecord[] = all; + if (all.length > 1) { + const txn = randomUUID(); + lines = [ + ...all.map((record) => ({ ...record, txn })), + { t: 'commit', at: Date.now(), txn }, + ]; + } mkdirSync(dirname(this.path), { recursive: true }); - const temporary = `${this.path}.${process.pid}.${randomUUID()}.tmp`; - writeFileSync(temporary, `${JSON.stringify(document, null, 2)}\n`, { mode: 0o600 }); - renameSync(temporary, this.path); + if (journal.repair && 'truncateTo' in journal.repair) truncateSync(this.path, journal.repair.truncateTo); + const prefix = journal.repair && 'terminate' in journal.repair ? '\n' : ''; + const fd = openSync(this.path, 'a', 0o600); + try { + writeSync(fd, prefix + lines.map((record) => JSON.stringify(record)).join('\n') + '\n'); + fsyncSync(fd); + } finally { + closeSync(fd); + } + if (legacy) this.retireLegacyFile(); + } + + private retireLegacyFile(): void { + if (!existsSync(this.legacyPath)) return; + try { + renameSync(this.legacyPath, `${this.legacyPath}.migrated-v2`); + } catch (error) { + // The import is already recorded (by hash), so the file is inert; the + // next write retries the rename. + console.error( + `[discord-awareness] imported ${basename(this.legacyPath)} but could not rename it: ` + + `${error instanceof Error ? error.message : String(error)}`, + ); + } } } -function setDesired(ref: DiscordAwarenessEntry, desired: boolean): void { - const changed = ref.desired !== desired; - ref.desired = desired; - if (ref.markerPresent === desired) { - ref.deliveryStatus = 'applied'; - return; +// --------------------------------------------------------------------------- +// Fold and legacy import +// --------------------------------------------------------------------------- + +function fold(stored: StoredRecord[]): JournalState { + const state: JournalState = { batches: new Map(), ops: new Map(), importedSources: new Set() }; + // Records of a transaction apply, in order, at its commit; an uncommitted + // transaction (a torn or interrupted append) never applies. + const open = new Map(); + const records: JournalRecord[] = []; + for (const record of stored) { + if (record.t === 'commit') { + records.push(...(open.get(record.txn) ?? [])); + open.delete(record.txn); + } else if (record.txn) { + const buffered = open.get(record.txn); + if (buffered) buffered.push(record); + else open.set(record.txn, [record]); + } else { + records.push(record); + } } - const action: DiscordAwarenessAction = desired ? 'add' : 'remove'; - if (changed || ref.lastAction !== action || ref.deliveryStatus !== 'permanent-failure') { - ref.deliveryStatus = 'pending'; + for (const record of records) { + switch (record.t) { + case 'batch': + if (!state.batches.has(record.batch.id)) { + state.batches.set(record.batch.id, { ...structuredClone(record.batch), status: 'prepared' }); + } + break; + case 'activated': { + const batch = state.batches.get(record.batchId); + if (batch && batch.status !== 'discarded') batch.status = 'active'; + break; + } + case 'discarded': { + const batch = state.batches.get(record.batchId); + if (batch && batch.status === 'prepared') batch.status = 'discarded'; + break; + } + case 'held': { + const batch = state.batches.get(record.batchId); + if (batch && !batch.cancelled) { + batch.status = 'held'; + batch.held = { reason: record.reason, at: record.at, releaseActions: structuredClone(record.releaseActions) }; + } + break; + } + case 'released': { + const batch = state.batches.get(record.batchId); + if (batch && batch.status === 'held') { + batch.status = 'active'; + delete batch.held; + batch.released = { at: record.at, ...(record.by ? { by: record.by } : {}) }; + } + break; + } + case 'cancelled': { + const batch = state.batches.get(record.batchId); + if (batch && !batch.cancelled) { + if (batch.status === 'held') { + batch.status = 'active'; + delete batch.held; + } + batch.cancelled = { at: record.at, ...(record.by ? { by: record.by } : {}) }; + } + break; + } + case 'requested': + if (!state.ops.has(record.opId)) { + state.ops.set(record.opId, { + opId: record.opId, + key: { ...record.key }, + action: record.action, + cause: { ...record.cause }, + requestedAt: record.at, + attempts: [], + }); + } + break; + case 'op-cancelled': { + const op = state.ops.get(record.opId); + if (op && !op.cancelled) op.cancelled = { at: record.at, reason: record.reason }; + break; + } + case 'dispatching': { + const op = state.ops.get(record.opId); + if (op && !op.attempts.some((attempt) => attempt.attempt === record.attempt)) { + op.attempts.push({ + attempt: record.attempt, + ...(record.dispatchId ? { dispatchId: record.dispatchId } : {}), + dispatchedAt: record.at, + }); + } + break; + } + case 'outcome': { + const op = state.ops.get(record.opId); + const attempt = op?.attempts.find((candidate) => candidate.attempt === record.attempt); + if (attempt && attempt.outcome === undefined) { + attempt.outcome = record.outcome; + attempt.outcomeAt = record.at; + if (record.permanent) attempt.permanent = true; + if (record.error !== undefined) attempt.error = record.error; + } + break; + } + case 'suppression-complete': { + const batch = state.batches.get(record.batchId); + if (batch) batch.suppressionComplete = true; + break; + } + case 'import-complete': + state.importedSources.add(record.sha256); + break; + default: + break; + } } + return state; +} + +interface LegacyEntry extends DiscordAwarenessRef { + desired?: boolean; + markerPresent?: boolean; + deliveryStatus?: 'pending' | 'applied' | 'permanent-failure'; + attempts?: number; + lastAction?: DiscordAwarenessAction; + lastAttemptAt?: number; + lastError?: string; } -function isTargetActive( - activeName: string, - targetName: string, - branches: BranchDescriptor[], -): boolean { - if (activeName === targetName) return true; - const byName = new Map(branches.map((branch) => [branch.name, branch])); - const byId = new Map(branches.map((branch) => [branch.id, branch])); - let cursor = byName.get(activeName); - const visited = new Set(); - while (cursor?.parentId && !visited.has(cursor.id)) { - visited.add(cursor.id); - cursor = byId.get(cursor.parentId); - if (cursor?.name === targetName) return true; - } - return false; +interface LegacyBatch { + id: string; + status: 'prepared' | 'active' | 'pending'; + agentName: string; + sourceBranch: string; + targetBranch: string; + emoji: string; + createdAt: number; + refs: LegacyEntry[]; + activationPolicy?: 'target-branch' | 'explicit'; + suppressionIntervals?: DiscordSuppressionInterval[]; } -function findEntry( - document: DiscordAwarenessOutboxDocument, - batchId: string, - ref: DiscordAwarenessRef, -): DiscordAwarenessEntry | undefined { - return document.batches - .find((batch) => batch.id === batchId) - ?.refs.find((candidate) => refKey(candidate) === refKey(ref)); +/** + * Translate a version 1 or 2 ledger into journal records with deterministic + * identities, ending with an `import-complete` record carrying the file's + * hash. What the old ledger recorded is kept as it was recorded; nothing is + * inferred about Discord. Pending work, and batches still prepared, are held + * for an operator: they were queued before marks were an explicit choice. + */ +function importLegacy(parsed: unknown, path: string, sha256: string): JournalRecord[] { + const document = parsed as { version?: unknown; batches?: unknown }; + if (!document || (document.version !== 1 && document.version !== 2) || !Array.isArray(document.batches)) { + throw new Error(`Invalid Discord awareness ledger ${path}`); + } + const at = Date.now(); + const records: JournalRecord[] = []; + for (const raw of document.batches as LegacyBatch[]) { + if (!raw || typeof raw.id !== 'string' || !Array.isArray(raw.refs)) { + throw new Error(`Invalid Discord awareness ledger ${path}`); + } + const legacyActive = raw.status === 'active' || raw.status === 'pending'; + const refs = raw.refs.map((entry) => ({ + serverId: entry.serverId, + channelId: entry.channelId, + messageId: entry.messageId, + })); + records.push({ + t: 'batch', + at: raw.createdAt, + batch: { + id: raw.id, + agentName: raw.agentName, + sourceBranch: raw.sourceBranch, + targetBranch: raw.targetBranch, + emoji: raw.emoji, + createdAt: raw.createdAt, + scope: 'legacy', + refs, + ...(raw.activationPolicy ? { activationPolicy: raw.activationPolicy } : {}), + ...(raw.suppressionIntervals?.length ? { suppressionIntervals: raw.suppressionIntervals } : {}), + }, + }); + if (legacyActive) records.push({ t: 'activated', at: raw.createdAt, batchId: raw.id }); + const releaseActions: DiscordAwarenessReleaseAction[] = []; + for (const entry of raw.refs) { + const key = { serverId: entry.serverId, channelId: entry.channelId, messageId: entry.messageId, emoji: raw.emoji }; + // v1 entries carry no delivery state: an active v1 batch was all pending. + const status = entry.deliveryStatus ?? (raw.status === 'pending' ? 'pending' : 'applied'); + const desired = entry.desired ?? raw.status === 'pending'; + if (!legacyActive || status === 'pending') { + if (!legacyActive || desired || entry.markerPresent) { + releaseActions.push({ + ...key, + action: desired || !legacyActive ? 'add' : 'remove', + ...(entry.attempts ? { priorAttempts: entry.attempts } : {}), + ...(entry.lastError ? { priorError: entry.lastError } : {}), + }); + } + continue; + } + if (!entry.attempts || !entry.lastAction) continue; + const opId = `v2:${raw.id}:${markKey(key)}`; + const attemptAt = entry.lastAttemptAt ?? raw.createdAt; + records.push({ t: 'requested', at: raw.createdAt, opId, key, action: entry.lastAction, cause: { kind: 'import', batchId: raw.id } }); + records.push({ t: 'dispatching', at: attemptAt, opId, attempt: 1 }); + records.push(status === 'applied' + ? { t: 'outcome', at: attemptAt, opId, attempt: 1, outcome: 'confirmed' } + : { + t: 'outcome', + at: attemptAt, + opId, + attempt: 1, + outcome: 'failed', + permanent: true, + ...(entry.lastError ? { error: `recorded by an earlier ledger: ${entry.lastError}` } : {}), + }); + } + if (releaseActions.length > 0) { + records.push({ + t: 'held', + at, + batchId: raw.id, + reason: legacyActive + ? 'queued by an earlier release, before marks were an explicit choice' + : 'prepared by an earlier release, before marks were an explicit choice', + releaseActions, + }); + } + } + records.push({ t: 'import-complete', at, source: 'v2', sha256 }); + return records; } +// --------------------------------------------------------------------------- +// Keys +// --------------------------------------------------------------------------- + function dedupeRefs(refs: DiscordAwarenessRef[]): DiscordAwarenessRef[] { const deduped = new Map(); for (const ref of refs) { if (!ref.serverId || !ref.channelId || !ref.messageId) continue; - deduped.set(refKey(ref), { ...ref }); + deduped.set(refKey(ref), { serverId: ref.serverId, channelId: ref.channelId, messageId: ref.messageId }); } return [...deduped.values()]; } -function refKey(ref: DiscordAwarenessRef): string { - return `${ref.serverId}\0${ref.channelId}\0${ref.messageId}`; -} - -function markerKey(emoji: string, ref: DiscordAwarenessRef): string { - return `${emoji}\0${refKey(ref)}`; +function uniqueKeys(keys: Array): Array { + const unique = new Map(); + for (const key of keys) unique.set(markKey(key), key); + return [...unique.values()]; } -function isEntry(value: unknown): value is DiscordAwarenessEntry { - if (!value || typeof value !== 'object') return false; - const entry = value as Partial; - return typeof entry.serverId === 'string' - && typeof entry.channelId === 'string' - && typeof entry.messageId === 'string' - && typeof entry.desired === 'boolean' - && typeof entry.markerPresent === 'boolean' - && (entry.deliveryStatus === 'pending' - || entry.deliveryStatus === 'applied' - || entry.deliveryStatus === 'permanent-failure') - && typeof entry.attempts === 'number' - && (entry.lastAction === undefined || entry.lastAction === 'add' || entry.lastAction === 'remove') - && (entry.lastAttemptAt === undefined || typeof entry.lastAttemptAt === 'number') - && (entry.lastError === undefined || typeof entry.lastError === 'string'); -} - -function isDocumentV2(value: unknown): value is DiscordAwarenessOutboxDocument { - if (!value || typeof value !== 'object') return false; - const document = value as { version?: unknown; batches?: unknown }; - if (document.version !== 2 || !Array.isArray(document.batches)) return false; - return document.batches.every((batch) => { - if (!batch || typeof batch !== 'object') return false; - const candidate = batch as Partial; - return typeof candidate.id === 'string' - && (candidate.status === 'prepared' || candidate.status === 'active') - && typeof candidate.agentName === 'string' - && typeof candidate.sourceBranch === 'string' - && typeof candidate.targetBranch === 'string' - && typeof candidate.emoji === 'string' - && typeof candidate.createdAt === 'number' - && Array.isArray(candidate.refs) - && (candidate.activationPolicy === undefined - || candidate.activationPolicy === 'target-branch' - || candidate.activationPolicy === 'explicit') - && (candidate.suppressionIntervals === undefined - || (Array.isArray(candidate.suppressionIntervals) - && candidate.suppressionIntervals.every((interval) => - !!interval - && typeof interval.fromId === 'string' - && typeof interval.toId === 'string'))) - && candidate.refs.every(isEntry); - }); +function refKey(ref: DiscordAwarenessRef): string { + return `${ref.serverId}\0${ref.channelId}\0${ref.messageId}`; } -interface LegacyDocument { - version: 1; - batches: Array<{ - id: string; - status: 'prepared' | 'pending'; - agentName: string; - sourceBranch: string; - targetBranch: string; - emoji: string; - createdAt: number; - refs: DiscordAwarenessRef[]; - activationPolicy?: 'target-branch' | 'explicit'; - }>; +function markKey(key: DiscordAwarenessRef & { emoji: string }): string { + return `${key.emoji}\0${refKey(key)}`; } -function isDocumentV1(value: unknown): value is LegacyDocument { - if (!value || typeof value !== 'object') return false; - const document = value as { version?: unknown; batches?: unknown }; - if (document.version !== 1 || !Array.isArray(document.batches)) return false; - return document.batches.every((batch) => { - if (!batch || typeof batch !== 'object') return false; - const candidate = batch as LegacyDocument['batches'][number]; - return typeof candidate.id === 'string' - && (candidate.status === 'prepared' || candidate.status === 'pending') - && typeof candidate.agentName === 'string' - && typeof candidate.sourceBranch === 'string' - && typeof candidate.targetBranch === 'string' - && typeof candidate.emoji === 'string' - && typeof candidate.createdAt === 'number' - && Array.isArray(candidate.refs) - && candidate.refs.every((ref) => !!ref - && typeof ref.serverId === 'string' - && typeof ref.channelId === 'string' - && typeof ref.messageId === 'string'); - }); +/** Unresolved requests among these operations' attempts, one per dispatch. */ +function unresolvedDispatches( + ops: DiscordAwarenessOp[], + unresolved: (op: DiscordAwarenessOp, attempt: DiscordAwarenessAttempt) => boolean, +): number { + const dispatches = new Set(); + for (const op of ops) { + for (const attempt of op.attempts) { + if (unresolved(op, attempt)) dispatches.add(attempt.dispatchId ?? `${op.opId}#${attempt.attempt}`); + } + } + return dispatches.size; } -function migrateV1(document: LegacyDocument): DiscordAwarenessOutboxDocument { - return { - version: 2, - batches: document.batches.map((batch) => ({ - ...batch, - status: batch.status === 'pending' ? 'active' : 'prepared', - refs: batch.refs.map((ref) => ({ - ...ref, - desired: batch.status === 'pending', - markerPresent: false, - deliveryStatus: batch.status === 'pending' ? 'pending' : 'applied', - attempts: 0, - })), - })), - }; +function emptyCounts(): Record { + return { requested: 0, dispatching: 0, confirmed: 0, failed: 0, unknown: 0, cancelled: 0 }; } diff --git a/src/recovery/offline-branch.ts b/src/recovery/offline-branch.ts index 6c2a202c8..7045d4392 100644 --- a/src/recovery/offline-branch.ts +++ b/src/recovery/offline-branch.ts @@ -5,7 +5,10 @@ import { DiscordAwarenessOutbox, defaultDiscordAwarenessOutboxPath, extractDiscordAwarenessRefs, + selectDiscordAwarenessRefs, + type DiscordAwarenessMarks, type DiscordAwarenessRef, + type DiscordMessageMetadataCarrier, } from './discord-awareness-outbox.js'; export interface OfflineRecoveryBranchOptions { @@ -28,6 +31,11 @@ export interface OfflineRecoveryBranchOptions { discordServerId?: string; outboxPath?: string; emoji?: string; + /** + * Awareness marks on the removed Discord messages: an explicit publication + * choice, `none` by default. The recovery itself stays local either way. + */ + marks?: DiscordAwarenessMarks; dryRun?: boolean; } @@ -37,8 +45,18 @@ export interface OfflineRecoveryBranchResult { targetBranch: string; messagesRemoved: number; messagesSuppressed: number; + /** Removed messages that carry a Discord address. */ + discordAddressable: number; + /** The publication scope recorded for this recovery. */ + marksScope: 'none' | 'addressed' | 'all'; + /** Marks requested (queued once the branch is activated); not delivered. */ discordMarkersQueued: number; + /** The refs that will be marked (empty unless marks were chosen). */ refs: DiscordAwarenessRef[]; + /** Removed addressable messages left unmarked. */ + unmarked: number; + /** Authorized refs (marks.refs) this recovery does not remove. */ + notRemoved: number; outboxPath: string; } @@ -81,7 +99,7 @@ export async function createOfflineRecoveryBranch( let target: StoredWindowMessage; let targetIndex: number; let removedCount: number; - let refs: DiscordAwarenessRef[]; + let carriers: DiscordMessageMetadataCarrier[]; if (options.messageId) { // Search backward in bounded windows: recovery anchors are normally near @@ -98,12 +116,7 @@ export async function createOfflineRecoveryBranch( target = located.message; targetIndex = located.index; removedCount = total - located.index - 1; - refs = collectDiscordRefs( - contextManager, - located.index + 1, - total, - options.discordServerId, - ); + carriers = collectDiscordCarriers(contextManager, located.index + 1, total); } else if (options.contextId) { const located = findInternalMessageFromTail(contextManager, total, options.contextId.trim()); if (!located) { @@ -112,12 +125,7 @@ export async function createOfflineRecoveryBranch( target = located.message; targetIndex = located.index; removedCount = total - located.index - 1; - refs = collectDiscordRefs( - contextManager, - located.index + 1, - total, - options.discordServerId, - ); + carriers = collectDiscordCarriers(contextManager, located.index + 1, total); } else { // Count mode remains for compatibility. Read only the target plus suffix, // avoiding full-history materialization and attachment blob inflation. @@ -132,7 +140,7 @@ export async function createOfflineRecoveryBranch( target = countTarget; targetIndex = total - count! - 1; removedCount = count!; - refs = extractDiscordAwarenessRefs(discarded, options.discordServerId); + carriers = discarded.map(addressingOnly); } validateAnchorBoundary(contextManager, targetIndex, total, target); @@ -142,7 +150,6 @@ export async function createOfflineRecoveryBranch( targetIndex, options.suppressMessageIds ?? [], options.suppressRanges ?? [], - options.discordServerId, ); validateRemovalIntegrity( contextManager, @@ -157,7 +164,11 @@ export async function createOfflineRecoveryBranch( ...suppression.intervals, ], ); - refs = dedupeDiscordRefs([...refs, ...suppression.refs]); + carriers = [...carriers, ...suppression.carriers]; + const addressable = extractDiscordAwarenessRefs(carriers, options.discordServerId).length; + const marks = options.marks ?? 'none'; + const selection = selectDiscordAwarenessRefs(carriers, marks, options.discordServerId); + const refs = selection.refs; const sourceBranch = store.currentBranch().name; const targetBranch = options.branchName @@ -174,8 +185,12 @@ export async function createOfflineRecoveryBranch( targetBranch, messagesRemoved: removedCount, messagesSuppressed: suppression.messageCount, + discordAddressable: addressable, + marksScope: marks === 'none' ? 'none' : marks.scope, discordMarkersQueued: refs.length, refs, + unmarked: selection.unmarked, + notRemoved: selection.notRemoved, outboxPath, }; if (options.dryRun) return result; @@ -186,6 +201,9 @@ export async function createOfflineRecoveryBranch( sourceBranch, targetBranch, refs, + scope: marks === 'none' ? 'none' : marks.scope, + unmarked: selection.unmarked, + notRemoved: selection.notRemoved, emoji: options.emoji ?? DEFAULT_DISCORD_AWARENESS_EMOJI, // Merely seeing targetBranch active does not prove branch-local // suppressions finished. Only the recovery operation may activate this @@ -368,8 +386,7 @@ function buildSuppressionPlan( targetIndex: number, rawMessageIds: string[], rawRanges: Array<{ fromMessageId: string; toMessageId: string }>, - forcedServerId?: string, -): { intervals: SuppressionInterval[]; refs: DiscordAwarenessRef[]; messageCount: number } { +): { intervals: SuppressionInterval[]; carriers: DiscordMessageMetadataCarrier[]; messageCount: number } { const messageIds = rawMessageIds.map(normalizeDiscordMessageId); const ranges = rawRanges.map((range) => ({ fromMessageId: normalizeDiscordMessageId(range.fromMessageId), @@ -379,7 +396,7 @@ function buildSuppressionPlan( ...messageIds, ...ranges.flatMap((range) => [range.fromMessageId, range.toMessageId]), ]); - if (wanted.size === 0) return { intervals: [], refs: [], messageCount: 0 }; + if (wanted.size === 0) return { intervals: [], carriers: [], messageCount: 0 }; const locations = new Map(); for (let offset = 0; offset <= targetIndex; offset += RECOVERY_SCAN_WINDOW) { @@ -466,45 +483,48 @@ function buildSuppressionPlan( } } - const refs: DiscordAwarenessRef[] = []; + const carriers: DiscordMessageMetadataCarrier[] = []; for (const interval of merged) { - refs.push(...collectDiscordRefs( - contextManager, - interval.start, - interval.end + 1, - forcedServerId, - )); + carriers.push(...collectDiscordCarriers(contextManager, interval.start, interval.end + 1)); } return { intervals: merged, - refs: dedupeDiscordRefs(refs), + carriers, messageCount: merged.reduce((sum, interval) => sum + interval.end - interval.start + 1, 0), }; } -function collectDiscordRefs( +/** + * The addressing metadata of removed messages, read in bounded windows: only + * the Discord address and the MCPL tags (for `addressed` selection) are kept, + * never content. A sharded message may repeat its metadata across adjacent + * records; selection dedupes by address. + */ +function collectDiscordCarriers( contextManager: ContextManager, start: number, end: number, - forcedServerId?: string, -): DiscordAwarenessRef[] { - const refs: DiscordAwarenessRef[] = []; +): DiscordMessageMetadataCarrier[] { + const carriers: DiscordMessageMetadataCarrier[] = []; for (let offset = start; offset < end; offset += RECOVERY_SCAN_WINDOW) { const messages = contextManager.getMessageWindow( offset, Math.min(RECOVERY_SCAN_WINDOW, end - offset), { resolveBlobs: false }, ).messages; - refs.push(...extractDiscordAwarenessRefs(messages, forcedServerId)); + carriers.push(...messages.map(addressingOnly)); } - // A sharded message may repeat Discord metadata across adjacent records. - return dedupeDiscordRefs(refs); + return carriers; } -function dedupeDiscordRefs(refs: DiscordAwarenessRef[]): DiscordAwarenessRef[] { - const deduped = new Map(); - for (const ref of refs) { - deduped.set(`${ref.serverId}\0${ref.channelId}\0${ref.messageId}`, ref); - } - return [...deduped.values()]; +function addressingOnly(message: { metadata?: Record }): DiscordMessageMetadataCarrier { + const metadata = message.metadata ?? {}; + return { + metadata: { + serverId: metadata.serverId, + channelId: metadata.channelId, + messageId: metadata.messageId, + tags: metadata.tags, + }, + }; } diff --git a/src/recovery/recover-cli.ts b/src/recovery/recover-cli.ts index e77556a15..99c642c0e 100644 --- a/src/recovery/recover-cli.ts +++ b/src/recovery/recover-cli.ts @@ -1,6 +1,11 @@ #!/usr/bin/env node import { createOfflineRecoveryBranch } from './offline-branch.js'; +import { + DiscordAwarenessOutbox, + defaultDiscordAwarenessOutboxPath, + type DiscordAwarenessMarks, +} from './discord-awareness-outbox.js'; interface CliOptions { storePath?: string; @@ -15,16 +20,21 @@ interface CliOptions { discordServerId?: string; outboxPath?: string; emoji?: string; + marks: 'none' | 'addressed' | 'all'; + awareness?: { verb: 'list' | 'cancel' | 'retract' | 'release'; target?: string }; dryRun: boolean; } function usage(): string { return `Usage: agent-framework-recover --store --agent [options] + agent-framework-recover --store --awareness -Creates and activates a Chronicle recovery branch while the normal host is down, -then queues Discord reactions for addressable messages removed from awareness. -Message content is never compiled or written to the recovery outbox. +Creates and activates a Chronicle recovery branch while the normal host is +down. The recovery is local to the resident: by default no Discord reaction is +placed. --marks chooses to mark removed Discord messages with an awareness +reaction (queued here, delivered when the host and its Discord MCPL server +next connect). Message content is never compiled or written to the journal. Options: --message-id Last Discord message the agent should still see @@ -35,16 +45,30 @@ Options: --branch Recovery branch name (generated by default) --namespace Context namespace (default: agents/) --discord-server Override/inject the Discord MCPL server id - --outbox Override the branch-independent outbox path + --marks + Awareness marks on removed Discord messages + (default none; addressed = mentions, replies to + the bot and DMs; all = every addressable message) + --outbox Override the awareness journal path --emoji Marker reaction (default: 💤) --dry-run Inspect counts/addresses without writing --help Show this help +Awareness journal (host stopped): + --awareness list Each batch: scope, counts by status, holds + --awareness cancel Stop a batch's unsent marks (never removes any) + --awareness retract + Queue removal of this bot's marks (through each + ref's configured MCPL route) on a batch's refs, + or on every ref an add was ever sent for + --awareness release Queue a held batch's recorded operations + +Queued work is delivered when the host next connects to the Discord server. The agent host must be stopped before running this command.`; } function parseArgs(args: string[]): CliOptions { - const options: CliOptions = { dryRun: false, suppressMessageIds: [], suppressRanges: [] }; + const options: CliOptions = { dryRun: false, suppressMessageIds: [], suppressRanges: [], marks: 'none' }; for (let i = 0; i < args.length; i++) { const arg = args[i]; const value = () => { @@ -80,6 +104,25 @@ function parseArgs(args: string[]): CliOptions { case '--discord-server': options.discordServerId = value(); break; case '--outbox': options.outboxPath = value(); break; case '--emoji': options.emoji = value(); break; + case '--marks': { + const marks = value(); + if (marks !== 'none' && marks !== 'addressed' && marks !== 'all') { + throw new Error('--marks must be none, addressed or all'); + } + options.marks = marks; + break; + } + case '--awareness': { + const verb = value(); + if (verb === 'list') { + options.awareness = { verb }; + } else if (verb === 'cancel' || verb === 'retract' || verb === 'release') { + options.awareness = { verb, target: value() }; + } else { + throw new Error('--awareness must be list, cancel , retract or release '); + } + break; + } case '--dry-run': options.dryRun = true; break; case '--help': case '-h': @@ -91,8 +134,31 @@ function parseArgs(args: string[]): CliOptions { return options; } +function awareness(options: CliOptions): void { + const outbox = new DiscordAwarenessOutbox( + options.outboxPath ?? defaultDiscordAwarenessOutboxPath(options.storePath!), + ); + const { verb, target } = options.awareness!; + const by = 'agent-framework-recover'; + let result: unknown; + switch (verb) { + case 'list': result = { journal: outbox.path, batches: outbox.view() }; break; + case 'cancel': result = outbox.cancel(target!, by); break; + case 'retract': result = outbox.retract(target!, by); break; + case 'release': result = outbox.release(target!, by); break; + } + console.log(JSON.stringify(result, null, 2)); +} + async function main(): Promise { const options = parseArgs(process.argv.slice(2)); + if (options.awareness) { + if (!options.storePath && !options.outboxPath) { + throw new Error(`--awareness needs --store (or --outbox)\n\n${usage()}`); + } + awareness(options); + return; + } if ( !options.storePath || !options.agentName @@ -101,6 +167,9 @@ async function main(): Promise { throw new Error(`--store, --agent, and one anchor (--message-id, --context-id, or --messages) are required\n\n${usage()}`); } + // A CLI choice and its application are the same moment (the host is + // stopped), so the scope is evaluated now rather than frozen to refs. + const marks: DiscordAwarenessMarks = options.marks === 'none' ? 'none' : { scope: options.marks }; const result = await createOfflineRecoveryBranch({ storePath: options.storePath, agentName: options.agentName, @@ -114,6 +183,7 @@ async function main(): Promise { discordServerId: options.discordServerId, outboxPath: options.outboxPath, emoji: options.emoji, + marks, dryRun: options.dryRun, }); diff --git a/test/discord-awareness-delivery.test.ts b/test/discord-awareness-delivery.test.ts new file mode 100644 index 000000000..137afb045 --- /dev/null +++ b/test/discord-awareness-delivery.test.ts @@ -0,0 +1,207 @@ +/** + * Awareness-mark delivery never gates the resident. + * + * Before: queued marks held every MCPL data plane, at startup, reconnect and + * after surgery, until each reaction had been attempted; at Discord's pace a + * large batch left the resident deaf for many minutes, and an unreadable + * ledger aborted startup or recycled connections at runtime. Now delivery is + * a background drain over the awareness journal: traffic flows while marks + * are delivered; a route that is down leaves its work queued; the mandatory + * per-request deadline still turns a hung reaction into an unknown outcome. + * The journal is still read before startup, because it also carries the + * suppression resume journal that protects the resident's body. + */ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { fileURLToPath } from 'node:url'; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { AgentFramework } from '../src/framework.js'; +import { DiscordAwarenessOutbox } from '../src/recovery/discord-awareness-outbox.js'; +import { MockMembrane } from './helpers/mock-membrane.js'; + +const FIXTURE = fileURLToPath(new URL('./fixtures/surgery-marks-mcpl-server.mjs', import.meta.url)); + +function jsonl(path: string): Array> { + if (!existsSync(path)) return []; + return readFileSync(path, 'utf8').split('\n').filter(Boolean).map((line) => JSON.parse(line)); +} + +async function waitFor(description: string, predicate: () => boolean, timeoutMs = 5_000): Promise { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + if (predicate()) return; + await new Promise((resolve) => setTimeout(resolve, 10)); + } + assert.fail(`timed out waiting for ${description}`); +} + +function setup() { + const dir = mkdtempSync(join(tmpdir(), 'awareness-delivery-')); + const storePath = join(dir, 'store'); + // Outside the store: the store directory must not exist before Chronicle + // creates it, and these tests seed the journal before the first start. + const journal = join(dir, 'awareness', 'discord-awareness-journal.jsonl'); + const paths = { + calls: join(dir, 'calls.jsonl'), + events: join(dir, 'events.jsonl'), + hold: join(dir, 'release'), + probe: join(dir, 'probe'), + }; + const queueMarks = (messageIds: string[]) => { + const outbox = new DiscordAwarenessOutbox(journal); + const batch = outbox.prepare({ + agentName: 'resident', + sourceBranch: 'source', + targetBranch: 'main', + refs: messageIds.map((messageId) => ({ serverId: 'discord', channelId: 'discord:g1:c1', messageId })), + scope: 'all', + activationPolicy: 'explicit', + })!; + outbox.activate(batch.id); + return outbox; + }; + const config = (opts: { discord?: boolean; requestTimeoutMs?: number; awarenessDeadlineMs?: number } = {}) => ({ + storePath, + membrane: new MockMembrane().asMembrane(), + agents: [{ name: 'resident', model: 'test', systemPrompt: 'test' }], + modules: [], + maintenanceIntervalMs: 0, + discordAwarenessOutboxPath: journal, + ...(opts.awarenessDeadlineMs ? { discordAwarenessDeadlineMs: opts.awarenessDeadlineMs } : {}), + ...(opts.discord === false ? {} : { + mcplServers: [{ + id: 'discord', + command: process.execPath, + args: [FIXTURE], + ...(opts.requestTimeoutMs !== undefined ? { requestTimeoutMs: opts.requestTimeoutMs } : {}), + env: { CALLS_PATH: paths.calls, EVENTS_PATH: paths.events, HOLD_PATH: paths.hold, PROBE_PATH: paths.probe }, + enabledFeatureSets: ['chat'], + }], + }), + }); + return { dir, storePath, journal, paths, queueMarks, config }; +} + +test('startup does not wait for queued marks, and channel traffic flows while they are held', async () => { + const { dir, journal, paths, queueMarks, config } = setup(); + let framework: AgentFramework | undefined; + try { + queueMarks(['m1', 'm2']); + writeFileSync(paths.probe, '1'); // the server sends channel traffic as soon as it can + framework = await AgentFramework.create(config()); + await waitFor('the queued add to reach the server', () => jsonl(paths.calls).length >= 1); + await waitFor('the channel traffic to be answered', () => + jsonl(paths.events).some((event) => event.event === 'probe-answered')); + assert.equal( + jsonl(paths.events).filter((event) => event.event === 'reaction-answered').length, + 0, + 'answered while every reaction was still withheld', + ); + writeFileSync(paths.hold, '1'); + await waitFor('both adds confirmed', () => { + const outbox = new DiscordAwarenessOutbox(journal); + return outbox.operations().filter((op) => outbox.operationStatus(op) === 'confirmed').length === 2; + }); + } finally { + writeFileSync(paths.hold, '1'); + await framework?.stop(); + rmSync(dir, { recursive: true, force: true }); + } +}); + +test('the mandatory awareness deadline still bounds a hung reaction with requestTimeoutMs: 0', async () => { + const { dir, journal, paths, queueMarks, config } = setup(); + let framework: AgentFramework | undefined; + try { + queueMarks(['m1']); + framework = await AgentFramework.create(config({ requestTimeoutMs: 0, awarenessDeadlineMs: 150 })); + // Another reader sees an unanswered dispatch as unknown at once; wait + // for the drain's own outcome record. + await waitFor('the hung add to be recorded unknown', () => + new DiscordAwarenessOutbox(journal).operations().some((op) => op.attempts[0]?.outcome === 'unknown')); + const outbox = new DiscordAwarenessOutbox(journal); + const [op] = outbox.operations(); + assert.equal(op.attempts[0].outcome, 'unknown'); + assert.match(op.attempts[0].error ?? '', /did not respond/); + } finally { + writeFileSync(paths.hold, '1'); + await framework?.stop(); + rmSync(dir, { recursive: true, force: true }); + } +}); + +test('a route that is not connected leaves marks queued; they go out once it is', async () => { + const { dir, journal, paths, queueMarks, config } = setup(); + let framework: AgentFramework | undefined; + try { + queueMarks(['m1']); + const before = readFileSync(journal, 'utf8'); + framework = await AgentFramework.create(config({ discord: false })); + await framework.syncDiscordAwarenessMarkers(); + assert.equal(readFileSync(journal, 'utf8'), before, 'nothing recorded for a route that is down'); + await framework.stop(); + framework = undefined; + + writeFileSync(paths.hold, '1'); + framework = await AgentFramework.create(config()); + await waitFor('the queued add confirmed', () => { + const outbox = new DiscordAwarenessOutbox(journal); + return outbox.operations().every((op) => outbox.operationStatus(op) === 'confirmed'); + }); + assert.deepEqual(jsonl(paths.calls).map((call) => call.name), ['add_reaction']); + } finally { + writeFileSync(paths.hold, '1'); + await framework?.stop(); + rmSync(dir, { recursive: true, force: true }); + } +}); + +test('a corrupt journal stops startup, since it may hold a suppression the body needs resumed', async () => { + const { dir, journal, config } = setup(); + try { + mkdirSync(join(journal, '..'), { recursive: true }); + writeFileSync(journal, '{"t":"batch",\nnot json at all\n{"t":"commit","at":1,"txn":"x"}\n'); + await assert.rejects( + AgentFramework.create(config({ discord: false })), + /Discord awareness accounting failed during startup reconciliation: Corrupt Discord awareness journal/, + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test('a branch switch neither adds nor removes marks', async () => { + const { dir, journal, paths, config } = setup(); + let framework: AgentFramework | undefined; + try { + writeFileSync(paths.hold, '1'); + framework = await AgentFramework.create(config()); + const cm = framework.getAgent('resident')!.getContextManager(); + cm.addMessage('Operator', [{ type: 'text', text: 'kept' }], { serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'kept' }); + const tail = String(cm.addMessage('Operator', [{ type: 'text', text: 'tail' }], { + serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'tail', + })); + cm.addMessage('Member', [{ type: 'text', text: 'hi' }], { + serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'm1', tags: ['chat:addressed'], + }); + const result = await framework.rollbackToMessage('resident', { messageId: tail, marks: { scope: 'addressed' } }); + await waitFor('the add confirmed', () => + new DiscordAwarenessOutbox(journal).operations().some((op) => op.attempts[0]?.outcome === 'confirmed')); + // Switch back to the source and forward again: nothing is sent. + await cm.switchBranch(result.sourceBranch); + await framework.syncDiscordAwarenessMarkers(); + await cm.switchBranch(result.targetBranch); + await framework.syncDiscordAwarenessMarkers(); + await framework.stop(); + framework = await AgentFramework.create(config()); + await framework.syncDiscordAwarenessMarkers(); + assert.deepEqual(jsonl(paths.calls).map((call) => call.name), ['add_reaction']); + assert.equal(new DiscordAwarenessOutbox(journal).operations().length, 1); + } finally { + writeFileSync(paths.hold, '1'); + await framework?.stop(); + rmSync(dir, { recursive: true, force: true }); + } +}); diff --git a/test/discord-awareness-outbox.test.ts b/test/discord-awareness-outbox.test.ts index 4a35a21f5..ba8e1b9b6 100644 --- a/test/discord-awareness-outbox.test.ts +++ b/test/discord-awareness-outbox.test.ts @@ -1,15 +1,75 @@ +/** + * The Discord awareness-mark journal: an append-only record of what was + * asked and what was answered, never a model of what is on Discord. + * + * Contract (room-225's proposed upstream change, after a field incident in + * which a web-UI rollback queued 918 💤 reactions on other people's messages): + * marks are an explicit publication choice (default none); one-shot, so + * branch moves and restarts never derive operations from branch state; and + * cancel / retract / release are explicit operator acts whose receipts say + * what is known and what is not. + */ import { test } from 'node:test'; import assert from 'node:assert/strict'; -import { mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import { appendFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; import { join } from 'node:path'; import { tmpdir } from 'node:os'; import { DiscordAwarenessOutbox, extractDiscordAwarenessRefs, + selectDiscordAwarenessRefs, + type DiscordAwarenessRef, } from '../src/recovery/discord-awareness-outbox.js'; import { AgentFramework } from '../src/framework.js'; +import { McplRequestError } from '../src/mcpl/server-connection.js'; import { OperatorLog } from '../src/operator-log.js'; +const ref = (messageId: string, channel = 'c1'): DiscordAwarenessRef => + ({ serverId: 'discord', channelId: `discord:g1:${channel}`, messageId }); + +function withJournal(fn: (path: string, dir: string) => void | Promise): () => Promise { + return async () => { + const dir = mkdtempSync(join(tmpdir(), 'discord-awareness-')); + try { + await fn(join(dir, 'recovery', 'discord-awareness-journal.jsonl'), dir); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }; +} + +function activeBatch(outbox: DiscordAwarenessOutbox, refs: DiscordAwarenessRef[], extra: Record = {}) { + const batch = outbox.prepare({ + agentName: 'cairn', + sourceBranch: 'main', + targetBranch: 'rollback/cairn/1', + refs, + scope: 'all', + ...extra, + })!; + outbox.activate(batch.id); + return batch; +} + +/** Dispatch everything due, answering each dispatch with `answer`. */ +function drain( + outbox: DiscordAwarenessOutbox, + answer: (dispatch: { action: string; messageId: string }) => 'confirmed' | 'failed' | 'not-sent' | 'unknown', +): string[] { + const seen: string[] = []; + for (let pass = 0; pass < 10; pass++) { + const dispatches = outbox.pendingDispatches('discord'); + if (dispatches.length === 0) break; + for (const dispatch of dispatches) { + const attempts = outbox.recordDispatching(dispatch); + const outcome = answer({ action: dispatch.action, messageId: dispatch.key.messageId }); + seen.push(`${dispatch.action}:${dispatch.key.messageId}:${outcome}`); + outbox.recordOutcome(attempts, outcome, outcome === 'failed' ? { permanent: true, error: 'Unknown Message' } : {}); + } + } + return seen; +} + test('extractDiscordAwarenessRefs keeps only direct Discord addressing metadata', () => { const refs = extractDiscordAwarenessRefs([ { metadata: { serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'm1' } }, @@ -17,104 +77,423 @@ test('extractDiscordAwarenessRefs keeps only direct Discord addressing metadata' { metadata: { serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'm1' } }, { metadata: { serverId: 'discord', channelId: 'discord:g1:c1' } }, ]); - - assert.deepEqual(refs, [ - { serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'm1' }, - ]); + assert.deepEqual(refs, [ref('m1')]); }); -test('ledger survives restart and reverses completed markers on source branch', () => { - const dir = mkdtempSync(join(tmpdir(), 'discord-awareness-outbox-')); - const path = join(dir, 'outbox.json'); - try { - const first = new DiscordAwarenessOutbox(path); - const batch = first.prepare({ - agentName: 'cairn', - sourceBranch: 'main', - targetBranch: 'recovery/cairn/1', - refs: [ - { serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'm1' }, - { serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'm2' }, - ], - }); - assert.ok(batch); - assert.equal(first.pending('discord').length, 0, 'prepared work is not delivered early'); - - // Simulate the host restarting after Chronicle switched branches but - // before the recovery process activated its batch. - const reopened = new DiscordAwarenessOutbox(path); - assert.equal(reopened.activatePreparedForBranch('some-other-branch'), 0); - assert.equal(reopened.activatePreparedForBranch('recovery/cairn/1'), 1); - assert.deepEqual( - reopened.pending('discord').map((operation) => operation.ref.messageId), - ['m1', 'm2'], - ); - - reopened.recordSuccess(batch.id, batch.refs[0], 'add'); - assert.deepEqual(reopened.pending('discord').map((op) => op.ref.messageId), ['m2']); - reopened.recordSuccess(batch.id, batch.refs[1], 'add'); - assert.equal(reopened.pending('discord').length, 0); - - // Completed work remains durable and switching back queues removals. - reopened.reconcileForBranch('main'); - const removals = reopened.pending('discord'); - assert.deepEqual(removals.map((op) => op.action), ['remove', 'remove']); - for (const operation of removals) { - reopened.recordSuccess(operation.batchId, operation.ref, operation.action); - } - assert.equal(reopened.pending('discord').length, 0); - assert.equal(reopened.batches().length, 1); - assert.ok(reopened.batches()[0].refs.every((ref) => !ref.markerPresent)); +test('selection: none marks nothing; addressed means the chat:addressed tag; refs bind the set', () => { + const messages = [ + { metadata: { ...ref('a1'), tags: ['chat:addressed', 'chat:mention'] } }, + { metadata: { ...ref('b1'), tags: ['chat:ambient'] } }, + { metadata: { ...ref('b2') } }, + { metadata: { ...ref('a2', 'dm'), tags: ['chat:addressed', 'chat:dm'] } }, + ]; + const none = selectDiscordAwarenessRefs(messages, 'none'); + assert.deepEqual(none, { refs: [], addressable: 4, unmarked: 4, notRemoved: 0 }); - const stored = JSON.parse(readFileSync(path, 'utf8')) as { version: number; batches: unknown[] }; - assert.equal(stored.version, 2); - assert.equal(stored.batches.length, 1); - } finally { - rmSync(dir, { recursive: true, force: true }); - } + const addressed = selectDiscordAwarenessRefs(messages, { scope: 'addressed' }); + assert.deepEqual(addressed.refs.map((r) => r.messageId), ['a1', 'a2']); + assert.equal(addressed.unmarked, 2); + + const all = selectDiscordAwarenessRefs(messages, { scope: 'all' }); + assert.equal(all.refs.length, 4); + + // A frozen choice: only previewed refs, and never anything else removed. + const frozen = selectDiscordAwarenessRefs(messages, { scope: 'all', refs: [ref('a1'), ref('gone')] }); + assert.deepEqual(frozen.refs.map((r) => r.messageId), ['a1']); + assert.equal(frozen.notRemoved, 1, 'an authorized ref the surgery did not remove'); + assert.equal(frozen.unmarked, 3); + + // Scope still applies to authorized refs. + const frozenAddressed = selectDiscordAwarenessRefs(messages, { scope: 'addressed', refs: [ref('b1'), ref('a2', 'dm')] }); + assert.deepEqual(frozenAddressed.refs.map((r) => r.messageId), ['a2']); }); -test('explicit batches are not activated merely because their target branch is current', () => { - const dir = mkdtempSync(join(tmpdir(), 'discord-awareness-explicit-')); - const path = join(dir, 'outbox.json'); +test('a prepared batch delivers nothing; activation queues one add per ref, written ahead of dispatch', withJournal((path) => { + const outbox = new DiscordAwarenessOutbox(path); + const batch = outbox.prepare({ + agentName: 'cairn', sourceBranch: 'main', targetBranch: 'rollback/cairn/1', refs: [ref('m1'), ref('m2')], scope: 'all', + })!; + assert.equal(outbox.pendingDispatches('discord').length, 0, 'prepared work is not deliverable'); + assert.equal(outbox.activate(batch.id), 2); + const [first] = outbox.pendingDispatches('discord'); + assert.equal(first.action, 'add'); + assert.equal(first.key.emoji, '💤'); + + const attempts = outbox.recordDispatching(first); + // The write-ahead record is durable before the request could leave. + const journal = readFileSync(path, 'utf8'); + assert.match(journal, /"t":"dispatching"/); + // A fresh reader (another process) sees it as unknown, never as unsent. + const other = new DiscordAwarenessOutbox(path); + assert.equal(other.operations().find((op) => op.opId === first.opIds[0])!.attempts.length, 1); + assert.equal(other.operationStatus(other.operations().find((op) => op.opId === first.opIds[0])!), 'unknown'); + + outbox.recordOutcome(attempts, 'confirmed'); + assert.deepEqual(outbox.pendingDispatches('discord').map((d) => d.key.messageId), ['m2']); +})); + +test('startup derives nothing from ancestry: exact-target crash completion, everything else held', withJournal((path) => { + const outbox = new DiscordAwarenessOutbox(path); + const crashed = outbox.prepare({ + agentName: 'cairn', sourceBranch: 'main', targetBranch: 'rollback/cairn/1', refs: [ref('m1')], scope: 'all', + })!; + const elsewhere = outbox.prepare({ + agentName: 'cairn', sourceBranch: 'main', targetBranch: 'rollback/cairn/2', refs: [ref('m2')], scope: 'addressed', + })!; + const suppression = outbox.prepare({ + agentName: 'cairn', sourceBranch: 'main', targetBranch: 'rollback/cairn/1', refs: [ref('m3')], scope: 'all', + activationPolicy: 'explicit', suppressionIntervals: [{ fromId: 'i1', toId: 'i1' }], + })!; + const active = activeBatch(outbox, [ref('m4')]); + const [dispatch] = outbox.pendingDispatches('discord').filter((d) => d.key.messageId === 'm4'); + outbox.recordDispatching(dispatch); // the process dies with this request on the wire + + const restarted = new DiscordAwarenessOutbox(path); + const recovered = restarted.recoverAtStartup('rollback/cairn/1'); + assert.deepEqual(recovered.activated, [crashed.id]); + assert.deepEqual(recovered.held, [elsewhere.id]); + assert.equal(recovered.unknownAttempts, 1); + const status = new Map(restarted.batches().map((b) => [b.id, b.status])); + assert.equal(status.get(crashed.id), 'active'); + assert.equal(status.get(elsewhere.id), 'held'); + assert.equal(status.get(suppression.id), 'prepared', 'left for resumePreparedDiscordSuppressions'); + assert.equal(status.get(active.id), 'active'); + // The unanswered add is unknown, and as the newest op for its key it is + // retried (idempotent); the held batch's ref is not dispatched at all. + assert.deepEqual( + restarted.pendingDispatches('discord').map((d) => d.key.messageId).sort(), + ['m1', 'm4'], + ); + + // A later restart on a different branch changes nothing. + const again = new DiscordAwarenessOutbox(path).recoverAtStartup('main'); + assert.deepEqual(again, { activated: [], held: [suppression.id], unknownAttempts: 0 }); +})); + +test('cancel stops every further send of a batch, never removes, and says what may still land', withJournal((path) => { + const outbox = new DiscordAwarenessOutbox(path); + const batch = activeBatch(outbox, [ref('m1'), ref('m2'), ref('m3'), ref('m4')]); + const dispatches = outbox.pendingDispatches('discord'); + const byId = new Map(dispatches.map((d) => [d.key.messageId, d])); + outbox.recordOutcome(outbox.recordDispatching(byId.get('m1')!), 'confirmed'); + outbox.recordOutcome(outbox.recordDispatching(byId.get('m2')!), 'unknown', { error: 'did not respond' }); + const m3Attempts = outbox.recordDispatching(byId.get('m3')!); // on the wire right now + + const receipt = outbox.cancel(batch.id, 'operator'); + assert.deepEqual( + { cancelled: receipt.cancelled, inFlight: receipt.inFlight, unknown: receipt.unknown, confirmed: receipt.confirmed }, + { cancelled: 1, inFlight: 1, unknown: 1, confirmed: 1 }, + ); + // Nothing new is sent, no removal is queued, and the unknown add is not + // retried; the key with a request on the wire stays busy meanwhile. + assert.equal(outbox.pendingDispatches('discord').length, 0); + + // The in-flight request lands after the cancel: that is recorded as what + // happened, and it is still not followed by anything. + outbox.recordOutcome(m3Attempts, 'confirmed'); + const m3 = outbox.operations().find((op) => op.key.messageId === 'm3')!; + assert.equal(outbox.operationStatus(m3), 'confirmed'); + assert.equal(outbox.pendingDispatches('discord').length, 0); + + // Cancellation is durable: a restart or a branch move re-arms nothing. + const restarted = new DiscordAwarenessOutbox(path); + restarted.recoverAtStartup('some/other/branch'); + assert.ok(restarted.batches().find((b) => b.id === batch.id)?.cancelled); + assert.equal(restarted.pendingDispatches('discord').length, 0); + // Retract still reaches what may have landed: m1, m2 (unknown) and m3. + const retracted = restarted.retract(batch.id); + assert.equal(retracted.removalsQueued, 3); + assert.equal(retracted.keysWithUnresolvedAdds, 1); +})); + +test('cancelling a batch whose surgery has not activated it stops every mark, and its body can still resume', withJournal((path) => { + const outbox = new DiscordAwarenessOutbox(path); + const rollback = outbox.prepare({ + agentName: 'cairn', sourceBranch: 'main', targetBranch: 'rollback/cairn/1', refs: [ref('m1'), ref('m2')], scope: 'all', + })!; + const receipt = outbox.cancel(rollback.id, 'operator'); + assert.equal(receipt.cancelled, 2); + assert.throws(() => outbox.activate(rollback.id), /cancelled before activation/); + // Not activated by startup's crash completion, and not held either. + assert.deepEqual(outbox.recoverAtStartup('rollback/cairn/1'), { activated: [], held: [], unknownAttempts: 0 }); + assert.equal(outbox.pendingDispatches('discord').length, 0); + + const suppression = outbox.prepare({ + agentName: 'cairn', sourceBranch: 'main', targetBranch: 'suppress/cairn/1', refs: [ref('m3')], scope: 'all', + activationPolicy: 'explicit', suppressionIntervals: [{ fromId: 'i1', toId: 'i1' }], + })!; + outbox.cancel(suppression.id); + // The suppression's body still resumes on its branch; its marks never go. + assert.deepEqual(outbox.preparedSuppressionsForBranch('suppress/cairn/1').map((b) => b.id), [suppression.id]); + outbox.recordSuppressionComplete(suppression.id); + assert.deepEqual(outbox.preparedSuppressionsForBranch('suppress/cairn/1'), []); + assert.equal(outbox.pendingDispatches('discord').length, 0); +})); + +test('retract removes this bot\'s mark across batches, supersedes unsent adds, and discloses unresolved attempts', withJournal((path) => { + const outbox = new DiscordAwarenessOutbox(path); + const a = activeBatch(outbox, [ref('m1'), ref('m2'), ref('m3')]); + const b = activeBatch(outbox, [ref('m1')]); // another batch asked for the same reaction + // m1's adds share one dispatch (same key, same action, both requested). + const m1 = outbox.pendingDispatches('discord').find((d) => d.key.messageId === 'm1')!; + assert.equal(m1.opIds.length, 2); + const m1Attempts = outbox.recordDispatching(m1); + outbox.recordOutcome(m1Attempts, 'unknown', { error: 'did not respond within 75ms' }); + // m2's add never left the host; m3's was confirmed. + const m2 = outbox.pendingDispatches('discord').find((d) => d.key.messageId === 'm2')!; + outbox.recordOutcome(outbox.recordDispatching(m2), 'not-sent'); + const m3 = outbox.pendingDispatches('discord').find((d) => d.key.messageId === 'm3')!; + outbox.recordOutcome(outbox.recordDispatching(m3), 'confirmed'); + + const receipt = outbox.retract(a.id, 'operator'); + // m1 and m3 had adds leave the host; m2's never did, so nothing to remove. + assert.equal(receipt.removalsQueued, 2); + assert.equal(receipt.addsSuperseded, 1, "m2's unsent add is superseded"); + assert.equal(receipt.keysWithUnresolvedAdds, 1, 'm1: a timed-out add may land after the removal'); + assert.equal(receipt.unresolvedAddAttempts, 1, "one request carried both batches' adds"); + + const dispatches = outbox.pendingDispatches('discord'); + assert.deepEqual(dispatches.map((d) => `${d.action}:${d.key.messageId}`).sort(), ['remove:m1', 'remove:m3']); + + // The removal is confirmed, and the earlier unknown add stays unresolved: + // a confirmation of a different request never resolves it. + drain(outbox, () => 'confirmed'); + const views = new Map(outbox.view().map((v) => [v.id, v])); + assert.equal(views.get(a.id)!.adds.unknown, 1); + assert.equal(views.get(a.id)!.removals.confirmed, 2); + assert.equal(views.get(b.id)!.unresolvedAttempts, 1); +})); + +test('a retry confirming an unknown add never erases the unresolved first attempt', withJournal((path) => { + const outbox = new DiscordAwarenessOutbox(path); + const batch = activeBatch(outbox, [ref('m1')]); + const first = outbox.pendingDispatches('discord')[0]; + outbox.recordOutcome(outbox.recordDispatching(first), 'unknown'); + // Still the newest op for its key: retried (idempotent), as attempt 2. + const retry = outbox.pendingDispatches('discord')[0]; + assert.deepEqual(retry.opIds, first.opIds); + const retryAttempts = outbox.recordDispatching(retry); + assert.equal(retryAttempts[0].attempt, 2); + outbox.recordOutcome(retryAttempts, 'confirmed'); + assert.equal(outbox.pendingDispatches('discord').length, 0); + assert.equal(outbox.view()[0].unresolvedAttempts, 1, 'attempt 1 is still unknown'); + + const receipt = outbox.retract(batch.id); + assert.equal(receipt.keysWithUnresolvedAdds, 1); + assert.equal(receipt.unresolvedAddAttempts, 1); +})); + +test('a new request reaches Discord even when an earlier one got the same answer', withJournal((path) => { + const outbox = new DiscordAwarenessOutbox(path); + const first = activeBatch(outbox, [ref('m1')]); + drain(outbox, () => 'confirmed'); + outbox.retract(first.id); + drain(outbox, () => 'confirmed'); + // Someone else's tool may have reacted again since; a fresh, explicit + // placement is sent rather than skipped on the strength of old receipts. + activeBatch(outbox, [ref('m1')]); + assert.deepEqual(outbox.pendingDispatches('discord').map((d) => d.action), ['add']); +})); + +test('retract supersedes an add that never left the host instead of churning the reaction', withJournal((path) => { + const outbox = new DiscordAwarenessOutbox(path); + const batch = activeBatch(outbox, [ref('m2'), ref('m3')]); + // m3's add was sent and confirmed; m2's has not been sent yet. + const m3 = outbox.pendingDispatches('discord').find((d) => d.key.messageId === 'm3')!; + outbox.recordOutcome(outbox.recordDispatching(m3), 'confirmed'); + const receipt = outbox.retract(batch.id); + assert.equal(receipt.addsSuperseded, 1); + assert.equal(receipt.removalsQueued, 1, 'nothing was ever sent for m2, so there is nothing to remove'); + assert.deepEqual(outbox.pendingDispatches('discord').map((d) => `${d.action}:${d.key.messageId}`), ['remove:m3']); + const m2 = outbox.operations().find((op) => op.key.messageId === 'm2')!; + assert.equal(m2.cancelled?.reason, 'superseded'); +})); + +test('release queues a held batch\'s recorded operations, explicitly', withJournal((path) => { + const outbox = new DiscordAwarenessOutbox(path); + const batch = outbox.prepare({ + agentName: 'cairn', sourceBranch: 'main', targetBranch: 'rollback/cairn/9', refs: [ref('m1'), ref('m2')], scope: 'addressed', + })!; + outbox.recoverAtStartup('main'); + assert.equal(outbox.batches()[0].status, 'held'); + assert.equal(outbox.pendingDispatches('discord').length, 0); + const receipt = outbox.release(batch.id, 'operator'); + assert.deepEqual({ adds: receipt.addsQueued, removals: receipt.removalsQueued }, { adds: 2, removals: 0 }); + assert.equal(outbox.batches()[0].status, 'active'); + assert.equal(outbox.pendingDispatches('discord').length, 2); + assert.throws(() => outbox.release(batch.id), /not held/); +})); + +test('a torn or uncommitted append applies none of its records', withJournal((path) => { + const outbox = new DiscordAwarenessOutbox(path); + const batch = outbox.prepare({ + agentName: 'cairn', sourceBranch: 'main', targetBranch: 'rollback/cairn/1', refs: [ref('m1'), ref('m2')], scope: 'all', + })!; + outbox.activate(batch.id); + const committed = readFileSync(path, 'utf8'); + // Simulate an activation whose append was cut off before its commit. + const lines = committed.trimEnd().split('\n'); + const activation = lines.filter((line) => line.includes('"txn"')); + assert.ok(activation.length >= 3, 'activation is a transaction'); + writeFileSync(path, lines.filter((line) => !line.includes('"txn"') && !line.includes('"t":"commit"')).join('\n') + '\n'); + appendFileSync(path, activation.slice(0, -1).join('\n') + '\n' + '{"t":"requested","at":1,"op'); + const reread = new DiscordAwarenessOutbox(path); + assert.equal(reread.batches()[0].status, 'prepared', 'the activation never committed'); + assert.equal(reread.pendingDispatches('discord').length, 0); + assert.equal(reread.operations().length, 0); +})); + +test('an append after a torn tail repairs it instead of corrupting the journal', withJournal((path) => { + const outbox = new DiscordAwarenessOutbox(path); + const batch = activeBatch(outbox, [ref('m1'), ref('m2')]); + // A single-record append cut off mid-line (the process died writing it). + appendFileSync(path, '{"t":"dispatching","at":1,"opId":"x","att'); + const reread = new DiscordAwarenessOutbox(path); + assert.equal(reread.pendingDispatches('discord').length, 2, 'the torn record never applied'); + // The next append must not share the torn line. + const dispatch = reread.pendingDispatches('discord')[0]; + reread.recordOutcome(reread.recordDispatching(dispatch), 'confirmed'); + const again = new DiscordAwarenessOutbox(path); + assert.equal(again.pendingDispatches('discord').length, 1); + assert.ok(!readFileSync(path, 'utf8').includes('"att{'), 'the fragment was truncated away'); + + // A complete record that only lost its newline is kept, then terminated. + const lines = readFileSync(path, 'utf8'); + writeFileSync(path, lines.trimEnd()); + const kept = new DiscordAwarenessOutbox(path); + assert.equal(kept.pendingDispatches('discord').length, 1); + kept.cancel(batch.id); + assert.equal(new DiscordAwarenessOutbox(path).pendingDispatches('discord').length, 0); + assert.equal(new DiscordAwarenessOutbox(path).batches()[0].cancelled !== undefined, true); +})); + +test('v2 import keeps what the old ledger recorded, holds pending work, and happens once', withJournal((path, dir) => { + const legacyPath = join(dir, 'recovery', 'discord-awareness-outbox.json'); + const entry = (messageId: string, fields: Record) => ({ ...ref(messageId), attempts: 1, lastAction: 'add', ...fields }); + const legacy = { + version: 2, + batches: [ + { + // The household's shape: an active target-branch batch, some marks + // applied, the rest marked permanent-failure by hand after shutdown. + id: 'dd34b043', status: 'active', agentName: 'resident', sourceBranch: 'main', + targetBranch: 'rollback/resident/1791358473307', emoji: '💤', createdAt: 1791358473400, + activationPolicy: 'target-branch', + refs: [ + entry('a1', { desired: true, markerPresent: true, deliveryStatus: 'applied' }), + entry('a2', { desired: true, markerPresent: true, deliveryStatus: 'applied' }), + entry('p1', { desired: true, markerPresent: false, deliveryStatus: 'permanent-failure', lastError: 'Cannot send request: connection to "discord" is closed' }), + ], + }, + { + // An interrupted drain that nobody edited: its pending work is held. + id: 'pending-batch', status: 'active', agentName: 'resident', sourceBranch: 'main', + targetBranch: 'rollback/resident/2', emoji: '💤', createdAt: 1791358473500, + refs: [ + entry('q1', { desired: true, markerPresent: false, deliveryStatus: 'pending', lastError: 'Cannot send request' }), + entry('q2', { desired: true, markerPresent: true, deliveryStatus: 'applied' }), + ], + }, + ], + }; + mkdirSync(join(dir, 'recovery'), { recursive: true }); + writeFileSync(legacyPath, JSON.stringify(legacy)); + + const outbox = new DiscordAwarenessOutbox(path); + // Reading imports in memory only; nothing is dispatched from an import. + assert.equal(outbox.pendingDispatches('discord').length, 0); + const views = new Map(outbox.view().map((v) => [v.id, v])); + assert.equal(views.get('dd34b043')!.adds.confirmed, 2); + assert.equal(views.get('dd34b043')!.adds.failed, 1); + assert.equal(views.get('pending-batch')!.status, 'held'); + assert.equal(views.get('pending-batch')!.held!.releaseActions, 1); + + // The first write persists the import (with its hash) and retires the file. + const receipt = outbox.retract('dd34b043', 'operator'); + assert.equal(receipt.removalsQueued, 3, 'every add that left the host is removed; the import cannot tell which were sent'); + assert.equal(existsSync(legacyPath), false); + assert.equal(existsSync(`${legacyPath}.migrated-v2`), true); + + // A crash between the import append and the rename re-reads the same + // file: its hash is recorded, so nothing is imported or queued twice. + writeFileSync(legacyPath, JSON.stringify(legacy)); + const again = new DiscordAwarenessOutbox(path); + assert.equal(again.batches().filter((b) => b.id === 'dd34b043').length, 1); + assert.equal(again.pendingDispatches('discord').length, 3); + again.cancel('pending-batch'); + assert.equal(existsSync(legacyPath), false, 'the next write finishes the rename'); +})); + +test('framework drain classifies outcomes from what reached the server', async () => { + const dir = mkdtempSync(join(tmpdir(), 'discord-awareness-drain-')); try { - const outbox = new DiscordAwarenessOutbox(path); - const batch = outbox.prepare({ - agentName: 'cairn', - sourceBranch: 'main', - targetBranch: 'recovery/cairn/suppressed', - activationPolicy: 'explicit', - refs: [ - { serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'm1' }, - ], - })!; + const outbox = new DiscordAwarenessOutbox(join(dir, 'journal.jsonl')); + activeBatch(outbox, [ref('m1'), ref('m2'), ref('m3'), ref('m4'), ref('m5')]); + const calls: string[] = []; + let connected = true; + const connection = { + get isConnected() { return connected; }, + sendToolsCallWithDeadline: async (name: string, args: Record) => { + calls.push(`${name}:${args.messageId}`); + if (args.messageId === 'm2') return { isError: true, content: [{ type: 'text', text: 'Unknown Message' }] }; + if (args.messageId === 'm3') throw new McplRequestError('Cannot send request: connection to "discord" is closed', 'not-sent'); + if (args.messageId === 'm4') throw new McplRequestError('did not respond', 'no-response'); + return { content: [{ type: 'text', text: 'Reaction applied' }] }; + }, + }; + const framework = Object.create(AgentFramework.prototype) as any; + framework.discordAwarenessOutbox = outbox; + framework.discordAwarenessDrains = new Map(); + framework.discordAwarenessDeadlineMs = 1000; + framework.mcplServerRegistry = { getServer: () => connection }; + const originalError = console.error; + console.error = () => {}; + try { + await framework.drainDiscordAwarenessOutbox('discord'); + } finally { + console.error = originalError; + } + assert.deepEqual(calls, ['add_reaction:m1', 'add_reaction:m2', 'add_reaction:m3', 'add_reaction:m4', 'add_reaction:m5']); + const status = new Map(outbox.operations().map((op) => [op.key.messageId, outbox.operationStatus(op)])); + assert.equal(status.get('m1'), 'confirmed'); + assert.equal(status.get('m2'), 'failed', 'Discord said the message is gone: final'); + assert.equal(status.get('m3'), 'requested', 'never written: still queued'); + assert.equal(status.get('m4'), 'unknown', 'written, no answer: unknown'); + assert.equal(status.get('m5'), 'confirmed'); + + // A disconnected route leaves its work queued without writing anything. + connected = false; + const before = readFileSync(join(dir, 'journal.jsonl'), 'utf8'); + console.error = () => {}; + try { + await framework.drainDiscordAwarenessOutbox('discord'); + } finally { + console.error = originalError; + } + assert.equal(readFileSync(join(dir, 'journal.jsonl'), 'utf8'), before); - assert.equal(outbox.activatePreparedForBranch('recovery/cairn/suppressed'), 0); - assert.equal(outbox.pending('discord').length, 0); - outbox.activate(batch.id); - assert.equal(outbox.pending('discord').length, 1); + // Back online: the unsent and the unknown (newest for their keys) go again. + connected = true; + calls.length = 0; + await framework.drainDiscordAwarenessOutbox('discord'); + assert.deepEqual(calls.sort(), ['add_reaction:m3', 'add_reaction:m4']); } finally { rmSync(dir, { recursive: true, force: true }); } }); -test('framework resumes an interrupted suppression before activating its markers', async () => { +test('framework resumes an interrupted suppression before activating its marks', async () => { const dir = mkdtempSync(join(tmpdir(), 'discord-awareness-resume-')); - const path = join(dir, 'outbox.json'); try { - const outbox = new DiscordAwarenessOutbox(path); + const outbox = new DiscordAwarenessOutbox(join(dir, 'journal.jsonl')); const batch = outbox.prepare({ agentName: 'cairn', sourceBranch: 'main', targetBranch: 'recovery/cairn/suppressed', activationPolicy: 'explicit', - suppressionIntervals: [ - { fromId: 'i1', toId: 'i2' }, - { fromId: 'i3', toId: 'i3' }, - ], - refs: [ - { serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'm1' }, - ], + suppressionIntervals: [{ fromId: 'i1', toId: 'i2' }, { fromId: 'i3', toId: 'i3' }], + refs: [ref('m1')], + scope: 'all', })!; const present = new Set(['i1', 'i2']); // i3 interval committed before crash. const removals: string[] = []; @@ -127,157 +506,153 @@ test('framework resumes an interrupted suppression before activating its markers }; const framework = Object.create(AgentFramework.prototype) as any; framework.discordAwarenessOutbox = outbox; - framework.store = { - currentBranch: () => ({ name: 'recovery/cairn/suppressed' }), - listBranches: () => [], - }; + framework.store = { currentBranch: () => ({ name: 'recovery/cairn/suppressed' }) }; framework.agents = new Map([['cairn', { getContextManager: () => cm }]]); await framework.resumePreparedDiscordSuppressions(); assert.deepEqual(removals, ['i1..i2']); assert.equal(outbox.batches().find((candidate) => candidate.id === batch.id)?.status, 'active'); - assert.deepEqual(outbox.pending('discord').map((operation) => operation.ref.messageId), ['m1']); + assert.deepEqual(outbox.pendingDispatches('discord').map((d) => d.key.messageId), ['m1']); } finally { rmSync(dir, { recursive: true, force: true }); } }); -test('framework drain records failures per ref and continues past permanent failures', async () => { - const dir = mkdtempSync(join(tmpdir(), 'discord-awareness-drain-')); - const path = join(dir, 'outbox.json'); - try { - const outbox = new DiscordAwarenessOutbox(path); - const batch = outbox.prepare({ - agentName: 'cairn', - sourceBranch: 'main', - targetBranch: 'recovery/cairn/2', - refs: [ - { serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'm1' }, - { serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'm2' }, - { serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'm3' }, - { serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'm4' }, - ], - })!; - outbox.activate(batch.id); - - const calls: Array<{ name: string; args: Record }> = []; - let retryableFailure = true; - const connection = { - isConnected: true, - sendToolsCallWithDeadline: async (name: string, args: Record) => { - calls.push({ name, args }); - if (args.messageId === 'm2') { - return { isError: true, content: [{ type: 'text', text: 'Unknown Message' }] }; - } - if (args.messageId === 'm3' && retryableFailure) throw new Error('Discord unavailable'); - return { content: [{ type: 'text', text: 'Reaction added' }] }; - }, - }; - const framework = Object.create(AgentFramework.prototype) as any; - framework.discordAwarenessOutbox = outbox; - framework.discordAwarenessDrains = new Map(); - framework.mcplServerRegistry = { getServer: () => connection }; - - const originalError = console.error; - console.error = () => {}; - try { - await framework.drainDiscordAwarenessOutbox('discord'); - } finally { - console.error = originalError; - } - assert.deepEqual(calls[0], { - name: 'add_reaction', - args: { channelId: 'c1', messageId: 'm1', emoji: '💤' }, - }); - assert.deepEqual(outbox.pending('discord').map((operation) => operation.ref.messageId), ['m3']); - assert.deepEqual(calls.map((call) => call.args.messageId), ['m1', 'm2', 'm3', 'm4']); - const entries = outbox.batches()[0].refs; - assert.equal(entries.find((entry) => entry.messageId === 'm2')?.deliveryStatus, 'permanent-failure'); - assert.equal(entries.find((entry) => entry.messageId === 'm3')?.deliveryStatus, 'pending'); - assert.equal(entries.find((entry) => entry.messageId === 'm4')?.deliveryStatus, 'applied'); - - retryableFailure = false; - await framework.drainDiscordAwarenessOutbox('discord'); - assert.equal(outbox.pending('discord').length, 0); - - outbox.reconcileForBranch('main'); - calls.length = 0; - await framework.drainDiscordAwarenessOutbox('discord'); - assert.deepEqual(calls.map((call) => [call.name, call.args.messageId]), [ - ['remove_reaction', 'm1'], - ['remove_reaction', 'm3'], - ['remove_reaction', 'm4'], - ]); - assert.ok(outbox.batches()[0].refs.every((entry) => !entry.markerPresent)); - } finally { - rmSync(dir, { recursive: true, force: true }); - } -}); - -test('message-granular undo prepares markers before switching branches', async () => { +test('message-granular host/command undo places no marks unless the command asks', async () => { const dir = mkdtempSync(join(tmpdir(), 'discord-awareness-undo-')); - const path = join(dir, 'outbox.json'); try { const messages = [ - { id: 'i0', metadata: { serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'm0' } }, - { id: 'i1', metadata: { serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'm1' } }, - { id: 'i2', metadata: { serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'm2' } }, + { id: 'i0', metadata: { ...ref('m0') } }, + { id: 'i1', metadata: { ...ref('m1'), tags: ['chat:addressed'] } }, + { id: 'i2', metadata: { ...ref('m2'), tags: ['chat:ambient'] } }, + { id: 'i3', metadata: { ...ref('m3') } }, ]; - const calls: string[] = []; let currentBranch = 'main'; const contextManager = { getAllMessages: () => messages, - // Windowed reads — what the live rollback path uses so it never - // re-inflates every attachment on the branch. getMessageCount: () => messages.length, getMessageWindow: (offset: number, limit: number) => ({ messages: messages.slice(offset, offset + limit), startIndex: offset, totalCount: messages.length, }), - branchAt: (_id: string, name: string) => { calls.push(`branch:${name}`); return name; }, - switchBranch: async (name: string) => { calls.push(`switch:${name}`); currentBranch = name; }, + branchAt: (_id: string, name: string) => name, + switchBranch: async (name: string) => { currentBranch = name; }, }; const framework = Object.create(AgentFramework.prototype) as any; - framework.agents = new Map([['cairn', { - state: { status: 'idle' }, - getContextManager: () => contextManager, - }]]); - framework.store = { - currentBranch: () => ({ name: currentBranch }), - listBranches: () => [ - { id: 'b-main', name: 'main' }, - { id: 'b-undo', name: currentBranch, parentId: 'b-main' }, - ], - }; - framework.discordAwarenessOutbox = new DiscordAwarenessOutbox(path); + framework.agents = new Map([['cairn', { state: { status: 'idle' }, getContextManager: () => contextManager }]]); + framework.store = { currentBranch: () => ({ name: currentBranch }) }; + framework.discordAwarenessOutbox = new DiscordAwarenessOutbox(join(dir, 'journal.jsonl')); framework.discordAwarenessEmoji = '🫥'; framework.discordAwarenessDrains = new Map(); framework.mcplServerRegistry = null; framework.moduleRegistry = { getModule: () => null }; framework.lastVisiblePreview = async () => null; - // Live rollback reserves the store: it mints turn tokens for every agent - // and flushes deferred writers on release — give the stub those fields. framework.activeTurnTokens = new Map(); framework.nextTurnToken = 1; framework.deferredMessages = []; framework.pendingAssistantBlocks = new Map(); framework.operatorLog = new OperatorLog(undefined); + const originalError = console.error; + console.error = () => {}; + try { + const silent = await framework.handleHostCommand('discord', { command: 'undo', agentName: 'cairn', messages: 1 }); + assert.equal(silent.ok, true); + assert.deepEqual(silent.markers, { scope: 'none', unmarked: 1, notRemoved: 0, status: 'none', queued: 0 }); + assert.equal(framework.discordAwarenessOutbox.pendingDispatches('discord').length, 0); + + currentBranch = 'main'; + const marked = await framework.handleHostCommand('discord', { + command: 'undo', agentName: 'cairn', messages: 3, marks: 'addressed', + }); + assert.equal(marked.markers.status, 'queued'); + assert.equal(marked.markers.queued, 1); + assert.equal(marked.markers.unmarked, 2); + const dispatches = framework.discordAwarenessOutbox.pendingDispatches('discord'); + assert.deepEqual(dispatches.map((d: { key: { messageId: string; emoji: string } }) => `${d.key.messageId}${d.key.emoji}`), ['m1🫥']); + + // A mistyped scope is refused, never guessed. + currentBranch = 'main'; + const typo = await framework.handleHostCommand('discord', { + command: 'undo', agentName: 'cairn', messages: 1, marks: 'adressed', + }); + assert.equal(typo.ok, false); + assert.match(typo.error, /marks must be none, addressed or all/); - const result = await framework.handleHostCommand('discord', { - command: 'undo', agentName: 'cairn', messages: 2, - }); - assert.equal(result.ok, true); - assert.deepEqual(result.removedRefs.map((ref: { messageId: string }) => ref.messageId), ['m1', 'm2']); - assert.deepEqual(calls.map((call) => call.split('/')[0]), ['branch:undo-msgs', 'switch:undo-msgs']); - assert.deepEqual( - framework.discordAwarenessOutbox.pending('discord').map((operation: { ref: { messageId: string } }) => operation.ref.messageId), - ['m1', 'm2'], - ); - assert.ok(framework.discordAwarenessOutbox.pending('discord').every( - (operation: { emoji: string }) => operation.emoji === '🫥', - )); + // The journal's controls are reachable through host/command too. + const listed = await framework.handleHostCommand('discord', { command: 'marks', action: 'list' }); + assert.equal(listed.ok, true); + assert.equal(listed.awareness.length, 1); + const cancelled = await framework.handleHostCommand('discord', { + command: 'marks', action: 'cancel', batchId: listed.awareness[0].id, requesterName: 'op', + }); + assert.equal(cancelled.ok, true); + assert.equal(cancelled.awareness.cancelled, 1); + assert.equal(framework.discordAwarenessOutbox.pendingDispatches('discord').length, 0); + const missing = await framework.handleHostCommand('discord', { command: 'marks', action: 'retract' }); + assert.equal(missing.ok, false); + } finally { + console.error = originalError; + } + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test('host/command hide marks only on request, through the journal, routed by the issuing server for old records', async () => { + const dir = mkdtempSync(join(tmpdir(), 'discord-awareness-hide-')); + try { + const messages = [ + { id: 'i0', metadata: { ...ref('m0') } }, + { id: 'i1', metadata: { channelId: 'discord:g1:c1', messageId: 'm1', tags: ['chat:addressed'] } }, // pre-serverId record + { id: 'i2', metadata: { ...ref('m2'), tags: ['chat:ambient'] } }, + { id: 'i3', metadata: { ...ref('m3') } }, + ]; + const removed: string[] = []; + const contextManager = { + getAllMessages: () => messages.filter((m) => !removed.includes(m.id)), + removeMessage: (id: string) => { removed.push(id); }, + removeMessages: (from: string, to: string) => { + const ids = messages.map((m) => m.id); + for (const id of ids.slice(ids.indexOf(from), ids.indexOf(to) + 1)) removed.push(id); + }, + }; + const framework = Object.create(AgentFramework.prototype) as any; + framework.agents = new Map([['cairn', { state: { status: 'idle' }, getContextManager: () => contextManager }]]); + framework.store = { currentBranch: () => ({ name: 'main' }) }; + framework.discordAwarenessOutbox = new DiscordAwarenessOutbox(join(dir, 'journal.jsonl')); + framework.discordAwarenessEmoji = '💤'; + framework.discordAwarenessDrains = new Map(); + framework.mcplServerRegistry = null; + framework.lastVisiblePreview = async () => null; + framework.operatorLog = new OperatorLog(undefined); + const originalError = console.error; + console.error = () => {}; + try { + const silent = await framework.handleHostCommand('discord', { command: 'hide', agentName: 'cairn', fromMessageId: 'm3' }); + assert.equal(silent.ok, true); + assert.equal(silent.markers.status, 'none'); + assert.equal(silent.markers.unmarked, 1); + assert.equal(framework.discordAwarenessOutbox.batches().length, 0); + + const marked = await framework.handleHostCommand('discord', { + command: 'hide', agentName: 'cairn', fromMessageId: 'm1', toMessageId: 'm2', marks: 'addressed', + }); + assert.equal(marked.ok, true); + assert.equal(marked.hidden, 2); + assert.equal(marked.markers.status, 'queued'); + assert.equal(marked.markers.queued, 1); + assert.equal(marked.markers.unmarked, 1); + const [dispatch] = framework.discordAwarenessOutbox.pendingDispatches('discord'); + assert.deepEqual(dispatch.key, { serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'm1', emoji: '💤' }); + // Explicit and already applied: a restart on this branch changes nothing. + const restarted = new DiscordAwarenessOutbox(join(dir, 'journal.jsonl')); + assert.deepEqual(restarted.recoverAtStartup('main'), { activated: [], held: [], unknownAttempts: 0 }); + assert.equal(restarted.batches()[0].status, 'active'); + } finally { + console.error = originalError; + } } finally { rmSync(dir, { recursive: true, force: true }); } diff --git a/test/fixtures/awareness-barrier-mcpl-server.mjs b/test/fixtures/awareness-barrier-mcpl-server.mjs deleted file mode 100644 index a16b342b3..000000000 --- a/test/fixtures/awareness-barrier-mcpl-server.mjs +++ /dev/null @@ -1,276 +0,0 @@ -import { appendFileSync, existsSync, readFileSync, writeFileSync } from 'node:fs'; - -const statusPath = process.env.STATUS_PATH; -const ledgerPath = process.env.LEDGER_PATH; -const releasePath = process.env.RELEASE_PATH; -const crashPath = process.env.CRASH_PATH; -const generationPath = process.env.GENERATION_PATH; -const listChangePath = process.env.LIST_CHANGE_PATH; -const failReaction = process.env.FAIL_REACTION === '1'; -const permanentReaction = process.env.PERMANENT_REACTION === '1'; -const failInitialGeneration = process.env.FAIL_INITIAL_GENERATION === '1'; -const nestedListChangeGeneration = Number(process.env.NESTED_LIST_CHANGE_GENERATION ?? 0); -const dataMethod = process.env.DATA_METHOD ?? 'push/event'; -const serverId = process.env.SERVER_ID ?? 'discord'; -const arrivalPath = process.env.ARRIVAL_PATH; -const waitForArrivalPath = process.env.WAIT_FOR_ARRIVAL_PATH; - -let generation = 1; -if (generationPath) { - try { - generation = Number(readFileSync(generationPath, 'utf8')) + 1; - } catch { - // First process for this test. - } - writeFileSync(generationPath, String(generation)); -} - -const log = (event, extra = {}) => { - if (!statusPath) return; - appendFileSync(statusPath, JSON.stringify({ event, generation, serverId, ...extra }) + '\n'); -}; -const send = (message) => process.stdout.write(JSON.stringify(message) + '\n'); -const request = (id, method, params) => send({ jsonrpc: '2.0', id, method, params }); -const reply = (id, result) => send({ jsonrpc: '2.0', id, result }); -const channel = (suffix) => ({ - id: `${serverId}:guild:${suffix}`, - type: serverId, - label: suffix, - direction: 'bidirectional', -}); - -if (failInitialGeneration && generation === 1) { - log('initial-connect-failure'); - process.exit(9); -} - -let initialized = false; -let registered = false; -let registrationReceived = false; -let pendingToolRequest = null; -let reactionQueued = false; -let pendingControl = false; -let listChangeTriggered = false; -let reactionOrdinal = 0; -let buf = ''; - -function ledgerDeliveryStatus() { - if (!ledgerPath) return 'missing'; - try { - const document = JSON.parse(readFileSync(ledgerPath, 'utf8')); - return document.batches?.[0]?.refs?.[0]?.deliveryStatus ?? 'missing'; - } catch { - return 'unreadable'; - } -} - -function maybeReplyToReaction() { - if (!pendingToolRequest || pendingControl || failReaction) return; - if (releasePath) { - if (!existsSync(releasePath)) return; - const releaseValue = readFileSync(releasePath, 'utf8').trim(); - const releaseOrdinal = Number(releaseValue); - if (Number.isFinite(releaseOrdinal) && releaseOrdinal < pendingToolRequest.ordinal) return; - } - const requestId = pendingToolRequest.id; - pendingToolRequest = null; - log('reaction-response'); - reply(requestId, permanentReaction - ? { isError: true, content: [{ type: 'text', text: 'Unknown Message' }] } - : { content: [{ type: 'text', text: 'Reaction applied' }] }); -} - -function serviceReaction(message) { - if (waitForArrivalPath && !existsSync(waitForArrivalPath)) { - pendingToolRequest = message; - return; - } - if (!registered) { - pendingToolRequest = message; - if (!reactionQueued) { - reactionQueued = true; - log('reaction-queued-before-registration'); - } - return; - } - reactionOrdinal++; - pendingToolRequest = { ...message, ordinal: reactionOrdinal }; - pendingControl = true; - log('reaction-call', { name: message.params?.name, ordinal: reactionOrdinal }); - request(201, 'channels/register', { channels: [channel('control-during-barrier')] }); - if ( - nestedListChangeGeneration === generation - && reactionOrdinal === 1 - && !listChangeTriggered - ) { - listChangeTriggered = true; - log('tools-list-changed-notification'); - send({ jsonrpc: '2.0', method: 'notifications/tools/list_changed', params: {} }); - sendDataRequest(302, 'list-change'); - } -} - -function maybeFinishRegistration() { - if (!registrationReceived || registered) return; - if (pendingToolRequest && waitForArrivalPath && !existsSync(waitForArrivalPath)) return; - if (pendingToolRequest && !reactionQueued) serviceReaction(pendingToolRequest); - registered = true; - log('registration-response'); - const queued = pendingToolRequest; - pendingToolRequest = null; - if (queued) serviceReaction(queued); -} - -function sendDataRequest(id, phase) { - let params; - if (dataMethod === 'inference/request') { - params = { - featureSet: 'chat', - messages: [{ role: 'user', content: `must wait for ${phase} awareness` }], - }; - } else if (dataMethod === 'channels/incoming') { - params = { - messages: [{ - channelId: `${serverId}:guild:startup`, - messageId: `${serverId}-${phase}-incoming-${generation}`, - author: { id: 'human', name: 'Human' }, - timestamp: new Date().toISOString(), - content: [{ type: 'text', text: `must wait for ${phase} awareness` }], - }], - }; - } else if (dataMethod === 'host/command') { - params = { command: 'barrier-probe', agentName: 'assistant' }; - } else { - params = { - featureSet: 'chat', - eventId: `${serverId}-${phase}-push-${generation}`, - timestamp: new Date().toISOString(), - payload: { content: [{ type: 'text', text: `must wait for ${phase} awareness` }] }, - }; - } - log('data-request-sent', { method: dataMethod, phase }); - if (arrivalPath) writeFileSync(arrivalPath, `${serverId}:${dataMethod}:${phase}`); - request(id, dataMethod, params); -} - -function beginServerTraffic() { - if (initialized) return; - initialized = true; - log('initialized'); - request(200, 'channels/register', { channels: [channel('startup')] }); - // The reconnect generation carries an inference-bearing event in the same - // registration window. Initial-start tests use generation 1 as well. - sendDataRequest(202, 'startup'); -} - -function handle(message) { - if (message.method === 'initialize') { - reply(message.id, { - capabilities: { - experimental: { - mcpl: { - version: '0.5', - // §5.4/§6.2: the grant is computed from this advertisement, and - // admission denies what it cannot contain. This fixture SENDS - // channels/register, channels/incoming and inference/request, so - // it must advertise them — with only pushEvents declared, the - // 0.5 host correctly rejected the released data requests - // (accepted:false), which is exactly what PR #79's ubuntu CI - // caught on the inference/request + channels/incoming matrix. - pushEvents: true, - inferenceRequest: true, - channels: { register: true, incoming: true }, - featureSets: { - chat: { description: 'chat', uses: ['pushEvents'] }, - }, - }, - }, - }, - }); - return; - } - if (message.method === 'featureSets/update') { - // MCPL 0.5 (§5.3/§6.7): the host sends initial policy as a Request and - // activates the grant only on this degradation receipt. Without it the - // host times out (15s), the grant stays empty, and every push/channel - // in these tests is rejected fail-closed. - if (message.id !== undefined && message.id !== null) { - reply(message.id, { accepted: true }); - } - return; - } - if (message.method === 'notifications/initialized') { - beginServerTraffic(); - return; - } - if (message.method === 'tools/list') { - log('tools-list'); - reply(message.id, { - tools: [ - { name: 'add_reaction', description: 'add', inputSchema: { type: 'object' } }, - { name: 'remove_reaction', description: 'remove', inputSchema: { type: 'object' } }, - ], - }); - return; - } - if (message.method === 'tools/call') { - serviceReaction(message); - return; - } - if (message.id === 200) { - registrationReceived = true; - maybeFinishRegistration(); - return; - } - if (message.id === 201) { - pendingControl = false; - log('control-response-during-barrier'); - maybeReplyToReaction(); - return; - } - if (message.id === 202 || message.id === 302) { - const event = message.id === 202 ? 'push-response' : 'list-change-push-response'; - const accepted = message.result?.accepted - ?? message.result?.results?.[0]?.accepted - ?? (message.error == null); - log(event, { - method: dataMethod, - accepted, - ledgerStatus: ledgerDeliveryStatus(), - }); - } -} - -process.stdin.on('data', (chunk) => { - buf += chunk.toString('utf8'); - let newline; - while ((newline = buf.indexOf('\n')) >= 0) { - const line = buf.slice(0, newline); - buf = buf.slice(newline + 1); - if (!line.trim()) continue; - try { - handle(JSON.parse(line)); - } catch (error) { - log('server-error', { message: error instanceof Error ? error.message : String(error) }); - } - } -}); - -const timer = setInterval(() => { - if (pendingToolRequest && !registered && waitForArrivalPath && existsSync(waitForArrivalPath)) { - serviceReaction(pendingToolRequest); - maybeFinishRegistration(); - } - maybeReplyToReaction(); - if (listChangePath && existsSync(listChangePath) && !listChangeTriggered) { - listChangeTriggered = true; - log('tools-list-changed-notification'); - send({ jsonrpc: '2.0', method: 'notifications/tools/list_changed', params: {} }); - sendDataRequest(302, 'list-change'); - } - if (crashPath && existsSync(crashPath) && generation === 1) { - log('crashing'); - process.exit(7); - } -}, 5); -timer.unref(); diff --git a/test/mcpl-awareness-barrier.test.ts b/test/mcpl-awareness-barrier.test.ts deleted file mode 100644 index 9cb27c5d3..000000000 --- a/test/mcpl-awareness-barrier.test.ts +++ /dev/null @@ -1,798 +0,0 @@ -import { test } from 'node:test'; -import assert from 'node:assert/strict'; -import { fileURLToPath } from 'node:url'; -import { - mkdirSync, - mkdtempSync, - readFileSync, - rmSync, - writeFileSync, -} from 'node:fs'; -import { join } from 'node:path'; - -import { AgentFramework } from '../src/framework.js'; -import { McplServerConnection } from '../src/mcpl/server-connection.js'; -import { CapabilityGrant } from '../src/mcpl/capability-grant.js'; -import type { McplServerConfig } from '../src/mcpl/types.js'; -import { DiscordAwarenessOutbox } from '../src/recovery/discord-awareness-outbox.js'; -import { MockMembrane } from './helpers/mock-membrane.js'; - -const FIXTURE = fileURLToPath(new URL('./fixtures/awareness-barrier-mcpl-server.mjs', import.meta.url)); -const TEST_ROOT = join(process.cwd(), '.test-tmp'); - -interface StatusRecord { - event: string; - generation: number; - serverId?: string; - accepted?: boolean; - ledgerStatus?: string; - method?: string; - ordinal?: number; -} - -function records(path: string): StatusRecord[] { - try { - return readFileSync(path, 'utf8') - .split('\n') - .filter(Boolean) - .map((line) => JSON.parse(line) as StatusRecord); - } catch { - return []; - } -} - -async function waitFor( - description: string, - predicate: () => boolean, - timeoutMs = 3_000, -): Promise { - const deadline = Date.now() + timeoutMs; - while (Date.now() < deadline) { - if (predicate()) return; - await new Promise((resolve) => setTimeout(resolve, 10)); - } - assert.fail(`timed out waiting for ${description}`); -} - -function preparePending(outboxPath: string): DiscordAwarenessOutbox { - const outbox = new DiscordAwarenessOutbox(outboxPath); - const batch = outbox.prepare({ - agentName: 'assistant', - sourceBranch: 'source', - targetBranch: 'main', - activationPolicy: 'explicit', - refs: [{ serverId: 'discord', channelId: 'discord:guild:startup', messageId: 'message-1' }], - }); - assert.ok(batch); - outbox.activate(batch.id); - return outbox; -} - -function frameworkConfig( - dir: string, - extraEnv: Record = {}, - requestTimeoutMs = 1_000, - serverOverrides: Partial = {}, - awarenessDeadlineMs = 1_000, -) { - const outboxPath = join(dir, 'awareness.json'); - return { - outboxPath, - config: { - storePath: join(dir, 'store'), - discordAwarenessOutboxPath: outboxPath, - discordAwarenessDeadlineMs: awarenessDeadlineMs, - membrane: new MockMembrane().asMembrane(), - agents: [{ name: 'assistant', model: 'test', systemPrompt: 'test' }], - modules: [], - mcplServers: [{ - id: 'discord', - command: process.execPath, - args: [FIXTURE], - requestTimeoutMs, - env: extraEnv, - enabledFeatureSets: ['chat'], - ...serverOverrides, - }], - }, - }; -} - -function twoServerFrameworkConfig( - dir: string, - order: 'heartbeat-first' | 'discord-first', - dataMethod: typeof DATA_METHODS[number], - runtime = false, -) { - const outboxPath = join(dir, 'awareness.json'); - const statusPath = join(dir, 'status.jsonl'); - const releasePath = join(dir, 'release'); - const listChangePath = join(dir, 'list-change'); - const arrivalPath = join(dir, 'heartbeat-arrival'); - const server = (id: 'heartbeat' | 'discord'): McplServerConfig => ({ - id, - command: process.execPath, - args: [FIXTURE], - // The host/command matrix leg models the legitimate slash-command relay: - // host-owned authority is the operator's config decision (PR #79 - // blocker 9), so the test config makes it, exactly as the fleet's - // discord config will at rollout. - allowHostCommands: true, - requestTimeoutMs: 0, - env: { - STATUS_PATH: statusPath, - LEDGER_PATH: outboxPath, - SERVER_ID: id, - DATA_METHOD: dataMethod, - ...(id === 'discord' - ? { RELEASE_PATH: releasePath, WAIT_FOR_ARRIVAL_PATH: arrivalPath } - : { ARRIVAL_PATH: arrivalPath }), - ...(runtime && id === 'heartbeat' ? { LIST_CHANGE_PATH: listChangePath } : {}), - }, - enabledFeatureSets: ['chat'], - }); - const servers = order === 'heartbeat-first' - ? [server('heartbeat'), server('discord')] - : [server('discord'), server('heartbeat')]; - return { - outboxPath, - statusPath, - releasePath, - listChangePath, - config: { - storePath: join(dir, 'store'), - discordAwarenessOutboxPath: outboxPath, - discordAwarenessDeadlineMs: 1_000, - membrane: new MockMembrane().asMembrane(), - agents: [{ name: 'assistant', model: 'test', systemPrompt: 'test' }], - modules: [], - mcplServers: servers, - }, - }; -} - -const DATA_METHODS = [ - 'push/event', - 'inference/request', - 'channels/incoming', - 'host/command', -] as const; - -for (const order of ['heartbeat-first', 'discord-first'] as const) { - for (const dataMethod of DATA_METHODS) test( - `startup globally gates ${dataMethod} with ${order} config`, - async () => { - mkdirSync(TEST_ROOT, { recursive: true }); - const dir = mkdtempSync(join(TEST_ROOT, 'awareness-startup-')); - const { outboxPath, statusPath, releasePath, config } = twoServerFrameworkConfig( - dir, - order, - dataMethod, - ); - const outbox = preparePending(outboxPath); - let framework: AgentFramework | undefined; - let createSettled = false; - const creating = AgentFramework.create(config).then((created) => { - framework = created; - createSettled = true; - return created; - }); - - try { - await waitFor('registration and a second control response', () => { - const events = records(statusPath) - .filter((record) => record.serverId === 'discord') - .map((record) => record.event); - return events.includes('registration-response') - && events.includes('control-response-during-barrier'); - }); - - const discordOrdering = records(statusPath) - .filter((record) => record.serverId === 'discord') - .map((record) => record.event) - .filter((event) => [ - 'reaction-queued-before-registration', - 'registration-response', - 'reaction-call', - ].includes(event)); - assert.deepEqual(discordOrdering, [ - 'reaction-queued-before-registration', - 'registration-response', - 'reaction-call', - ]); - const allStatus = records(statusPath); - const heartbeatArrival = allStatus.findIndex((record) => - record.serverId === 'heartbeat' && record.event === 'data-request-sent'); - const discordDrain = allStatus.findIndex((record) => - record.serverId === 'discord' && record.event === 'reaction-queued-before-registration'); - assert.ok( - heartbeatArrival >= 0 && heartbeatArrival < discordDrain, - 'non-Discord data arrives before Discord registration/drain starts', - ); - - assert.equal(createSettled, false, 'framework readiness must wait for the marker outcome'); - assert.equal( - records(statusPath).some((record) => - record.serverId === 'heartbeat' && record.event === 'push-response'), - false, - `non-Discord ${dataMethod} must remain held while the Discord marker is pending`, - ); - assert.equal(outbox.pending('discord').length, 1); - - writeFileSync(releasePath, 'release'); - await creating; - await waitFor('gated push response', () => - records(statusPath).some((record) => - record.serverId === 'heartbeat' && record.event === 'push-response')); - - const status = records(statusPath); - const push = status.find((record) => - record.serverId === 'heartbeat' && record.event === 'push-response'); - assert.equal(push?.method, dataMethod); - assert.equal(push?.accepted, true); - assert.equal(push?.ledgerStatus, 'applied', 'push released only after durable success'); - assert.equal(outbox.pending('discord').length, 0); - assert.equal(status.filter((record) => - record.serverId === 'discord' && record.event === 'reaction-call').length, 1); - } finally { - await framework?.stop(); - rmSync(dir, { recursive: true, force: true }); - } - }, - ); -} - -test('mandatory awareness deadline survives requestTimeoutMs: 0 and durably releases startup', async () => { - mkdirSync(TEST_ROOT, { recursive: true }); - const dir = mkdtempSync(join(TEST_ROOT, 'awareness-timeout-')); - const statusPath = join(dir, 'status.jsonl'); - const { outboxPath, config } = frameworkConfig(dir, { - STATUS_PATH: statusPath, - LEDGER_PATH: join(dir, 'awareness.json'), - FAIL_REACTION: '1', - }, 0, {}, 60); - const outbox = preparePending(outboxPath); - let framework: AgentFramework | undefined; - - try { - framework = await AgentFramework.create(config); - await waitFor('push response after timeout accounting', () => - records(statusPath).some((record) => record.event === 'push-response')); - - const entry = outbox.batches()[0]!.refs[0]!; - assert.equal(entry.deliveryStatus, 'pending'); - assert.equal(entry.attempts, 1); - assert.match(entry.lastError ?? '', /did not respond to tools\/call/); - assert.match(entry.lastError ?? '', /outcome is unknown/i); - assert.equal(records(statusPath).filter((record) => record.event === 'reaction-call').length, 1); - const push = records(statusPath).find((record) => record.event === 'push-response'); - assert.equal(push?.ledgerStatus, 'pending', 'failure was durable before inference release'); - } finally { - await framework?.stop(); - rmSync(dir, { recursive: true, force: true }); - } -}); - -test('permanent marker failure is durably accounted before startup data is released', async () => { - mkdirSync(TEST_ROOT, { recursive: true }); - const dir = mkdtempSync(join(TEST_ROOT, 'awareness-permanent-')); - const statusPath = join(dir, 'status.jsonl'); - const { outboxPath, config } = frameworkConfig(dir, { - STATUS_PATH: statusPath, - LEDGER_PATH: join(dir, 'awareness.json'), - PERMANENT_REACTION: '1', - }); - const outbox = preparePending(outboxPath); - let framework: AgentFramework | undefined; - - try { - framework = await AgentFramework.create(config); - await waitFor('data response after permanent failure accounting', () => - records(statusPath).some((record) => record.event === 'push-response')); - - const entry = outbox.batches()[0]!.refs[0]!; - assert.equal(entry.deliveryStatus, 'permanent-failure'); - assert.equal(entry.attempts, 1); - assert.match(entry.lastError ?? '', /Unknown Message/); - const response = records(statusPath).find((record) => record.event === 'push-response'); - assert.equal(response?.ledgerStatus, 'permanent-failure'); - } finally { - await framework?.stop(); - rmSync(dir, { recursive: true, force: true }); - } -}); - -test('initial reconnect stub durably records unavailable work, then retries behind a fresh gate', async () => { - mkdirSync(TEST_ROOT, { recursive: true }); - const dir = mkdtempSync(join(TEST_ROOT, 'awareness-initial-unavailable-')); - const statusPath = join(dir, 'status.jsonl'); - const releasePath = join(dir, 'release'); - const generationPath = join(dir, 'generation'); - const { outboxPath, config } = frameworkConfig(dir, { - STATUS_PATH: statusPath, - LEDGER_PATH: join(dir, 'awareness.json'), - RELEASE_PATH: releasePath, - GENERATION_PATH: generationPath, - FAIL_INITIAL_GENERATION: '1', - }, 1_000, { - reconnect: true, - reconnectIntervalMs: 100, - reconnectMaxIntervalMs: 100, - }); - const outbox = preparePending(outboxPath); - let framework: AgentFramework | undefined; - - try { - framework = await AgentFramework.create(config); - const deferred = outbox.batches()[0]!.refs[0]!; - assert.equal(deferred.deliveryStatus, 'pending'); - assert.equal(deferred.attempts, 1, 'ready state must include a durable unavailable attempt'); - assert.match(deferred.lastError ?? '', /not attempted.*connection unavailable/i); - - await waitFor('successful reconnect registration and awareness call', () => { - const second = records(statusPath).filter((record) => record.generation === 2); - return second.some((record) => record.event === 'registration-response') - && second.some((record) => record.event === 'reaction-call'); - }); - assert.equal( - records(statusPath).some((record) => - record.generation === 2 && record.event === 'push-response'), - false, - 'reconnect data remains behind the newly installed awareness gate', - ); - - writeFileSync(releasePath, 'release'); - await waitFor('reconnected data after durable marker success', () => - records(statusPath).some((record) => - record.generation === 2 && record.event === 'push-response')); - assert.equal(outbox.pending('discord').length, 0); - assert.equal(outbox.batches()[0]!.refs[0]!.attempts, 2); - } finally { - await framework?.stop(); - rmSync(dir, { recursive: true, force: true }); - } -}); - -test('tools/list_changed installs the data gate before a following push', async () => { - mkdirSync(TEST_ROOT, { recursive: true }); - const dir = mkdtempSync(join(TEST_ROOT, 'awareness-list-change-')); - const statusPath = join(dir, 'status.jsonl'); - const releasePath = join(dir, 'release'); - const listChangePath = join(dir, 'list-change'); - const { outboxPath, config } = frameworkConfig(dir, { - STATUS_PATH: statusPath, - LEDGER_PATH: join(dir, 'awareness.json'), - RELEASE_PATH: releasePath, - LIST_CHANGE_PATH: listChangePath, - }); - const outbox = new DiscordAwarenessOutbox(outboxPath); - let framework: AgentFramework | undefined; - - try { - framework = await AgentFramework.create(config); - await waitFor('initial zero-pending push response', () => - records(statusPath).some((record) => record.event === 'push-response')); - - const batch = outbox.prepare({ - agentName: 'assistant', - sourceBranch: 'source', - targetBranch: 'main', - activationPolicy: 'explicit', - refs: [{ serverId: 'discord', channelId: 'discord:guild:startup', messageId: 'message-list' }], - }); - assert.ok(batch); - outbox.activate(batch.id); - writeFileSync(listChangePath, 'change'); - - await waitFor('control response while list-change barrier is pending', () => { - const status = records(statusPath); - return status.some((record) => record.event === 'tools-list-changed-notification') - && status.some((record) => record.event === 'control-response-during-barrier'); - }); - assert.equal( - records(statusPath).some((record) => record.event === 'list-change-push-response'), - false, - ); - - writeFileSync(releasePath, 'release'); - await waitFor('list-change gated push response', () => - records(statusPath).some((record) => record.event === 'list-change-push-response')); - const push = records(statusPath).find((record) => record.event === 'list-change-push-response'); - assert.equal(push?.ledgerStatus, 'applied'); - assert.equal(outbox.pending('discord').length, 0); - } finally { - await framework?.stop(); - rmSync(dir, { recursive: true, force: true }); - } -}); - -for (const dataMethod of DATA_METHODS) test( - `runtime awareness generation globally gates non-Discord ${dataMethod}`, - async () => { - mkdirSync(TEST_ROOT, { recursive: true }); - const dir = mkdtempSync(join(TEST_ROOT, 'awareness-runtime-global-')); - const { - outboxPath, - statusPath, - releasePath, - listChangePath, - config, - } = twoServerFrameworkConfig(dir, 'discord-first', dataMethod, true); - const outbox = new DiscordAwarenessOutbox(outboxPath); - let framework: AgentFramework | undefined; - - try { - framework = await AgentFramework.create(config); - await waitFor('both initial data responses', () => { - const initial = records(statusPath).filter((record) => record.event === 'push-response'); - return initial.some((record) => record.serverId === 'discord') - && initial.some((record) => record.serverId === 'heartbeat'); - }); - - const batch = outbox.prepare({ - agentName: 'assistant', - sourceBranch: 'source', - targetBranch: 'main', - activationPolicy: 'explicit', - refs: [{ - serverId: 'discord', - channelId: 'discord:guild:startup', - messageId: `runtime-global-${dataMethod}`, - }], - }); - assert.ok(batch); - outbox.activate(batch.id); - writeFileSync(listChangePath, 'change'); - - await waitFor('Discord marker call for heartbeat generation', () => - records(statusPath).some((record) => - record.serverId === 'discord' && record.event === 'reaction-call')); - assert.equal( - records(statusPath).some((record) => - record.serverId === 'heartbeat' && record.event === 'list-change-push-response'), - false, - `fresh heartbeat ${dataMethod} must wait for Discord accounting`, - ); - - writeFileSync(releasePath, 'release'); - await waitFor('globally gated runtime response', () => - records(statusPath).some((record) => - record.serverId === 'heartbeat' && record.event === 'list-change-push-response')); - const response = records(statusPath).find((record) => - record.serverId === 'heartbeat' && record.event === 'list-change-push-response'); - assert.equal(response?.method, dataMethod); - assert.equal(response?.ledgerStatus, 'applied'); - assert.equal(outbox.pending('discord').length, 0); - } finally { - await framework?.stop(); - rmSync(dir, { recursive: true, force: true }); - } - }, -); - -test('reconnect pauses fresh inbound data but permits registration and awareness tool service', async () => { - mkdirSync(TEST_ROOT, { recursive: true }); - const dir = mkdtempSync(join(TEST_ROOT, 'awareness-reconnect-')); - const statusPath = join(dir, 'status.jsonl'); - const releasePath = join(dir, 'release'); - const crashPath = join(dir, 'crash'); - const generationPath = join(dir, 'generation'); - const { outboxPath, config } = frameworkConfig(dir, { - STATUS_PATH: statusPath, - LEDGER_PATH: join(dir, 'awareness.json'), - RELEASE_PATH: releasePath, - CRASH_PATH: crashPath, - GENERATION_PATH: generationPath, - }, 1_000, { - reconnect: true, - reconnectIntervalMs: 20, - reconnectMaxIntervalMs: 20, - }); - const outbox = new DiscordAwarenessOutbox(outboxPath); - let framework: AgentFramework | undefined; - - try { - // No startup marker: this path must release push responders synchronously. - framework = await AgentFramework.create(config); - await waitFor('initial synchronous push response', () => - records(statusPath).some((record) => record.generation === 1 && record.event === 'push-response')); - - const batch = outbox.prepare({ - agentName: 'assistant', - sourceBranch: 'source', - targetBranch: 'main', - activationPolicy: 'explicit', - refs: [{ serverId: 'discord', channelId: 'discord:guild:startup', messageId: 'message-2' }], - }); - assert.ok(batch); - outbox.activate(batch.id); - writeFileSync(crashPath, 'crash'); - - await waitFor('reconnect control plane during pending barrier', () => { - const second = records(statusPath).filter((record) => record.generation === 2); - return second.some((record) => record.event === 'registration-response') - && second.some((record) => record.event === 'control-response-during-barrier'); - }); - assert.equal( - records(statusPath).some((record) => record.generation === 2 && record.event === 'push-response'), - false, - 'fresh reconnect data must not pass the new gate', - ); - - writeFileSync(releasePath, 'release'); - await waitFor('reconnect gated push response', () => - records(statusPath).some((record) => record.generation === 2 && record.event === 'push-response')); - const push = records(statusPath).find( - (record) => record.generation === 2 && record.event === 'push-response', - ); - assert.equal(push?.ledgerStatus, 'applied'); - assert.equal(outbox.pending('discord').length, 0); - } finally { - await framework?.stop(); - rmSync(dir, { recursive: true, force: true }); - } -}); - -test('startup fails explicitly when awareness reconciliation throws', async () => { - mkdirSync(TEST_ROOT, { recursive: true }); - const dir = mkdtempSync(join(TEST_ROOT, 'awareness-reconcile-failure-')); - const statusPath = join(dir, 'status.jsonl'); - const { outboxPath, config } = frameworkConfig(dir, { - STATUS_PATH: statusPath, - LEDGER_PATH: join(dir, 'awareness.json'), - }); - preparePending(outboxPath); - - const prototype = DiscordAwarenessOutbox.prototype as any; - const original = prototype.reconcileForBranch; - let calls = 0; - prototype.reconcileForBranch = function (...args: unknown[]) { - calls++; - if (calls === 2) throw new Error('injected reconcile failure'); - return original.apply(this, args); - }; - try { - await assert.rejects( - AgentFramework.create(config), - /awareness accounting failed.*injected reconcile failure/i, - ); - assert.equal( - records(statusPath).some((record) => record.event === 'push-response'), - false, - 'failed startup never reports a released data plane', - ); - } finally { - prototype.reconcileForBranch = original; - rmSync(dir, { recursive: true, force: true }); - } -}); - -test('startup fails explicitly when the awareness ledger read throws', async () => { - mkdirSync(TEST_ROOT, { recursive: true }); - const dir = mkdtempSync(join(TEST_ROOT, 'awareness-read-failure-')); - const { outboxPath, config } = frameworkConfig(dir); - preparePending(outboxPath); - - const prototype = DiscordAwarenessOutbox.prototype as any; - const original = prototype.pending; - prototype.pending = () => { throw new Error('injected read failure'); }; - try { - await assert.rejects( - AgentFramework.create(config), - /awareness accounting failed.*injected read failure/i, - ); - } finally { - prototype.pending = original; - rmSync(dir, { recursive: true, force: true }); - } -}); - -for (const writeMethod of ['recordSuccess', 'recordFailure'] as const) test( - `startup fails explicitly when ${writeMethod} cannot persist`, - async () => { - mkdirSync(TEST_ROOT, { recursive: true }); - const dir = mkdtempSync(join(TEST_ROOT, `awareness-${writeMethod}-failure-`)); - const statusPath = join(dir, 'status.jsonl'); - const { outboxPath, config } = frameworkConfig(dir, { - STATUS_PATH: statusPath, - LEDGER_PATH: join(dir, 'awareness.json'), - ...(writeMethod === 'recordFailure' ? { PERMANENT_REACTION: '1' } : {}), - }); - preparePending(outboxPath); - - const prototype = DiscordAwarenessOutbox.prototype as any; - const original = prototype[writeMethod]; - prototype[writeMethod] = () => { throw new Error(`injected ${writeMethod} write failure`); }; - try { - await assert.rejects( - AgentFramework.create(config), - new RegExp(`awareness accounting failed.*injected ${writeMethod} write failure`, 'i'), - ); - assert.equal( - records(statusPath).some((record) => record.event === 'push-response'), - false, - ); - } finally { - prototype[writeMethod] = original; - rmSync(dir, { recursive: true, force: true }); - } - }, -); - -test('runtime reconciliation failure recycles the connection and keeps fresh data gated', async () => { - mkdirSync(TEST_ROOT, { recursive: true }); - const dir = mkdtempSync(join(TEST_ROOT, 'awareness-runtime-accounting-')); - const statusPath = join(dir, 'status.jsonl'); - const releasePath = join(dir, 'release'); - const listChangePath = join(dir, 'list-change'); - const generationPath = join(dir, 'generation'); - const { outboxPath, config } = frameworkConfig(dir, { - STATUS_PATH: statusPath, - LEDGER_PATH: join(dir, 'awareness.json'), - RELEASE_PATH: releasePath, - LIST_CHANGE_PATH: listChangePath, - GENERATION_PATH: generationPath, - }, 1_000, { - reconnect: true, - reconnectIntervalMs: 20, - reconnectMaxIntervalMs: 20, - }); - const outbox = new DiscordAwarenessOutbox(outboxPath); - let framework: AgentFramework | undefined; - - try { - framework = await AgentFramework.create(config); - await waitFor('initial response', () => - records(statusPath).some((record) => record.event === 'push-response')); - const batch = outbox.prepare({ - agentName: 'assistant', - sourceBranch: 'source', - targetBranch: 'main', - activationPolicy: 'explicit', - refs: [{ serverId: 'discord', channelId: 'discord:guild:startup', messageId: 'runtime-fail' }], - })!; - outbox.activate(batch.id); - - const prototype = DiscordAwarenessOutbox.prototype as any; - const original = prototype.reconcileForBranch; - let injected = false; - prototype.reconcileForBranch = function (...args: unknown[]) { - if (!injected) { - injected = true; - throw new Error('injected runtime reconcile failure'); - } - return original.apply(this, args); - }; - try { - writeFileSync(listChangePath, 'change'); - await waitFor('replacement connection awareness call', () => - records(statusPath).some((record) => - record.generation === 2 && record.event === 'reaction-call')); - } finally { - prototype.reconcileForBranch = original; - } - - assert.equal( - records(statusPath).some((record) => - record.generation === 2 && record.event === 'push-response'), - false, - ); - writeFileSync(releasePath, 'release'); - await waitFor('replacement connection data release', () => - records(statusPath).some((record) => - record.generation === 2 && record.event === 'push-response')); - assert.equal(outbox.pending('discord').length, 0); - } finally { - await framework?.stop(); - rmSync(dir, { recursive: true, force: true }); - } -}); - -function deferred() { - let resolve!: (value: T) => void; - let reject!: (error: unknown) => void; - const promise = new Promise((res, rej) => { resolve = res; reject = rej; }); - return { promise, resolve, reject }; -} - -test('a control flush that installs a newer generation cannot be released by the old barrier', async () => { - const oldDrain = deferred(); - const newDrain = deferred(); - const framework = Object.create(AgentFramework.prototype) as any; - framework.discordAwarenessBarrier = null; - framework.discordAwarenessBarrierGeneration = 0; - const drains = [oldDrain.promise, newDrain.promise]; - framework.beginDiscordAwarenessBarrier = () => ({ - requiresBarrier: true, - promise: drains.shift(), - }); - - let readyCalls = 0; - let newer: any; - const connection = { - id: 'discord', - pauseDataPlane: () => {}, - ready: () => { readyCalls++; }, - readyControlPlane: () => { - if (newer) return; - newer = framework.installMcplDataPlaneGate(); - framework.releaseMcplDataPlaneGate(newer); - }, - }; - framework.mcplServerRegistry = { getAllServers: () => [connection] }; - - const older = framework.installMcplDataPlaneGate(); - framework.releaseMcplDataPlaneGate(older); - assert.equal(newer.generation, older.generation + 1); - - oldDrain.resolve({ status: 'delivered', delivered: 1, failed: 0 }); - await older.promise; - assert.equal(framework.completeMcplDataPlaneGate(older), false); - assert.equal(readyCalls, 0, 'old completion leaves fresh data held'); - - newDrain.resolve({ status: 'delivered', delivered: 1, failed: 0 }); - await newer.promise; - assert.equal(framework.completeMcplDataPlaneGate(newer), true); - assert.equal(readyCalls, 1); -}); - -test('overlapping reconnect/list-change generations release only the newest barrier', async () => { - const reconnectDrain = deferred(); - const listChangeDrain = deferred(); - const framework = Object.create(AgentFramework.prototype) as any; - framework.discordAwarenessBarrier = null; - framework.discordAwarenessBarrierGeneration = 0; - const drains = [reconnectDrain.promise, listChangeDrain.promise]; - framework.beginDiscordAwarenessBarrier = () => ({ - requiresBarrier: true, - promise: drains.shift(), - }); - let readyCalls = 0; - const connection = { - id: 'discord', - pauseDataPlane: () => {}, - readyControlPlane: () => {}, - ready: () => { readyCalls++; }, - }; - framework.mcplServerRegistry = { getAllServers: () => [connection] }; - - const reconnectBarrier = framework.installMcplDataPlaneGate(); - framework.releaseMcplDataPlaneGate(reconnectBarrier); - const listChangeBarrier = framework.installMcplDataPlaneGate(); - framework.releaseMcplDataPlaneGate(listChangeBarrier); - - reconnectDrain.resolve({ status: 'delivered', delivered: 1, failed: 0 }); - await reconnectBarrier.promise; - framework.completeMcplDataPlaneGate(reconnectBarrier); - assert.equal(readyCalls, 0, 'reconnect completion cannot bypass overlapping list-change work'); - - listChangeDrain.resolve({ status: 'delivered', delivered: 1, failed: 0 }); - await listChangeBarrier.promise; - framework.completeMcplDataPlaneGate(listChangeBarrier); - assert.equal(readyCalls, 1); -}); - -test('synchronous zero-pending flush rechecks the gate after a nested list change', () => { - const connection = new (McplServerConnection as any)( - 'discord', null, null, - ) as McplServerConnection; - // This test exercises plane-flush mechanics, not authorization: admission - // now enforces the grant (§5.4), so model the post-policy state or the - // push is correctly discarded before it can count. - connection.establishGrant(new CapabilityGrant(new Set(['pushEvents']), [])); - let dataDeliveries = 0; - connection.on('tools-list-changed', () => connection.pauseDataPlane()); - connection.on('push-event', () => { dataDeliveries++; }); - - connection.emit('tools-list-changed'); - connection.emit('push-event', {}); - connection.ready(); - assert.equal( - dataDeliveries, - 0, - 'the nested list-change pause must retain later items from the same ready() snapshot', - ); - - connection.ready(); - assert.equal(dataDeliveries, 1); -}); diff --git a/test/mcpl-reconnect-reregistration.test.ts b/test/mcpl-reconnect-reregistration.test.ts index a7627a77f..35a1fadd6 100644 --- a/test/mcpl-reconnect-reregistration.test.ts +++ b/test/mcpl-reconnect-reregistration.test.ts @@ -131,14 +131,11 @@ async function makeHarness() { getAllServers: () => [connection], getServer: (id: string) => id === connection.id ? connection : null, }; - fw.discordAwarenessBarrier = null; - fw.discordAwarenessBarrierGeneration = 0; fw.wireMcplEvents(connection); await fw.registerMcplServerFeatures(config, connection); - // The push handler responds synchronously when no awareness barrier is - // up, but during reconnect the §5.3 policy round-trip holds the barrier - // across a microtask — so the harness settles the loop once before + // During reconnect the §5.3 policy round-trip spans a microtask before + // the data plane reopens — so the harness settles the loop once before // asserting. (Was 'must respond synchronously' pre-0.5.) const sendPush = async (eventId: string): Promise => { let result: PushEventResult | undefined; diff --git a/test/offline-recovery-branch.test.ts b/test/offline-recovery-branch.test.ts index c514bc3d5..8b34e9aff 100644 --- a/test/offline-recovery-branch.test.ts +++ b/test/offline-recovery-branch.test.ts @@ -30,6 +30,7 @@ test('offline recovery branches without compiling and queues discarded Discord r store.close(); const result = await createOfflineRecoveryBranch({ + marks: { scope: 'all' }, storePath, agentName: 'cairn', messageId: 'm-safe', @@ -54,9 +55,9 @@ test('offline recovery branches without compiling and queues discarded Discord r const batches = new DiscordAwarenessOutbox( defaultDiscordAwarenessOutboxPath(storePath), - ).pending('discord'); + ).pendingDispatches('discord'); assert.equal(batches.length, 2); - assert.deepEqual(batches.map((operation) => operation.ref.messageId), ['m-toxic-1', 'm-toxic-2']); + assert.deepEqual(batches.map((dispatch) => dispatch.key.messageId), ['m-toxic-1', 'm-toxic-2']); // The outbox is metadata-only: quarantined text must never leak to it. const rawOutbox = await import('node:fs').then(({ readFileSync }) => @@ -91,6 +92,7 @@ test('offline recovery can branch at the current message and suppress an exact l store.close(); const result = await createOfflineRecoveryBranch({ + marks: { scope: 'all' }, storePath, agentName: 'cairn', messageId: 'm-current', @@ -150,6 +152,7 @@ test('offline recovery suppresses inclusive ranges in context order', async () = store.close(); const result = await createOfflineRecoveryBranch({ + marks: { scope: 'all' }, storePath, agentName: 'cairn', messageId: 'm-current', @@ -179,10 +182,10 @@ test('offline recovery suppresses inclusive ranges in context order', async () = const batches = new DiscordAwarenessOutbox( defaultDiscordAwarenessOutboxPath(storePath), - ).pending('discord'); + ).pendingDispatches('discord'); assert.equal(batches.length, 3); assert.deepEqual( - batches.map((operation) => operation.ref.messageId), + batches.map((dispatch) => dispatch.key.messageId), ['m-from', 'm-inside', 'm-to'], ); } finally { @@ -207,6 +210,7 @@ test('offline recovery accepts an exact internal context ID anchor', async () => store.close(); const result = await createOfflineRecoveryBranch({ + marks: { scope: 'all' }, storePath, agentName: 'cairn', contextId: String(coherentAssistantId), @@ -268,3 +272,56 @@ test('suppression rejects a range that splits a tool exchange', async () => { rmSync(dir, { recursive: true, force: true }); } }); + +test('offline recovery is local by default; addressed marks only messages that addressed the agent', async () => { + const dir = mkdtempSync(join(tmpdir(), 'offline-recovery-marks-')); + const storePath = join(dir, 'agent.chronicle'); + try { + const seed = async () => { + const store = JsStore.openOrCreate({ path: storePath }); + const cm = await ContextManager.open({ store, namespace: 'agents/cairn' }); + cm.addMessage('user', [{ type: 'text', text: 'safe' }], { + serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'm-safe', + }); + cm.addMessage('user', [{ type: 'text', text: 'hey @cairn' }], { + serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'm-addressed', tags: ['chat:addressed', 'chat:mention'], + }); + cm.addMessage('user', [{ type: 'text', text: 'ambient chatter' }], { + serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'm-ambient', tags: ['chat:ambient'], + }); + cm.close(); + store.close(); + }; + await seed(); + const local = await createOfflineRecoveryBranch({ + storePath, agentName: 'cairn', messageId: 'm-safe', branchName: 'recovery/cairn/local', + }); + assert.equal(local.marksScope, 'none'); + assert.equal(local.discordAddressable, 2); + assert.equal(local.discordMarkersQueued, 0); + assert.equal(local.unmarked, 2); + const outbox = new DiscordAwarenessOutbox(defaultDiscordAwarenessOutboxPath(storePath)); + assert.equal(outbox.pendingDispatches('discord').length, 0, 'nothing is queued for Discord'); + assert.equal(outbox.batches().length, 0); + + // A dry run with addressed shows exactly what would be marked. + rmSync(storePath, { recursive: true, force: true }); + await seed(); + const preview = await createOfflineRecoveryBranch({ + storePath, agentName: 'cairn', messageId: 'm-safe', marks: { scope: 'addressed' }, dryRun: true, + }); + assert.deepEqual(preview.refs.map((ref) => ref.messageId), ['m-addressed']); + const marked = await createOfflineRecoveryBranch({ + storePath, agentName: 'cairn', messageId: 'm-safe', marks: { scope: 'addressed' }, branchName: 'recovery/cairn/marked', + }); + assert.equal(marked.discordMarkersQueued, 1); + assert.equal(marked.unmarked, 1); + assert.deepEqual( + new DiscordAwarenessOutbox(defaultDiscordAwarenessOutboxPath(storePath)).pendingDispatches('discord') + .map((dispatch) => dispatch.key.messageId), + ['m-addressed'], + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); diff --git a/test/operator-surgery.test.ts b/test/operator-surgery.test.ts index aeb162b59..f6180a632 100644 --- a/test/operator-surgery.test.ts +++ b/test/operator-surgery.test.ts @@ -89,7 +89,8 @@ describe('live operator surgery', () => { assert.equal(entry.agent, 'scout'); assert.deepEqual(entry.requester, { via: 'webui', name: 'antra' }); assert.equal(entry.note, 'bad turn'); - assert.deepEqual(entry.params, { messageId: ids[2] }); + // The marks choice is recorded with the operator's other parameters. + assert.deepEqual(entry.params, { messageId: ids[2], marks: 'none' }); assert.equal(entry.result?.messagesRemoved, 2); assert.equal(entry.result?.targetBranch, r.targetBranch); assert.ok(typeof entry.at === 'string' && !Number.isNaN(Date.parse(entry.at))); @@ -297,7 +298,7 @@ describe('live operator surgery', () => { const entry = readLog(framework.getOperatorLogPath()!).find((e) => e.kind === 'suppress'); assert.ok(entry); - assert.deepEqual(entry.params, { messageIds: [ids[1], ids[3]] }); + assert.deepEqual(entry.params, { messageIds: [ids[1], ids[3]], marks: 'none' }); assert.deepEqual(new Set(entry.result?.removedIds as string[]), new Set([ids[1], ids[3]])); }); diff --git a/test/surgery-marks-receipt.test.ts b/test/surgery-marks-receipt.test.ts index f2a1b4c1b..674d08f90 100644 --- a/test/surgery-marks-receipt.test.ts +++ b/test/surgery-marks-receipt.test.ts @@ -1,17 +1,22 @@ /** - * Live surgery reports its body change apart from the Discord awareness marks - * it schedules, and does not wait on Discord to accept them. + * Live surgery and Discord awareness marks, end to end against a fake + * Discord MCPL server. * * Field incident (2026-10-07): a web-UI rollback removed 918 Discord messages - * from a resident's context. rollbackToMessage awaited the serial reaction - * drain inside its store reservation, so the operator's surgery-result never - * arrived within the client's 60 s, the store stayed reserved, and every MCPL - * data plane stayed paused for the whole drain. These tests hold the fake - * server's reaction replies to model that drain. + * from a resident's context, and the framework queued a 💤 reaction on every + * one of them, mostly other people's ambient messages in a busy channel. The + * rollback awaited the serial reaction drain inside its store reservation and + * held every MCPL data plane until it finished. * - * The receipt is a scheduling statement (`markers`), never a delivery claim, - * and a ledger failure after the body change landed is reported as - * `not-scheduled` instead of a failed surgery an operator might retry. + * The contract these tests pin: + * - a surgery is local unless the operator chooses marks (`marks`, default + * `none`); `addressed` covers messages tagged chat:addressed; a choice + * with `refs` marks only those refs, never later arrivals; + * - the surgery returns with a marker-scheduling receipt, never a delivery + * claim, and never fails an applied body change over marker bookkeeping; + * - delivery runs in the background and never holds MCPL traffic; + * - cancel and retract are explicit, and their receipts disclose requests + * whose outcome is unknown. */ import { describe, it, beforeEach, afterEach } from 'node:test'; import assert from 'node:assert/strict'; @@ -19,6 +24,7 @@ import { fileURLToPath } from 'node:url'; import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; +import { JsStore } from '@animalabs/chronicle'; import { AgentFramework, DiscordAwarenessOutbox } from '../src/index.js'; import type { OperatorLogEntry } from '../src/index.js'; import { MockMembrane } from './helpers/mock-membrane.js'; @@ -54,7 +60,7 @@ async function settlesWithin(promise: Promise, ms: number, what: string): } } -describe('surgery marker receipt', () => { +describe('surgery and awareness marks', () => { let dir: string; let storePath: string; let callsPath: string; @@ -64,35 +70,35 @@ describe('surgery marker receipt', () => { let framework: AgentFramework; const cm = () => framework.getAgent('resident')!.getContextManager(); - const outboxPath = () => join(storePath, 'recovery', 'discord-awareness-outbox.json'); - const ledger = () => new DiscordAwarenessOutbox(outboxPath()).batches(); + const outbox = () => new DiscordAwarenessOutbox(join(storePath, 'recovery', 'discord-awareness-journal.jsonl')); + const answered = () => jsonl(eventsPath).filter((e) => e.event === 'reaction-answered').length; + const calls = () => jsonl<{ name: string; args: { messageId: string; emoji: string } }>(callsPath); - async function start(withDiscord: boolean): Promise { + async function start(withDiscord: boolean, awarenessDeadlineMs?: number): Promise { framework = await AgentFramework.create({ storePath, membrane: new MockMembrane().asMembrane(), agents: [{ name: 'resident', model: 'test-model', systemPrompt: 'You are a resident.' }], modules: [], maintenanceIntervalMs: 0, + ...(awarenessDeadlineMs ? { discordAwarenessDeadlineMs: awarenessDeadlineMs } : {}), ...(withDiscord ? { mcplServers: [{ id: 'discord', command: process.execPath, args: [FIXTURE], - env: { - CALLS_PATH: callsPath, - EVENTS_PATH: eventsPath, - HOLD_PATH: holdPath, - PROBE_PATH: probePath, - }, + env: { CALLS_PATH: callsPath, EVENTS_PATH: eventsPath, HOLD_PATH: holdPath, PROBE_PATH: probePath }, enabledFeatureSets: ['chat'], }], } : {}), }); } - /** A kept message, the rollback target, then `n` addressable messages. */ - function seed(n: number): { tail: string; removed: string[] } { + /** + * A kept message, the rollback target, then `n` messages from a busy + * channel; those whose index is in `addressed` addressed the resident. + */ + function seed(n: number, addressed: number[] = []): { tail: string; removed: string[] } { cm().addMessage('Operator', [{ type: 'text', text: 'kept' }], { serverId: 'discord', channelId: 'discord:g1:dm', messageId: 'kept', }); @@ -101,8 +107,11 @@ describe('surgery marker receipt', () => { })); const removed: string[] = []; for (let i = 0; i < n; i++) { - removed.push(String(cm().addMessage(`Member${i}`, [{ type: 'text', text: `ambient ${i}` }], { - serverId: 'discord', channelId: 'discord:g1:c1', messageId: `amb-${i}`, + removed.push(String(cm().addMessage(`Member${i}`, [{ type: 'text', text: `message ${i}` }], { + serverId: 'discord', + channelId: 'discord:g1:c1', + messageId: `amb-${i}`, + tags: addressed.includes(i) ? ['chat:addressed', 'chat:mention'] : ['chat:ambient'], }))); } return { tail, removed }; @@ -123,144 +132,114 @@ describe('surgery marker receipt', () => { rmSync(dir, { recursive: true, force: true }); }); - it('rollback returns once the batch is active, before Discord has accepted any reaction', async () => { + it('a rollback is local by default: no reaction, and the receipt counts what stayed unmarked', async () => { await start(true); - const { tail } = seed(3); - - const result = await settlesWithin( - framework.rollbackToMessage('resident', { messageId: tail }), - 3_000, - 'rollbackToMessage with reactions held', - ); - - assert.equal(result.messagesRemoved, 3); - assert.equal(result.markers.status, 'queued'); - assert.equal(result.markers.queued, 3); - const [batch] = ledger(); - assert.equal(batch.status, 'active', 'the batch is durably active before the call returns'); - assert.equal(result.markers.status === 'queued' && result.markers.batchId, batch.id); - assert.equal( - jsonl(eventsPath).filter((e) => e.event === 'reaction-answered').length, - 0, - 'no reaction had been accepted when the rollback returned', - ); + const { tail } = seed(5, [3]); + writeFileSync(holdPath, '1'); + const result = await framework.rollbackToMessage('resident', { messageId: tail }); + assert.equal(result.messagesRemoved, 5); + assert.equal(result.removedRefs.length, 5); + assert.deepEqual(result.markers, { scope: 'none', unmarked: 5, notRemoved: 0, status: 'none', queued: 0 }); + await framework.syncDiscordAwarenessMarkers(); + assert.equal(calls().length, 0); + assert.equal(outbox().batches().length, 0); + const entry = framework.getOperatorLog({ limit: 5 }).find((e) => e.kind === 'rollback'); + assert.equal(entry?.params?.marks, 'none'); + }); - // Delivery continues in the background and completes once Discord answers. + it('addressed marks only the messages that addressed the resident', async () => { + await start(true); + const { tail } = seed(6, [1, 4]); writeFileSync(holdPath, '1'); - await waitFor('all three reactions answered', () => - jsonl(eventsPath).filter((e) => e.event === 'reaction-answered').length === 3); + const preview = framework.previewSurgeryMarks('resident', { rollbackTo: tail }); + assert.equal(preview.messagesRemoved, 6); + assert.equal(preview.addressable, 6); + assert.equal(preview.scopes.addressed.count, 2); + assert.deepEqual(preview.scopes.addressed.channels, [{ channelId: 'discord:g1:c1', count: 2 }]); + assert.equal(preview.scopes.all.count, 6); + + const result = await framework.rollbackToMessage('resident', { messageId: tail, marks: { scope: 'addressed' } }); + assert.equal(result.markers.status, 'queued'); + assert.equal(result.markers.queued, 2); + assert.equal(result.markers.unmarked, 4); + await waitFor('two reactions answered', () => answered() === 2); await framework.syncDiscordAwarenessMarkers(); - const entries = ledger()[0].refs; - assert.deepEqual(entries.map((ref) => ref.deliveryStatus), ['applied', 'applied', 'applied']); - assert.deepEqual( - jsonl<{ name: string; args: { emoji: string } }>(callsPath).map((call) => `${call.name}:${call.args.emoji}`), - ['add_reaction:💤', 'add_reaction:💤', 'add_reaction:💤'], - ); + assert.deepEqual(calls().map((c) => `${c.name}:${c.args.messageId}:${c.args.emoji}`).sort(), [ + 'add_reaction:amb-1:💤', + 'add_reaction:amb-4:💤', + ]); + }); - const logged = new Map( - framework.getOperatorLog({ limit: 10 }).map((entry) => [entry.kind, entry]), - ); - assert.deepEqual(logged.get('rollback')?.result?.markers, result.markers); + it('a choice bound to previewed refs never widens to messages that arrive before it applies', async () => { + await start(true); + const { tail } = seed(3, [0, 2]); + writeFileSync(holdPath, '1'); + const preview = framework.previewSurgeryMarks('resident', { rollbackTo: tail }); + const authorized = preview.scopes.addressed.refs; + assert.equal(authorized.length, 2); + // An addressed message arrives while the operator is deciding. + cm().addMessage('Late', [{ type: 'text', text: '@resident are you there?' }], { + serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'late-1', tags: ['chat:addressed'], + }); + const result = await framework.rollbackToMessage('resident', { + messageId: tail, + marks: { scope: 'addressed', refs: authorized }, + }); + assert.equal(result.messagesRemoved, 4, 'the rollback still removes everything after its anchor'); + assert.equal(result.markers.queued, 2); + assert.equal(result.markers.unmarked, 2, 'the late arrival is removed locally, unmarked'); + await waitFor('two reactions answered', () => answered() === 2); + assert.ok(!calls().some((c) => c.args.messageId === 'late-1')); }); - it('releases the store while delivery runs, but keeps MCPL traffic behind the delivery gate', async () => { + it('returns once marks are scheduled, before Discord has accepted any, and never holds channel traffic', async () => { await start(true); - const { removed } = seed(4); + const { tail, removed } = seed(4); - await settlesWithin( - framework.rollbackToMessage('resident', { messageId: removed[1] }), + const result = await settlesWithin( + framework.rollbackToMessage('resident', { messageId: tail, marks: { scope: 'all' } }), 3_000, 'rollbackToMessage with reactions held', ); - await waitFor('the first reaction call to reach the server', () => jsonl(callsPath).length >= 1); + assert.equal(result.markers.status, 'queued'); + assert.equal(result.markers.queued, 4); + assert.equal(answered(), 0, 'no reaction had been accepted when the rollback returned'); + await waitFor('the first reaction call to reach the server', () => calls().length >= 1); - // The store reservation is released: a second surgery is admitted while - // the first one's reactions are still unanswered. + // The store is free: a second surgery is admitted during delivery. + cm().addMessage('Someone', [{ type: 'text', text: 'new' }], { + serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'after-1', + }); + const fresh = cm().getAllMessages().at(-1)!; const second = await settlesWithin( - framework.suppressMessages('resident', { messageIds: [removed[0]] }), + framework.suppressMessages('resident', { messageIds: [String(fresh.id)] }), 3_000, 'a second surgery during delivery', ); - assert.equal(second.messagesRemoved, 1); - assert.equal(second.markers.status, 'queued'); + assert.equal(second.markers.status, 'none'); - // The gate was installed before release: ordinary channel traffic sent - // during delivery is not answered until the reactions are. + // Channel traffic sent while reactions are held is answered anyway. writeFileSync(probePath, '1'); - await waitFor('the probe to be sent', () => jsonl(eventsPath).some((e) => e.event === 'probe-sent')); - await new Promise((resolve) => setTimeout(resolve, 200)); - assert.equal( - jsonl(eventsPath).some((e) => e.event === 'probe-answered'), - false, - 'channel traffic waits behind the delivery gate', - ); + await waitFor('the probe to be answered while reactions are still held', () => + jsonl(eventsPath).some((e) => e.event === 'probe-answered')); + assert.equal(answered(), 0, 'the probe was not waiting behind the marks'); writeFileSync(holdPath, '1'); - await waitFor('the probe to be answered after delivery', () => - jsonl(eventsPath).some((e) => e.event === 'probe-answered')); - // The server's own event order, not timestamps (two events can share a - // millisecond), and every expected reaction by id: one compared only with - // the reactions that happened to be answered would pass if the second - // surgery's batch were never delivered. - const events = jsonl<{ event: string; messageId?: string }>(eventsPath); - const probeIndex = events.findIndex((e) => e.event === 'probe-answered'); - const answeredBeforeProbe = events.slice(0, probeIndex) - .filter((e) => e.event === 'reaction-answered') - .map((e) => e.messageId) - .sort(); + await waitFor('all four adds confirmed in the journal', () => { + const journal = outbox(); + return journal.operations().filter((op) => journal.operationStatus(op) === 'confirmed').length === 4; + }); + // Exactly the rolled-back messages, nothing from the later suppression. assert.deepEqual( - answeredBeforeProbe, - ['amb-0', 'amb-2', 'amb-3'], - 'both surgeries\' reactions are answered before channel traffic is released', + outbox().operations().map((op) => `${op.action}:${op.key.messageId}`).sort(), + ['add:amb-0', 'add:amb-1', 'add:amb-2', 'add:amb-3'], ); - }); + assert.equal(removed.length, 4); - it('restoring the source while delivery runs keeps traffic gated until the running drain is done', async () => { - await start(true); - const { tail } = seed(3); - - const rollback = await settlesWithin( - framework.rollbackToMessage('resident', { messageId: tail }), - 3_000, - 'rollbackToMessage with reactions held', - ); - await waitFor('the first reaction call to reach the server', () => jsonl(callsPath).length >= 1); - - // The operator restores the source the way the branch-switch API does - // (store switch, then synchronization) while the first add_reaction still - // waits for its reply. Reconciliation makes every mark undesired, and the - // held one is not present yet, so the ledger shows nothing pending: only - // the running drain knows it will still send its saved adds. - framework.getStore().switchBranch(rollback.sourceBranch); - const restored = framework.syncDiscordAwarenessMarkers(); - - writeFileSync(probePath, '1'); - await waitFor('the probe to be sent', () => jsonl(eventsPath).some((e) => e.event === 'probe-sent')); - await new Promise((resolve) => setTimeout(resolve, 200)); - assert.equal( - jsonl(eventsPath).some((e) => e.event === 'probe-answered'), - false, - 'channel traffic waits for the running drain even though nothing appears pending', - ); - - writeFileSync(holdPath, '1'); - await settlesWithin(restored, 5_000, "the restore's synchronization"); - await waitFor('the probe to be answered after delivery', () => - jsonl(eventsPath).some((e) => e.event === 'probe-answered')); - - // Every reaction request the drain made, including the removals the - // restore needs, was answered before channel traffic was released, and - // the restore leaves no mark behind. - const events = jsonl<{ event: string }>(eventsPath); - const probeIndex = events.findIndex((e) => e.event === 'probe-answered'); - const answered = (from: number, to?: number) => - events.slice(from, to).filter((e) => e.event === 'reaction-answered').length; - assert.equal(answered(0, probeIndex), jsonl(callsPath).length, 'every reaction call answered before the probe'); - assert.equal(answered(probeIndex), 0, 'no reaction answered after the probe'); - assert.deepEqual( - ledger()[0].refs.map((ref) => [ref.messageId, ref.desired, ref.markerPresent, ref.deliveryStatus]), - [['amb-0', false, false, 'applied'], ['amb-1', false, false, 'applied'], ['amb-2', false, false, 'applied']], + const logged = new Map( + framework.getOperatorLog({ limit: 10 }).map((entry) => [entry.kind, entry]), ); + assert.deepEqual(logged.get('rollback')?.result?.markers, result.markers); }); it('a rollback whose marker bookkeeping fails after the switch reports the body as applied', async () => { @@ -272,29 +251,25 @@ describe('surgery marker receipt', () => { }; let result: Awaited>; try { - result = await framework.rollbackToMessage('resident', { messageId: tail }); + result = await framework.rollbackToMessage('resident', { messageId: tail, marks: { scope: 'all' } }); } finally { DiscordAwarenessOutbox.prototype.activate = original; } assert.equal(cm().currentBranch().name, result.targetBranch, 'the rollback stands'); - assert.equal(result.messagesRemoved, 2); assert.equal(result.markers.status, 'not-scheduled'); assert.match(result.markers.status === 'not-scheduled' ? result.markers.error : '', /injected ledger failure/); const entry = framework.getOperatorLog({ limit: 10 }).find((e) => e.kind === 'rollback'); assert.equal(entry?.error, undefined, 'logged as an applied rollback, not a failure'); - assert.equal((entry?.result?.markers as { status?: string } | undefined)?.status, 'not-scheduled'); - // "Not scheduled" holds across the reconciliation delivery runs and - // across a restart: the batch was retired, so nothing can promote it. + // "Not scheduled" holds across delivery and a full restart. writeFileSync(holdPath, '1'); await framework.syncDiscordAwarenessMarkers(); - assert.deepEqual(ledger(), []); await framework.stop(); await start(true); await framework.syncDiscordAwarenessMarkers(); - assert.deepEqual(ledger(), []); - assert.equal(jsonl(callsPath).length, 0, 'no reaction was ever sent'); + assert.equal(outbox().operations().length, 0); + assert.equal(calls().length, 0, 'no reaction was ever sent'); }); it('reports unresolved bookkeeping when the batch can be neither activated nor retired', async () => { @@ -310,19 +285,16 @@ describe('surgery marker receipt', () => { }; let result: Awaited>; try { - result = await framework.rollbackToMessage('resident', { messageId: tail }); + result = await framework.rollbackToMessage('resident', { messageId: tail, marks: { scope: 'all' } }); } finally { DiscordAwarenessOutbox.prototype.activate = activate; DiscordAwarenessOutbox.prototype.discard = discard; } assert.equal(cm().currentBranch().name, result.targetBranch, 'the rollback stands'); - // The receipt promises nothing about this batch: it is still in the - // ledger, and a reconciliation may promote and deliver it. assert.equal(result.markers.status, 'unresolved'); - const batches = ledger(); + const batches = outbox().batches(); assert.equal(batches.length, 1); assert.equal(result.markers.status === 'unresolved' && result.markers.batchId, batches[0].id); - assert.match(result.markers.status === 'unresolved' ? result.markers.error : '', /injected ledger failure/); }); it('a suppression whose marker bookkeeping fails after the last redaction stands', async () => { @@ -334,24 +306,73 @@ describe('surgery marker receipt', () => { }; let result: Awaited>; try { - result = await framework.suppressMessages('resident', { messageIds: [removed[1]] }); + result = await framework.suppressMessages('resident', { messageIds: [removed[1]], marks: { scope: 'all' } }); } finally { DiscordAwarenessOutbox.prototype.activate = original; } - assert.equal(cm().currentBranch().name, result.targetBranch, 'the suppression is not undone'); - assert.deepEqual(result.removedIds, [removed[1]]); assert.ok(!cm().getAllMessages().some((m) => String(m.id) === removed[1])); assert.equal(result.markers.status, 'not-scheduled'); - assert.deepEqual(ledger(), [], 'every interval committed, so the retired journal loses nothing'); + assert.equal(outbox().batches().every((batch) => batch.status === 'discarded' || batch.status === 'active'), true); + }); + + it('cancel stops unsent marks without removing any; retract removes this bot\'s marks and discloses an unknown add', async () => { + // A short awareness deadline turns a withheld reply into "no response". + await start(true, 150); + const { tail } = seed(3); + const result = await framework.rollbackToMessage('resident', { messageId: tail, marks: { scope: 'all' } }); + assert.equal(result.markers.status, 'queued'); + const batchId = result.markers.status === 'queued' ? result.markers.batchId : ''; + // The first add goes out and is never answered within the deadline. + await waitFor('the first add to time out as unknown', () => + outbox().operations().some((op) => outbox().operationStatus(op) === 'unknown')); + + const cancelled = framework.cancelDiscordAwareness(batchId, { requester: { via: 'test', name: 'operator' } }); + assert.equal(cancelled.confirmed, 0); + assert.equal(cancelled.unknown + cancelled.inFlight + cancelled.cancelled, 3); + assert.ok(cancelled.unknown + cancelled.inFlight >= 1, 'the timed-out add may still land'); + + // Discord finally answers the withheld request: the late add lands. + writeFileSync(holdPath, '1'); + await waitFor('the withheld add to be answered', () => answered() >= 1); + const sentBeforeRetract = calls().length; + assert.ok(calls().every((c) => c.name === 'add_reaction')); + + const retracted = framework.retractDiscordAwareness(batchId, { requester: { via: 'test', name: 'operator' } }); + assert.ok(retracted.removalsQueued >= 1); + assert.ok(retracted.keysWithUnresolvedAdds >= 1, 'the unknown add is disclosed, not assumed absent'); + await waitFor('the removals to be sent', () => + calls().slice(sentBeforeRetract).filter((c) => c.name === 'remove_reaction').length === retracted.removalsQueued); + const kinds = framework.getOperatorLog({ limit: 10 }).map((e) => e.kind); + assert.ok(kinds.includes('awareness-cancel') && kinds.includes('awareness-retract')); + }); + + it('marks chosen where no awareness journal exists are reported as not scheduled, not as unwanted', async () => { + // A caller-supplied store and no storePath: no default journal location. + framework = await AgentFramework.create({ + store: JsStore.openOrCreate({ path: storePath }), + membrane: new MockMembrane().asMembrane(), + agents: [{ name: 'resident', model: 'test-model', systemPrompt: 'You are a resident.' }], + modules: [], + maintenanceIntervalMs: 0, + operatorLogPath: false, + }); + const { tail } = seed(2, [0]); + const result = await framework.rollbackToMessage('resident', { messageId: tail, marks: { scope: 'addressed' } }); + assert.equal(result.messagesRemoved, 2); + assert.equal(result.markers.status, 'not-scheduled'); + assert.match(result.markers.status === 'not-scheduled' ? result.markers.error : '', /no awareness journal/); + assert.equal(result.markers.unmarked, 1); + const local = await framework.rollbackToMessage('resident', { messageId: String(cm().getAllMessages()[0].id) }); + assert.equal(local.markers.status, 'none', 'nothing chosen is still none'); }); - it('a surgery that removes nothing addressable schedules no marks', async () => { + it('a surgery that removes nothing addressable schedules no marks even when marks are chosen', async () => { await start(false); const keep = String(cm().addMessage('Operator', [{ type: 'text', text: 'kept' }], {})); cm().addMessage('resident', [{ type: 'text', text: 'local only' }], {}); - const result = await framework.rollbackToMessage('resident', { messageId: keep }); - assert.deepEqual(result.markers, { status: 'none', queued: 0 }); - assert.equal(ledger().length, 0); + const result = await framework.rollbackToMessage('resident', { messageId: keep, marks: { scope: 'all' } }); + assert.deepEqual(result.markers, { scope: 'all', unmarked: 0, notRemoved: 0, status: 'none', queued: 0 }); + assert.equal(outbox().batches().length, 0); }); }); From 6bb0c61e201f6eca7f7131861383cb977534beae Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Wed, 7 Oct 2026 07:43:33 -0400 Subject: [PATCH 0029/1112] docs(awareness): document opt-in, one-shot marks, operator controls and the journal README: the offline-recovery section now says recovery is local by default and explains --marks addressed|all, --dry-run counts and the journal path; the old ledger/projection/global-generation paragraph is replaced by --awareness controls; a new "Live surgery and Discord awareness marks" section covers marks/preview/frozen refs, the receipt statuses (including not-scheduled when no journal exists), background non-gating delivery and the deadline/emoji config, one-shot semantics with unknown outcomes and held batches, the operator controls, host/command undo/hide marks and the marks verb, and the one-time v2 import. Changelog: breaking (default none + who must act/migration/unchanged; the DiscordAwarenessOutbox API), changed (one-shot; delivery no longer gates MCPL; corrupt journal still stops startup), added (preview + controls on every surface). #246's fixed fragment drops "behind the same MCPL data-plane gate as before", which this branch makes untrue. --- README.md | 103 ++++++++++++++---- .../fix-surgery-marks-receipt.fixed.md | 19 ++-- changelog.d/surgery-marks-controls.added.md | 18 +++ changelog.d/surgery-marks-one-shot.changed.md | 17 +++ changelog.d/surgery-marks-opt-in.breaking.md | 35 ++++++ 5 files changed, 159 insertions(+), 33 deletions(-) create mode 100644 changelog.d/surgery-marks-controls.added.md create mode 100644 changelog.d/surgery-marks-one-shot.changed.md create mode 100644 changelog.d/surgery-marks-opt-in.breaking.md diff --git a/README.md b/README.md index 9edfeb41b..d3d9d359c 100644 --- a/README.md +++ b/README.md @@ -161,15 +161,18 @@ agent-framework-recover \ The specified Discord message is the last message the agent will still see; everything after it is left on the old branch. The command scans in bounded -windows, never compiles message content, activates -the new Chronicle branch, and writes only Discord `{serverId, channelId, -messageId}` metadata to -`/recovery/discord-awareness-outbox.json`. When the host and that -agent's `discord-mcpl` bot reconnect, each addressable discarded message is -marked with 💤. Delivery state is recorded per message: retryable failures -remain queued, permanent deleted/inaccessible-message failures remain in the -audit ledger without blocking later markers. Portal and other non-Discord -records are ignored. +windows, never compiles message content, and activates the new Chronicle +branch. By default the recovery is local to the resident: nothing is posted to +Discord. To let the people involved see that the agent no longer has their +messages, choose awareness marks explicitly with `--marks addressed` (removed +messages that mentioned or replied to the bot, and DMs) or `--marks all` +(every removed Discord message). The output counts what will be marked and +what stays unmarked, and `--dry-run` shows exactly which addresses each choice +covers. Only `{serverId, channelId, messageId}` metadata is written, to the +awareness journal `/recovery/discord-awareness-journal.jsonl`, and the +marks are delivered when the host and that agent's `discord-mcpl` bot next +connect (see [Live surgery and Discord awareness marks](#live-surgery-and-discord-awareness-marks)). +Portal and other non-Discord records are ignored. When the safe point is an assistant/tool-side record rather than a Discord message, use its exact ContextManager message ID: @@ -202,7 +205,7 @@ agent-framework-recover \ `--suppress-range ..` suppresses the inclusive context interval between two Discord messages, including intervening agent/tool entries. These removals exist only on the recovery branch; the source branch remains intact. -Suppressed Discord messages are queued for the same 💤 awareness marker. +Suppressed Discord messages follow the same `--marks` choice. Selections that split a sharded body group or a tool-use/tool-result exchange are rejected. The suppression plan is journaled before the branch switch; if the recovery process is interrupted between interval removals, framework @@ -212,17 +215,10 @@ For old stores whose messages lack `metadata.serverId`, supply normal agent host must be stopped while this command has the Chronicle store open. -The marker sidecar is a retained operation ledger, not a delete-on-success -queue. Switching back to the source branch queues removal of the bot's marker; -returning to the recovery branch queues it again. Initial MCPL events remain -buffered until the ledger has been reconciled. Startup, reconnect, runtime -list-change, and online undo use one framework-global generation: every MCPL -data plane waits while all control planes remain live for registration and -marker service. Awareness marker calls also have a mandatory deadline that is -independent of `requestTimeoutMs` (including when that value is `0`); configure -it with `discordAwarenessDeadlineMs` (default 10000ms, clamped to 50..60000ms). -Configure the online marker with `discordAwarenessEmoji`; the offline CLI -accepts `--emoji`. +With the host stopped, the same command inspects and controls the awareness +journal: `--awareness list`, `--awareness cancel `, +`--awareness retract ` and `--awareness release ` behave as +the live controls described below. ### MCPL (MCP Live) @@ -324,6 +320,71 @@ servers granted `allowHostCommands`. The HTTP host verbs accept an optional shared secret (`ApiServerConfig.adminToken`, sent as `x-admin-token`) for deployments that front the port with a proxy. +### Live surgery and Discord awareness marks + +`rollbackToMessage(agent, { messageId })` forks the agent's context at a +message and switches to the fork; `suppressMessages(agent, { messageIds })` +forks at the head and removes the chosen messages on the fork. The source +branch stays intact either way. Removing Discord messages from an agent's +context is local to that agent unless the operator also chooses to mark them: + +```typescript +const preview = framework.previewSurgeryMarks('cairn', { rollbackTo: messageId }); +// preview.scopes.addressed / .all: count, per-channel counts and refs +const result = await framework.rollbackToMessage('cairn', { + messageId, + marks: { scope: 'addressed', refs: preview.scopes.addressed.refs }, +}); +// result.markers: { status: 'none' | 'queued' | 'not-scheduled' | 'unresolved', ... } +``` + +- **`marks`** is `'none'` (the default), or `{ scope: 'addressed' | 'all', refs? }`. + `addressed` covers messages tagged `chat:addressed` (mentions, replies to + the bot, DMs). Passing a preview's `refs` binds the choice to exactly that + set: messages that arrive before the surgery applies are removed locally but + never marked. The choice is recorded in the operator log. +- **The surgery returns once marks are scheduled**, never after Discord has + accepted them. `result.markers` says whether they were scheduled (`queued` + with a count and batch id), not chosen or not in scope (`none`), chosen but + never to be delivered (`not-scheduled`: the framework has no awareness + journal, having neither `storePath` nor `discordAwarenessOutboxPath`, or + recording the batch failed after the body change and the batch was retired; + `error` says which), or + `unresolved` (neither activation nor retirement could be recorded; it names + a batch that may still be delivered). Every receipt also counts the removed + messages left unmarked. Marker bookkeeping never fails or undoes an applied + rollback or suppression. +- **Delivery runs in the background** and never holds MCPL traffic or turns. + A route that is not connected keeps its work queued until it connects. + Each reaction call has a mandatory deadline independent of + `requestTimeoutMs` (including `0`): `discordAwarenessDeadlineMs` (default + 10000ms, clamped to 50..60000ms). The emoji is `discordAwarenessEmoji` + (offline: `--emoji`). +- **Marks are one-shot.** Switching branches, undo/redo and restarts never add + or remove a mark. The journal is an append-only history of requests and + their attempts, written ahead of each dispatch. A request written and never + answered is recorded as `unknown` and is never resolved by a later + confirmation of a different attempt. A batch whose surgery was interrupted + before its branch switch was recorded is held at startup until an operator + releases it. +- **Operator controls:** `listDiscordAwareness()`; + `cancelDiscordAwareness(batch)` stops a batch's further sends and retries + and never removes a reaction (its receipt counts requests in flight or + unknown, which may still land); `retractDiscordAwareness(batch | 'all')` + removes this bot's reaction, through each address's configured MCPL route, + from a batch's messages or from every message an add was ever sent for, and + its receipt discloses earlier adds whose outcome is unknown, any of which + may land after the removal; `releaseDiscordAwareness(batch)` queues a held + batch. Each is recorded in the operator log. +- **Over `host/command`** (servers granted `allowHostCommands`): message-granular + `undo` takes `marks: 'none' | 'addressed' | 'all'` (default `none`; any + other value is refused), and the `marks` verb takes `action: 'list' | + 'cancel' | 'retract' | 'release'` with a `batchId` (`all` for retract). + +A pre-journal awareness ledger (`discord-awareness-outbox.json`) is imported +once on first use: its recorded outcomes are kept, its undelivered work is +held for an explicit release, and the old file is renamed `.migrated-v2`. + ## Observability Subscribe to trace events for logging, UI updates, or debugging: diff --git a/changelog.d/fix-surgery-marks-receipt.fixed.md b/changelog.d/fix-surgery-marks-receipt.fixed.md index 20abfe420..0fc3c82a3 100644 --- a/changelog.d/fix-surgery-marks-receipt.fixed.md +++ b/changelog.d/fix-surgery-marks-receipt.fixed.md @@ -5,15 +5,10 @@ operator got no answer, for the whole serial drain. At the reported pace of about one reaction per second, 918 queued marks would take roughly 15 minutes; the household's client timed out after 60 seconds. Delivery now - continues in the background, behind the same MCPL data-plane gate as - before. A gate installed while a drain is still running (restoring the - source branch mid-delivery, say) now waits for that drain even when the - switch leaves nothing pending, so traffic stays held through the drain's - remaining reactions and the removals they need. The result's new - `markers` field (`none` | `queued` | `not-scheduled` | `unresolved`) says - whether marks were scheduled, never that Discord accepted them. If the - ledger write after the switch fails, the rollback that already applied is - no longer reported as failed: the batch is retired (`not-scheduled`), or, - if even that write fails, the result says the batch may still be delivered - (`unresolved`). A suppression whose redactions all committed is no longer - undone by that failure. + continues in the background. The result's new `markers` field (`none` | + `queued` | `not-scheduled` | `unresolved`) says whether marks were scheduled, never + that Discord accepted them. If the ledger write after the switch fails, the + rollback that already applied is no longer reported as failed: the batch is + retired (`not-scheduled`), or, if even that write fails, the result says + the batch may still be delivered (`unresolved`). A suppression whose + redactions all committed is no longer undone by that failure. diff --git a/changelog.d/surgery-marks-controls.added.md b/changelog.d/surgery-marks-controls.added.md new file mode 100644 index 000000000..2746480cc --- /dev/null +++ b/changelog.d/surgery-marks-controls.added.md @@ -0,0 +1,18 @@ +- Operator controls for awareness marks: + - `previewSurgeryMarks(agent, { rollbackTo } | { suppress })` shows what a + surgery would remove and which messages each scope would mark, per + channel. + - `listDiscordAwareness()` lists the journal's batches. + - `cancelDiscordAwareness(batch)` stops a batch's unsent requests and + retries, and never removes a reaction. Its receipt counts requests in + flight or unknown, which may still land. + - `retractDiscordAwareness(batch | 'all')` queues removal of this bot's + reaction through each message's configured MCPL route. Its receipt + discloses earlier adds whose outcome is unknown, which may land after the + removal. + - `releaseDiscordAwareness(batch)` queues a held batch. + + Each control is recorded in the operator log. The same controls are + available as the `host/command` `marks` verb (`action: list | cancel | + retract | release`, `batchId`) and, with the host stopped, as + `agent-framework-recover --awareness list|cancel|retract|release`. diff --git a/changelog.d/surgery-marks-one-shot.changed.md b/changelog.d/surgery-marks-one-shot.changed.md new file mode 100644 index 000000000..1a62549b3 --- /dev/null +++ b/changelog.d/surgery-marks-one-shot.changed.md @@ -0,0 +1,17 @@ +- Awareness marks are one-shot. Switching branches, undo/redo and restarts no + longer add or remove marks; previously a mark followed the branch, removed + when the source branch became active and re-added when the recovery branch + did. Each request is written ahead of its dispatch. A request that went out + and got no answer is recorded as `unknown`, and a later confirmation of a + different attempt never resolves it. A batch whose surgery was interrupted + before its branch switch was recorded is held at startup until an operator + releases it. +- Awareness delivery no longer holds MCPL traffic. Previously every MCPL data + plane waited at startup, at reconnect, at a tools list change and after each + surgery until every queued mark had been attempted, which at Discord's pace + could leave the resident deaf for minutes. Delivery now runs in the + background. A route that is not connected keeps its work queued until it + connects, and a delivery failure raises an ops alert instead of recycling + connections. A journal that cannot be read still stops startup, because it + also carries the suppression-resume record that protects the agent's + context. diff --git a/changelog.d/surgery-marks-opt-in.breaking.md b/changelog.d/surgery-marks-opt-in.breaking.md new file mode 100644 index 000000000..0bcc98841 --- /dev/null +++ b/changelog.d/surgery-marks-opt-in.breaking.md @@ -0,0 +1,35 @@ +- **Hosts and operator surfaces that remove Discord messages from an agent's + context:** awareness marks are now an explicit publication choice, and the + default is none. Previously every live rollback or suppression, + message-granular `host/command` undo, and offline `agent-framework-recover` + run queued a 💤 reaction on every removed Discord message, including other + people's ambient messages in a busy channel. In the incident behind this + change, one web-UI rollback queued 918 of them. + - **Who needs to act:** a surface that should keep offering marks passes the + operator's choice: `marks: { scope: 'addressed' | 'all', refs? }` on + `rollbackToMessage`/`suppressMessages`, `marks: 'addressed' | 'all'` on + `host/command` `undo` and `hide`, and `--marks addressed|all` on the CLI. + `addressed` covers messages tagged `chat:addressed` (mentions, replies to + the bot, DMs). Show `previewSurgeryMarks()` before the choice, and pass a + scope's `refs` back to bind the choice to exactly the previewed messages. + Any other `host/command` `marks` value is refused rather than guessed. + - **Unchanged:** the body change itself (fork, switch, redaction), the + receipt's `status` values, the emoji (`discordAwarenessEmoji`) and the + reaction deadline (`discordAwarenessDeadlineMs`). Marks already on Discord + are left alone; nothing is removed automatically. + - Receipts (`markers`) now also carry `scope`, `unmarked` and `notRemoved`. + Marks that were chosen but can't be recorded, because there is no journal + (no `storePath` or `discordAwarenessOutboxPath`), are reported as + `not-scheduled` rather than `none`. + - No sibling version is required: a host that ignores the new option simply + places no marks. The connectome-host and discord-mcpl companions add the + choice to their surfaces. +- **Code using `DiscordAwarenessOutbox` directly:** the awareness ledger is now + an append-only journal of requests and attempts, + `/recovery/discord-awareness-journal.jsonl`. `pending()` and + `reconcileForBranch()` are gone. Use `pendingDispatches()`, + `recordDispatching()`/`recordOutcome()`, `view()`, and + `cancel()`/`retract()`/`release()`. A previous `discord-awareness-outbox.json` + is imported once on first use: its recorded outcomes are kept, its undelivered + work is held for an explicit release, and the file is renamed + `.migrated-v2`. From 6e9e51b5c01c61819484c882284fd2f54a79d3cf Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Wed, 7 Oct 2026 08:00:43 -0400 Subject: [PATCH 0030/1112] fix(awareness): bound outside error text before it is journaled, logged or returned June (#43747) pointed out that message-only logging is not bounded logging: an MCPL tool result's error text or a filesystem error has no length limit. On this lane it is also persisted: every failed or unknown outcome's error lands in the awareness journal, which is re-read on each operation. boundDiscordAwarenessText (500 chars: head + tail with the omitted length, never splitting a surrogate pair) is applied in recordOutcome before journaling; in the drain after permanence is classified on the full text and before the log line; and on the receipt/opsAlert errors of activation/retire failures, hide-mark preparation failures and background-delivery failures. Test: a 1 MB mixed-width string with an emoji near the cut stays <= 500 chars with no lone surrogate, and the journal holds only the bounded copy. --- src/framework.ts | 13 +++++++++---- src/recovery/discord-awareness-outbox.ts | 20 +++++++++++++++++++- test/discord-awareness-outbox.test.ts | 15 +++++++++++++++ 3 files changed, 43 insertions(+), 5 deletions(-) diff --git a/src/framework.ts b/src/framework.ts index 8ea76cbc8..a40b7f56d 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -407,6 +407,7 @@ import { defaultDiscordAwarenessOutboxPath, extractDiscordAwarenessRefs, selectDiscordAwarenessRefs, + boundDiscordAwarenessText, type DiscordAwarenessBatch, type DiscordAwarenessBatchView, type DiscordAwarenessCancelReceipt, @@ -6542,13 +6543,15 @@ export class AgentFramework { try { this.discordAwarenessOutbox!.activate(batch.id); } catch (error) { - const detail = error instanceof Error ? error.message : String(error); + const detail = boundDiscordAwarenessText(error instanceof Error ? error.message : String(error)); let retired = false; let retireDetail = ''; try { retired = this.discordAwarenessOutbox!.discard(batch.id); } catch (discardError) { - retireDetail = discardError instanceof Error ? discardError.message : String(discardError); + retireDetail = boundDiscordAwarenessText( + discardError instanceof Error ? discardError.message : String(discardError), + ); } console.error( `[discord-awareness] ${verb} agent=${agentName}: body applied, but marker batch ${batch.id} ` + @@ -6612,7 +6615,7 @@ export class AgentFramework { }) ?? null; } catch (error) { // The redaction stands; nothing was recorded, so nothing will be sent. - const detail = error instanceof Error ? error.message : String(error); + const detail = boundDiscordAwarenessText(error instanceof Error ? error.message : String(error)); this.opsAlert( 'discord-awareness-not-scheduled', agentName, @@ -6638,7 +6641,7 @@ export class AgentFramework { ): void { if (!this.discordAwarenessOutbox) return; this.syncDiscordAwarenessMarkers().catch((error) => { - const detail = error instanceof Error ? error.message : String(error); + const detail = boundDiscordAwarenessText(error instanceof Error ? error.message : String(error)); console.error(`[discord-awareness] delivery after ${occasion} failed: ${detail}`); this.opsAlert( 'discord-awareness-delivery', @@ -13700,7 +13703,9 @@ export class AgentFramework { outcome = 'unknown'; } } + // Classify on the full text, then keep only a bounded copy. const permanent = outcome === 'failed' && isPermanentDiscordReactionFailure(detail ?? ''); + if (detail !== undefined) detail = boundDiscordAwarenessText(detail); try { outbox.recordOutcome(attempts, outcome, { ...(permanent ? { permanent: true } : {}), diff --git a/src/recovery/discord-awareness-outbox.ts b/src/recovery/discord-awareness-outbox.ts index cde3dfc3a..c75d93c4c 100644 --- a/src/recovery/discord-awareness-outbox.ts +++ b/src/recovery/discord-awareness-outbox.ts @@ -39,6 +39,23 @@ import { createHash, randomUUID } from 'node:crypto'; export const DEFAULT_DISCORD_AWARENESS_EMOJI = '💤'; /** The journal file under a store, next to the other recovery state. */ +/** + * Bound text that came from elsewhere (a Discord error body, a filesystem + * error) before it is journaled, logged or returned in a receipt: the journal + * is re-read on every operation, and a server's error text has no length + * limit of its own. Head and tail are kept, with the omitted length stated; + * a cut never splits a surrogate pair. + */ +export function boundDiscordAwarenessText(text: string, max = 500): string { + if (text.length <= max) return text; + const tailLength = Math.min(100, Math.floor(max / 4)); + let head = text.slice(0, max - tailLength - 40); + let tail = text.slice(text.length - tailLength); + if (/[\uD800-\uDBFF]$/.test(head)) head = head.slice(0, -1); + if (/^[\uDC00-\uDFFF]/.test(tail)) tail = tail.slice(1); + return `${head} … [${text.length - head.length - tail.length} chars omitted] … ${tail}`; +} + export function defaultDiscordAwarenessOutboxPath(storePath: string): string { return join(storePath, 'recovery', 'discord-awareness-journal.jsonl'); } @@ -733,6 +750,7 @@ export class DiscordAwarenessOutbox { detail: { permanent?: boolean; error?: string } = {}, ): void { const at = Date.now(); + const error = detail.error !== undefined ? boundDiscordAwarenessText(detail.error) : undefined; try { this.append(attempts.map(({ opId, attempt }) => ({ t: 'outcome' as const, @@ -741,7 +759,7 @@ export class DiscordAwarenessOutbox { attempt, outcome, ...(outcome === 'failed' && detail.permanent ? { permanent: true } : {}), - ...(detail.error !== undefined ? { error: detail.error } : {}), + ...(error !== undefined ? { error } : {}), }))); } finally { for (const { opId } of attempts) this.inFlight.delete(opId); diff --git a/test/discord-awareness-outbox.test.ts b/test/discord-awareness-outbox.test.ts index ba8e1b9b6..4ba916680 100644 --- a/test/discord-awareness-outbox.test.ts +++ b/test/discord-awareness-outbox.test.ts @@ -16,6 +16,7 @@ import { join } from 'node:path'; import { tmpdir } from 'node:os'; import { DiscordAwarenessOutbox, + boundDiscordAwarenessText, extractDiscordAwarenessRefs, selectDiscordAwarenessRefs, type DiscordAwarenessRef, @@ -324,6 +325,20 @@ test('release queues a held batch\'s recorded operations, explicitly', withJourn assert.throws(() => outbox.release(batch.id), /not held/); })); +test('outside error text is bounded before it is journaled', withJournal((path) => { + assert.equal(boundDiscordAwarenessText('short'), 'short'); + const huge = `${'x'.repeat(300)}${'界'.repeat(500_000)}😀tail`; + const bounded = boundDiscordAwarenessText(huge); + assert.ok(bounded.length <= 500, `bounded to ${bounded.length}`); + assert.match(bounded, /chars omitted/); + assert.ok(!/[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(? { const outbox = new DiscordAwarenessOutbox(path); const batch = outbox.prepare({ From 8e4146d33bca66cb907e4ecf68b0bcd844af64ac Mon Sep 17 00:00:00 2001 From: Petra-1012 Date: Wed, 7 Oct 2026 08:50:13 -0400 Subject: [PATCH 0031/1112] fix(drafts): keep uncertainty across attempts; resends own and re-check drafts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tessa-974's review of 351 (room-203 #44135, #44214; probe /tmp/tessa-974-draft-state-probe.mjs): 1. Uncertainty was cleared by a later failure: unknown, then a confirmed retry that failed, read `held`, and a third resend went out without confirmation. A draft is now `unconfirmed` while ANY attempt has an unknown or unrecorded outcome and none was confirmed; uncertainAttempt() names that attempt, and refusals, the list line and a failed retry's receipt cite it (destination, time, reason) rather than "the last one". 2. The up-front batch check went stale across an await: a batch [A,B] waiting on A let a concurrent resend of B go out, then published B again; a dismissal of B meanwhile was ignored. A resend now claims every draft of its batch before its first await (a concurrent resend or {{unsent}} of any of them is refused while the batch owns it), and re-reads each draft from the current projection at its own dispatch boundary, so a dismissal or delivery since the batch began stops it there (also after a journal reconcile replaced the projection). 3. The crash catch-up said "They were not sent" of an unconfirmed draft. It and the list now render each draft's actual state: "held (not sent)", "held (every attempt failed: nothing was posted)" or "UNCONFIRMED — an attempt to … may have been posted". 4. bounceProse expanded `{{unsent}}` from an unconfirmed (or in-flight) latest bounce into a fresh held draft when the envelope's target failed, which then resent without confirmation. Such a draft is never copied: it stays held as itself, only the resident's other words become the new draft, and the bounce notice says so. Privacy boundary, agreed with Tessa (#44155, #44214): drafts are never published except by their resident's explicit resend (or {{unsent}}), only that resident can list or act on them, and notices go into the resident's own history, which residents sharing one message slot (#197) already share. The tool description, notices, module doc, explicit-routing doc and changelog now say that instead of "private". Tests (prose-drafts.test.ts, 22): unknown then failed stays unconfirmed and refuses an unconfirmed third resend; the catch-up renders held and unconfirmed drafts truthfully; an at-risk bounce is not copied into a new draft and {{unsent}} then means only the fresh words; and, through the real drafts handler with publish completions under the test's control, a concurrent resend of a queued draft is refused with B published once, and a draft dismissed while the batch waits never reaches publish. Suite (--test-concurrency=4, load ~7-12): 1329/1330 (#196 vector 34c's wall-clock bound at 30.7 s) + code-execution 36/36. Composition with Greta-1013's 376 companion (6ff476d, membrane#101 pack) merged cleanly: 30/30 incl. her xml-tool-call-notices and a composed text-only tool_attempt split check (room-220 #44367). Co-Authored-By: Claude Opus 5.5 --- changelog.d/prose-drafts.added.md | 14 +- docs/explicit-prose-routing.md | 3 +- src/framework.ts | 252 +++++++++++++++++++----------- src/prose-drafts.ts | 32 +++- test/prose-drafts.test.ts | 148 +++++++++++++++++- 5 files changed, 337 insertions(+), 112 deletions(-) diff --git a/changelog.d/prose-drafts.added.md b/changelog.d/prose-drafts.added.md index a115c52fe..45af20a6a 100644 --- a/changelog.d/prose-drafts.added.md +++ b/changelog.d/prose-drafts.added.md @@ -1,6 +1,6 @@ - **Held prose drafts: suppressed speech is kept, not destroyed.** Plain - speech that is not sent is now held as a private draft with its exact - words, instead of surviving only as a count: speech suppressed by an + speech that is not sent is now held as a draft with its exact words, + instead of surviving only as a count: speech suppressed by an explicit send in the same round (locus and hybrid modes, mid-turn, trailing and text-only, native and XML tool modes), hybrid prose left without a destination after a failed envelope, and explicit/hybrid bounces (which @@ -15,13 +15,19 @@ stream presents mid-turn messages, and the turn-end `[delivered]` receipt names every draft held in the turn ("held as drafts d-…"); a draft no notice reached (a crash, an aborted turn) is named at the next turn's start. +- A draft is never published except by its resident's explicit resend (or + `{{unsent}}`), and only that resident can list or act on it. Its notices go + into the resident's own history, which residents sharing one message slot + already share, as they share the assistant turn the words came from. - New `drafts` tool (whenever MCPL channels are configured): `list`, `read`, `resend` and `dismiss`, scoped to the calling resident. `resend` publishes drafts verbatim, in order, to one destination through the normal publish path, journaling each attempt durably before the request leaves; it counts as an explicit send. A delivered draft returns its historical receipt - instead of sending again; a draft whose last attempt's outcome is unknown - needs `confirmDuplicate: true`. Unused fields may be passed as null. + instead of sending again; a draft with any attempt whose outcome is + unknown (and none confirmed) needs `confirmDuplicate: true`, and a resend + owns its drafts until it finishes, re-checking each before it is sent. + Unused fields may be passed as null. - `ChannelRegistry.deliverSpeech` (routeSpeech with its `PublishOutcome`) and `ChannelRegistry.resolveDestination` are new; routeSpeech is unchanged. - The bounce notice now goes to the bouncing resident rather than the primary. diff --git a/docs/explicit-prose-routing.md b/docs/explicit-prose-routing.md index 1c428316d..b719b400c 100644 --- a/docs/explicit-prose-routing.md +++ b/docs/explicit-prose-routing.md @@ -36,7 +36,8 @@ A prose segment's FIRST line may carry a routing prefix: ## The bounce (unprefixed prose) Unprefixed prose with no sticky target is **never delivered**. It is held as a -private draft (src/prose-drafts.ts), and a system notice naming the draft tells +draft (src/prose-drafts.ts) — never published except by the agent's own +resend — and a system notice naming the draft tells the agent how to resend: `>>#channel {{unsent}}`, or the `drafts` tool by id. The notice requests inference so the resend can happen immediately — capped at 2 consecutive bounce-wakes per agent (then notices append without waking, diff --git a/src/framework.ts b/src/framework.ts index 4fdab2234..e927a2cbf 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -82,7 +82,7 @@ import { ProseStreamRouter } from './mcpl/prose-stream-router.js'; import { detectKnownToolWrapperProse } from './tool-wrapper-prose-guard.js'; import { InferenceRouter } from './mcpl/inference-router.js'; import { ChannelRegistry, type ChannelToolOrigin, type PublishDestination, type PublishOutcome } from './mcpl/channel-registry.js'; -import { ProseDraftStore, draftState, type Draft, type DraftReason, type DraftSource } from './prose-drafts.js'; +import { ProseDraftStore, draftState, uncertainAttempt, type Draft, type DraftReason, type DraftSource } from './prose-drafts.js'; import { ConversationRouter, CONVERSATION_ROUTING_DEPRECATION_NOTICE } from './mcpl/conversation-router.js'; import { INBOUND_SOURCE_KEY, type InboundAcceptanceObserver, type InboundChannelSource, type InboundLane, type InboundSource, type InboundUnscopedSource } from './mcpl/inbound-source.js'; import { safeSlice } from './safe-slice.js'; @@ -1116,7 +1116,8 @@ export class AgentFramework { // ---- Held prose drafts (src/prose-drafts.ts) --------------------------- /** Every resident's held drafts: suppressed, bounced or ambiguous plain - * speech, kept privately until resent or dismissed. */ + * speech, never published except by their resident's resend, kept until + * resent or dismissed. */ private proseDrafts: ProseDraftStore; /** Drafts held during the current logical turn, for its `[delivered]` * receipt. Cleared each fresh turn; restarts keep it. */ @@ -4652,7 +4653,8 @@ export class AgentFramework { /** Synthesized drafts tool — present whenever MCPL channels are * configured. A resident's held drafts (src/prose-drafts.ts): plain speech - * that was not sent, kept privately until resent or dismissed. Nullable + * that was not sent, never published except by its resident's resend and + * kept until resent or dismissed. Nullable * fields, cast like save_recent_image's: a caller whose provider presents * every property as required can still say "not in use". */ private static readonly DRAFTS_TOOL = { @@ -4660,8 +4662,8 @@ export class AgentFramework { description: 'Your held drafts: plain speech of yours that was NOT sent — held back because an explicit send in the ' + 'same round holds plain speech back, because a routing prefix bounced, or because it had no destination. ' + - 'Drafts are private to you and stay until you resend or dismiss them; nothing is ever sent from here on ' + - 'its own. list: your open drafts, newest first. read: drafts in full, with every delivery attempt. ' + + 'Drafts are never published except by your own resend, only you can list or act on them, and they stay ' + + 'until you resend or dismiss them. list: your open drafts, newest first. read: drafts in full, with every delivery attempt. ' + 'resend: publish drafts verbatim, in the order given, to one destination ("#channel", "@person" or a ' + 'channel id) — an explicit send, like send_message; it stops at the first draft not confirmed delivered. ' + 'A draft already delivered returns its receipt instead of sending again. If an earlier attempt may ' + @@ -8828,7 +8830,7 @@ export class AgentFramework { } /** - * Hold plain speech as private drafts (src/prose-drafts.ts) instead of + * Hold plain speech as drafts (src/prose-drafts.ts) instead of * letting it vanish: journaled durably and recorded for this turn's * receipt. `notice: 'now'` also queues a private notice naming them, which * the resident hears at its next tool boundary — only when the live stream @@ -8916,7 +8918,7 @@ export class AgentFramework { return ( `[drafts] ${agentName}: not sent — ${many ? `${held.length} plain-speech segments` : 'a plain-speech segment'} ` + `held as draft${many ? 's' : ''} ${ids.join(', ')} — ${AgentFramework.DRAFT_REASON_TEXT[reason]}. ` + - `Drafts are private to you and stay until you act: drafts(action: "resend", draftIds: [${idList}], ` + + `Nothing publishes drafts but your own resend, and they stay until you act: drafts(action: "resend", draftIds: [${idList}], ` + `destination: "#channel") delivers ${many ? 'them' : 'it'} unchanged; drafts(action: "dismiss", ` + `draftIds: [${idList}]) sets ${many ? 'them' : 'it'} aside.` ); @@ -8939,20 +8941,26 @@ export class AgentFramework { /** One line per draft: id, state, when, why, size, preview. */ private draftLine(draft: Draft): string { - const state = draftState(draft); - const last = draft.attempts.at(-1); - let stateText: string = state; - if (state === 'unconfirmed' && last) { - stateText = `UNCONFIRMED — an attempt to ${AgentFramework.destinationText(last.destination)} at ` + - `${this.draftTime(last.at)} may have been posted`; - } else if (state === 'held' && last) { - stateText = `held (last attempt failed: nothing was posted)`; - } + const stateText = this.draftStateText(draft); return `${draft.id} · ${stateText} · held ${this.draftTime(draft.heldAt)} · ` + `${AgentFramework.DRAFT_REASON_SHORT[draft.reason]} · ${draft.text.length} chars · ` + AgentFramework.draftPreview(draft.text); } + /** A draft's state in words that claim only what is known. */ + private draftStateText(draft: Draft): string { + const state = draftState(draft); + if (state === 'unconfirmed') { + const risky = uncertainAttempt(draft)!; + return `UNCONFIRMED — an attempt to ${AgentFramework.destinationText(risky.destination)} at ` + + `${this.draftTime(risky.at)} may have been posted`; + } + if (state === 'held') { + return draft.attempts.length > 0 ? 'held (every attempt failed: nothing was posted)' : 'held (not sent)'; + } + return state; + } + /** A delivered draft's historical receipt. */ private draftReceipt(draft: Draft): string { const delivered = draft.attempts.find((a) => a.outcome?.status === 'delivered'); @@ -8971,13 +8979,13 @@ export class AgentFramework { const pending = this.proseDrafts.unnoticed(agent.name); if (pending.length === 0) return; const shown = pending.slice(0, 8).map((d) => - `${d.id} (held ${this.draftTime(d.heldAt)}, ${AgentFramework.DRAFT_REASON_SHORT[d.reason]}, ` + - `${d.text.length} chars: ${AgentFramework.draftPreview(d.text, 60)})`); + `${d.id} (${this.draftStateText(d)}; held ${this.draftTime(d.heldAt)}, ` + + `${AgentFramework.DRAFT_REASON_SHORT[d.reason]}, ${d.text.length} chars: ${AgentFramework.draftPreview(d.text, 60)})`); const more = pending.length > shown.length ? ` · and ${pending.length - shown.length} more` : ''; const text = `[drafts] ${agent.name}: ${pending.length === 1 ? 'a held draft' : `${pending.length} held drafts`} of yours ` + `${pending.length === 1 ? 'has' : 'have'} not been named to you yet: ${shown.join(' · ')}${more}. ` + - 'They were not sent. drafts(action: "list") shows your open drafts; resend or dismiss them by id.'; + 'drafts(action: "list") shows your open drafts; resend or dismiss them by id.'; try { const mid = agent.getContextManager().addMessage( 'user', @@ -9161,87 +9169,114 @@ export class AgentFramework { // Refuse up front, before anything is published, rather than half-way. for (const d of drafts) { if (this.draftsInFlight.has(`${agentName}\u0000${d.id}`)) { - return refuse(`${d.id} is being sent right now; wait for that result before sending it again. Nothing was sent.`); - } - const state = draftState(d); - if (state === 'dismissed') { - return refuse(`${d.id} was dismissed, so it can't be resent. Nothing was sent.`); - } - if (state === 'unconfirmed' && !confirmDuplicate) { - const last = d.attempts.at(-1)!; - return refuse( - `${d.id}'s last attempt (to ${AgentFramework.destinationText(last.destination)} at ${this.draftTime(last.at)}) ` + - `may already have been posted: ${last.outcome?.reason ?? 'its outcome was never recorded'}. ` + - 'Check that channel; to send it again anyway, resend with confirmDuplicate: true. Nothing was sent.', - ); + return refuse(`${d.id} is being sent right now (or is queued in a resend in progress); wait for that result before sending it again. Nothing was sent.`); } + const refusal = this.resendRefusal(d, confirmDuplicate); + if (refusal) return refuse(`${refusal} Nothing was sent.`); } + // Claim every draft of the batch before the first await: a concurrent + // resend of any of them is refused while this batch owns it. Each draft + // is still re-read and re-checked at its own dispatch boundary below, so + // a dismissal (or a delivery) since the batch began is honoured. + const claims = drafts.map((d) => `${agentName}\u0000${d.id}`); + for (const claim of claims) this.draftsInFlight.add(claim); const lines: string[] = []; let stopped = false; - for (const d of drafts) { - if (stopped) { - lines.push(`${d.id}: not attempted (an earlier draft was not confirmed delivered).`); - continue; - } - if (draftState(d) === 'delivered') { - lines.push(`${this.draftReceipt(d)} — not sent again.`); - continue; - } - let attemptId: string; - try { - attemptId = this.proseDrafts.beginAttempt(agentName, d.id, destination, 'resend', draftState(d) === 'unconfirmed'); - } catch (err) { - lines.push(`${d.id}: not sent — the attempt could not be recorded durably (${err instanceof Error ? err.message : String(err)}).`); - stopped = true; - continue; - } - const flight = `${agentName}\u0000${d.id}`; - this.draftsInFlight.add(flight); - let outcome: PublishOutcome; - try { - outcome = await registry.publish(agentName, d.text, { serverId: destination.serverId, channelId: destination.channelId }); - } finally { - this.draftsInFlight.delete(flight); - } - let recorded = true; - try { - this.proseDrafts.recordOutcome(agentName, d.id, attemptId, outcome); - } catch (err) { - recorded = false; - console.error(`[drafts] ${agentName}: outcome of ${d.id} not recorded:`, err); - } - this.emitTrace({ - type: 'prose:draft-resent', - agentName, - draftId: d.id, - status: outcome.status, - serverId: outcome.destination?.serverId ?? destination.serverId, - channelId: outcome.destination?.channelId ?? destination.channelId, - ...(outcome.messageId ? { messageId: outcome.messageId } : {}), - }); - const where = AgentFramework.destinationText(outcome.destination ?? destination); - const unrecorded = recorded ? '' : ' (this outcome could not be recorded; the draft will read as unconfirmed)'; - if (outcome.status === 'delivered') { - lines.push(`${d.id}: delivered to ${where}${outcome.messageId ? `, message ${outcome.messageId}` : ''}${unrecorded}.`); - let engaged = this.turnEngagedChannels.get(agentName); - if (!engaged) { - engaged = new Set(); - this.turnEngagedChannels.set(agentName, engaged); + try { + for (const queued of drafts) { + if (stopped) { + lines.push(`${queued.id}: not attempted (an earlier draft was not confirmed delivered).`); + continue; } - engaged.add(destination.channelId); - continue; + // The current projection, not the batch's snapshot (it may have been + // rebuilt by a reconcile, or changed while an earlier draft was sent). + const d = this.proseDrafts.get(agentName, queued.id); + if (!d) { + lines.push(`${queued.id}: no longer found among your drafts — not sent.`); + stopped = true; + continue; + } + if (draftState(d) === 'delivered') { + lines.push(`${this.draftReceipt(d)} — not sent again.`); + continue; + } + const refusal = this.resendRefusal(d, confirmDuplicate); + if (refusal) { + lines.push(`${refusal} It changed while this resend was waiting — not sent.`); + stopped = true; + continue; + } + let attemptId: string; + try { + attemptId = this.proseDrafts.beginAttempt(agentName, d.id, destination, 'resend', draftState(d) === 'unconfirmed'); + } catch (err) { + lines.push(`${d.id}: not sent — the attempt could not be recorded durably (${err instanceof Error ? err.message : String(err)}).`); + stopped = true; + continue; + } + const outcome = await registry.publish(agentName, d.text, { serverId: destination.serverId, channelId: destination.channelId }); + let recorded = true; + try { + this.proseDrafts.recordOutcome(agentName, d.id, attemptId, outcome); + } catch (err) { + recorded = false; + console.error(`[drafts] ${agentName}: outcome of ${d.id} not recorded:`, err); + } + this.emitTrace({ + type: 'prose:draft-resent', + agentName, + draftId: d.id, + status: outcome.status, + serverId: outcome.destination?.serverId ?? destination.serverId, + channelId: outcome.destination?.channelId ?? destination.channelId, + ...(outcome.messageId ? { messageId: outcome.messageId } : {}), + }); + const where = AgentFramework.destinationText(outcome.destination ?? destination); + const unrecorded = recorded ? '' : ' (this outcome could not be recorded; the draft will read as unconfirmed)'; + if (outcome.status === 'delivered') { + lines.push(`${d.id}: delivered to ${where}${outcome.messageId ? `, message ${outcome.messageId}` : ''}${unrecorded}.`); + let engaged = this.turnEngagedChannels.get(agentName); + if (!engaged) { + engaged = new Set(); + this.turnEngagedChannels.set(agentName, engaged); + } + engaged.add(destination.channelId); + continue; + } + stopped = true; + const after = this.proseDrafts.get(agentName, d.id); + const stillUncertain = after ? uncertainAttempt(after) : undefined; + lines.push(outcome.status === 'unknown' + ? `${d.id}: delivery to ${where} NOT confirmed — it may or may not have been posted: ${outcome.reason ?? 'no valid receipt'}. ` + + `Check the channel before sending it again (that needs confirmDuplicate: true)${unrecorded}.` + : `${d.id}: not sent to ${where} — ${outcome.reason ?? 'refused'}. Nothing was posted by this attempt${unrecorded}.` + + (stillUncertain + ? ` An earlier attempt (to ${AgentFramework.destinationText(stillUncertain.destination)} at ` + + `${this.draftTime(stillUncertain.at)}) may still have been posted, so the draft stays unconfirmed.` + : ' The draft stays held.')); } - stopped = true; - lines.push(outcome.status === 'unknown' - ? `${d.id}: delivery to ${where} NOT confirmed — it may or may not have been posted: ${outcome.reason ?? 'no valid receipt'}. ` + - `Check the channel before sending it again (that needs confirmDuplicate: true)${unrecorded}.` - : `${d.id}: not sent to ${where} — ${outcome.reason ?? 'refused'}. Nothing was posted; the draft stays held${unrecorded}.`); + } finally { + for (const claim of claims) this.draftsInFlight.delete(claim); } const allDelivered = !stopped; return allDelivered ? ok(lines.join('\n')) : refuse(lines.join('\n')); } + /** Why a draft can't be resent now (dismissed, or possibly already posted + * without the resident's confirmation), or undefined when it can. */ + private resendRefusal(d: Draft, confirmDuplicate: boolean): string | undefined { + const state = draftState(d); + if (state === 'dismissed') return `${d.id} was dismissed, so it can't be resent.`; + if (state === 'unconfirmed' && !confirmDuplicate) { + const risky = uncertainAttempt(d)!; + return `${d.id}'s attempt to ${AgentFramework.destinationText(risky.destination)} at ${this.draftTime(risky.at)} ` + + `may already have been posted: ${risky.outcome?.reason ?? 'its outcome was never recorded'}. ` + + 'Check that channel; to send it again anyway, resend with confirmDuplicate: true.'; + } + return undefined; + } + /** Record a successful plain-prose delivery for this turn's receipt. */ private recordProseDelivery( agentName: string, @@ -9464,12 +9499,22 @@ export class AgentFramework { */ private takeUnsent(agent: Agent, prefix: '>>' | '>>>'): { draft?: Draft; refused?: true } { const draft = this.proseDrafts.latestBounce(agent.name); - if (!draft || draftState(draft) !== 'unconfirmed') return { draft }; - const last = draft.attempts.at(-1)!; + if (!draft) return {}; + if (this.draftsInFlight.has(`${agent.name}\u0000${draft.id}`)) { + const busy = `[prose-routing] {{unsent}} is draft ${draft.id}, which is being sent right now. Nothing was sent; wait for that result.`; + try { + this.addMessage('user', [{ type: 'text', text: busy }], { system: true, kind: 'prose-bounce' }, { forAgent: agent.name }); + } catch (err) { + console.error('[drafts] in-flight-unsent notice failed:', err); + } + return { draft, refused: true }; + } + if (draftState(draft) !== 'unconfirmed') return { draft }; + const risky = uncertainAttempt(draft)!; const text = - `[prose-routing] {{unsent}} is draft ${draft.id}, and its last delivery attempt (to ` + - `${AgentFramework.destinationText(last.destination)} at ${this.draftTime(last.at)}) may already have been ` + - `posted: ${last.outcome?.reason ?? 'its outcome was never recorded'}. Nothing was sent. Check that channel; ` + + `[prose-routing] {{unsent}} is draft ${draft.id}, and an attempt to deliver it (to ` + + `${AgentFramework.destinationText(risky.destination)} at ${this.draftTime(risky.at)}) may already have been ` + + `posted: ${risky.outcome?.reason ?? 'its outcome was never recorded'}. Nothing was sent. Check that channel; ` + `to send it again anyway use drafts(action: "resend", draftIds: ["${draft.id}"], destination: "#channel", confirmDuplicate: true).`; try { this.addMessage('user', [{ type: 'text', text }], { system: true, kind: 'prose-bounce' }, { forAgent: agent.name }); @@ -9704,11 +9749,24 @@ export class AgentFramework { ): void { const name = agent.name; const previous = this.proseDrafts.latestBounce(name); + // A latest bounce that may already have been posted (unconfirmed, or in + // flight) is never copied into a fresh draft: that copy would resend + // without the confirmation its own uncertainty requires. It stays held as + // itself; only the resident's other words become a new draft. + const previousAtRisk = previous !== undefined + && (draftState(previous) === 'unconfirmed' || this.draftsInFlight.has(`${name}\u0000${previous.id}`)); let draft: Draft | undefined; + let keptRisky: Draft | undefined; if (previous && text.trim() === '{{unsent}}') { draft = previous; } else { - const expanded = text.includes('{{unsent}}') ? text.replaceAll('{{unsent}}', previous?.text ?? '') : text; + const usesUnsent = text.includes('{{unsent}}'); + if (usesUnsent && previousAtRisk) keptRisky = previous; + const expanded = !usesUnsent + ? text + : previousAtRisk + ? text.replaceAll('{{unsent}}', '').trim() + : text.replaceAll('{{unsent}}', previous?.text ?? ''); // The bounce notice below names the draft, so no separate held notice. draft = this.holdProseDrafts(agent, [expanded], 'bounced', { round: hold.round, note: reason, notice: 'later' })[0]; } @@ -9721,8 +9779,12 @@ export class AgentFramework { `drafts(action: "resend", draftIds: ["${draft.id}"], destination: "#channel"). ` + `"${prefix}skip_reply {{unsent}}" sets it aside.` : 'It could not be held as a draft; the words remain in your history.'; + const risky = keptRisky + ? ` {{unsent}} referred to draft ${keptRisky.id}, which may already have been posted, so it was not copied: ` + + `it is still held as ${keptRisky.id}; resend it with the drafts tool and confirmDuplicate if you mean to.` + : ''; const notice = - `[prose-routing] Your text (${text.length} chars) was not delivered — ${reason}.${cand} ${kept}` + + `[prose-routing] Your text (${text.length} chars) was not delivered — ${reason}.${cand} ${kept}${risky}` + (prefix === '>>' ? ' The prose_help tool shows the full syntax.' : ''); try { // Through framework.addMessage, NOT the context manager directly: while diff --git a/src/prose-drafts.ts b/src/prose-drafts.ts index 57e18706f..2ef8a76d9 100644 --- a/src/prose-drafts.ts +++ b/src/prose-drafts.ts @@ -8,7 +8,14 @@ * unresolvable routing prefix bounces. Before drafts, the held words were * kept only as a count (or, in explicit mode, as one in-memory latest-wins * clipboard): the law was right, and the penalty was amnesia. A draft keeps - * the exact words, privately, until the resident resends or dismisses them. + * the exact words until the resident resends or dismisses them. + * + * The boundary is publication and agency, not visibility: a draft is never + * published except by its resident's explicit resend (or `{{unsent}}`), and + * only that resident can list or act on it. Its notices go into the + * resident's own history — which residents that share one message slot + * (#197) share, as they already share the assistant turn the words came + * from. * * Durability is outside the conversation. Drafts live in a RecordJournal of * typed Chronicle records, which do not follow branch switches: undo, @@ -93,9 +100,11 @@ export interface Draft { } /** - * - `held`: never attempted, or the last attempt definitely failed — resend freely; - * - `unconfirmed`: the last attempt's outcome is unknown (or was never - * recorded): it may have been posted, so a resend needs explicit confirmation; + * - `held`: never attempted, or every attempt definitely failed — resend freely; + * - `unconfirmed`: some attempt's outcome is unknown (or was never + * recorded), and none was confirmed: it may have been posted, so a resend + * needs explicit confirmation. A later attempt that fails does not resolve + * an earlier one that may have landed — the uncertainty stays; * - `delivered`: an attempt was confirmed — its receipt is historical; * - `dismissed`: the resident set it aside. */ @@ -104,9 +113,18 @@ export type DraftState = 'held' | 'unconfirmed' | 'delivered' | 'dismissed'; export function draftState(draft: Draft): DraftState { if (draft.attempts.some((a) => a.outcome?.status === 'delivered')) return 'delivered'; if (draft.dismissedAt !== undefined) return 'dismissed'; - const last = draft.attempts.at(-1); - if (last && (!last.outcome || last.outcome.status === 'unknown')) return 'unconfirmed'; - return 'held'; + return uncertainAttempt(draft) ? 'unconfirmed' : 'held'; +} + +/** The most recent attempt that may have been posted (unknown or unrecorded + * outcome), when no attempt was confirmed. */ +export function uncertainAttempt(draft: Draft): DraftAttempt | undefined { + if (draft.attempts.some((a) => a.outcome?.status === 'delivered')) return undefined; + for (let i = draft.attempts.length - 1; i >= 0; i--) { + const attempt = draft.attempts[i]!; + if (!attempt.outcome || attempt.outcome.status === 'unknown') return attempt; + } + return undefined; } type DraftEntry = diff --git a/test/prose-drafts.test.ts b/test/prose-drafts.test.ts index f4c0e1e3d..639064f72 100644 --- a/test/prose-drafts.test.ts +++ b/test/prose-drafts.test.ts @@ -420,7 +420,7 @@ describe('held prose drafts, end to end', () => { h.command({ op: 'publish-mode', mode: 'not-delivered' }); await h.turn([createMockResponse([resend('r1')], 'tool_use'), createMockResponse([])]); - assert.match(h.toolResults()[0]!, /not sent to .* Nothing was posted; the draft stays held/); + assert.match(h.toolResults()[0]!, /not sent to .* Nothing was posted by this attempt\. The draft stays held\./); assert.equal(draftState(h.store().get('scout', draft!.id)!), 'held'); h.command({ op: 'publish-mode', mode: 'no-receipt' }); @@ -444,6 +444,32 @@ describe('held prose drafts, end to end', () => { } }); + it('an unknown outcome is not cleared by a later failure: the draft stays unconfirmed', async () => { + const h = await harness(); + try { + await h.turn([ + createMockResponse([text('maybe already out'), explicitSend('s1')], 'tool_use'), + createMockResponse([]), + ]); + const [draft] = h.drafts(); + const resend = (id: string, confirm?: boolean) => + drafts(id, { action: 'resend', draftIds: [draft!.id], destination: ROOM, ...(confirm ? { confirmDuplicate: true } : {}) }); + h.command({ op: 'publish-mode', mode: 'no-receipt' }); + await h.turn([createMockResponse([resend('r1')], 'tool_use'), createMockResponse([])]); + h.command({ op: 'publish-mode', mode: 'not-delivered' }); + await h.turn([createMockResponse([resend('r2', true)], 'tool_use'), createMockResponse([])]); + assert.match(h.toolResults()[0]!, /Nothing was posted by this attempt\. An earlier attempt \(to .*\) may still have been posted, so the draft stays unconfirmed/); + assert.equal(draftState(h.store().get('scout', draft!.id)!), 'unconfirmed'); + h.command({ op: 'publish-mode', mode: 'delivered' }); + const before = h.publishes().length; + await h.turn([createMockResponse([resend('r3')], 'tool_use'), createMockResponse([])]); + assert.match(h.toolResults()[0]!, /may already have been posted: .*delivery uncertain.*confirmDuplicate: true\. Nothing was sent/); + assert.equal(h.publishes().length, before, 'no third publish without confirmation'); + } finally { + await h.close(); + } + }); + it('June\'s cases: a rollback to before a confirmed resend returns the receipt, no new publish; a dismissal survives it', async () => { const h = await harness(); try { @@ -499,14 +525,20 @@ describe('held prose drafts, end to end', () => { } }); - it('a draft held in a turn that crashed is named by the next turn\'s catch-up notice', async () => { + it('drafts held in a turn that crashed are named by the next turn\'s catch-up notice, each by its actual state', async () => { const h = await harness(); try { - // Held directly, as if the turn that held it died before its receipt. + // Held directly, as if the turn that held it died before its receipt; + // the second also died mid-send, its attempt never resolved. h.store().hold('scout', [{ text: 'orphaned words', source }], 'explicit-send'); + const midSend = h.store().hold('scout', [{ text: 'half-sent words', source }], 'explicit-send').held[0]!; + h.store().beginAttempt('scout', midSend.id, dest(), 'resend', false); await h.turn([createMockResponse([])]); const notice = h.texts().find((t) => t.startsWith('[drafts]'))!; - assert.match(notice, /^\[drafts\] scout: a held draft of yours has not been named to you yet: d-[a-z2-9]{5} .*"orphaned words"/); + assert.match(notice, /^\[drafts\] scout: 2 held drafts of yours have not been named to you yet: /); + assert.match(notice, /d-[a-z2-9]{5} \(held \(not sent\); held .*"orphaned words"\)/); + assert.match(notice, new RegExp(`${midSend.id} \\(UNCONFIRMED — an attempt to #room \\(Guild One\\) \\(${ROOM}\\) at .* may have been posted; .*"half-sent words"`)); + assert.doesNotMatch(notice, /They were not sent/); await h.turn([createMockResponse([])]); assert.equal(h.texts().filter((t) => t.startsWith('[drafts]')).length, 1, 'named once'); } finally { @@ -532,6 +564,31 @@ describe('held prose drafts, end to end', () => { } }); + it('explicit mode: a bounce that may already have been posted is never copied into a fresh draft', async () => { + const h = await harness({ agents: [{ name: 'scout', proseRouting: 'explicit' }] }); + try { + await h.turn([createMockResponse([text('risky words')])]); + const [risky] = h.drafts(); + h.command({ op: 'publish-mode', mode: 'no-receipt' }); + await h.turn([createMockResponse([text(`>>${ROOM} {{unsent}}`)])]); + assert.equal(draftState(h.store().get('scout', risky!.id)!), 'unconfirmed'); + h.command({ op: 'publish-mode', mode: 'delivered' }); + // The destination does not resolve, so the envelope bounces: its other + // words are held, the possibly-posted text is not copied with them. + await h.turn([createMockResponse([text('>>#nowhere {{unsent}} and a postscript')])]); + const fresh = h.drafts().find((d) => d.id !== risky!.id)!; + assert.equal(fresh.text, 'and a postscript'); + assert.ok(h.texts().some((t) => t.includes(`{{unsent}} referred to draft ${risky!.id}, which may already have been posted, so it was not copied`))); + // And {{unsent}} itself now means the fresh draft, never the risky one. + const before = h.publishes().length; + await h.turn([createMockResponse([text(`>>${ROOM} {{unsent}}`)])]); + assert.deepEqual(h.publishes().slice(before).map((p) => p.text), ['and a postscript']); + assert.equal(draftState(h.store().get('scout', risky!.id)!), 'unconfirmed', 'still needs its own confirmation'); + } finally { + await h.close(); + } + }); + it('resend is an explicit send: prose beside it is held, not auto-routed', async () => { const h = await harness(); try { @@ -564,7 +621,7 @@ describe('held prose drafts, end to end', () => { await h.turn([createMockResponse([resend('a'), resend('b')], 'tool_use'), createMockResponse([])]); const [first, second] = h.toolResults(); assert.match(first!, /delivered to/); - assert.match(second!, /is being sent right now; wait for that result/); + assert.match(second!, /is being sent right now \(or is queued in a resend in progress\); wait for that result/); assert.equal(h.publishes().filter((p) => p.text === 'once only').length, 1); } finally { await h.close(); @@ -601,3 +658,84 @@ describe('held prose drafts, end to end', () => { } }); }); + +// --------------------------------------------------------------------------- +// Concurrent resends through the actual drafts handler, with publish +// completions under the test's control. +// --------------------------------------------------------------------------- + +describe('drafts resend ownership', () => { + const setup = async () => { + const dir = mkdtempSync(join(tmpdir(), 'prose-drafts-own-')); + const framework = await AgentFramework.create({ + storePath: join(dir, 'store'), + membrane: new MockMembrane().asMembrane(), + agents: [{ name: 'scout', model: 'test-model', systemPrompt: 'You are scout.' }], + modules: [], + }); + const pending: Array<{ text: string; resolve: (o: unknown) => void }> = []; + const stub: Record = { + resolveProseTarget: (spec: string) => ({ channelId: spec }), + resolveDestination: ({ channelId }: { channelId: string }) => ({ destination: { serverId: 'discord', channelId } }), + publish: (_agent: string, text: string, target: { channelId: string }) => + new Promise((resolve) => pending.push({ + text, + resolve: (o) => resolve({ destination: { serverId: 'discord', channelId: target.channelId }, at: Date.now(), ...(o as object) }), + })), + getChannelTools: () => [], + }; + (framework as unknown as { channelRegistry: unknown }).channelRegistry = new Proxy(stub, { + get: (t, prop: string) => (prop in t ? t[prop] : () => undefined), + }); + const internals = framework as unknown as { + proseDrafts: ProseDraftStore; + handleDraftsTool(agent: string, input: unknown): Promise<{ success: boolean; data?: Array<{ text: string }>; error?: string }>; + }; + const say = (r: { success: boolean; data?: Array<{ text: string }>; error?: string }) => + r.success ? r.data!.map((b) => b.text).join('') : r.error!; + const tick = () => new Promise((r) => setTimeout(r, 5)); + const [a, b] = internals.proseDrafts.hold('scout', [{ text: 'A words', source }, { text: 'B words', source }], 'explicit-send').held; + return { + dir, framework, internals, pending, say, tick, a: a!, b: b!, + close: async () => { await framework.stop(); rmSync(dir, { recursive: true, force: true }); }, + }; + }; + + it('a batch owns its drafts: a concurrent resend of a queued draft is refused, and nothing is sent twice', async () => { + const t = await setup(); + try { + const batch = t.internals.handleDraftsTool('scout', { action: 'resend', draftIds: [t.a.id, t.b.id], destination: 'chan' }); + await t.tick(); + assert.deepEqual(t.pending.map((p) => p.text), ['A words'], 'the batch is waiting on A'); + const concurrent = t.say(await t.internals.handleDraftsTool('scout', { action: 'resend', draftIds: [t.b.id], destination: 'chan' })); + assert.match(concurrent, new RegExp(`${t.b.id} is being sent right now \\(or is queued in a resend in progress\\)`)); + t.pending.shift()!.resolve({ status: 'delivered', messageId: 'm-a' }); + await t.tick(); + assert.deepEqual(t.pending.map((p) => p.text), ['B words']); + t.pending.shift()!.resolve({ status: 'delivered', messageId: 'm-b' }); + const result = t.say(await batch); + assert.match(result, /delivered to \(chan\), message m-a/); + assert.match(result, /delivered to \(chan\), message m-b/); + assert.equal(t.pending.length, 0, 'B was published exactly once'); + } finally { + await t.close(); + } + }); + + it('a draft dismissed while the batch waits is not published', async () => { + const t = await setup(); + try { + const batch = t.internals.handleDraftsTool('scout', { action: 'resend', draftIds: [t.a.id, t.b.id], destination: 'chan' }); + await t.tick(); + const dismissed = t.say(await t.internals.handleDraftsTool('scout', { action: 'dismiss', draftIds: [t.b.id] })); + assert.match(dismissed, new RegExp(`${t.b.id}: dismissed`)); + t.pending.shift()!.resolve({ status: 'delivered', messageId: 'm-a' }); + const result = t.say(await batch); + assert.match(result, new RegExp(`${t.b.id} was dismissed, so it can't be resent\\. It changed while this resend was waiting — not sent\\.`)); + assert.equal(t.pending.length, 0, 'B never reached publish'); + assert.equal(draftState(t.internals.proseDrafts.get('scout', t.b.id)!), 'dismissed'); + } finally { + await t.close(); + } + }); +}); From 191ab09073eaf0924b44ac8ee1bad5feabb59654 Mon Sep 17 00:00:00 2001 From: Petra-1012 Date: Wed, 7 Oct 2026 09:17:37 -0400 Subject: [PATCH 0032/1112] fix(drafts): a re-bounce keeps the whole text and its risk; notices durable first Tessa-974's follow-up review of 397be39 (room-203 #44505, #44548, #44612): 1. Re-bounce of {{unsent}} from a possibly-posted draft. 397be39 stripped the token and held only the resident's other words, which changed the message being recovered (the notice still promised it "unchanged", and the next {{unsent}} could send just the prefix). Now the whole authored attempt is held, words of the risky draft included, and it carries that draft's duplication risk: Draft.inheritedRisk {draftId, destination?, at?, reason}. A draft with inherited risk reads `unconfirmed` until a confirmed delivery (a failed attempt does not clear it), so it can be resent only with confirmDuplicate, and {{unsent}} refuses it exactly as it refuses an unconfirmed draft. riskText() words both kinds of risk for refusals, the list, read, the {{unsent}} refusal and a failed retry's receipt; the bounce notice says the new draft includes the risky draft's words. 2. A noticed record could outlive its notice: the notice is conversation state, buffered until the store syncs, while a journal record reaches the OS at once, so a kill between them left drafts marked noticed with no notice in the history and no catch-up. markNoticed now appends with afterCommittedState (sync first, then append, then durable sync), and the turn's drafts are settled in driveStream's finally after the deferred flush, marking only drafts a stored message names (metadata.draftIds on the receipt, held-draft and bounce notices). A draft whose notice never reached the history stays unnoticed for the next turn's catch-up. Tests (prose-drafts.test.ts, 25): markNoticed's call order is sync, append, sync; inherited risk stays unconfirmed through a failed attempt until a delivery; settlement marks only the draft a stored notice names; the explicit re-bounce holds "risky words and a postscript" with the inherited risk, {{unsent}} refuses it, and a confirmed resend publishes the whole text while the original stays unconfirmed. Suite (--test-concurrency=4): 1333/1333 + code-execution 36/36. Co-Authored-By: Claude Opus 5.5 --- src/framework.ts | 137 ++++++++++++++++++++++++++------------ src/prose-drafts.ts | 35 ++++++++-- test/prose-drafts.test.ts | 89 ++++++++++++++++++++++--- 3 files changed, 206 insertions(+), 55 deletions(-) diff --git a/src/framework.ts b/src/framework.ts index e927a2cbf..dc3248a3b 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -82,7 +82,7 @@ import { ProseStreamRouter } from './mcpl/prose-stream-router.js'; import { detectKnownToolWrapperProse } from './tool-wrapper-prose-guard.js'; import { InferenceRouter } from './mcpl/inference-router.js'; import { ChannelRegistry, type ChannelToolOrigin, type PublishDestination, type PublishOutcome } from './mcpl/channel-registry.js'; -import { ProseDraftStore, draftState, uncertainAttempt, type Draft, type DraftReason, type DraftSource } from './prose-drafts.js'; +import { ProseDraftStore, draftState, uncertainAttempt, type Draft, type DraftReason, type DraftSource, type InheritedRisk } from './prose-drafts.js'; import { ConversationRouter, CONVERSATION_ROUTING_DEPRECATION_NOTICE } from './mcpl/conversation-router.js'; import { INBOUND_SOURCE_KEY, type InboundAcceptanceObserver, type InboundChannelSource, type InboundLane, type InboundSource, type InboundUnscopedSource } from './mcpl/inbound-source.js'; import { safeSlice } from './safe-slice.js'; @@ -8840,12 +8840,14 @@ export class AgentFramework { */ private holdProseDrafts( agent: Agent, - texts: Array, + texts: Array, reason: DraftReason, opts: { round?: number; note?: string; notice: 'now' | 'later' }, ): Draft[] { const segments = texts - .map((t) => (typeof t === 'string' ? { text: t, note: opts.note } : { text: t.text, note: t.note ?? opts.note })) + .map((t) => (typeof t === 'string' + ? { text: t, note: opts.note } + : { text: t.text, note: t.note ?? opts.note, inheritedRisk: t.inheritedRisk })) .filter((t) => t.text.trim().length > 0); if (segments.length === 0) return []; const turn = this.logicalTurnToolCalls.get(agent)?.turnToken ?? 0; @@ -8857,9 +8859,10 @@ export class AgentFramework { } const { held, notHeld } = this.proseDrafts.hold( agent.name, - segments.map(({ text, note }, segment) => ({ + segments.map(({ text, note, inheritedRisk }, segment) => ({ text, ...(note ? { note } : {}), + ...(inheritedRisk ? { inheritedRisk } : {}), source: { branch, turn, round: opts.round ?? 0, segment } satisfies DraftSource, })), reason, @@ -8951,9 +8954,12 @@ export class AgentFramework { private draftStateText(draft: Draft): string { const state = draftState(draft); if (state === 'unconfirmed') { - const risky = uncertainAttempt(draft)!; - return `UNCONFIRMED — an attempt to ${AgentFramework.destinationText(risky.destination)} at ` + - `${this.draftTime(risky.at)} may have been posted`; + const risky = uncertainAttempt(draft); + if (risky) { + return `UNCONFIRMED — an attempt to ${AgentFramework.destinationText(risky.destination)} at ` + + `${this.draftTime(risky.at)} may have been posted`; + } + return `UNCONFIRMED — it includes the words of ${draft.inheritedRisk!.draftId}, which may already have been posted`; } if (state === 'held') { return draft.attempts.length > 0 ? 'held (every attempt failed: nothing was posted)' : 'held (not sent)'; @@ -8961,6 +8967,21 @@ export class AgentFramework { return state; } + /** Why an unconfirmed draft may already have been posted, in full. */ + private riskText(draft: Draft): string { + const risky = uncertainAttempt(draft); + if (risky) { + return `${draft.id}'s attempt to ${AgentFramework.destinationText(risky.destination)} at ` + + `${this.draftTime(risky.at)} may already have been posted: ${risky.outcome?.reason ?? 'its outcome was never recorded'}`; + } + const inherited = draft.inheritedRisk!; + const where = inherited.destination + ? `its attempt to ${AgentFramework.destinationText(inherited.destination)}` + + (inherited.at !== undefined ? ` at ${this.draftTime(inherited.at)}` : '') + : 'it'; + return `${draft.id} includes the words of ${inherited.draftId}, and ${where} may already have been posted: ${inherited.reason}`; + } + /** A delivered draft's historical receipt. */ private draftReceipt(draft: Draft): string { const delivered = draft.attempts.find((a) => a.outcome?.status === 'delivered'); @@ -8999,21 +9020,42 @@ export class AgentFramework { } } - /** Turn end: every draft held this turn has now been named (receipt or notice). */ + /** + * Logical turn end, after the turn's deferred notices are flushed: record + * as noticed exactly the drafts a stored message names (the receipt, a + * held-draft notice, a bounce notice). One whose notice never reached the + * history stays unnoticed, for the next turn's catch-up. + */ private settleTurnDrafts(agent: Agent): void { const held = this.turnDrafts.get(agent.name); this.turnDrafts.delete(agent.name); this.turnDraftFailures.delete(agent.name); this.turnProsePrivate.delete(agent.name); if (!held?.length) return; + const named = this.draftsNamedInHistory(agent, held.map((d) => d.id)); + if (named.length === 0) return; try { - this.proseDrafts.markNoticed(agent.name, held.map((d) => d.id)); + this.proseDrafts.markNoticed(agent.name, named); } catch (err) { // Unmarked drafts are named again by the next turn's catch-up. console.error('[drafts] could not record notice:', err); } } + /** Which of these drafts a stored notice or receipt names (scanning the + * recent history, where this turn's notices are). */ + private draftsNamedInHistory(agent: Agent, ids: string[]): string[] { + const wanted = new Set(ids); + const named = new Set(); + const messages = agent.getContextManager().getAllMessages(); + for (let i = messages.length - 1, scanned = 0; i >= 0 && scanned < 500 && named.size < wanted.size; i--, scanned++) { + const draftIds = (messages[i]!.metadata as { draftIds?: unknown } | undefined)?.draftIds; + if (!Array.isArray(draftIds)) continue; + for (const id of draftIds) if (typeof id === 'string' && wanted.has(id)) named.add(id); + } + return ids.filter((id) => named.has(id)); + } + /** Page size for drafts(action: "list"). */ private static readonly DRAFTS_PAGE = 10; /** Drafts with a delivery attempt in flight, `${agent}\u0000${id}`: a @@ -9093,6 +9135,7 @@ export class AgentFramework { const lines = [ `${d.id} — ${draftState(d)}; held ${this.draftTime(d.heldAt)} (${AgentFramework.DRAFT_REASON_TEXT[d.reason]})` + (d.note ? `; ${d.note}` : ''), + ...(d.inheritedRisk && draftState(d) === 'unconfirmed' && !uncertainAttempt(d) ? [` ${this.riskText(d)}`] : []), ...d.attempts.map((a) => { const o = a.outcome; const status = !o @@ -9246,14 +9289,13 @@ export class AgentFramework { } stopped = true; const after = this.proseDrafts.get(agentName, d.id); - const stillUncertain = after ? uncertainAttempt(after) : undefined; + const stillAtRisk = after !== undefined && draftState(after) === 'unconfirmed'; lines.push(outcome.status === 'unknown' ? `${d.id}: delivery to ${where} NOT confirmed — it may or may not have been posted: ${outcome.reason ?? 'no valid receipt'}. ` + `Check the channel before sending it again (that needs confirmDuplicate: true)${unrecorded}.` : `${d.id}: not sent to ${where} — ${outcome.reason ?? 'refused'}. Nothing was posted by this attempt${unrecorded}.` + - (stillUncertain - ? ` An earlier attempt (to ${AgentFramework.destinationText(stillUncertain.destination)} at ` + - `${this.draftTime(stillUncertain.at)}) may still have been posted, so the draft stays unconfirmed.` + (stillAtRisk + ? ` The draft stays unconfirmed: ${this.riskText(after!)}.` : ' The draft stays held.')); } } finally { @@ -9269,10 +9311,7 @@ export class AgentFramework { const state = draftState(d); if (state === 'dismissed') return `${d.id} was dismissed, so it can't be resent.`; if (state === 'unconfirmed' && !confirmDuplicate) { - const risky = uncertainAttempt(d)!; - return `${d.id}'s attempt to ${AgentFramework.destinationText(risky.destination)} at ${this.draftTime(risky.at)} ` + - `may already have been posted: ${risky.outcome?.reason ?? 'its outcome was never recorded'}. ` + - 'Check that channel; to send it again anyway, resend with confirmDuplicate: true.'; + return `${this.riskText(d)}. Check that channel; to send it again anyway, resend with confirmDuplicate: true.`; } return undefined; } @@ -9354,7 +9393,11 @@ export class AgentFramework { const mid = agent.getContextManager().addMessage( 'user', [{ type: 'text', text }], - { system: true, kind: 'delivery-receipt' } as MessageMetadata, + { + system: true, + kind: 'delivery-receipt', + ...(drafts.length > 0 ? { draftIds: drafts.map((d) => d.id) } : {}), + } as MessageMetadata, ); this.emitTrace({ type: 'message:added', messageId: mid, source: 'delivery-receipt' }); } catch (err) { @@ -9510,11 +9553,8 @@ export class AgentFramework { return { draft, refused: true }; } if (draftState(draft) !== 'unconfirmed') return { draft }; - const risky = uncertainAttempt(draft)!; const text = - `[prose-routing] {{unsent}} is draft ${draft.id}, and an attempt to deliver it (to ` + - `${AgentFramework.destinationText(risky.destination)} at ${this.draftTime(risky.at)}) may already have been ` + - `posted: ${risky.outcome?.reason ?? 'its outcome was never recorded'}. Nothing was sent. Check that channel; ` + + `[prose-routing] {{unsent}} is draft ${draft.id}: ${this.riskText(draft)}. Nothing was sent. Check that channel; ` + `to send it again anyway use drafts(action: "resend", draftIds: ["${draft.id}"], destination: "#channel", confirmDuplicate: true).`; try { this.addMessage('user', [{ type: 'text', text }], { system: true, kind: 'prose-bounce' }, { forAgent: agent.name }); @@ -9756,19 +9796,32 @@ export class AgentFramework { const previousAtRisk = previous !== undefined && (draftState(previous) === 'unconfirmed' || this.draftsInFlight.has(`${name}\u0000${previous.id}`)); let draft: Draft | undefined; - let keptRisky: Draft | undefined; + let inherited: InheritedRisk | undefined; if (previous && text.trim() === '{{unsent}}') { draft = previous; } else { const usesUnsent = text.includes('{{unsent}}'); - if (usesUnsent && previousAtRisk) keptRisky = previous; - const expanded = !usesUnsent - ? text - : previousAtRisk - ? text.replaceAll('{{unsent}}', '').trim() - : text.replaceAll('{{unsent}}', previous?.text ?? ''); + const expanded = usesUnsent ? text.replaceAll('{{unsent}}', previous?.text ?? '') : text; + if (usesUnsent && previous && previousAtRisk) { + // The whole authored attempt is kept, words of the risky draft + // included, and it carries that draft's duplication risk: it can be + // resent only with confirmDuplicate, like the draft it copies. + const risky = uncertainAttempt(previous); + inherited = { + draftId: previous.id, + ...(risky ? { destination: risky.destination, at: risky.at } : {}), + reason: risky + ? risky.outcome?.reason ?? 'its outcome was never recorded' + : 'it was being sent when these words were held', + }; + } // The bounce notice below names the draft, so no separate held notice. - draft = this.holdProseDrafts(agent, [expanded], 'bounced', { round: hold.round, note: reason, notice: 'later' })[0]; + draft = this.holdProseDrafts( + agent, + [{ text: expanded, ...(inherited ? { inheritedRisk: inherited } : {}) }], + 'bounced', + { round: hold.round, note: reason, notice: 'later' }, + )[0]; } const streak = (this.proseBounceStreaks.get(name) ?? 0) + 1; this.proseBounceStreaks.set(name, streak); @@ -9779,9 +9832,9 @@ export class AgentFramework { `drafts(action: "resend", draftIds: ["${draft.id}"], destination: "#channel"). ` + `"${prefix}skip_reply {{unsent}}" sets it aside.` : 'It could not be held as a draft; the words remain in your history.'; - const risky = keptRisky - ? ` {{unsent}} referred to draft ${keptRisky.id}, which may already have been posted, so it was not copied: ` + - `it is still held as ${keptRisky.id}; resend it with the drafts tool and confirmDuplicate if you mean to.` + const risky = inherited && draft + ? ` It includes the words of draft ${inherited.draftId}, which may already have been posted, so ${draft.id} ` + + 'can be resent only with the drafts tool and confirmDuplicate: true, after checking that channel.' : ''; const notice = `[prose-routing] Your text (${text.length} chars) was not delivered — ${reason}.${cand} ${kept}${risky}` + @@ -11363,10 +11416,6 @@ export class AgentFramework { if (!trigger?.suppressProse && turnProseRouting !== 'explicit') { this.appendProseDeliveryReceipt(agent); } - // The turn's drafts have now been named (receipt, held notice or - // bounce notice); a crash before this leaves them for the next - // turn's catch-up. - this.settleTurnDrafts(agent); // Explicit-prose `!` continuation: a prose segment this turn asked // to keep going (`>>#x !` / `>>private !`) — start another turn @@ -11537,10 +11586,6 @@ export class AgentFramework { await turnSpeechChain; this.appendProseDeliveryReceipt(agent); } - if (cancelKind === 'turn_ended') { - await turnSpeechChain; - this.settleTurnDrafts(agent); - } return; } } @@ -11852,6 +11897,16 @@ export class AgentFramework { } this.ackDeferredWrites(); } + + // The logical turn has ended and its notices are written: record the + // drafts they name as noticed (the store is synced first). + if (frameReachedTerminal && ownsPhysicalStream) { + try { + this.settleTurnDrafts(agent); + } catch (err) { + console.error('[drafts] turn-end settlement failed:', err); + } + } } } diff --git a/src/prose-drafts.ts b/src/prose-drafts.ts index 2ef8a76d9..b00409bc0 100644 --- a/src/prose-drafts.ts +++ b/src/prose-drafts.ts @@ -83,12 +83,29 @@ export interface DraftAttempt { outcome?: DraftAttemptOutcome; } +/** + * A draft held with words copied from another draft that may already have + * been posted (`{{unsent}}` re-bounced while that draft was unconfirmed or in + * flight). The new draft keeps the whole authored text and carries the same + * duplication risk: it reads `unconfirmed` until a confirmed delivery. + */ +export interface InheritedRisk { + /** The draft whose words were copied in. */ + draftId: string; + /** Its uncertain attempt, when there was one (absent: it was in flight). */ + destination?: PublishDestination; + at?: number; + reason: string; +} + export interface Draft { id: string; agent: string; /** The held words, byte for byte. */ text: string; reason: DraftReason; + /** Words copied in from a draft that may already have been posted. */ + inheritedRisk?: InheritedRisk; /** Short context for the resident: the bounce reason, or the competing conversations. */ note?: string; heldAt: number; @@ -113,7 +130,7 @@ export type DraftState = 'held' | 'unconfirmed' | 'delivered' | 'dismissed'; export function draftState(draft: Draft): DraftState { if (draft.attempts.some((a) => a.outcome?.status === 'delivered')) return 'delivered'; if (draft.dismissedAt !== undefined) return 'dismissed'; - return uncertainAttempt(draft) ? 'unconfirmed' : 'held'; + return uncertainAttempt(draft) || draft.inheritedRisk ? 'unconfirmed' : 'held'; } /** The most recent attempt that may have been posted (unknown or unrecorded @@ -189,9 +206,9 @@ export class ProseDraftStore { * leaves the journal unreconciled: rebuild from it before rethrowing, so * the projection never disagrees with what the journal holds. */ - private write(entry: DraftEntry, durable: boolean): void { + private write(entry: DraftEntry, durable: boolean, afterCommittedState = false): void { try { - this.journal.append(entry, { durable }); + this.journal.append(entry, { durable, afterCommittedState }); } catch (err) { if (this.journal.needsReconcile) this.reload(); throw err; @@ -225,7 +242,7 @@ export class ProseDraftStore { */ hold( agent: string, - segments: Array<{ text: string; source: DraftSource; note?: string }>, + segments: Array<{ text: string; source: DraftSource; note?: string; inheritedRisk?: InheritedRisk }>, reason: DraftReason, note?: string, ): { held: Draft[]; notHeld?: { count: number; error: string } } { @@ -239,6 +256,7 @@ export class ProseDraftStore { text: segment.text, reason, ...(segmentNote ? { note: segmentNote } : {}), + ...(segment.inheritedRisk ? { inheritedRisk: segment.inheritedRisk } : {}), heldAt: Date.now(), source: segment.source, }; @@ -269,9 +287,16 @@ export class ProseDraftStore { return this.open(agent).filter((d) => d.noticedAt === undefined).reverse(); } + /** + * Record that a notice naming these drafts is in the resident's history. + * The notice is conversation state, buffered until the store syncs, while + * this record reaches the OS at once: so the store is synced FIRST + * (afterCommittedState), and a notice that never became durable can't + * leave a record that suppresses the crash catch-up. + */ markNoticed(agent: string, ids: string[]): void { if (ids.length === 0) return; - this.write({ op: 'noticed', agent, ids, at: Date.now() }, true); + this.write({ op: 'noticed', agent, ids, at: Date.now() }, true, true); } dismiss(agent: string, id: string): void { diff --git a/test/prose-drafts.test.ts b/test/prose-drafts.test.ts index 639064f72..fb0e34f4b 100644 --- a/test/prose-drafts.test.ts +++ b/test/prose-drafts.test.ts @@ -110,6 +110,41 @@ describe('ProseDraftStore', () => { } }); + it('records a notice only after the history holding it is synced (afterCommittedState)', () => { + const { dir, store, drafts } = open(); + try { + const { held } = drafts.hold('a', [{ text: 'x', source }], 'explicit-send'); + const calls: string[] = []; + const real = { sync: store.sync.bind(store), appendJson: store.appendJson.bind(store) }; + (store as unknown as { sync: () => void }).sync = () => { calls.push('sync'); real.sync(); }; + (store as unknown as { appendJson: (t: string, p: unknown) => unknown }).appendJson = (t, p) => { calls.push(`append:${t}`); return real.appendJson(t, p); }; + drafts.markNoticed('a', [held[0]!.id]); + assert.deepEqual(calls, ['sync', `append:${PROSE_DRAFT_RECORD_TYPE}`, 'sync']); + assert.ok(drafts.get('a', held[0]!.id)!.noticedAt); + } finally { + store.close(); + rmSync(dir, { recursive: true, force: true }); + } + }); + + it('an inherited duplication risk keeps a draft unconfirmed until a confirmed delivery', () => { + const { dir, store, drafts } = open(); + try { + const risky = { draftId: 'd-zzzzz', destination: dest(), at: 1, reason: 'no valid receipt' }; + const [copy] = drafts.hold('a', [{ text: 'risky words and more', source, inheritedRisk: risky }], 'bounced').held; + assert.equal(draftState(copy!), 'unconfirmed'); + const failed = drafts.beginAttempt('a', copy!.id, dest(), 'resend', true); + drafts.recordOutcome('a', copy!.id, failed, { status: 'failed', at: 2, destination: dest() }); + assert.equal(draftState(drafts.get('a', copy!.id)!), 'unconfirmed', 'a failed attempt does not clear it'); + const ok = drafts.beginAttempt('a', copy!.id, dest(), 'resend', true); + drafts.recordOutcome('a', copy!.id, ok, { status: 'delivered', at: 3, destination: dest() }); + assert.equal(draftState(drafts.get('a', copy!.id)!), 'delivered'); + } finally { + store.close(); + rmSync(dir, { recursive: true, force: true }); + } + }); + it('{{unsent}} is the latest bounce while it is open, latest-wins', () => { const { dir, store, drafts } = open(); try { @@ -458,7 +493,7 @@ describe('held prose drafts, end to end', () => { await h.turn([createMockResponse([resend('r1')], 'tool_use'), createMockResponse([])]); h.command({ op: 'publish-mode', mode: 'not-delivered' }); await h.turn([createMockResponse([resend('r2', true)], 'tool_use'), createMockResponse([])]); - assert.match(h.toolResults()[0]!, /Nothing was posted by this attempt\. An earlier attempt \(to .*\) may still have been posted, so the draft stays unconfirmed/); + assert.match(h.toolResults()[0]!, /Nothing was posted by this attempt\. The draft stays unconfirmed: d-[a-z2-9]{5}'s attempt to .* may already have been posted: .*delivery uncertain/); assert.equal(draftState(h.store().get('scout', draft!.id)!), 'unconfirmed'); h.command({ op: 'publish-mode', mode: 'delivered' }); const before = h.publishes().length; @@ -564,7 +599,7 @@ describe('held prose drafts, end to end', () => { } }); - it('explicit mode: a bounce that may already have been posted is never copied into a fresh draft', async () => { + it('explicit mode: re-bouncing {{unsent}} of a possibly-posted draft keeps the whole text and its risk', async () => { const h = await harness({ agents: [{ name: 'scout', proseRouting: 'explicit' }] }); try { await h.turn([createMockResponse([text('risky words')])]); @@ -573,17 +608,22 @@ describe('held prose drafts, end to end', () => { await h.turn([createMockResponse([text(`>>${ROOM} {{unsent}}`)])]); assert.equal(draftState(h.store().get('scout', risky!.id)!), 'unconfirmed'); h.command({ op: 'publish-mode', mode: 'delivered' }); - // The destination does not resolve, so the envelope bounces: its other - // words are held, the possibly-posted text is not copied with them. + // The destination does not resolve, so the envelope bounces: the whole + // authored attempt is held, and it carries the copied draft's risk. await h.turn([createMockResponse([text('>>#nowhere {{unsent}} and a postscript')])]); const fresh = h.drafts().find((d) => d.id !== risky!.id)!; - assert.equal(fresh.text, 'and a postscript'); - assert.ok(h.texts().some((t) => t.includes(`{{unsent}} referred to draft ${risky!.id}, which may already have been posted, so it was not copied`))); - // And {{unsent}} itself now means the fresh draft, never the risky one. + assert.equal(fresh.text, 'risky words and a postscript', 'nothing substituted or truncated'); + assert.equal(fresh.inheritedRisk?.draftId, risky!.id); + assert.equal(draftState(fresh), 'unconfirmed'); + assert.ok(h.texts().some((t) => t.includes(`It includes the words of draft ${risky!.id}, which may already have been posted, so ${fresh.id} can be resent only with the drafts tool and confirmDuplicate: true`))); + // {{unsent}} now means the fresh draft, and it is refused unconfirmed. const before = h.publishes().length; await h.turn([createMockResponse([text(`>>${ROOM} {{unsent}}`)])]); - assert.deepEqual(h.publishes().slice(before).map((p) => p.text), ['and a postscript']); - assert.equal(draftState(h.store().get('scout', risky!.id)!), 'unconfirmed', 'still needs its own confirmation'); + assert.equal(h.publishes().length, before, 'neither draft goes out without confirmation'); + assert.ok(h.texts().some((t) => t.startsWith(`[prose-routing] {{unsent}} is draft ${fresh.id}: ${fresh.id} includes the words of ${risky!.id}`))); + await h.turn([createMockResponse([drafts('r1', { action: 'resend', draftIds: [fresh.id], destination: ROOM, confirmDuplicate: true })], 'tool_use'), createMockResponse([])]); + assert.deepEqual(h.publishes().slice(before).map((p) => p.text), ['risky words and a postscript']); + assert.equal(draftState(h.store().get('scout', risky!.id)!), 'unconfirmed', 'the original still needs its own confirmation'); } finally { await h.close(); } @@ -664,6 +704,37 @@ describe('held prose drafts, end to end', () => { // completions under the test's control. // --------------------------------------------------------------------------- +describe('draft notices and turn-end settlement', () => { + it('marks as noticed only the drafts a stored message names', async () => { + const dir = mkdtempSync(join(tmpdir(), 'prose-drafts-settle-')); + const framework = await AgentFramework.create({ + storePath: join(dir, 'store'), + membrane: new MockMembrane().asMembrane(), + agents: [{ name: 'scout', model: 'test-model', systemPrompt: 'You are scout.' }], + modules: [], + }); + try { + const internals = framework as unknown as { + proseDrafts: ProseDraftStore; + turnDrafts: Map; + settleTurnDrafts(agent: unknown): void; + }; + const agent = framework.getAgent('scout')!; + const [named, lost] = internals.proseDrafts.hold('scout', [{ text: 'named', source }, { text: 'lost', source }], 'explicit-send').held; + // Only the first draft's notice reached the history. + agent.getContextManager().addMessage('user', [{ type: 'text', text: '[drafts] …' }], { system: true, kind: 'prose-drafts', draftIds: [named!.id] } as never); + internals.turnDrafts.set('scout', [named, lost]); + internals.settleTurnDrafts(agent); + assert.ok(internals.proseDrafts.get('scout', named!.id)!.noticedAt); + assert.equal(internals.proseDrafts.get('scout', lost!.id)!.noticedAt, undefined, 'left for the catch-up'); + assert.deepEqual(internals.proseDrafts.unnoticed('scout').map((d) => d.id), [lost!.id]); + } finally { + await framework.stop(); + rmSync(dir, { recursive: true, force: true }); + } + }); +}); + describe('drafts resend ownership', () => { const setup = async () => { const dir = mkdtempSync(join(tmpdir(), 'prose-drafts-own-')); From 6c5f054e8305c7e58bf63a31c60935d375d5a2dc Mon Sep 17 00:00:00 2001 From: Petra-1012 Date: Wed, 7 Oct 2026 09:34:09 -0400 Subject: [PATCH 0033/1112] fix(drafts): owner-scoped notice identity; true risk evidence; idempotent receipts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tessa-974's review of 4f6974d (room-203 #44739, #44759; probe /tmp/tessa-974-draft-state-probe.mjs, logs 2987/96 and 2987/98): 1. Notice identity must include its resident. Draft ids are unique only per resident, and residents can share one history; settlement accepted any message's metadata.draftIds, so another resident's notice for the same id marked this resident's draft noticed. Every notice and receipt that names drafts now carries draftOwner (held-draft notice, catch-up, receipt, bounce notice), and settlement matches owner and id. 2. Inherited risk inherited again: re-bouncing a draft whose risk was itself inherited (no attempts of its own) recorded the false reason "it was being sent when these words were held" and lost the destination and time. InheritedRisk gains sourceDraftId; a chain carries the original uncertain attempt's destination, time and reason, and riskText says "d-3 includes the words of d-2, which itself includes the words of d-1, and d-1's attempt to … may already have been posted". The in-flight reason is used only when the copied draft really was in flight with no other risk. 3. A repeated resend of delivered drafts failed in destination resolution once the channel was gone. A request whose drafts are all delivered now returns their historical receipts before any destination is parsed or resolved; a mixed batch still needs a destination that resolves. Tests (prose-drafts.test.ts, 28): a same-id draft of another resident is not settled by that resident's notice; a two-step re-bounce keeps the original attempt as evidence; an all-delivered resend returns receipts after the destination stops resolving, while a mixed batch still refuses it. Suite (--test-concurrency=4): 1336/1336 + code-execution 36/36. Co-Authored-By: Claude Opus 5.5 --- src/framework.ts | 62 +++++++++++++++++++---------- src/prose-drafts.ts | 6 ++- test/prose-drafts.test.ts | 84 ++++++++++++++++++++++++++++++++++++++- 3 files changed, 130 insertions(+), 22 deletions(-) diff --git a/src/framework.ts b/src/framework.ts index dc3248a3b..c462c1c55 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -8889,7 +8889,7 @@ export class AgentFramework { this.addMessage( 'user', [{ type: 'text', text: this.draftsHeldNotice(agent.name, held, reason) }], - { system: true, kind: 'prose-drafts', draftIds: held.map((d) => d.id) } as MessageMetadata, + { system: true, kind: 'prose-drafts', draftOwner: agent.name, draftIds: held.map((d) => d.id) } as MessageMetadata, { forAgent: agent.name }, ); } catch (err) { @@ -8975,11 +8975,13 @@ export class AgentFramework { `${this.draftTime(risky.at)} may already have been posted: ${risky.outcome?.reason ?? 'its outcome was never recorded'}`; } const inherited = draft.inheritedRisk!; + const owner = inherited.sourceDraftId ?? inherited.draftId; + const via = owner !== inherited.draftId ? `, which itself includes the words of ${owner}` : ''; const where = inherited.destination - ? `its attempt to ${AgentFramework.destinationText(inherited.destination)}` + + ? `${owner}'s attempt to ${AgentFramework.destinationText(inherited.destination)}` + (inherited.at !== undefined ? ` at ${this.draftTime(inherited.at)}` : '') - : 'it'; - return `${draft.id} includes the words of ${inherited.draftId}, and ${where} may already have been posted: ${inherited.reason}`; + : owner; + return `${draft.id} includes the words of ${inherited.draftId}${via}, and ${where} may already have been posted: ${inherited.reason}`; } /** A delivered draft's historical receipt. */ @@ -9011,7 +9013,7 @@ export class AgentFramework { const mid = agent.getContextManager().addMessage( 'user', [{ type: 'text', text }], - { system: true, kind: 'prose-drafts', draftIds: pending.map((d) => d.id) } as MessageMetadata, + { system: true, kind: 'prose-drafts', draftOwner: agent.name, draftIds: pending.map((d) => d.id) } as MessageMetadata, ); this.emitTrace({ type: 'message:added', messageId: mid, source: 'prose-drafts' }); this.proseDrafts.markNoticed(agent.name, pending.map((d) => d.id)); @@ -9042,16 +9044,17 @@ export class AgentFramework { } } - /** Which of these drafts a stored notice or receipt names (scanning the - * recent history, where this turn's notices are). */ + /** Which of these drafts a stored notice or receipt OF THIS RESIDENT names + * (draft ids are unique only per resident, and residents can share one + * history), scanning the recent history, where this turn's notices are. */ private draftsNamedInHistory(agent: Agent, ids: string[]): string[] { const wanted = new Set(ids); const named = new Set(); const messages = agent.getContextManager().getAllMessages(); for (let i = messages.length - 1, scanned = 0; i >= 0 && scanned < 500 && named.size < wanted.size; i--, scanned++) { - const draftIds = (messages[i]!.metadata as { draftIds?: unknown } | undefined)?.draftIds; - if (!Array.isArray(draftIds)) continue; - for (const id of draftIds) if (typeof id === 'string' && wanted.has(id)) named.add(id); + const meta = messages[i]!.metadata as { draftIds?: unknown; draftOwner?: unknown } | undefined; + if (meta?.draftOwner !== agent.name || !Array.isArray(meta.draftIds)) continue; + for (const id of meta.draftIds) if (typeof id === 'string' && wanted.has(id)) named.add(id); } return ids.filter((id) => named.has(id)); } @@ -9183,6 +9186,13 @@ export class AgentFramework { } // resend + // Idempotence first: a request whose drafts were all delivered returns + // their historical receipts, whether or not the former destination is + // still registered or resolvable — nothing needs sending, so nothing + // needs authorizing. + if (drafts.every((d) => draftState(d) === 'delivered')) { + return ok(drafts.map((d) => `${this.draftReceipt(d)} — not sent again.`).join('\n')); + } const registry = this.channelRegistry; if (!registry) return refuse('No channels are configured, so there is nowhere to resend to.'); const home = this.conversationAgentHomes.get(agentName); @@ -9396,7 +9406,7 @@ export class AgentFramework { { system: true, kind: 'delivery-receipt', - ...(drafts.length > 0 ? { draftIds: drafts.map((d) => d.id) } : {}), + ...(drafts.length > 0 ? { draftOwner: agent.name, draftIds: drafts.map((d) => d.id) } : {}), } as MessageMetadata, ); this.emitTrace({ type: 'message:added', messageId: mid, source: 'delivery-receipt' }); @@ -9805,15 +9815,27 @@ export class AgentFramework { if (usesUnsent && previous && previousAtRisk) { // The whole authored attempt is kept, words of the risky draft // included, and it carries that draft's duplication risk: it can be - // resent only with confirmDuplicate, like the draft it copies. + // resent only with confirmDuplicate, like the draft it copies. The + // evidence is the risk's own: the copied draft's uncertain attempt, + // else the risk that draft itself inherited, else (truly) that it + // was in flight when copied. const risky = uncertainAttempt(previous); - inherited = { - draftId: previous.id, - ...(risky ? { destination: risky.destination, at: risky.at } : {}), - reason: risky - ? risky.outcome?.reason ?? 'its outcome was never recorded' - : 'it was being sent when these words were held', - }; + inherited = risky + ? { + draftId: previous.id, + destination: risky.destination, + at: risky.at, + reason: risky.outcome?.reason ?? 'its outcome was never recorded', + } + : previous.inheritedRisk + ? { + draftId: previous.id, + sourceDraftId: previous.inheritedRisk.sourceDraftId ?? previous.inheritedRisk.draftId, + ...(previous.inheritedRisk.destination ? { destination: previous.inheritedRisk.destination } : {}), + ...(previous.inheritedRisk.at !== undefined ? { at: previous.inheritedRisk.at } : {}), + reason: previous.inheritedRisk.reason, + } + : { draftId: previous.id, reason: 'it was being sent when these words were held' }; } // The bounce notice below names the draft, so no separate held notice. draft = this.holdProseDrafts( @@ -9850,7 +9872,7 @@ export class AgentFramework { const id = this.addMessage( 'user', [{ type: 'text', text: notice }], - { system: true, kind: 'prose-bounce', ...(draft ? { draftIds: [draft.id] } : {}) } as MessageMetadata, + { system: true, kind: 'prose-bounce', ...(draft ? { draftOwner: name, draftIds: [draft.id] } : {}) } as MessageMetadata, { forAgent: name }, ); if (id) this.emitTrace({ type: 'message:added', messageId: id, source: 'prose-bounce' }); diff --git a/src/prose-drafts.ts b/src/prose-drafts.ts index b00409bc0..2735fd689 100644 --- a/src/prose-drafts.ts +++ b/src/prose-drafts.ts @@ -92,7 +92,11 @@ export interface DraftAttempt { export interface InheritedRisk { /** The draft whose words were copied in. */ draftId: string; - /** Its uncertain attempt, when there was one (absent: it was in flight). */ + /** The draft whose attempt is the uncertain one, when the copied draft + * itself carried inherited risk (a chain of re-bounces): the evidence is + * that original attempt's, never re-invented. */ + sourceDraftId?: string; + /** That uncertain attempt, when there was one (absent: it was in flight). */ destination?: PublishDestination; at?: number; reason: string; diff --git a/test/prose-drafts.test.ts b/test/prose-drafts.test.ts index fb0e34f4b..663706489 100644 --- a/test/prose-drafts.test.ts +++ b/test/prose-drafts.test.ts @@ -722,7 +722,7 @@ describe('draft notices and turn-end settlement', () => { const agent = framework.getAgent('scout')!; const [named, lost] = internals.proseDrafts.hold('scout', [{ text: 'named', source }, { text: 'lost', source }], 'explicit-send').held; // Only the first draft's notice reached the history. - agent.getContextManager().addMessage('user', [{ type: 'text', text: '[drafts] …' }], { system: true, kind: 'prose-drafts', draftIds: [named!.id] } as never); + agent.getContextManager().addMessage('user', [{ type: 'text', text: '[drafts] …' }], { system: true, kind: 'prose-drafts', draftOwner: 'scout', draftIds: [named!.id] } as never); internals.turnDrafts.set('scout', [named, lost]); internals.settleTurnDrafts(agent); assert.ok(internals.proseDrafts.get('scout', named!.id)!.noticedAt); @@ -735,6 +735,66 @@ describe('draft notices and turn-end settlement', () => { }); }); +describe('draft identity and risk evidence', () => { + const setupFramework = async () => { + const dir = mkdtempSync(join(tmpdir(), 'prose-drafts-ident-')); + const framework = await AgentFramework.create({ + storePath: join(dir, 'store'), + membrane: new MockMembrane().asMembrane(), + agents: [{ name: 'scout', model: 'test-model', systemPrompt: 'scout' }, { name: 'other', model: 'test-model', systemPrompt: 'other' }], + modules: [], + }); + const internals = framework as unknown as { + proseDrafts: ProseDraftStore; + turnDrafts: Map; + settleTurnDrafts(agent: unknown): void; + bounceProse(agent: unknown, text: string, reason: string): void; + riskText(draft: unknown): string; + }; + return { dir, framework, internals, close: async () => { await framework.stop(); rmSync(dir, { recursive: true, force: true }); } }; + }; + + it('a notice names drafts of its own resident only: the same id under another resident is not settled by it', async () => { + const t = await setupFramework(); + try { + (t.internals.proseDrafts as unknown as { newId: (agent: string) => string }).newId = () => 'd-same2'; + const [mine] = t.internals.proseDrafts.hold('scout', [{ text: 'scout words', source }], 'explicit-send').held; + t.internals.proseDrafts.hold('other', [{ text: 'other words', source }], 'explicit-send'); + // Only the OTHER resident's notice for "d-same2" is in the shared history. + t.framework.getAgent('other')!.getContextManager().addMessage('user', [{ type: 'text', text: '[drafts] other: …' }], + { system: true, kind: 'prose-drafts', draftOwner: 'other', draftIds: ['d-same2'] } as never); + t.internals.turnDrafts.set('scout', [mine]); + t.internals.settleTurnDrafts(t.framework.getAgent('scout')!); + assert.equal(t.internals.proseDrafts.get('scout', 'd-same2')!.noticedAt, undefined); + } finally { + await t.close(); + } + }); + + it('a re-bounce of inherited risk keeps the original attempt as its evidence', async () => { + const t = await setupFramework(); + try { + const agent = t.framework.getAgent('scout')!; + const [original] = t.internals.proseDrafts.hold('scout', [{ text: 'first', source }], 'bounced').held; + t.internals.proseDrafts.beginAttempt('scout', original!.id, dest(), 'unsent-token', false); // never resolved + t.internals.bounceProse(agent, '{{unsent}} second', 'no such channel'); + const copy = t.internals.proseDrafts.open('scout').find((d) => d.text === 'first second')!; + assert.equal(copy.inheritedRisk?.draftId, original!.id); + t.internals.bounceProse(agent, '{{unsent}} third', 'no such channel'); + const again = t.internals.proseDrafts.open('scout').find((d) => d.text === 'first second third')!; + assert.equal(again.inheritedRisk?.draftId, copy.id); + assert.equal(again.inheritedRisk?.sourceDraftId, original!.id); + assert.deepEqual(again.inheritedRisk?.destination, dest()); + assert.equal(again.inheritedRisk?.reason, 'its outcome was never recorded'); + const text = t.internals.riskText(again); + assert.match(text, new RegExp(`${again.id} includes the words of ${copy.id}, which itself includes the words of ${original!.id}, and ${original!.id}'s attempt to #room \\(Guild One\\)`)); + assert.doesNotMatch(text, /being sent/); + } finally { + await t.close(); + } + }); +}); + describe('drafts resend ownership', () => { const setup = async () => { const dir = mkdtempSync(join(tmpdir(), 'prose-drafts-own-')); @@ -772,6 +832,28 @@ describe('drafts resend ownership', () => { }; }; + it('an all-delivered resend returns its receipts even when the destination no longer resolves', async () => { + const t = await setup(); + try { + const sending = t.internals.handleDraftsTool('scout', { action: 'resend', draftIds: [t.a.id], destination: 'chan' }); + await t.tick(); + t.pending.shift()!.resolve({ status: 'delivered', messageId: 'm-a' }); + assert.match(t.say(await sending), /delivered to \(chan\), message m-a/); + // The channel is gone: destination resolution would now fail. + const registry = (t.framework as unknown as { channelRegistry: Record }).channelRegistry; + registry.resolveProseTarget = () => ({ error: 'no channel matches' }); + registry.resolveDestination = () => ({ error: 'no registered channel' }); + const again = t.say(await t.internals.handleDraftsTool('scout', { action: 'resend', draftIds: [t.a.id], destination: '#gone' })); + assert.match(again, new RegExp(`${t.a.id}: delivered — confirmed at .* message m-a .*not sent again`)); + assert.equal(t.pending.length, 0); + // A mixed batch still needs a destination that resolves. + const mixed = t.say(await t.internals.handleDraftsTool('scout', { action: 'resend', draftIds: [t.a.id, t.b.id], destination: '#gone' })); + assert.match(mixed, /Destination "#gone" did not resolve/); + } finally { + await t.close(); + } + }); + it('a batch owns its drafts: a concurrent resend of a queued draft is refused, and nothing is sent twice', async () => { const t = await setup(); try { From 799a16b88535bdf176455b7a387f3353f69f0d5b Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Wed, 7 Oct 2026 09:37:31 -0400 Subject: [PATCH 0034/1112] refactor(awareness): keep the journal in the store (RecordJournal); fix June's review findings Storage (June #44070): the awareness journal moves from an fsynced JSONL file to typed Chronicle records on the shared RecordJournal (Basil, 2153e3d, merged as a shared base; Hugo's helper approval room-220 #41860). One entry = one group of records that apply together (replaces txn/commit records, the parser and torn-tail repair). The outbox keeps reduced state in memory, checkpoints every 256 entries (constructor seam for tests), reloads after an ambiguous append (RecordJournal unreconciled guard). Constructor is now (store, {legacyPath?, checkpointEvery?}); discordAwarenessOutboxPath names only a pre-journal JSON ledger to import; every framework has a journal, so the "marks chosen but no journal" receipt branch is gone. Crash ordering this fixes: Chronicle buffers branch-state commits until store.sync(), so a JSONL activation could outlive the switch it asserted. Activation, discard and suppression-complete now append with afterCommittedState (sync first) + durable; batch prepare, dispatch write-ahead, startup reconciliation and operator acts are durable; outcomes are not (a lost outcome reads as unknown and is retried). Review findings, each with a regression: - #44236 cancel vs a running drain: the drain no longer iterates a snapshot. claimDispatch(serverId, skip) recomputes due dispatches and writes the chosen one ahead in one synchronous step; recordDispatching narrows a stale dispatch to still-due ops and returns [] when none remain. - #44283/#44349/#44458 legacy uncertainty: v2 import no longer synthesizes attempts or presence. Each entry with attempts becomes legacy-evidence facts (attempts, lastAction, lastStatus, bounded lastError, lastErrorKind pre-write-refusal|discord-answer|other, outcomesUnrecorded = earlier attempts + last unless applied/pre-write/Discord answer). Retract now queues a removal for every selected key whatever history says (idempotent removal covers absence), supersedes due adds, and discloses native unresolved requests (per dispatch) and keysWithLegacyUncertainty; cancel reports legacyOutcomesUnrecorded; the view reports legacy facts. - #44493 release vs unfinished body: only activation (surgery or its resume) certifies a suppression body; release does not. The resume selector takes every non-discarded explicit suppression with an incomplete body on its exact target, whatever its marks state; prepared ones are crash-completed, others get suppression-complete only. - #44517 cancel history: cancel receipt gains unresolvedAttempts across all attempts (a timeout then a confirmed retry, or then a permanent failure). - #44551 offline CLI separates body from marks (settleActivation: activate, else retire, else unresolved; result.markers); barrier on every record that certifies or retires a body; turn-based host/command undo honours marks via an addressing-metadata diff of the agent's context. - #44585 a newer opposite request supersedes due retries (retryable failures and pre-write refusals included) of the old one, symmetric; history kept; the in-flight request still settles first. - #44609 failure paths restore the source first and retire the prepared batch only when the store is confirmed back on it; otherwise it stays as the unfinished body's record and the error says so; a failed retire is reported and alerted, never masking the surgery's error. - #44638 cancel(target) also takes a retract request id (request-cancelled); the view lists retract groups (kind 'retract') so the id survives restart. host/command marks verb param is now `target`. - Kit/Agnes-1009 (room-203 #44126): host/command hide takes reserveStoreForSurgery around its removal and marks (agent-busy otherwise). Tests: discord-awareness-outbox 24, surgery-marks-receipt 14 (real store + fake Discord server: cancel during delivery sends exactly one request; restore-unconfirmed keeps the batch; failed retire reported; prepared -> held -> release -> target-startup resumes the body), delivery 5, offline 7 (body stands when activation fails), operator-surgery 18: 68/68. --- README.md | 87 +- changelog.d/surgery-marks-controls.added.md | 19 +- changelog.d/surgery-marks-one-shot.changed.md | 10 +- changelog.d/surgery-marks-opt-in.breaking.md | 37 +- src/framework.ts | 332 +++++--- src/index.ts | 4 + src/operator-log.ts | 8 +- src/recovery/discord-awareness-outbox.ts | 784 +++++++++++------ src/recovery/offline-branch.ts | 41 +- src/recovery/recover-cli.ts | 43 +- test/discord-awareness-delivery.test.ts | 67 +- test/discord-awareness-outbox.test.ts | 792 +++++++++--------- test/offline-recovery-branch.test.ts | 79 +- test/surgery-marks-receipt.test.ts | 113 ++- 14 files changed, 1496 insertions(+), 920 deletions(-) diff --git a/README.md b/README.md index d3d9d359c..1cba189ae 100644 --- a/README.md +++ b/README.md @@ -169,9 +169,11 @@ messages that mentioned or replied to the bot, and DMs) or `--marks all` (every removed Discord message). The output counts what will be marked and what stays unmarked, and `--dry-run` shows exactly which addresses each choice covers. Only `{serverId, channelId, messageId}` metadata is written, to the -awareness journal `/recovery/discord-awareness-journal.jsonl`, and the -marks are delivered when the host and that agent's `discord-mcpl` bot next -connect (see [Live surgery and Discord awareness marks](#live-surgery-and-discord-awareness-marks)). +awareness journal kept in the store itself, and the marks are delivered when +the host and that agent's `discord-mcpl` bot next connect (see +[Live surgery and Discord awareness marks](#live-surgery-and-discord-awareness-marks)). +If the branch is made but the marks can't be recorded, the output's `markers` +says so and the branch stands. Portal and other non-Discord records are ignored. When the safe point is an assistant/tool-side record rather than a Discord @@ -216,7 +218,7 @@ normal agent host must be stopped while this command has the Chronicle store open. With the host stopped, the same command inspects and controls the awareness -journal: `--awareness list`, `--awareness cancel `, +journal: `--awareness list`, `--awareness cancel `, `--awareness retract ` and `--awareness release ` behave as the live controls described below. @@ -345,15 +347,13 @@ const result = await framework.rollbackToMessage('cairn', { never marked. The choice is recorded in the operator log. - **The surgery returns once marks are scheduled**, never after Discord has accepted them. `result.markers` says whether they were scheduled (`queued` - with a count and batch id), not chosen or not in scope (`none`), chosen but - never to be delivered (`not-scheduled`: the framework has no awareness - journal, having neither `storePath` nor `discordAwarenessOutboxPath`, or - recording the batch failed after the body change and the batch was retired; - `error` says which), or - `unresolved` (neither activation nor retirement could be recorded; it names - a batch that may still be delivered). Every receipt also counts the removed - messages left unmarked. Marker bookkeeping never fails or undoes an applied - rollback or suppression. + with a count and batch id), not chosen or not in scope (`none`), not to be + delivered because recording the batch failed after the body change and it + was retired (`not-scheduled`), or `unresolved` (neither activation nor + retirement could be recorded; it names a batch that may still be + delivered). Every receipt also counts the removed messages left unmarked. + Marker bookkeeping never fails or undoes an applied rollback or + suppression. - **Delivery runs in the background** and never holds MCPL traffic or turns. A route that is not connected keeps its work queued until it connects. Each reaction call has a mandatory deadline independent of @@ -362,28 +362,45 @@ const result = await framework.rollbackToMessage('cairn', { (offline: `--emoji`). - **Marks are one-shot.** Switching branches, undo/redo and restarts never add or remove a mark. The journal is an append-only history of requests and - their attempts, written ahead of each dispatch. A request written and never - answered is recorded as `unknown` and is never resolved by a later - confirmation of a different attempt. A batch whose surgery was interrupted - before its branch switch was recorded is held at startup until an operator - releases it. -- **Operator controls:** `listDiscordAwareness()`; - `cancelDiscordAwareness(batch)` stops a batch's further sends and retries - and never removes a reaction (its receipt counts requests in flight or - unknown, which may still land); `retractDiscordAwareness(batch | 'all')` - removes this bot's reaction, through each address's configured MCPL route, - from a batch's messages or from every message an add was ever sent for, and - its receipt discloses earlier adds whose outcome is unknown, any of which - may land after the removal; `releaseDiscordAwareness(batch)` queues a held - batch. Each is recorded in the operator log. -- **Over `host/command`** (servers granted `allowHostCommands`): message-granular - `undo` takes `marks: 'none' | 'addressed' | 'all'` (default `none`; any - other value is refused), and the `marks` verb takes `action: 'list' | - 'cancel' | 'retract' | 'release'` with a `batchId` (`all` for retract). - -A pre-journal awareness ledger (`discord-awareness-outbox.json`) is imported -once on first use: its recorded outcomes are kept, its undelivered work is -held for an explicit release, and the old file is renamed `.migrated-v2`. + their attempts, kept as typed records in the Chronicle store: it survives + rollbacks, branch deletion and a killed process, and lives with its store. + Each request is written ahead of its dispatch, and a dispatch is admitted + from the journal as it is at that moment, so a cancel takes effect even + while an earlier request is on the wire. A request written and never + answered is recorded as `unknown`, and a later confirmation of a different + attempt never resolves it. A record that certifies a body change (an + activation, a completed suppression, a retirement) is written only after + that change is synced. A batch whose surgery was interrupted before its + branch switch was recorded is held at startup until an operator releases + it; an interrupted suppression's redactions are resumed whenever its branch + is active at startup, whatever its marks' state. +- **Operator controls,** each recorded in the operator log: + - `listDiscordAwareness()` lists batches and retract requests; + - `cancelDiscordAwareness(batch | retractRequest)` stops all further sends + and retries of a batch's marks or a retract's removals, and never removes + or undoes anything. Its receipt counts requests in flight or unknown, + including earlier attempts later answered, any of which may still land; + - `retractDiscordAwareness(batch | 'all')` queues removal of this bot's + reaction, through each address's configured MCPL route, for every message + of the batch (or every message any batch or imported ledger recorded an + add for), whatever history says: removing an absent reaction does + nothing. It supersedes adds not yet sent, and its receipt discloses + earlier requests whose outcome is unknown and imported history that + leaves outcomes unrecorded; + - `releaseDiscordAwareness(batch)` queues a held batch. +- **Over `host/command`** (servers granted `allowHostCommands`): `undo` (by + messages or by turns) and `hide` take `marks: 'none' | 'addressed' | 'all'` + (default `none`; any other value is refused) and return `markers`. `hide` + holds the store like every surgery. The `marks` verb takes `action: 'list' + | 'cancel' | 'retract' | 'release'` with a `target` (a batch id; for cancel + also a retract request id; for retract also `all`). + +A pre-journal awareness ledger (`discord-awareness-outbox.json` under the +store, or `discordAwarenessOutboxPath`) is imported once on first use and the +file renamed `.migrated-v2`. What it recorded about each message is kept as +evidence (attempt count, last action, status and error, and how many attempt +outcomes it leaves unrecorded), never as a claim about what is on Discord. +Its undelivered work is held for an explicit release. ## Observability diff --git a/changelog.d/surgery-marks-controls.added.md b/changelog.d/surgery-marks-controls.added.md index 2746480cc..28be5e31f 100644 --- a/changelog.d/surgery-marks-controls.added.md +++ b/changelog.d/surgery-marks-controls.added.md @@ -2,17 +2,20 @@ - `previewSurgeryMarks(agent, { rollbackTo } | { suppress })` shows what a surgery would remove and which messages each scope would mark, per channel. - - `listDiscordAwareness()` lists the journal's batches. - - `cancelDiscordAwareness(batch)` stops a batch's unsent requests and - retries, and never removes a reaction. Its receipt counts requests in - flight or unknown, which may still land. + - `listDiscordAwareness()` lists the journal's batches and retract + requests. + - `cancelDiscordAwareness(batch | retractRequest)` stops all further sends + and retries of a batch's marks, or of a retract's removals, and never + removes or undoes anything. Its receipt counts requests in flight or + unknown, including earlier attempts later answered, which may still land. - `retractDiscordAwareness(batch | 'all')` queues removal of this bot's - reaction through each message's configured MCPL route. Its receipt - discloses earlier adds whose outcome is unknown, which may land after the - removal. + reaction, through each message's configured MCPL route, for every + selected message, whatever history says. It supersedes adds not yet + sent, and its receipt discloses requests whose outcome is unknown and + imported history that leaves outcomes unrecorded. - `releaseDiscordAwareness(batch)` queues a held batch. Each control is recorded in the operator log. The same controls are available as the `host/command` `marks` verb (`action: list | cancel | - retract | release`, `batchId`) and, with the host stopped, as + retract | release`, `target`) and, with the host stopped, as `agent-framework-recover --awareness list|cancel|retract|release`. diff --git a/changelog.d/surgery-marks-one-shot.changed.md b/changelog.d/surgery-marks-one-shot.changed.md index 1a62549b3..043a702d6 100644 --- a/changelog.d/surgery-marks-one-shot.changed.md +++ b/changelog.d/surgery-marks-one-shot.changed.md @@ -1,11 +1,17 @@ - Awareness marks are one-shot. Switching branches, undo/redo and restarts no longer add or remove marks; previously a mark followed the branch, removed when the source branch became active and re-added when the recovery branch - did. Each request is written ahead of its dispatch. A request that went out + did. Each request is written ahead of its dispatch, and a dispatch is + admitted from the journal as it is at that moment. A request that went out and got no answer is recorded as `unknown`, and a later confirmation of a different attempt never resolves it. A batch whose surgery was interrupted before its branch switch was recorded is held at startup until an operator - releases it. + releases it. An interrupted suppression's redactions are resumed whenever + its branch is active at startup, independent of what happened to its marks. +- A journal record that certifies a body change (a marks activation, a + completed suppression, a retired batch) is now written only after that + change is synced to the store. Previously a crash could keep marks active + for a rollback the store had lost. - Awareness delivery no longer holds MCPL traffic. Previously every MCPL data plane waited at startup, at reconnect, at a tools list change and after each surgery until every queued mark had been attempted, which at Discord's pace diff --git a/changelog.d/surgery-marks-opt-in.breaking.md b/changelog.d/surgery-marks-opt-in.breaking.md index 0bcc98841..0b403f5ff 100644 --- a/changelog.d/surgery-marks-opt-in.breaking.md +++ b/changelog.d/surgery-marks-opt-in.breaking.md @@ -8,28 +8,33 @@ - **Who needs to act:** a surface that should keep offering marks passes the operator's choice: `marks: { scope: 'addressed' | 'all', refs? }` on `rollbackToMessage`/`suppressMessages`, `marks: 'addressed' | 'all'` on - `host/command` `undo` and `hide`, and `--marks addressed|all` on the CLI. - `addressed` covers messages tagged `chat:addressed` (mentions, replies to - the bot, DMs). Show `previewSurgeryMarks()` before the choice, and pass a - scope's `refs` back to bind the choice to exactly the previewed messages. - Any other `host/command` `marks` value is refused rather than guessed. + `host/command` `undo` (by messages or by turns) and `hide`, and + `--marks addressed|all` on the CLI. `addressed` covers messages tagged + `chat:addressed` (mentions, replies to the bot, DMs). Show + `previewSurgeryMarks()` before the choice, and pass a scope's `refs` back + to bind the choice to exactly the previewed messages. Any other + `host/command` `marks` value is refused rather than guessed. A surface + that reacted to `hide`'s `hiddenRefs` itself should stop when the result + carries `markers`: the framework marked through its journal. - **Unchanged:** the body change itself (fork, switch, redaction), the receipt's `status` values, the emoji (`discordAwarenessEmoji`) and the reaction deadline (`discordAwarenessDeadlineMs`). Marks already on Discord are left alone; nothing is removed automatically. - Receipts (`markers`) now also carry `scope`, `unmarked` and `notRemoved`. - Marks that were chosen but can't be recorded, because there is no journal - (no `storePath` or `discordAwarenessOutboxPath`), are reported as - `not-scheduled` rather than `none`. + - `host/command` `hide` now holds the store like every surgery, so it is + refused (`agent-busy`) while any agent sharing the store is mid-turn. - No sibling version is required: a host that ignores the new option simply places no marks. The connectome-host and discord-mcpl companions add the choice to their surfaces. - **Code using `DiscordAwarenessOutbox` directly:** the awareness ledger is now - an append-only journal of requests and attempts, - `/recovery/discord-awareness-journal.jsonl`. `pending()` and - `reconcileForBranch()` are gone. Use `pendingDispatches()`, - `recordDispatching()`/`recordOutcome()`, `view()`, and - `cancel()`/`retract()`/`release()`. A previous `discord-awareness-outbox.json` - is imported once on first use: its recorded outcomes are kept, its undelivered - work is held for an explicit release, and the file is renamed - `.migrated-v2`. + an append-only journal of typed records in the Chronicle store + (`af:discord-awareness`, on the shared `RecordJournal`), so the constructor + takes the store: `new DiscordAwarenessOutbox(store, { legacyPath? })`. + `pending()` and `reconcileForBranch()` are gone. Use `claimDispatch()` (or + `pendingDispatches()` and `recordDispatching()`), `recordOutcome()`, + `view()`, `cancel()`, `retract()`, `release()` and `settleActivation()`. + `discordAwarenessOutboxPath` now only names a previous JSON ledger to + import. Such a file (by default `/recovery/discord-awareness-outbox.json`) + is imported once on first use and renamed `.migrated-v2`. What it recorded + is kept as evidence, never as synthesized attempts, and its undelivered + work is held for an explicit release. diff --git a/src/framework.ts b/src/framework.ts index a40b7f56d..fb58eea51 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -408,8 +408,9 @@ import { extractDiscordAwarenessRefs, selectDiscordAwarenessRefs, boundDiscordAwarenessText, + isPermanentDiscordReactionFailure, type DiscordAwarenessBatch, - type DiscordAwarenessBatchView, + type DiscordAwarenessView, type DiscordAwarenessCancelReceipt, type DiscordAwarenessMarks, type DiscordAwarenessReleaseReceipt, @@ -754,10 +755,6 @@ function hostCommandRequester(serverId: string, params: HostCommandParams): Oper }; } -function isPermanentDiscordReactionFailure(message: string): boolean { - return /unknown message|unknown channel|missing access|missing permissions|missing permission|cannot access|channel .* not found|message .* not found/i - .test(message); -} /** * Default error policy - retry with exponential backoff. @@ -904,8 +901,9 @@ interface HostCommandParams { marks?: string; /** For the `marks` command: `list`, `cancel`, `retract` or `release`. */ action?: string; - /** For the `marks` command: the batch to act on (`all` for retract). */ - batchId?: string; + /** For the `marks` command: a batch id; for cancel also a retract request + * id; for retract also `all`. */ + target?: string; /** For the `hide` command: Discord message id of the (first) message to * remove. With `toMessageId`, removes the inclusive range between them. */ fromMessageId?: string; @@ -1550,11 +1548,12 @@ export class AgentFramework { } } - const discordAwarenessOutboxPath = config.discordAwarenessOutboxPath - ?? (config.storePath ? defaultDiscordAwarenessOutboxPath(config.storePath) : undefined); - const discordAwarenessOutbox = discordAwarenessOutboxPath - ? new DiscordAwarenessOutbox(discordAwarenessOutboxPath) - : null; + // The awareness journal lives in the store (typed records); a + // pre-journal JSON ledger, if one is found, is imported once. + const discordAwarenessOutbox = new DiscordAwarenessOutbox(store, { + legacyPath: config.discordAwarenessOutboxPath + ?? (config.storePath ? defaultDiscordAwarenessOutboxPath(config.storePath) : undefined), + }); const operatorLogPath = config.operatorLogPath === false ? undefined @@ -5088,7 +5087,7 @@ export class AgentFramework { /** For the `marks` command: the journal's batches (`list`) or the * action's receipt. */ awareness?: - | DiscordAwarenessBatchView[] + | DiscordAwarenessView[] | DiscordAwarenessCancelReceipt | DiscordAwarenessRetractReceipt | DiscordAwarenessReleaseReceipt; @@ -5135,20 +5134,20 @@ export class AgentFramework { // The awareness journal's operator controls, for surfaces that act // through host/command (the web UI calls the framework methods). const by = { requester: hostCommandRequester(serverId, params) }; - const batchId = params.batchId; + const target = params.target; try { switch (params.action) { case 'list': return { ok: true, awareness: this.listDiscordAwareness() }; case 'cancel': - if (!batchId) return { ok: false, error: 'marks cancel needs batchId' }; - return { ok: true, awareness: this.cancelDiscordAwareness(batchId, by) }; + if (!target) return { ok: false, error: 'marks cancel needs target (a batch or retract request id)' }; + return { ok: true, awareness: this.cancelDiscordAwareness(target, by) }; case 'retract': - if (!batchId) return { ok: false, error: 'marks retract needs batchId (or all)' }; - return { ok: true, awareness: this.retractDiscordAwareness(batchId, by) }; + if (!target) return { ok: false, error: 'marks retract needs target (a batch id, or all)' }; + return { ok: true, awareness: this.retractDiscordAwareness(target, by) }; case 'release': - if (!batchId) return { ok: false, error: 'marks release needs batchId' }; - return { ok: true, awareness: this.releaseDiscordAwareness(batchId, by) }; + if (!target) return { ok: false, error: 'marks release needs target (a held batch id)' }; + return { ok: true, awareness: this.releaseDiscordAwareness(target, by) }; default: return { ok: false, error: 'marks needs action list, cancel, retract or release' }; } @@ -5287,6 +5286,18 @@ export class AgentFramework { }; } + // Like every surgery, a redaction holds the whole store: no turn (of + // any agent sharing it) may start while messages are being removed. + let release: () => void; + try { + release = this.reserveStoreForSurgery('hide', agentName); + } catch (error) { + return { + ok: false, + error: error instanceof Error ? error.message : String(error), + ...(error instanceof OperatorActionError ? { code: error.code } : {}), + }; + } try { if (params.toMessageId) { const toIdx = byDiscordId(params.toMessageId); @@ -5297,7 +5308,7 @@ export class AgentFramework { const refs = refsIn(lo, hi); const rangeIds = all.slice(lo, hi + 1).map((m) => String(m.id)); cm.removeMessages(all[lo].id, all[hi].id); - const markers = this.scheduleHideMarks(agentName, serverId, all.slice(lo, hi + 1), hideMarks); + const markers = this.scheduleAppliedMarks('hide', agentName, serverId, all.slice(lo, hi + 1), hideMarks); console.error( `[host-command] hide agent=${agentName} range removed=${hi - lo + 1} ` + `(${params.fromMessageId}..${params.toMessageId}) by=${params.requesterName ?? params.requesterId ?? 'unknown'} (server=${serverId})`, @@ -5331,7 +5342,7 @@ export class AgentFramework { const refs = refsIn(fromIdx, fromIdx); const hiddenId = String(all[fromIdx].id); cm.removeMessage(all[fromIdx].id); - const markers = this.scheduleHideMarks(agentName, serverId, [all[fromIdx]], hideMarks); + const markers = this.scheduleAppliedMarks('hide', agentName, serverId, [all[fromIdx]], hideMarks); console.error( `[host-command] hide agent=${agentName} removed=1 (${params.fromMessageId}) ` + `by=${params.requesterName ?? params.requesterId ?? 'unknown'} (server=${serverId})`, @@ -5352,6 +5363,8 @@ export class AgentFramework { }; } catch (error) { return { ok: false, error: error instanceof Error ? error.message : String(error) }; + } finally { + release(); } } @@ -5402,6 +5415,12 @@ export class AgentFramework { } const requested = Math.max(1, Math.min(20, Math.floor(params.turns ?? 1))); + const turnMarks: DiscordAwarenessMarks = params.marks === 'addressed' || params.marks === 'all' + ? { scope: params.marks } + : 'none'; + // What the undone turns remove from this agent's context is whatever is + // in it now and not after: addressing metadata only, never content. + const before = this.addressingSnapshot(agentName); let undone = 0; try { for (let i = 0; i < requested; i++) { @@ -5423,12 +5442,48 @@ export class AgentFramework { ); if (undone === 0) { - return { ok: true, undone: 0, requested, lastVisible: null }; + return { + ok: true, + undone: 0, + requested, + markers: { scope: marksScope(turnMarks) as SurgeryMarkerReceipt['scope'], unmarked: 0, notRemoved: 0, status: 'none', queued: 0 }, + lastVisible: null, + }; } - // Turn undo is a branch move: awareness marks are one-shot and never - // follow branch state, so there is nothing to reconcile here. - return { ok: true, undone, requested, lastVisible: await this.lastVisiblePreview(agentName) }; + // Marks are a one-shot choice about the messages these turns removed; + // the branch move itself never adds or removes any. + const after = new Set(this.addressingSnapshot(agentName).keys()); + const removed = [...before].filter(([id]) => !after.has(id)).map(([, carrier]) => carrier); + const markers = this.scheduleAppliedMarks('undo', agentName, serverId, removed, turnMarks); + return { ok: true, undone, requested, markers, lastVisible: await this.lastVisiblePreview(agentName) }; + } + + /** + * The addressing metadata of every message in an agent's context, by + * message id, read in bounded windows without resolving blobs: the Discord + * address and MCPL tags only, never content. + */ + private addressingSnapshot(agentName: string): Map }> { + const snapshot = new Map }>(); + const cm = this.agents.get(agentName)?.getContextManager(); + if (!cm) return snapshot; + const total = cm.getMessageCount(); + for (let offset = 0; offset < total; offset += 500) { + const { messages } = cm.getMessageWindow(offset, Math.min(500, total - offset), { resolveBlobs: false }); + for (const message of messages) { + const metadata = (message.metadata ?? {}) as Record; + snapshot.set(String(message.id), { + metadata: { + serverId: metadata.serverId, + channelId: metadata.channelId, + messageId: metadata.messageId, + tags: metadata.tags, + }, + }); + } + } + return snapshot; } /** @@ -5990,6 +6045,37 @@ export class AgentFramework { }; } + /** + * After a failed surgery has put the source back: retire its prepared + * awareness batch, but only when the store is confirmed on the source. If + * it isn't, the batch stays prepared as the evidence of an unfinished body + * (startup resumes or holds it). Never throws: the caller is already + * reporting the surgery's failure; returns a clause to append to it. + */ + private retireAfterRestore( + batch: DiscordAwarenessBatch | null, + sourceBranch: string, + agentName: string, + ): string { + if (!batch || !this.discordAwarenessOutbox) return ''; + if (this.store.currentBranch().name !== sourceBranch) { + return `; awareness batch ${batch.id} kept, since the source branch could not be confirmed`; + } + try { + this.discordAwarenessOutbox.discard(batch.id); + return ''; + } catch (error) { + const detail = boundDiscordAwarenessText(error instanceof Error ? error.message : String(error)); + this.opsAlert( + 'discord-awareness-unresolved', + agentName, + `A failed surgery's awareness batch ${batch.id} could not be retired; startup will hold it: ${detail}`, + { data: { batchId: batch.id } }, + ); + return `; awareness batch ${batch.id} could not be retired (${detail})`; + } + } + /** * After a failed switch or redaction: put the store back on `sourceBranch` * if it moved, and return a sentence saying what actually happened. Never @@ -6107,11 +6193,11 @@ export class AgentFramework { // initialization, so a rejection here can leave the store on the // new branch with an uninitialized strategy. Retire the batch and // go back to the source; keep the branch for diagnosis. - if (markerBatch) this.discordAwarenessOutbox!.discard(markerBatch.id); const restored = await this.restoreSourceBranch(cm, sourceBranch, targetBranch, agentName); + const retired = this.retireAfterRestore(markerBatch, sourceBranch, agentName); throw new OperatorActionError( 'failed', - `Rollback failed; ${restored}: ${error instanceof Error ? error.message : String(error)}`, + `Rollback failed; ${restored}${retired}: ${error instanceof Error ? error.message : String(error)}`, { cause: error }, ); } @@ -6122,7 +6208,7 @@ export class AgentFramework { scope: marksScope(opts.marks) as SurgeryMarkerReceipt['scope'], unmarked: selection.unmarked, notRemoved: selection.notRemoved, - }, selection.refs.length); + }); // Starts delivery without awaiting it (the first reaction request may // already be on the wire when this returns); it never gates traffic. this.deliverDiscordAwarenessInBackground('rollback', agentName); @@ -6242,25 +6328,24 @@ export class AgentFramework { activationPolicy: 'explicit', suppressionIntervals: ordered.map((iv) => ({ fromId: iv.fromId, toId: iv.toId })), }) ?? null; - // Anything short of activation retires the batch: a prepared explicit - // batch is resumed at startup whenever its exact target branch is - // active — e.g. after an operator opens the failed fork to look — and - // an un-completable resume there aborts framework start. - const retireBatch = (): void => { - if (markerBatch) this.discordAwarenessOutbox!.discard(markerBatch.id); - }; + // A failure short of activation puts the source back first, and only + // then retires the batch: a prepared explicit batch is resumed at + // startup whenever its exact target branch is active (e.g. after an + // operator opens the failed fork to look), and an un-completable + // resume there aborts framework start. If the source can't be + // confirmed, the batch stays as the unfinished body's evidence. try { createdBranch = await cm.fork(targetBranch); } catch (error) { // fork() switches the chronicle branch before awaiting strategy // initialization; a rejection can leave the store on the new - // branch. Retire the batch and go back to the source. - retireBatch(); + // branch. Go back to the source, then retire the batch. const restored = await this.restoreSourceBranch(cm, sourceBranch, targetBranch, agentName); + const retired = this.retireAfterRestore(markerBatch, sourceBranch, agentName); throw new OperatorActionError( 'failed', - `Suppression failed creating ${targetBranch}; ${restored}: ${error instanceof Error ? error.message : String(error)}`, + `Suppression failed creating ${targetBranch}; ${restored}${retired}: ${error instanceof Error ? error.message : String(error)}`, { cause: error }, ); } @@ -6273,11 +6358,11 @@ export class AgentFramework { // A partially suppressed branch is not safe to serve from. Keep it // for diagnosis, put the agent back on the untouched source — and // say honestly whether that restore happened. - retireBatch(); const restored = await this.restoreSourceBranch(cm, sourceBranch, createdBranch, agentName); + const retired = this.retireAfterRestore(markerBatch, sourceBranch, agentName); throw new OperatorActionError( 'failed', - `Suppression failed on ${createdBranch}; ${restored}: ${error instanceof Error ? error.message : String(error)}`, + `Suppression failed on ${createdBranch}; ${restored}${retired}: ${error instanceof Error ? error.message : String(error)}`, { cause: error }, ); } @@ -6291,7 +6376,7 @@ export class AgentFramework { scope: marksScope(opts.marks) as SurgeryMarkerReceipt['scope'], unmarked: selection.unmarked, notRemoved: selection.notRemoved, - }, selection.refs.length); + }); // Starts delivery without awaiting it (the first reaction request may // already be on the wire when this returns); it never gates traffic. this.deliverDiscordAwarenessInBackground('suppress', agentName); @@ -6427,22 +6512,23 @@ export class AgentFramework { }; } - /** The awareness journal's batches, for operator surfaces. */ - listDiscordAwareness(): DiscordAwarenessBatchView[] { + /** The awareness journal's batches and retract requests, for operator surfaces. */ + listDiscordAwareness(): DiscordAwarenessView[] { return this.discordAwarenessOutbox?.view() ?? []; } /** - * Stop a batch's awareness marks that have not been sent. Never removes a - * reaction: requests already on the wire, or whose outcome is unknown, are - * reported and may still land; retract explicitly to remove them. + * Stop all further sends and retries of a batch's marks, or of a retract + * request's removals (by its request id). Never removes a reaction and + * never undoes one: requests already on the wire, or whose outcome is + * unknown, are reported and may still land. */ cancelDiscordAwareness( - batchId: string, + target: string, opts: { requester?: OperatorRequester; note?: string } = {}, ): DiscordAwarenessCancelReceipt { - return this.awarenessOperatorAction('awareness-cancel', { batchId }, opts, () => - this.requireDiscordAwarenessOutbox().cancel(batchId, describeRequester(opts.requester))); + return this.awarenessOperatorAction('awareness-cancel', { target }, opts, () => + this.requireDiscordAwarenessOutbox().cancel(target, describeRequester(opts.requester))); } /** @@ -6519,72 +6605,48 @@ export class AgentFramework { */ private activateSurgeryMarkers( batch: DiscordAwarenessBatch | null, - verb: 'rollback' | 'suppress' | 'hide', + verb: 'rollback' | 'suppress' | 'hide' | 'undo', agentName: string, facts: SurgeryMarkerFacts, - selected: number, ): SurgeryMarkerReceipt { - if (!batch) { - if (selected > 0 && !this.discordAwarenessOutbox) { - // Chosen and in scope, but there is nowhere to record them: say so - // rather than report that none were wanted. - console.error( - `[discord-awareness] ${verb} agent=${agentName}: ${selected} mark(s) chosen, but no awareness journal is configured`, - ); - return { - ...facts, - status: 'not-scheduled', - queued: 0, - error: 'no awareness journal is configured (no storePath or discordAwarenessOutboxPath)', - }; - } - return { ...facts, status: 'none', queued: 0 }; - } - try { - this.discordAwarenessOutbox!.activate(batch.id); - } catch (error) { - const detail = boundDiscordAwarenessText(error instanceof Error ? error.message : String(error)); - let retired = false; - let retireDetail = ''; - try { - retired = this.discordAwarenessOutbox!.discard(batch.id); - } catch (discardError) { - retireDetail = boundDiscordAwarenessText( - discardError instanceof Error ? discardError.message : String(discardError), - ); - } - console.error( - `[discord-awareness] ${verb} agent=${agentName}: body applied, but marker batch ${batch.id} ` + - `could not be activated (${detail}); ` + - (retired ? 'batch retired' : `batch left prepared${retireDetail ? ` (retire failed: ${retireDetail})` : ''}`), - ); - if (batch.refs.length === 0) return { ...facts, status: 'none', queued: 0 }; - this.opsAlert( - retired ? 'discord-awareness-not-scheduled' : 'discord-awareness-unresolved', - agentName, - retired - ? `${verb} applied, but its ${batch.refs.length} awareness mark(s) were not scheduled ` + - `(batch ${batch.id} retired): ${detail}` - : `${verb} applied, but awareness batch ${batch.id} (${batch.refs.length} mark(s)) could be ` + - `neither activated nor retired; it may still be delivered: ${detail}`, - { data: { batchId: batch.id, retired } }, - ); - return retired - ? { ...facts, status: 'not-scheduled', queued: 0, error: detail } - : { ...facts, status: 'unresolved', queued: 0, batchId: batch.id, error: detail }; - } + if (!batch || !this.discordAwarenessOutbox) return { ...facts, status: 'none', queued: 0 }; + const settled = this.discordAwarenessOutbox.settleActivation(batch.id); + // A suppression-only batch carries no marks: its activation records the + // body complete, and the receipt has nothing to report. if (batch.refs.length === 0) return { ...facts, status: 'none', queued: 0 }; - return { ...facts, status: 'queued', queued: batch.refs.length, batchId: batch.id }; + if (settled.status === 'queued') { + return { ...facts, status: 'queued', queued: settled.queued, batchId: batch.id }; + } + const retired = settled.status === 'not-scheduled'; + console.error( + `[discord-awareness] ${verb} agent=${agentName}: body applied, but marker batch ${batch.id} ` + + `could not be activated (${settled.error}); ${retired ? 'batch retired' : 'batch left prepared'}`, + ); + this.opsAlert( + retired ? 'discord-awareness-not-scheduled' : 'discord-awareness-unresolved', + agentName, + retired + ? `${verb} applied, but its ${batch.refs.length} awareness mark(s) were not scheduled ` + + `(batch ${batch.id} retired): ${settled.error}` + : `${verb} applied, but awareness batch ${batch.id} (${batch.refs.length} mark(s)) could be ` + + `neither activated nor retired; it may still be delivered: ${settled.error}`, + { data: { batchId: batch.id, retired } }, + ); + return retired + ? { ...facts, status: 'not-scheduled', queued: 0, error: settled.error } + : { ...facts, status: 'unresolved', queued: 0, batchId: batch.id, error: settled.error }; } /** - * Marks for an in-place `hide`, scheduled after its redaction committed. - * The batch targets the current branch and is explicit (activated here); - * a crash between its two journal records leaves it prepared for exactly - * this branch, which startup completes. Old records without a serverId are + * Marks for a change already applied in place (an in-place `hide`, or a + * turn undo), scheduled after it. The batch targets the current branch and + * is explicit (activated here, after the change is synced); a crash + * between its two journal records leaves it prepared for exactly this + * branch, which startup completes. Old records without a serverId are * routed through the server that issued the command. */ - private scheduleHideMarks( + private scheduleAppliedMarks( + verb: 'hide' | 'undo', agentName: string, serverId: string, removed: Array<{ metadata?: Record }>, @@ -6619,12 +6681,12 @@ export class AgentFramework { this.opsAlert( 'discord-awareness-not-scheduled', agentName, - `hide applied, but its ${selection.refs.length} awareness mark(s) were not scheduled: ${detail}`, + `${verb} applied, but its ${selection.refs.length} awareness mark(s) were not scheduled: ${detail}`, ); return { ...facts, status: 'not-scheduled', queued: 0, error: detail }; } - const receipt = this.activateSurgeryMarkers(batch, 'hide', agentName, facts, selection.refs.length); - if (receipt.status === 'queued') this.deliverDiscordAwarenessInBackground('hide', agentName); + const receipt = this.activateSurgeryMarkers(batch, verb, agentName, facts); + if (receipt.status === 'queued') this.deliverDiscordAwarenessInBackground(verb, agentName); return receipt; } @@ -6636,7 +6698,7 @@ export class AgentFramework { * the queued work stays in the journal for the next pass. */ private deliverDiscordAwarenessInBackground( - occasion: 'rollback' | 'suppress' | 'hide' | 'retract' | 'release' | 'startup' | 'connect' | 'list-change' | 'reconnect', + occasion: 'rollback' | 'suppress' | 'hide' | 'undo' | 'retract' | 'release' | 'startup' | 'connect' | 'list-change' | 'reconnect', agentName: string, ): void { if (!this.discordAwarenessOutbox) return; @@ -13600,15 +13662,16 @@ export class AgentFramework { if (interval.fromId === interval.toId) cm.removeMessage(from.id); else cm.removeMessages(from.id, to.id); } - if (batch.status === 'held' || batch.cancelled) { - // Body integrity only: its marks were held at an earlier startup (and - // are released explicitly, never by a branch becoming active), or an - // operator cancelled them. - this.discordAwarenessOutbox.recordSuppressionComplete(batch.id); - console.error(`[discord-awareness] completed held suppression ${batch.id} (marks stay held)`); - } else { + if (batch.status === 'prepared' && !batch.cancelled) { + // Crash completion of the surgery itself: body, then its marks. this.discordAwarenessOutbox.activate(batch.id); console.error(`[discord-awareness] resumed suppression batch ${batch.id}`); + } else { + // Body integrity only. Its marks are the operator's: held at an + // earlier startup, released, or cancelled; a branch becoming active + // never changes them. + this.discordAwarenessOutbox.recordSuppressionComplete(batch.id); + console.error(`[discord-awareness] completed suppression body ${batch.id} (marks ${batch.status})`); } } } @@ -13648,33 +13711,32 @@ export class AgentFramework { let delivered = 0; let failed = 0; const attempted = new Set(); + const signature = (dispatch: { opIds: string[]; action: string }) => + `${dispatch.opIds.join(',')}\0${dispatch.action}`; let announced = false; while (true) { - const dispatches = this.readDiscordAwarenessDispatches(serverId).filter((dispatch) => { - const key = `${dispatch.opIds.join(',')}\0${dispatch.action}`; - if (attempted.has(key)) return false; - attempted.add(key); - return true; - }); - if (dispatches.length === 0) break; + // One dispatch at a time, chosen and written ahead in one step from + // the journal as it is now: a cancel or retract made while the + // previous request was on the wire takes effect here. + let claimed: ReturnType; + try { + claimed = outbox.claimDispatch(serverId, (dispatch) => attempted.has(signature(dispatch))); + } catch (error) { + throw new DiscordAwarenessAccountingError('dispatch write-ahead', error); + } + if (!claimed) break; + const { dispatch, attempts } = claimed; + attempted.add(signature(dispatch)); if (!announced) { announced = true; - console.error( - `[discord-awareness] ${serverId}: delivering ${dispatches.length} queued mark operation(s)`, - ); + console.error(`[discord-awareness] ${serverId}: delivering queued mark operations`); } - for (const dispatch of dispatches) { + { const ref = dispatch.key; const channelId = ref.channelId.startsWith('discord:') ? ref.channelId.split(':').at(-1)! : ref.channelId; const tool = dispatch.action === 'add' ? 'add_reaction' : 'remove_reaction'; - let attempts: Array<{ opId: string; attempt: number }>; - try { - attempts = outbox.recordDispatching(dispatch); - } catch (error) { - throw new DiscordAwarenessAccountingError('dispatch write-ahead', error); - } let outcome: 'confirmed' | 'failed' | 'not-sent' | 'unknown'; let detail: string | undefined; try { diff --git a/src/index.ts b/src/index.ts index fe873b1ce..50ae955b2 100644 --- a/src/index.ts +++ b/src/index.ts @@ -132,6 +132,7 @@ export { DEFAULT_DISCORD_AWARENESS_EMOJI, defaultDiscordAwarenessOutboxPath, extractDiscordAwarenessRefs, + isPermanentDiscordReactionFailure, selectDiscordAwarenessRefs, } from './recovery/discord-awareness-outbox.js'; export type { @@ -140,6 +141,9 @@ export type { DiscordAwarenessBatch, DiscordAwarenessBatchRecord, DiscordAwarenessBatchView, + DiscordAwarenessLegacyEvidence, + DiscordAwarenessRetractView, + DiscordAwarenessView, DiscordAwarenessCancelReceipt, DiscordAwarenessDispatch, DiscordAwarenessMarks, diff --git a/src/operator-log.ts b/src/operator-log.ts index 173192c26..ef7e87bb7 100644 --- a/src/operator-log.ts +++ b/src/operator-log.ts @@ -144,11 +144,9 @@ export interface SurgeryMarkerFacts { * - `queued` — one add request per mark is durably recorded and delivery has * started. It is not a delivery claim: per-request outcomes stay in the * awareness journal. - * - `not-scheduled` — marks were chosen and in scope, but none will be - * delivered: either this framework has no awareness journal (no storePath - * or discordAwarenessOutboxPath), or the body change landed and recording - * the batch as active failed, so it was retired from the journal. `error` - * says which. + * - `not-scheduled` — the body change landed, but recording the batch as + * active failed, so it was retired from the journal: none of its marks will + * be delivered. * - `unresolved` — the journal could record neither the activation nor the * retirement. `batchId` names a retained batch whose scheduling outcome is * unresolved: any later reconciliation or restart that can read it may diff --git a/src/recovery/discord-awareness-outbox.ts b/src/recovery/discord-awareness-outbox.ts index c75d93c4c..f7e6592d7 100644 --- a/src/recovery/discord-awareness-outbox.ts +++ b/src/recovery/discord-awareness-outbox.ts @@ -1,16 +1,8 @@ -import { - closeSync, - existsSync, - fsyncSync, - mkdirSync, - openSync, - readFileSync, - renameSync, - truncateSync, - writeSync, -} from 'node:fs'; -import { basename, dirname, join } from 'node:path'; +import { existsSync, readFileSync, renameSync } from 'node:fs'; +import { basename, join } from 'node:path'; import { createHash, randomUUID } from 'node:crypto'; +import type { JsStore } from '@animalabs/chronicle'; +import { RecordJournal, type AppendOptions } from '../record-journal.js'; /** * Discord awareness marks: a reaction (💤 by default) that an operator may @@ -34,16 +26,22 @@ import { createHash, randomUUID } from 'node:crypto'; * server id is the configured route to a bot, not the bot's identity: if the * route is re-pointed at another bot, retained operations act as that bot, * and nothing here can detect it. + * + * The journal lives in the Chronicle store as typed records (RecordJournal): + * it survives branch switches, rollbacks, `deleteBranch` and a killed + * process, and lives and dies with its store. Each append is one record + * holding a group of journal records that apply together. A record that + * asserts a branch change (an activation after a switch or redaction) is + * appended only after that state is synced, so it can never outlive it. */ export const DEFAULT_DISCORD_AWARENESS_EMOJI = '💤'; -/** The journal file under a store, next to the other recovery state. */ /** * Bound text that came from elsewhere (a Discord error body, a filesystem * error) before it is journaled, logged or returned in a receipt: the journal - * is re-read on every operation, and a server's error text has no length - * limit of its own. Head and tail are kept, with the omitted length stated; + * is kept for the store's life, and a server's error text has no length limit + * of its own. Head and tail are kept, with the omitted length stated; * a cut never splits a surrogate pair. */ export function boundDiscordAwarenessText(text: string, max = 500): string { @@ -56,8 +54,24 @@ export function boundDiscordAwarenessText(text: string, max = 500): string { return `${head} … [${text.length - head.length - tail.length} chars omitted] … ${tail}`; } +/** + * Where a pre-journal awareness ledger (the JSON file earlier releases kept + * under a store) is found, to import once. + */ +/** + * A Discord answer that the reaction can never succeed: the message or + * channel is gone or inaccessible. Retrying cannot help. + */ +export function isPermanentDiscordReactionFailure(message: string): boolean { + return /unknown message|unknown channel|missing access|missing permissions|missing permission|cannot access|channel .* not found|message .* not found/i + .test(message); +} + +/** AF's own refusal to write a request on a closed connection. */ +const PRE_WRITE_REFUSAL = /^Cannot send request:/; + export function defaultDiscordAwarenessOutboxPath(storePath: string): string { - return join(storePath, 'recovery', 'discord-awareness-journal.jsonl'); + return join(storePath, 'recovery', LEGACY_LEDGER_NAME); } /** The pre-journal (version 1 and 2) ledger file name, imported once. */ @@ -129,7 +143,8 @@ export type DiscordAwarenessBatchStatus = 'prepared' | 'active' | 'held' | 'disc export interface DiscordAwarenessBatch extends DiscordAwarenessBatchRecord { status: DiscordAwarenessBatchStatus; - /** A held suppression whose body was completed at a later startup. */ + /** A suppression whose body is recorded complete (by its activation, or by + * a later startup's resume). */ suppressionComplete?: boolean; held?: { reason: string; at: number; releaseActions: DiscordAwarenessReleaseAction[] }; cancelled?: { at: number; by?: string }; @@ -172,7 +187,7 @@ export interface DiscordAwarenessOp { cause: DiscordAwarenessOpCause; requestedAt: number; attempts: DiscordAwarenessAttempt[]; - cancelled?: { at: number; reason: 'batch-cancelled' | 'superseded' }; + cancelled?: { at: number; reason: 'batch-cancelled' | 'request-cancelled' | 'superseded' | 'imported' }; } /** One dispatch: the head operation for a reaction key, plus any consecutive @@ -185,7 +200,9 @@ export interface DiscordAwarenessDispatch { /** Receipt for cancel(). Nothing is ever removed from Discord by a cancel. */ export interface DiscordAwarenessCancelReceipt { - batchId: string; + /** The batch id or retract request id that was cancelled. */ + target: string; + kind: 'batch' | 'retract'; /** Requests that will now never be sent (for a batch not yet activated, * every mark it would have requested). */ cancelled: number; @@ -197,12 +214,19 @@ export interface DiscordAwarenessCancelReceipt { unknown: number; /** Operations Discord already confirmed (cancel does not undo them). */ confirmed: number; + /** Requests of this batch or retract whose outcome is unresolved (on the + * wire or unknown), counted per request sent and including earlier + * attempts of operations whose last attempt was answered: any may land. */ + unresolvedAttempts: number; + /** Imported (pre-journal) attempts on this batch whose outcome the old + * ledger does not establish: any may have landed. */ + legacyOutcomesUnrecorded: number; } /** Receipt for retract(). */ export interface DiscordAwarenessRetractReceipt { requestId: string; - /** Removals queued (one per reaction key). */ + /** Removals queued: one per selected reaction key, whatever history says. */ removalsQueued: number; /** Add requests for those keys that had not been sent, now superseded. */ addsSuperseded: number; @@ -212,6 +236,32 @@ export interface DiscordAwarenessRetractReceipt { /** Unresolved earlier add requests on those keys (one per request sent, * however many batches' operations it carried). */ unresolvedAddAttempts: number; + /** Keys whose imported (pre-journal) history leaves attempt outcomes + * unrecorded: any of those may have had an effect. */ + keysWithLegacyUncertainty: number; +} + +/** + * What a pre-journal ledger recorded about one batch ref, kept as facts. It + * is evidence for operators and receipts, never a claim about what is on + * Discord now, and it never causes a dispatch. + */ +export interface DiscordAwarenessLegacyEvidence { + batchId: string; + key: DiscordAwarenessRef & { emoji: string }; + /** Attempts the old ledger counted (it kept only the last one's details). */ + attempts: number; + lastAction: DiscordAwarenessAction; + /** The old ledger's status after the last attempt. */ + lastStatus: 'pending' | 'applied' | 'permanent-failure'; + lastError?: string; + /** What the last error was: AF's refusal before writing a request, a + * Discord answer (message/channel gone or inaccessible), or anything else. */ + lastErrorKind?: 'pre-write-refusal' | 'discord-answer' | 'other'; + /** Recorded attempts whose outcome the old ledger does not establish: + * every attempt before the last, and the last unless it was applied, + * refused before writing, or answered by Discord. */ + outcomesUnrecorded: number; } export interface DiscordAwarenessReleaseReceipt { @@ -222,6 +272,7 @@ export interface DiscordAwarenessReleaseReceipt { /** Per-batch view for operators: what was asked and what is known. */ export interface DiscordAwarenessBatchView { + kind: 'batch'; id: string; status: DiscordAwarenessBatchStatus; scope: DiscordAwarenessBatchRecord['scope']; @@ -240,10 +291,37 @@ export interface DiscordAwarenessBatchView { adds: Record; /** Removals requested (by any retract) on this batch's refs, by status. */ removals: Record; - /** Attempts on this batch's refs whose outcome is unknown or on the wire. */ + /** Requests on this batch's refs whose outcome is unknown or on the wire. */ unresolvedAttempts: number; + /** What a pre-journal ledger recorded about this batch's refs, if imported. */ + legacy?: { + entries: number; + /** Refs whose last recorded action was an add the old ledger saw applied. */ + lastAddConfirmed: number; + /** Refs whose last recorded action was a removal it saw applied. */ + lastRemoveConfirmed: number; + /** Recorded attempts whose outcome the old ledger does not establish. */ + outcomesUnrecorded: number; + }; } +/** One retract request: the removals it queued and what became of them. */ +export interface DiscordAwarenessRetractView { + kind: 'retract'; + /** The retract's request id (what cancel takes). */ + id: string; + /** The batch it retracted, or `all`. */ + target: string; + at: number; + by?: string; + cancelled?: { at: number; by?: string }; + removals: Record; + /** Removal requests whose outcome is unknown or on the wire. */ + unresolvedAttempts: number; +} + +export type DiscordAwarenessView = DiscordAwarenessBatchView | DiscordAwarenessRetractView; + /** Extract Discord addressing metadata without reading message content. */ export interface DiscordMessageMetadataCarrier { metadata?: Record; @@ -334,7 +412,7 @@ type JournalRecord = action: DiscordAwarenessAction; cause: DiscordAwarenessOpCause; } - | { t: 'op-cancelled'; at: number; opId: string; reason: 'batch-cancelled' | 'superseded' } + | { t: 'op-cancelled'; at: number; opId: string; reason: 'batch-cancelled' | 'request-cancelled' | 'superseded' | 'imported' } | { t: 'dispatching'; at: number; opId: string; attempt: number; dispatchId?: string } | { t: 'outcome'; @@ -346,49 +424,72 @@ type JournalRecord = error?: string; } | { t: 'retract'; at: number; requestId: string; target: string; by?: string } + | { t: 'request-cancelled'; at: number; requestId: string; by?: string } | { t: 'suppression-complete'; at: number; batchId: string } - | { t: 'import-complete'; at: number; source: 'v2'; sha256: string } - | { t: 'commit'; at: number; txn: string }; + | { t: 'legacy-evidence'; at: number; evidence: DiscordAwarenessLegacyEvidence } + | { t: 'import-complete'; at: number; source: 'v2'; sha256: string }; -/** - * Every multi-record append is one transaction: its records carry a `txn` - * id and only take effect when the closing `commit` record is present. A - * torn append (disk full, crash mid-write) therefore applies none of its - * records, never a prefix, so an activation can't half-queue its marks and a - * retract can't supersede adds without queuing its removals. - */ -type StoredRecord = JournalRecord & { txn?: string }; +/** One journal entry (one Chronicle record): records that apply together. */ +interface JournalEntryGroup { + records: JournalRecord[]; +} + +/** The reduced state as a checkpoint stores it. */ +interface JournalSnapshot { + batches: DiscordAwarenessBatch[]; + ops: DiscordAwarenessOp[]; + legacy: DiscordAwarenessLegacyEvidence[]; + retracts?: RetractRequest[]; + importedSources: string[]; +} + +export const DISCORD_AWARENESS_RECORD_TYPE = 'af:discord-awareness'; +/** Replay at open is bounded by a checkpoint every this many entries. */ +const CHECKPOINT_EVERY = 256; interface JournalState { batches: Map; /** Operations in request order. */ ops: Map; + /** Imported pre-journal evidence, by reaction key. */ + legacy: Map; + /** Retract requests, by request id. */ + retracts: Map; importedSources: Set; } +interface RetractRequest { + requestId: string; + target: string; + at: number; + by?: string; + cancelled?: { at: number; by?: string }; +} + /** - * The durable awareness-mark ledger: an append-only journal (one JSON record - * per line, each append fsynced) and the operations derived from it. + * The durable awareness-mark ledger: an append-only journal of typed records + * in the store, and the operations reduced from it. One instance per store is + * the journal's only writer (RecordJournal's single-writer rule): the running + * framework, or the offline CLI while the host is stopped. */ export class DiscordAwarenessOutbox { - readonly path: string; - private readonly legacyPath: string; + private readonly journal: RecordJournal; + private readonly legacyPath?: string; + private readonly checkpointEvery: number; + /** Reduced state; rebuilt from the journal when null. */ + private state: JournalState | null = null; /** Operations dispatched by this process and still awaiting an outcome. */ private readonly inFlight = new Set(); /** - * @param path the journal file. A path ending in `.json` names a pre-journal - * ledger (as an older `discordAwarenessOutboxPath` would): the journal then - * lives beside it as `.journal.jsonl` and the file itself is imported. + * @param store the Chronicle store the journal lives in. + * @param opts.legacyPath a pre-journal JSON ledger to import once, if present. + * @param opts.checkpointEvery entries between checkpoints (default 256). */ - constructor(path: string) { - if (path.endsWith('.json')) { - this.legacyPath = path; - this.path = `${path.slice(0, -'.json'.length)}.journal.jsonl`; - } else { - this.path = path; - this.legacyPath = join(dirname(path), LEGACY_LEDGER_NAME); - } + constructor(store: JsStore, opts: { legacyPath?: string; checkpointEvery?: number } = {}) { + this.journal = new RecordJournal(store, { type: DISCORD_AWARENESS_RECORD_TYPE }); + if (opts.legacyPath) this.legacyPath = opts.legacyPath; + this.checkpointEvery = Math.max(1, Math.floor(opts.checkpointEvery ?? CHECKPOINT_EVERY)); } // -- surgery side ----------------------------------------------------------- @@ -428,7 +529,8 @@ export class DiscordAwarenessOutbox { ? { suppressionIntervals: input.suppressionIntervals.map((interval) => ({ ...interval })) } : {}), }; - this.append([{ t: 'batch', at: batch.createdAt, batch }]); + // Durable before the surgery's switch, so startup can crash-complete it. + this.append([{ t: 'batch', at: batch.createdAt, batch }], { durable: true }); return { ...structuredClone(batch), status: 'prepared' }; } @@ -454,10 +556,45 @@ export class DiscordAwarenessOutbox { kind: 'batch', batchId, }, at)); - this.append(records); + // The activation asserts the body change just committed (the switch, or + // the last redaction): sync that first, so a crash can never leave marks + // active for a change the store lost. + this.append(records, { afterCommittedState: true, durable: true }); return batch.refs.length; } + /** + * Activate a batch after its body change landed, and if that cannot be + * recorded, retire it instead, so that what the caller reports is what + * the journal will do: + * - `queued`: the batch is active (including when the activation landed + * but its durability barrier failed: startup would crash-complete it); + * - `not-scheduled`: it was retired; none of its marks will be sent; + * - `unresolved`: neither could be recorded; a later startup that can read + * the batch may activate it. + * Never throws. + */ + settleActivation(batchId: string): + | { status: 'queued'; queued: number } + | { status: 'not-scheduled'; error: string } + | { status: 'unresolved'; error: string } { + let detail: string; + try { + this.activate(batchId); + return { status: 'queued', queued: this.load().batches.get(batchId)?.refs.length ?? 0 }; + } catch (error) { + detail = boundDiscordAwarenessText(error instanceof Error ? error.message : String(error)); + } + try { + const current = this.load().batches.get(batchId); + if (current?.status === 'active') return { status: 'queued', queued: current.refs.length }; + if (this.discard(batchId)) return { status: 'not-scheduled', error: detail }; + } catch (error) { + detail += `; retiring it also failed: ${boundDiscordAwarenessText(error instanceof Error ? error.message : String(error))}`; + } + return { status: 'unresolved', error: detail }; + } + /** * Retire a batch whose surgery did not complete. Only a prepared batch can * be discarded; returns false when nothing was retired. @@ -465,20 +602,26 @@ export class DiscordAwarenessOutbox { discard(batchId: string): boolean { const batch = this.load().batches.get(batchId); if (!batch || batch.status !== 'prepared') return false; - this.append([{ t: 'discarded', at: Date.now(), batchId }]); + // Retiring a batch retires its obligations (marks and any body resume): + // the branch state its surgery left (a restored source, or a completed + // body) is synced first, and the record is durable, so a crash can never + // keep the retirement while losing the state it relies on. + this.append([{ t: 'discarded', at: Date.now(), batchId }], { afterCommittedState: true, durable: true }); return true; } /** * Suppressions whose body may still need resuming on the active branch: - * explicit batches with intervals whose target is exactly that branch, - * prepared (crash completion: resume, then activate) or held at an earlier - * startup (resume the body only; marks stay held). Branch ancestry is never - * consulted. + * explicit batches with intervals whose target is exactly that branch and + * whose body was never recorded complete. Body recovery is independent of + * the marks: a prepared batch is crash-completed (resume, then activate); + * any other, held, released or cancelled, has only its body resumed. + * A batch its own surgery retired (discarded) is not resumed. Branch + * ancestry is never consulted. */ preparedSuppressionsForBranch(branchName: string): DiscordAwarenessBatch[] { return [...this.load().batches.values()] - .filter((batch) => (batch.status === 'prepared' || batch.status === 'held') + .filter((batch) => batch.status !== 'discarded' && !batch.suppressionComplete && batch.activationPolicy === 'explicit' && !!batch.suppressionIntervals?.length @@ -486,9 +629,11 @@ export class DiscordAwarenessOutbox { .map((batch) => structuredClone(batch)); } - /** Record that a held suppression's body was completed (its marks stay held). */ + /** Record that a suppression's body was completed by a resume that did not + * activate its marks (they were held, released or cancelled). */ recordSuppressionComplete(batchId: string): void { - this.append([{ t: 'suppression-complete', at: Date.now(), batchId }]); + // Certifies the redactions just made: only after they are synced. + this.append([{ t: 'suppression-complete', at: Date.now(), batchId }], { afterCommittedState: true, durable: true }); } /** @@ -552,7 +697,7 @@ export class DiscordAwarenessOutbox { held.push(batch.id); } } - if (records.length > 0) this.append(records); + if (records.length > 0) this.append(records, { durable: true }); return { activated, held, unknownAttempts }; } @@ -565,12 +710,15 @@ export class DiscordAwarenessOutbox { * dropped. Nothing is ever removed from Discord by a cancel: the receipt * reports what may already have landed, and retract removes it. */ - cancel(batchId: string, by?: string): DiscordAwarenessCancelReceipt { + cancel(target: string, by?: string): DiscordAwarenessCancelReceipt { const state = this.load(); + if (state.retracts.has(target)) return this.cancelRetract(state, target, by); + const batchId = target; const batch = state.batches.get(batchId); - if (!batch) throw new Error(`Discord awareness batch not found: ${batchId}`); + if (!batch) throw new Error(`Discord awareness batch or retract request not found: ${target}`); const receipt: DiscordAwarenessCancelReceipt = { - batchId, + target: batchId, + kind: 'batch', // A prepared batch (its surgery still running, or its bookkeeping // unresolved) has no requests yet: every one of its marks is stopped. cancelled: batch.status === 'prepared' && !batch.cancelled ? batch.refs.length : 0, @@ -578,6 +726,13 @@ export class DiscordAwarenessOutbox { inFlight: 0, unknown: 0, confirmed: 0, + unresolvedAttempts: unresolvedDispatches( + [...state.ops.values()].filter((op) => op.cause.batchId === batchId && op.action === 'add'), + (op, attempt) => this.attemptUnresolved(op, attempt), + ), + legacyOutcomesUnrecorded: [...state.legacy.values()].flat() + .filter((evidence) => evidence.batchId === batchId) + .reduce((sum, evidence) => sum + evidence.outcomesUnrecorded, 0), }; if (batch.cancelled) return receipt; const at = Date.now(); @@ -595,27 +750,67 @@ export class DiscordAwarenessOutbox { else continue; // failed or already cancelled records.push({ t: 'op-cancelled', at, opId: op.opId, reason: 'batch-cancelled' }); } - this.append(records); + this.append(records, { durable: true }); + return receipt; + } + + /** Stop a retract's removals that are still due; nothing sent is undone. */ + private cancelRetract(state: JournalState, requestId: string, by?: string): DiscordAwarenessCancelReceipt { + const retract = state.retracts.get(requestId)!; + const ops = [...state.ops.values()].filter((op) => op.cause.requestId === requestId); + const receipt: DiscordAwarenessCancelReceipt = { + target: requestId, + kind: 'retract', + cancelled: 0, + heldDropped: 0, + inFlight: 0, + unknown: 0, + confirmed: 0, + unresolvedAttempts: unresolvedDispatches(ops, (op, attempt) => this.attemptUnresolved(op, attempt)), + legacyOutcomesUnrecorded: 0, + }; + if (retract.cancelled) return receipt; + const at = Date.now(); + const records: JournalRecord[] = [{ t: 'request-cancelled', at, requestId, ...(by ? { by } : {}) }]; + for (const op of ops) { + const status = this.opStatus(op); + if (status === 'confirmed') { + receipt.confirmed++; + continue; + } + if (status === 'requested') receipt.cancelled++; + else if (status === 'dispatching') receipt.inFlight++; + else if (status === 'unknown') receipt.unknown++; + else continue; + records.push({ t: 'op-cancelled', at, opId: op.opId, reason: 'request-cancelled' }); + } + this.append(records, { durable: true }); return receipt; } /** * Remove this bot's mark (through the configured route) from a batch's refs, - * or from every ref any batch ever sent an add for. Acts across batches: a - * reaction is one per (route, message, emoji), whichever batches asked for - * it. Keys where no add attempt ever left the host are skipped. Add - * requests not yet sent for those keys are superseded. The removal is - * always sent, whatever earlier outcomes say. + * or from every key any batch requested or an imported ledger recorded an + * add for (`all`). A reaction is one per (route, message, emoji), whichever + * batches asked for it, so this acts across batches. A removal is queued for + * every selected key, whatever history says: history can't establish what + * is on Discord now, and removing an absent reaction does nothing. Add + * requests not yet sent for those keys are superseded. The receipt reports + * history only: requests whose outcome is unresolved, and imported history + * that leaves outcomes unrecorded, any of which may land or have landed. */ retract(target: string | 'all', by?: string): DiscordAwarenessRetractReceipt { const state = this.load(); let candidateKeys: Array; if (target === 'all') { - candidateKeys = uniqueKeys([...state.ops.values()].filter((op) => op.action === 'add').map((op) => op.key)); + candidateKeys = uniqueKeys([ + ...[...state.ops.values()].filter((op) => op.action === 'add').map((op) => op.key), + ...[...state.legacy.values()].flat().map((evidence) => evidence.key), + ]); } else { const batch = state.batches.get(target); if (!batch) throw new Error(`Discord awareness batch not found: ${target}`); - candidateKeys = batch.refs.map((ref) => ({ ...ref, emoji: batch.emoji })); + candidateKeys = uniqueKeys(batch.refs.map((ref) => ({ ...ref, emoji: batch.emoji }))); } const requestId = randomUUID(); const at = Date.now(); @@ -625,34 +820,27 @@ export class DiscordAwarenessOutbox { addsSuperseded: 0, keysWithUnresolvedAdds: 0, unresolvedAddAttempts: 0, + keysWithLegacyUncertainty: 0, }; - const toRemove: Array = []; const records: JournalRecord[] = [{ t: 'retract', at, requestId, target, ...(by ? { by } : {}) }]; + const toRemove: Array = []; for (const key of candidateKeys) { const adds = this.opsForKey(state, key).filter((op) => op.action === 'add'); - const left = adds.some((op) => op.attempts.some((attempt) => attempt.outcome !== 'not-sent')); - if (!left) { - // Nothing was ever sent for this key: stop its unsent adds, and - // there is no reaction of ours to remove. - for (const op of adds) { - if (this.opStatus(op) !== 'requested') continue; - records.push({ t: 'op-cancelled', at, opId: op.opId, reason: 'superseded' }); - receipt.addsSuperseded++; - } - continue; - } const unresolved = unresolvedDispatches(adds, (op, attempt) => this.attemptUnresolved(op, attempt)); if (unresolved > 0) { receipt.keysWithUnresolvedAdds++; receipt.unresolvedAddAttempts += unresolved; } + if ((state.legacy.get(markKey(key)) ?? []).some((evidence) => evidence.outcomesUnrecorded > 0)) { + receipt.keysWithLegacyUncertainty++; + } toRemove.push({ ...key, action: 'remove' }); } const requested = this.requestRecords(state, toRemove, undefined, { kind: 'retract', requestId }, at); - receipt.addsSuperseded += requested.filter((record) => record.t === 'op-cancelled').length; + receipt.addsSuperseded = requested.filter((record) => record.t === 'op-cancelled').length; receipt.removalsQueued = requested.filter((record) => record.t === 'requested').length; records.push(...requested); - this.append(records); + this.append(records, { durable: true }); return receipt; } @@ -671,7 +859,7 @@ export class DiscordAwarenessOutbox { kind: 'release', batchId, }, at)); - this.append(records); + this.append(records, { durable: true }); return { batchId, addsQueued: actions.filter((action) => action.action === 'add').length, @@ -700,49 +888,81 @@ export class DiscordAwarenessOutbox { } const dispatches: Array = []; for (const ops of byKey.values()) { - // One request per reaction key on the wire at a time, including a - // cancelled one still awaiting its outcome: opposite actions must not - // overtake each other. - if (ops.some((op) => op.attempts.at(-1)?.outcome === undefined && op.attempts.length > 0 - && this.inFlight.has(op.opId))) continue; - const live = ops.filter((op) => !op.cancelled); - let head: DiscordAwarenessOp | undefined; - for (let index = 0; index < live.length; index++) { - const op = live[index]; - const status = this.opStatus(op); - if (status === 'requested') { head = op; break; } - if (status === 'unknown' && index === live.length - 1) { head = op; break; } - } - if (!head) continue; - const opIds = [head.opId]; - const start = live.indexOf(head); - for (let index = start + 1; index < live.length; index++) { - const next = live[index]; - if (next.action !== head.action || this.opStatus(next) !== 'requested') break; - opIds.push(next.opId); - } - dispatches.push({ key: head.key, action: head.action, opIds, order: head.requestedAt }); + const head = this.headDispatch(ops); + if (head) dispatches.push(head); } return dispatches .sort((a, b) => a.order - b.order) .map(({ order: _order, ...dispatch }) => dispatch); } - /** Write-ahead: record that a dispatch is about to leave the host. */ + /** + * Choose a server's next due dispatch and write it ahead, in one step: + * eligibility is computed from the journal as it is now, so a cancel or + * retract made while an earlier request was on the wire is never + * overridden by an older list. `skip` excludes dispatches already tried + * in this pass. Returns null when nothing is due. + */ + claimDispatch( + serverId: string, + skip: (dispatch: DiscordAwarenessDispatch) => boolean = () => false, + ): { dispatch: DiscordAwarenessDispatch; attempts: Array<{ opId: string; attempt: number }> } | null { + for (const dispatch of this.pendingDispatches(serverId)) { + if (skip(dispatch)) continue; + const attempts = this.recordDispatching(dispatch); + if (attempts.length > 0) return { dispatch, attempts }; + } + return null; + } + + /** + * Write-ahead: record that a dispatch is about to leave the host. A + * dispatch computed earlier is narrowed to the operations still due for its + * key now; when none remain, nothing is written and no attempts are + * returned (do not send). + */ recordDispatching(dispatch: DiscordAwarenessDispatch): Array<{ opId: string; attempt: number }> { const state = this.load(); + const current = this.headDispatch(this.opsForKey(state, dispatch.key)); + if (!current || current.action !== dispatch.action) return []; + const opIds = dispatch.opIds.filter((opId) => current.opIds.includes(opId)); + if (opIds.length === 0) return []; const at = Date.now(); const dispatchId = randomUUID(); - const attempts = dispatch.opIds.map((opId) => { - const op = state.ops.get(opId); - if (!op) throw new Error(`Discord awareness operation not found: ${opId}`); - return { opId, attempt: op.attempts.length + 1 }; - }); - this.append(attempts.map(({ opId, attempt }) => ({ t: 'dispatching' as const, at, opId, attempt, dispatchId }))); + const attempts = opIds.map((opId) => ({ opId, attempt: state.ops.get(opId)!.attempts.length + 1 })); + this.append( + attempts.map(({ opId, attempt }) => ({ t: 'dispatching' as const, at, opId, attempt, dispatchId })), + { durable: true }, + ); for (const { opId } of attempts) this.inFlight.add(opId); return attempts; } + /** The due dispatch for one key's operations (in request order), if any. */ + private headDispatch(ops: DiscordAwarenessOp[]): (DiscordAwarenessDispatch & { order: number }) | null { + // One request per reaction key on the wire at a time, including a + // cancelled one still awaiting its outcome: opposite actions must not + // overtake each other. + if (ops.some((op) => op.attempts.length > 0 && op.attempts.at(-1)!.outcome === undefined + && this.inFlight.has(op.opId))) return null; + const live = ops.filter((op) => !op.cancelled); + let head: DiscordAwarenessOp | undefined; + for (let index = 0; index < live.length; index++) { + const op = live[index]; + const status = this.opStatus(op); + if (status === 'requested') { head = op; break; } + if (status === 'unknown' && index === live.length - 1) { head = op; break; } + } + if (!head) return null; + const opIds = [head.opId]; + for (let index = live.indexOf(head) + 1; index < live.length; index++) { + const next = live[index]; + if (next.action !== head.action || this.opStatus(next) !== 'requested') break; + opIds.push(next.opId); + } + return { key: { ...head.key }, action: head.action, opIds, order: head.requestedAt }; + } + /** Record the outcome of a dispatch's attempts. */ recordOutcome( attempts: Array<{ opId: string; attempt: number }>, @@ -782,9 +1002,9 @@ export class DiscordAwarenessOutbox { return this.opStatus(op); } - view(): DiscordAwarenessBatchView[] { + view(): DiscordAwarenessView[] { const state = this.load(); - const views: DiscordAwarenessBatchView[] = []; + const views: DiscordAwarenessView[] = []; for (const batch of state.batches.values()) { if (batch.status === 'discarded') continue; const keys = new Set(batch.refs.map((ref) => markKey({ ...ref, emoji: batch.emoji }))); @@ -798,7 +1018,9 @@ export class DiscordAwarenessOutbox { if (op.action === 'remove') removals[this.opStatus(op)]++; } const unresolvedAttempts = unresolvedDispatches(onKeys, (op, attempt) => this.attemptUnresolved(op, attempt)); + const evidence = [...state.legacy.values()].flat().filter((entry) => entry.batchId === batch.id); views.push({ + kind: 'batch', id: batch.id, status: batch.status, scope: batch.scope, @@ -818,6 +1040,31 @@ export class DiscordAwarenessOutbox { adds, removals, unresolvedAttempts, + ...(evidence.length > 0 + ? { + legacy: { + entries: evidence.length, + lastAddConfirmed: evidence.filter((entry) => entry.lastAction === 'add' && entry.lastStatus === 'applied').length, + lastRemoveConfirmed: evidence.filter((entry) => entry.lastAction === 'remove' && entry.lastStatus === 'applied').length, + outcomesUnrecorded: evidence.reduce((sum, entry) => sum + entry.outcomesUnrecorded, 0), + }, + } + : {}), + }); + } + for (const retract of state.retracts.values()) { + const ops = [...state.ops.values()].filter((op) => op.cause.requestId === retract.requestId); + const removals = emptyCounts(); + for (const op of ops) removals[this.opStatus(op)]++; + views.push({ + kind: 'retract', + id: retract.requestId, + target: retract.target, + at: retract.at, + ...(retract.by ? { by: retract.by } : {}), + ...(retract.cancelled ? { cancelled: retract.cancelled } : {}), + removals, + unresolvedAttempts: unresolvedDispatches(ops, (op, attempt) => this.attemptUnresolved(op, attempt)), }); } return views; @@ -853,10 +1100,11 @@ export class DiscordAwarenessOutbox { } /** - * Records for new requests. A request supersedes requests for the same - * key with the opposite action that were never sent (the newest explicit - * intent wins without churning the reaction); operations already sent are - * never touched. + * Records for new requests. A request supersedes the future of every + * request for the same key with the opposite action that is still due + * (never sent, refused before writing, or failed retryably): the newest + * explicit intent wins. Their attempts stay as history, and a request on + * the wire still settles before anything else for that key is sent. */ private requestRecords( state: JournalState, @@ -876,7 +1124,7 @@ export class DiscordAwarenessOutbox { }; for (const op of this.opsForKey(state, key)) { if (op.action === request.action || superseded.has(op.opId)) continue; - if (this.opStatus(op) !== 'requested' || op.attempts.some((attempt) => attempt.outcome !== 'not-sent')) continue; + if (this.opStatus(op) !== 'requested') continue; records.push({ t: 'op-cancelled', at, opId: op.opId, reason: 'superseded' }); superseded.add(op.opId); } @@ -885,146 +1133,130 @@ export class DiscordAwarenessOutbox { return records; } - private load(): JournalState { - const { records } = this.readJournal(); - const state = fold(records); - const legacy = this.pendingLegacyImport(state); - if (legacy?.records.length) return fold([...records, ...legacy.records]); - return state; - } - /** - * The journal's records, and how to repair an unterminated final line - * before the next append: a torn fragment (never applied) is truncated - * away; a complete record that only lost its newline is terminated. - * Without the repair, the next append would share that line and turn a - * harmless torn tail into corruption. + * The reduced state, rebuilt from the journal when there is none yet or an + * earlier write left the journal unreconciled (RecordJournal then requires + * a reload before writing again). The first load also imports a + * pre-journal ledger, once. */ - private readJournal(): { records: StoredRecord[]; repair?: { truncateTo: number } | { terminate: true } } { - let raw: Buffer; - try { - raw = readFileSync(this.path); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'ENOENT') return { records: [] }; - throw new Error( - `Could not read Discord awareness journal ${this.path}: ${error instanceof Error ? error.message : String(error)}`, - ); - } - const terminated = raw.length === 0 || raw[raw.length - 1] === 0x0a; - const lastNewline = raw.lastIndexOf(0x0a); - const body = terminated ? raw : raw.subarray(0, lastNewline + 1); - const lines = body.toString('utf8').split('\n'); - const records: StoredRecord[] = []; - for (let index = 0; index < lines.length; index++) { - const line = lines[index]; - if (!line.trim()) continue; - try { - records.push(JSON.parse(line) as StoredRecord); - } catch { - throw new Error(`Corrupt Discord awareness journal ${this.path} at line ${index + 1}`); + private load(): JournalState { + if (this.state && !this.journal.needsReconcile) return this.state; + const { snapshot, entries } = this.journal.load(); + const state = snapshot ? restoreState(snapshot) : emptyState(); + for (const { id, entry } of entries) { + if (!entry || !Array.isArray(entry.records)) { + throw new Error(`Corrupt Discord awareness journal entry ${id}`); } + applyRecords(state, entry.records); } - if (terminated) return { records }; - const tail = raw.subarray(lastNewline + 1).toString('utf8'); - try { - records.push(JSON.parse(tail) as StoredRecord); - return { records, repair: { terminate: true } }; - } catch { - // A torn append: none of it ever applied. - return { records, repair: { truncateTo: lastNewline + 1 } }; - } + this.state = state; + this.importLegacyLedger(state); + return state; } - /** - * A pre-journal ledger beside the journal: null when there is none; - * otherwise its import records, empty when its hash is already imported - * (a crash between the import and the rename), so the file is only renamed. - */ - private pendingLegacyImport(state: JournalState): { records: JournalRecord[]; sha256: string } | null { + /** Import a pre-journal ledger beside the store once (by hash), then rename it. */ + private importLegacyLedger(state: JournalState): void { + if (!this.legacyPath) return; let raw: string; try { raw = readFileSync(this.legacyPath, 'utf8'); } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'ENOENT') return null; + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return; throw new Error( `Could not read Discord awareness ledger ${this.legacyPath}: ${error instanceof Error ? error.message : String(error)}`, ); } const sha256 = createHash('sha256').update(raw).digest('hex'); - if (state.importedSources.has(sha256)) return { records: [], sha256 }; - let parsed: unknown; - try { - parsed = JSON.parse(raw); - } catch { - throw new Error(`Invalid Discord awareness ledger ${this.legacyPath}`); - } - return { records: importLegacy(parsed, this.legacyPath, sha256), sha256 }; - } - - private append(records: JournalRecord[]): void { - if (records.length === 0) return; - const journal = this.readJournal(); - const state = fold(journal.records); - const legacy = this.pendingLegacyImport(state); - const all: StoredRecord[] = legacy ? [...legacy.records, ...records] : records; - let lines: StoredRecord[] = all; - if (all.length > 1) { - const txn = randomUUID(); - lines = [ - ...all.map((record) => ({ ...record, txn })), - { t: 'commit', at: Date.now(), txn }, - ]; - } - mkdirSync(dirname(this.path), { recursive: true }); - if (journal.repair && 'truncateTo' in journal.repair) truncateSync(this.path, journal.repair.truncateTo); - const prefix = journal.repair && 'terminate' in journal.repair ? '\n' : ''; - const fd = openSync(this.path, 'a', 0o600); - try { - writeSync(fd, prefix + lines.map((record) => JSON.stringify(record)).join('\n') + '\n'); - fsyncSync(fd); - } finally { - closeSync(fd); + if (!state.importedSources.has(sha256)) { + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch { + throw new Error(`Invalid Discord awareness ledger ${this.legacyPath}`); + } + this.append(importLegacy(parsed, this.legacyPath, sha256), { durable: true }); } - if (legacy) this.retireLegacyFile(); - } - - private retireLegacyFile(): void { + // Imported (now or by an earlier process that stopped before renaming): + // the file is inert; move it out of the way. if (!existsSync(this.legacyPath)) return; try { renameSync(this.legacyPath, `${this.legacyPath}.migrated-v2`); } catch (error) { - // The import is already recorded (by hash), so the file is inert; the - // next write retries the rename. console.error( `[discord-awareness] imported ${basename(this.legacyPath)} but could not rename it: ` + `${error instanceof Error ? error.message : String(error)}`, ); } } + + /** + * Append one group of records as one journal entry and apply it. A failed + * append leaves the state to be rebuilt from the journal: whether it was + * written is unknown, and the next operation reloads before writing. + */ + private append(records: JournalRecord[], opts: AppendOptions = {}): void { + if (records.length === 0) return; + const state = this.load(); + try { + this.journal.append({ records }, opts); + } catch (error) { + this.state = null; + throw error; + } + applyRecords(state, records); + if (this.journal.entriesSinceCheckpoint >= this.checkpointEvery) { + try { + this.journal.checkpoint(snapshotState(state)); + } catch (error) { + // Replay just stays longer; the journal reloads before its next write. + this.state = null; + console.error( + `[discord-awareness] checkpoint failed: ${error instanceof Error ? error.message : String(error)}`, + ); + } + } + } } // --------------------------------------------------------------------------- // Fold and legacy import // --------------------------------------------------------------------------- -function fold(stored: StoredRecord[]): JournalState { - const state: JournalState = { batches: new Map(), ops: new Map(), importedSources: new Set() }; - // Records of a transaction apply, in order, at its commit; an uncommitted - // transaction (a torn or interrupted append) never applies. - const open = new Map(); - const records: JournalRecord[] = []; - for (const record of stored) { - if (record.t === 'commit') { - records.push(...(open.get(record.txn) ?? [])); - open.delete(record.txn); - } else if (record.txn) { - const buffered = open.get(record.txn); - if (buffered) buffered.push(record); - else open.set(record.txn, [record]); - } else { - records.push(record); - } - } +function emptyState(): JournalState { + return { batches: new Map(), ops: new Map(), legacy: new Map(), retracts: new Map(), importedSources: new Set() }; +} + +function snapshotState(state: JournalState): JournalSnapshot { + return { + batches: [...state.batches.values()].map((batch) => structuredClone(batch)), + ops: [...state.ops.values()].map((op) => structuredClone(op)), + legacy: [...state.legacy.values()].flat().map((evidence) => structuredClone(evidence)), + retracts: [...state.retracts.values()].map((retract) => structuredClone(retract)), + importedSources: [...state.importedSources], + }; +} + +function restoreState(snapshot: JournalSnapshot): JournalState { + const state: JournalState = { + batches: new Map(snapshot.batches.map((batch) => [batch.id, structuredClone(batch)])), + ops: new Map(snapshot.ops.map((op) => [op.opId, structuredClone(op)])), + legacy: new Map(), + retracts: new Map((snapshot.retracts ?? []).map((retract) => [retract.requestId, structuredClone(retract)])), + importedSources: new Set(snapshot.importedSources), + }; + for (const evidence of snapshot.legacy ?? []) addLegacyEvidence(state, structuredClone(evidence)); + return state; +} + +function addLegacyEvidence(state: JournalState, evidence: DiscordAwarenessLegacyEvidence): void { + const key = markKey(evidence.key); + const list = state.legacy.get(key); + if (list) list.push(evidence); + else state.legacy.set(key, [evidence]); +} + +/** Apply one entry's records, in order, to the reduced state. */ +function applyRecords(state: JournalState, records: JournalRecord[]): void { for (const record of records) { switch (record.t) { case 'batch': @@ -1034,7 +1266,13 @@ function fold(stored: StoredRecord[]): JournalState { break; case 'activated': { const batch = state.batches.get(record.batchId); - if (batch && batch.status !== 'discarded') batch.status = 'active'; + if (batch && batch.status !== 'discarded') { + batch.status = 'active'; + // Only a surgery (or its resume) activates a suppression, after its + // last redaction: activation records the body complete. Release + // does not. + if (batch.suppressionIntervals?.length) batch.suppressionComplete = true; + } break; } case 'discarded': { @@ -1114,6 +1352,26 @@ function fold(stored: StoredRecord[]): JournalState { if (batch) batch.suppressionComplete = true; break; } + case 'legacy-evidence': + addLegacyEvidence(state, structuredClone(record.evidence)); + break; + case 'retract': + if (!state.retracts.has(record.requestId)) { + state.retracts.set(record.requestId, { + requestId: record.requestId, + target: record.target, + at: record.at, + ...(record.by ? { by: record.by } : {}), + }); + } + break; + case 'request-cancelled': { + const retract = state.retracts.get(record.requestId); + if (retract && !retract.cancelled) { + retract.cancelled = { at: record.at, ...(record.by ? { by: record.by } : {}) }; + } + break; + } case 'import-complete': state.importedSources.add(record.sha256); break; @@ -1121,7 +1379,6 @@ function fold(stored: StoredRecord[]): JournalState { break; } } - return state; } interface LegacyEntry extends DiscordAwarenessRef { @@ -1148,11 +1405,13 @@ interface LegacyBatch { } /** - * Translate a version 1 or 2 ledger into journal records with deterministic - * identities, ending with an `import-complete` record carrying the file's - * hash. What the old ledger recorded is kept as it was recorded; nothing is - * inferred about Discord. Pending work, and batches still prepared, are held - * for an operator: they were queued before marks were an explicit choice. + * Translate a version 1 or 2 ledger into journal records, ending with an + * `import-complete` record carrying the file's hash. Batches keep their ids. + * What the old ledger recorded about each ref is kept as legacy evidence: + * facts (attempt count, last action, status and error), never synthesized + * attempts and never a judgment about what is on Discord. Pending work, and + * batches still prepared, are held for an operator: they were queued before + * marks were an explicit choice. Nothing imported is ever dispatched. */ function importLegacy(parsed: unknown, path: string, sha256: string): JournalRecord[] { const document = parsed as { version?: unknown; batches?: unknown }; @@ -1187,6 +1446,8 @@ function importLegacy(parsed: unknown, path: string, sha256: string): JournalRec ...(raw.suppressionIntervals?.length ? { suppressionIntervals: raw.suppressionIntervals } : {}), }, }); + // An active batch was activated by its surgery, after its body (an + // earlier release activated suppressions only after the last redaction). if (legacyActive) records.push({ t: 'activated', at: raw.createdAt, batchId: raw.id }); const releaseActions: DiscordAwarenessReleaseAction[] = []; for (const entry of raw.refs) { @@ -1200,27 +1461,13 @@ function importLegacy(parsed: unknown, path: string, sha256: string): JournalRec ...key, action: desired || !legacyActive ? 'add' : 'remove', ...(entry.attempts ? { priorAttempts: entry.attempts } : {}), - ...(entry.lastError ? { priorError: entry.lastError } : {}), + ...(entry.lastError ? { priorError: boundDiscordAwarenessText(entry.lastError) } : {}), }); } - continue; } - if (!entry.attempts || !entry.lastAction) continue; - const opId = `v2:${raw.id}:${markKey(key)}`; - const attemptAt = entry.lastAttemptAt ?? raw.createdAt; - records.push({ t: 'requested', at: raw.createdAt, opId, key, action: entry.lastAction, cause: { kind: 'import', batchId: raw.id } }); - records.push({ t: 'dispatching', at: attemptAt, opId, attempt: 1 }); - records.push(status === 'applied' - ? { t: 'outcome', at: attemptAt, opId, attempt: 1, outcome: 'confirmed' } - : { - t: 'outcome', - at: attemptAt, - opId, - attempt: 1, - outcome: 'failed', - permanent: true, - ...(entry.lastError ? { error: `recorded by an earlier ledger: ${entry.lastError}` } : {}), - }); + if (entry.attempts && entry.attempts > 0 && entry.lastAction) { + records.push({ t: 'legacy-evidence', at, evidence: legacyEvidence(raw.id, key, entry, status) }); + } } if (releaseActions.length > 0) { records.push({ @@ -1238,6 +1485,37 @@ function importLegacy(parsed: unknown, path: string, sha256: string): JournalRec return records; } +/** The facts a v2 entry recorded about its attempts, and what they leave open. */ +function legacyEvidence( + batchId: string, + key: DiscordAwarenessRef & { emoji: string }, + entry: LegacyEntry, + status: 'pending' | 'applied' | 'permanent-failure', +): DiscordAwarenessLegacyEvidence { + const attempts = entry.attempts!; + const lastError = entry.lastError; + const lastErrorKind = lastError === undefined ? undefined + : PRE_WRITE_REFUSAL.test(lastError) ? 'pre-write-refusal' + : isPermanentDiscordReactionFailure(lastError) ? 'discord-answer' + : 'other'; + // The last attempt's outcome is established when the ledger saw it + // applied, or its error proves it was refused before writing or answered + // by Discord. Earlier attempts' details were never kept. + const lastEstablished = status === 'applied' + || lastErrorKind === 'pre-write-refusal' + || (status === 'permanent-failure' && lastErrorKind === 'discord-answer'); + return { + batchId, + key: { ...key }, + attempts, + lastAction: entry.lastAction!, + lastStatus: status, + ...(lastError !== undefined ? { lastError: boundDiscordAwarenessText(lastError) } : {}), + ...(lastErrorKind ? { lastErrorKind } : {}), + outcomesUnrecorded: (attempts - 1) + (lastEstablished ? 0 : 1), + }; +} + // --------------------------------------------------------------------------- // Keys // --------------------------------------------------------------------------- diff --git a/src/recovery/offline-branch.ts b/src/recovery/offline-branch.ts index 7045d4392..26c467e49 100644 --- a/src/recovery/offline-branch.ts +++ b/src/recovery/offline-branch.ts @@ -1,4 +1,5 @@ import { JsStore } from '@animalabs/chronicle'; +import type { SurgeryMarkerReceipt } from '../operator-log.js'; import { ContextManager } from '@animalabs/context-manager'; import { DEFAULT_DISCORD_AWARENESS_EMOJI, @@ -29,6 +30,7 @@ export interface OfflineRecoveryBranchOptions { namespace?: string; /** Use for old message records that do not carry metadata.serverId. */ discordServerId?: string; + /** A pre-journal awareness ledger to import (default: under the store). */ outboxPath?: string; emoji?: string; /** @@ -49,15 +51,17 @@ export interface OfflineRecoveryBranchResult { discordAddressable: number; /** The publication scope recorded for this recovery. */ marksScope: 'none' | 'addressed' | 'all'; - /** Marks requested (queued once the branch is activated); not delivered. */ + /** Marks queued for delivery (planned, for a dry run); not delivered. */ discordMarkersQueued: number; + /** What became of the marks once the branch was made (absent on a dry + * run): scheduling only, never Discord acceptance. */ + markers?: SurgeryMarkerReceipt; /** The refs that will be marked (empty unless marks were chosen). */ refs: DiscordAwarenessRef[]; /** Removed addressable messages left unmarked. */ unmarked: number; /** Authorized refs (marks.refs) this recovery does not remove. */ notRemoved: number; - outboxPath: string; } /** @@ -176,8 +180,6 @@ export async function createOfflineRecoveryBranch( if (targetBranch === sourceBranch) { throw new Error('Recovery branch name must differ from the active source branch'); } - const outboxPath = options.outboxPath - ?? defaultDiscordAwarenessOutboxPath(options.storePath); const result: OfflineRecoveryBranchResult = { dryRun: options.dryRun === true, @@ -191,11 +193,14 @@ export async function createOfflineRecoveryBranch( refs, unmarked: selection.unmarked, notRemoved: selection.notRemoved, - outboxPath, }; if (options.dryRun) return result; - const outbox = new DiscordAwarenessOutbox(outboxPath); + // The journal lives in this store; the host is stopped, so this process + // is its only writer. + const outbox = new DiscordAwarenessOutbox(store, { + legacyPath: options.outboxPath ?? defaultDiscordAwarenessOutboxPath(options.storePath), + }); const batch = outbox.prepare({ agentName: options.agentName, sourceBranch, @@ -226,18 +231,36 @@ export async function createOfflineRecoveryBranch( if (interval.fromId === interval.toId) contextManager.removeMessage(interval.fromId); else contextManager.removeMessages(interval.fromId, interval.toId); } - if (batch) outbox.activate(batch.id); } catch (error) { // A partially suppressed branch is not safe to boot into. Preserve it // for diagnosis, but leave Chronicle on the untouched source branch and - // keep the explicit outbox batch non-deliverable. + // keep the explicit batch prepared: it is the unfinished body's record + // (startup holds its marks, and resumes the body only on its branch). if (store.currentBranch().name === createdBranch) { await contextManager.switchBranch(sourceBranch); } throw error; } - return result; + // The body change landed. Marker bookkeeping is reported apart from it + // and never undoes it. + const facts = { + scope: (marks === 'none' ? 'none' : marks.scope) as SurgeryMarkerReceipt['scope'], + unmarked: selection.unmarked, + notRemoved: selection.notRemoved, + }; + let markers: SurgeryMarkerReceipt = { ...facts, status: 'none', queued: 0 }; + if (batch) { + const settled = outbox.settleActivation(batch.id); + if (batch.refs.length > 0) { + markers = settled.status === 'queued' + ? { ...facts, status: 'queued', queued: settled.queued, batchId: batch.id } + : settled.status === 'not-scheduled' + ? { ...facts, status: 'not-scheduled', queued: 0, error: settled.error } + : { ...facts, status: 'unresolved', queued: 0, batchId: batch.id, error: settled.error }; + } + } + return { ...result, discordMarkersQueued: markers.queued, markers }; } finally { contextManager?.close(); store.close(); diff --git a/src/recovery/recover-cli.ts b/src/recovery/recover-cli.ts index 99c642c0e..9191cea2a 100644 --- a/src/recovery/recover-cli.ts +++ b/src/recovery/recover-cli.ts @@ -1,5 +1,6 @@ #!/usr/bin/env node +import { JsStore } from '@animalabs/chronicle'; import { createOfflineRecoveryBranch } from './offline-branch.js'; import { DiscordAwarenessOutbox, @@ -29,6 +30,7 @@ function usage(): string { return `Usage: agent-framework-recover --store --agent [options] agent-framework-recover --store --awareness + (cancel takes a batch id or a retract request id) Creates and activates a Chronicle recovery branch while the normal host is down. The recovery is local to the resident: by default no Discord reaction is @@ -49,14 +51,16 @@ Options: Awareness marks on removed Discord messages (default none; addressed = mentions, replies to the bot and DMs; all = every addressable message) - --outbox Override the awareness journal path + --outbox A pre-journal awareness ledger to import + (default: /recovery/discord-awareness-outbox.json) --emoji Marker reaction (default: 💤) --dry-run Inspect counts/addresses without writing --help Show this help Awareness journal (host stopped): --awareness list Each batch: scope, counts by status, holds - --awareness cancel Stop a batch's unsent marks (never removes any) + --awareness cancel Stop further sends of a batch's marks, or of a + retract's removals (never undoes anything) --awareness retract Queue removal of this bot's marks (through each ref's configured MCPL route) on a batch's refs, @@ -135,26 +139,33 @@ function parseArgs(args: string[]): CliOptions { } function awareness(options: CliOptions): void { - const outbox = new DiscordAwarenessOutbox( - options.outboxPath ?? defaultDiscordAwarenessOutboxPath(options.storePath!), - ); - const { verb, target } = options.awareness!; - const by = 'agent-framework-recover'; - let result: unknown; - switch (verb) { - case 'list': result = { journal: outbox.path, batches: outbox.view() }; break; - case 'cancel': result = outbox.cancel(target!, by); break; - case 'retract': result = outbox.retract(target!, by); break; - case 'release': result = outbox.release(target!, by); break; + // The journal lives in the store; with the host stopped, this process is + // its only writer. + const store = JsStore.openOrCreate({ path: options.storePath! }); + try { + const outbox = new DiscordAwarenessOutbox(store, { + legacyPath: options.outboxPath ?? defaultDiscordAwarenessOutboxPath(options.storePath!), + }); + const { verb, target } = options.awareness!; + const by = 'agent-framework-recover'; + let result: unknown; + switch (verb) { + case 'list': result = { store: options.storePath, batches: outbox.view() }; break; + case 'cancel': result = outbox.cancel(target!, by); break; + case 'retract': result = outbox.retract(target!, by); break; + case 'release': result = outbox.release(target!, by); break; + } + console.log(JSON.stringify(result, null, 2)); + } finally { + store.close(); } - console.log(JSON.stringify(result, null, 2)); } async function main(): Promise { const options = parseArgs(process.argv.slice(2)); if (options.awareness) { - if (!options.storePath && !options.outboxPath) { - throw new Error(`--awareness needs --store (or --outbox)\n\n${usage()}`); + if (!options.storePath) { + throw new Error(`--awareness needs --store\n\n${usage()}`); } awareness(options); return; diff --git a/test/discord-awareness-delivery.test.ts b/test/discord-awareness-delivery.test.ts index 137afb045..cdf39b48d 100644 --- a/test/discord-awareness-delivery.test.ts +++ b/test/discord-awareness-delivery.test.ts @@ -14,11 +14,12 @@ import { test } from 'node:test'; import assert from 'node:assert/strict'; import { fileURLToPath } from 'node:url'; -import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; +import { JsStore } from '@animalabs/chronicle'; import { AgentFramework } from '../src/framework.js'; -import { DiscordAwarenessOutbox } from '../src/recovery/discord-awareness-outbox.js'; +import { DISCORD_AWARENESS_RECORD_TYPE, DiscordAwarenessOutbox } from '../src/recovery/discord-awareness-outbox.js'; import { MockMembrane } from './helpers/mock-membrane.js'; const FIXTURE = fileURLToPath(new URL('./fixtures/surgery-marks-mcpl-server.mjs', import.meta.url)); @@ -40,35 +41,39 @@ async function waitFor(description: string, predicate: () => boolean, timeoutMs function setup() { const dir = mkdtempSync(join(tmpdir(), 'awareness-delivery-')); const storePath = join(dir, 'store'); - // Outside the store: the store directory must not exist before Chronicle - // creates it, and these tests seed the journal before the first start. - const journal = join(dir, 'awareness', 'discord-awareness-journal.jsonl'); const paths = { calls: join(dir, 'calls.jsonl'), events: join(dir, 'events.jsonl'), hold: join(dir, 'release'), probe: join(dir, 'probe'), }; + /** Queue marks in the store's journal while no host has it open. */ const queueMarks = (messageIds: string[]) => { - const outbox = new DiscordAwarenessOutbox(journal); - const batch = outbox.prepare({ - agentName: 'resident', - sourceBranch: 'source', - targetBranch: 'main', - refs: messageIds.map((messageId) => ({ serverId: 'discord', channelId: 'discord:g1:c1', messageId })), - scope: 'all', - activationPolicy: 'explicit', - })!; - outbox.activate(batch.id); - return outbox; + const store = JsStore.openOrCreate({ path: storePath }); + try { + const outbox = new DiscordAwarenessOutbox(store); + const batch = outbox.prepare({ + agentName: 'resident', + sourceBranch: 'source', + targetBranch: 'main', + refs: messageIds.map((messageId) => ({ serverId: 'discord', channelId: 'discord:g1:c1', messageId })), + scope: 'all', + activationPolicy: 'explicit', + })!; + outbox.activate(batch.id); + } finally { + store.close(); + } }; + /** The running framework's own journal instance. */ + const journal = (framework: AgentFramework): DiscordAwarenessOutbox => + (framework as unknown as { discordAwarenessOutbox: DiscordAwarenessOutbox }).discordAwarenessOutbox; const config = (opts: { discord?: boolean; requestTimeoutMs?: number; awarenessDeadlineMs?: number } = {}) => ({ storePath, membrane: new MockMembrane().asMembrane(), agents: [{ name: 'resident', model: 'test', systemPrompt: 'test' }], modules: [], maintenanceIntervalMs: 0, - discordAwarenessOutboxPath: journal, ...(opts.awarenessDeadlineMs ? { discordAwarenessDeadlineMs: opts.awarenessDeadlineMs } : {}), ...(opts.discord === false ? {} : { mcplServers: [{ @@ -101,7 +106,7 @@ test('startup does not wait for queued marks, and channel traffic flows while th ); writeFileSync(paths.hold, '1'); await waitFor('both adds confirmed', () => { - const outbox = new DiscordAwarenessOutbox(journal); + const outbox = journal(framework!); return outbox.operations().filter((op) => outbox.operationStatus(op) === 'confirmed').length === 2; }); } finally { @@ -117,12 +122,9 @@ test('the mandatory awareness deadline still bounds a hung reaction with request try { queueMarks(['m1']); framework = await AgentFramework.create(config({ requestTimeoutMs: 0, awarenessDeadlineMs: 150 })); - // Another reader sees an unanswered dispatch as unknown at once; wait - // for the drain's own outcome record. await waitFor('the hung add to be recorded unknown', () => - new DiscordAwarenessOutbox(journal).operations().some((op) => op.attempts[0]?.outcome === 'unknown')); - const outbox = new DiscordAwarenessOutbox(journal); - const [op] = outbox.operations(); + journal(framework!).operations().some((op) => op.attempts[0]?.outcome === 'unknown')); + const [op] = journal(framework).operations(); assert.equal(op.attempts[0].outcome, 'unknown'); assert.match(op.attempts[0].error ?? '', /did not respond/); } finally { @@ -137,17 +139,17 @@ test('a route that is not connected leaves marks queued; they go out once it is' let framework: AgentFramework | undefined; try { queueMarks(['m1']); - const before = readFileSync(journal, 'utf8'); framework = await AgentFramework.create(config({ discord: false })); + const before = JSON.stringify(journal(framework).operations()); await framework.syncDiscordAwarenessMarkers(); - assert.equal(readFileSync(journal, 'utf8'), before, 'nothing recorded for a route that is down'); + assert.equal(JSON.stringify(journal(framework).operations()), before, 'nothing recorded for a route that is down'); await framework.stop(); framework = undefined; writeFileSync(paths.hold, '1'); framework = await AgentFramework.create(config()); await waitFor('the queued add confirmed', () => { - const outbox = new DiscordAwarenessOutbox(journal); + const outbox = journal(framework!); return outbox.operations().every((op) => outbox.operationStatus(op) === 'confirmed'); }); assert.deepEqual(jsonl(paths.calls).map((call) => call.name), ['add_reaction']); @@ -159,13 +161,14 @@ test('a route that is not connected leaves marks queued; they go out once it is' }); test('a corrupt journal stops startup, since it may hold a suppression the body needs resumed', async () => { - const { dir, journal, config } = setup(); + const { dir, storePath, config } = setup(); try { - mkdirSync(join(journal, '..'), { recursive: true }); - writeFileSync(journal, '{"t":"batch",\nnot json at all\n{"t":"commit","at":1,"txn":"x"}\n'); + const store = JsStore.openOrCreate({ path: storePath }); + store.appendJson(DISCORD_AWARENESS_RECORD_TYPE, { records: 'not a record group' }); + store.close(); await assert.rejects( AgentFramework.create(config({ discord: false })), - /Discord awareness accounting failed during startup reconciliation: Corrupt Discord awareness journal/, + /Discord awareness accounting failed during startup reconciliation: Corrupt Discord awareness journal entry/, ); } finally { rmSync(dir, { recursive: true, force: true }); @@ -188,7 +191,7 @@ test('a branch switch neither adds nor removes marks', async () => { }); const result = await framework.rollbackToMessage('resident', { messageId: tail, marks: { scope: 'addressed' } }); await waitFor('the add confirmed', () => - new DiscordAwarenessOutbox(journal).operations().some((op) => op.attempts[0]?.outcome === 'confirmed')); + journal(framework!).operations().some((op) => op.attempts[0]?.outcome === 'confirmed')); // Switch back to the source and forward again: nothing is sent. await cm.switchBranch(result.sourceBranch); await framework.syncDiscordAwarenessMarkers(); @@ -198,7 +201,7 @@ test('a branch switch neither adds nor removes marks', async () => { framework = await AgentFramework.create(config()); await framework.syncDiscordAwarenessMarkers(); assert.deepEqual(jsonl(paths.calls).map((call) => call.name), ['add_reaction']); - assert.equal(new DiscordAwarenessOutbox(journal).operations().length, 1); + assert.equal(journal(framework).operations().length, 1); } finally { writeFileSync(paths.hold, '1'); await framework?.stop(); diff --git a/test/discord-awareness-outbox.test.ts b/test/discord-awareness-outbox.test.ts index 4ba916680..c3b4eb532 100644 --- a/test/discord-awareness-outbox.test.ts +++ b/test/discord-awareness-outbox.test.ts @@ -1,25 +1,30 @@ /** * The Discord awareness-mark journal: an append-only record of what was - * asked and what was answered, never a model of what is on Discord. + * asked and what was answered, kept as typed records in the Chronicle store + * (RecordJournal), never a model of what is on Discord. * * Contract (room-225's proposed upstream change, after a field incident in * which a web-UI rollback queued 918 💤 reactions on other people's messages): * marks are an explicit publication choice (default none); one-shot, so * branch moves and restarts never derive operations from branch state; and * cancel / retract / release are explicit operator acts whose receipts say - * what is known and what is not. + * what history establishes and what it leaves open. */ import { test } from 'node:test'; import assert from 'node:assert/strict'; -import { appendFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; import { join } from 'node:path'; import { tmpdir } from 'node:os'; +import { JsStore } from '@animalabs/chronicle'; import { + DISCORD_AWARENESS_RECORD_TYPE, DiscordAwarenessOutbox, boundDiscordAwarenessText, extractDiscordAwarenessRefs, selectDiscordAwarenessRefs, + type DiscordAwarenessBatchView, type DiscordAwarenessRef, + type DiscordAwarenessRetractView, } from '../src/recovery/discord-awareness-outbox.js'; import { AgentFramework } from '../src/framework.js'; import { McplRequestError } from '../src/mcpl/server-connection.js'; @@ -28,12 +33,24 @@ import { OperatorLog } from '../src/operator-log.js'; const ref = (messageId: string, channel = 'c1'): DiscordAwarenessRef => ({ serverId: 'discord', channelId: `discord:g1:${channel}`, messageId }); -function withJournal(fn: (path: string, dir: string) => void | Promise): () => Promise { +interface Harness { + store: JsStore; + dir: string; + legacyPath: string; + /** A fresh instance over the same store: what a restarted process reads. */ + reopen(): DiscordAwarenessOutbox; +} + +function withJournal(fn: (outbox: DiscordAwarenessOutbox, h: Harness) => void | Promise): () => Promise { return async () => { const dir = mkdtempSync(join(tmpdir(), 'discord-awareness-')); + const store = JsStore.openOrCreate({ path: join(dir, 'store') }); + const legacyPath = join(dir, 'recovery', 'discord-awareness-outbox.json'); + const h: Harness = { store, dir, legacyPath, reopen: () => new DiscordAwarenessOutbox(store, { legacyPath }) }; try { - await fn(join(dir, 'recovery', 'discord-awareness-journal.jsonl'), dir); + await fn(h.reopen(), h); } finally { + if (!store.isClosed()) store.close(); rmSync(dir, { recursive: true, force: true }); } }; @@ -52,25 +69,24 @@ function activeBatch(outbox: DiscordAwarenessOutbox, refs: DiscordAwarenessRef[] return batch; } -/** Dispatch everything due, answering each dispatch with `answer`. */ -function drain( +/** Claim and answer one dispatch for a message. */ +function answer( outbox: DiscordAwarenessOutbox, - answer: (dispatch: { action: string; messageId: string }) => 'confirmed' | 'failed' | 'not-sent' | 'unknown', -): string[] { - const seen: string[] = []; - for (let pass = 0; pass < 10; pass++) { - const dispatches = outbox.pendingDispatches('discord'); - if (dispatches.length === 0) break; - for (const dispatch of dispatches) { - const attempts = outbox.recordDispatching(dispatch); - const outcome = answer({ action: dispatch.action, messageId: dispatch.key.messageId }); - seen.push(`${dispatch.action}:${dispatch.key.messageId}:${outcome}`); - outbox.recordOutcome(attempts, outcome, outcome === 'failed' ? { permanent: true, error: 'Unknown Message' } : {}); - } - } - return seen; + messageId: string, + outcome: 'confirmed' | 'failed' | 'not-sent' | 'unknown', + detail: { permanent?: boolean; error?: string } = {}, +) { + const claimed = outbox.claimDispatch('discord', (d) => d.key.messageId !== messageId); + assert.ok(claimed, `a dispatch for ${messageId} is due`); + outbox.recordOutcome(claimed.attempts, outcome, detail); + return claimed; } +const batches = (outbox: DiscordAwarenessOutbox) => + outbox.view().filter((v): v is DiscordAwarenessBatchView => v.kind === 'batch'); +const retracts = (outbox: DiscordAwarenessOutbox) => + outbox.view().filter((v): v is DiscordAwarenessRetractView => v.kind === 'retract'); + test('extractDiscordAwarenessRefs keeps only direct Discord addressing metadata', () => { const refs = extractDiscordAwarenessRefs([ { metadata: { serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'm1' } }, @@ -88,68 +104,53 @@ test('selection: none marks nothing; addressed means the chat:addressed tag; ref { metadata: { ...ref('b2') } }, { metadata: { ...ref('a2', 'dm'), tags: ['chat:addressed', 'chat:dm'] } }, ]; - const none = selectDiscordAwarenessRefs(messages, 'none'); - assert.deepEqual(none, { refs: [], addressable: 4, unmarked: 4, notRemoved: 0 }); - + assert.deepEqual(selectDiscordAwarenessRefs(messages, 'none'), { refs: [], addressable: 4, unmarked: 4, notRemoved: 0 }); const addressed = selectDiscordAwarenessRefs(messages, { scope: 'addressed' }); assert.deepEqual(addressed.refs.map((r) => r.messageId), ['a1', 'a2']); assert.equal(addressed.unmarked, 2); - - const all = selectDiscordAwarenessRefs(messages, { scope: 'all' }); - assert.equal(all.refs.length, 4); - + assert.equal(selectDiscordAwarenessRefs(messages, { scope: 'all' }).refs.length, 4); // A frozen choice: only previewed refs, and never anything else removed. const frozen = selectDiscordAwarenessRefs(messages, { scope: 'all', refs: [ref('a1'), ref('gone')] }); assert.deepEqual(frozen.refs.map((r) => r.messageId), ['a1']); - assert.equal(frozen.notRemoved, 1, 'an authorized ref the surgery did not remove'); + assert.equal(frozen.notRemoved, 1); assert.equal(frozen.unmarked, 3); - // Scope still applies to authorized refs. const frozenAddressed = selectDiscordAwarenessRefs(messages, { scope: 'addressed', refs: [ref('b1'), ref('a2', 'dm')] }); assert.deepEqual(frozenAddressed.refs.map((r) => r.messageId), ['a2']); }); -test('a prepared batch delivers nothing; activation queues one add per ref, written ahead of dispatch', withJournal((path) => { - const outbox = new DiscordAwarenessOutbox(path); +test('the journal is typed records in the store: activation queues one add per ref, written ahead of dispatch', withJournal((outbox, h) => { const batch = outbox.prepare({ agentName: 'cairn', sourceBranch: 'main', targetBranch: 'rollback/cairn/1', refs: [ref('m1'), ref('m2')], scope: 'all', })!; assert.equal(outbox.pendingDispatches('discord').length, 0, 'prepared work is not deliverable'); assert.equal(outbox.activate(batch.id), 2); - const [first] = outbox.pendingDispatches('discord'); - assert.equal(first.action, 'add'); - assert.equal(first.key.emoji, '💤'); - - const attempts = outbox.recordDispatching(first); - // The write-ahead record is durable before the request could leave. - const journal = readFileSync(path, 'utf8'); - assert.match(journal, /"t":"dispatching"/); - // A fresh reader (another process) sees it as unknown, never as unsent. - const other = new DiscordAwarenessOutbox(path); - assert.equal(other.operations().find((op) => op.opId === first.opIds[0])!.attempts.length, 1); - assert.equal(other.operationStatus(other.operations().find((op) => op.opId === first.opIds[0])!), 'unknown'); - - outbox.recordOutcome(attempts, 'confirmed'); - assert.deepEqual(outbox.pendingDispatches('discord').map((d) => d.key.messageId), ['m2']); + assert.ok(h.store.getRecordIdsByType(DISCORD_AWARENESS_RECORD_TYPE).length >= 2); + + const claimed = outbox.claimDispatch('discord')!; + assert.equal(claimed.dispatch.action, 'add'); + // The write-ahead record is in the store before the request could leave: + // a fresh reader (a restarted process) sees the attempt, outcome unknown. + const other = h.reopen(); + const op = other.operations().find((candidate) => candidate.opId === claimed.attempts[0].opId)!; + assert.equal(op.attempts.length, 1); + assert.equal(other.operationStatus(op), 'unknown'); + + outbox.recordOutcome(claimed.attempts, 'confirmed'); + assert.deepEqual(outbox.pendingDispatches('discord').map((d) => d.key.messageId), [claimed.dispatch.key.messageId === 'm1' ? 'm2' : 'm1']); })); -test('startup derives nothing from ancestry: exact-target crash completion, everything else held', withJournal((path) => { - const outbox = new DiscordAwarenessOutbox(path); - const crashed = outbox.prepare({ - agentName: 'cairn', sourceBranch: 'main', targetBranch: 'rollback/cairn/1', refs: [ref('m1')], scope: 'all', - })!; - const elsewhere = outbox.prepare({ - agentName: 'cairn', sourceBranch: 'main', targetBranch: 'rollback/cairn/2', refs: [ref('m2')], scope: 'addressed', - })!; +test('startup derives nothing from ancestry: exact-target crash completion, everything else held', withJournal((outbox, h) => { + const crashed = outbox.prepare({ agentName: 'cairn', sourceBranch: 'main', targetBranch: 'rollback/cairn/1', refs: [ref('m1')], scope: 'all' })!; + const elsewhere = outbox.prepare({ agentName: 'cairn', sourceBranch: 'main', targetBranch: 'rollback/cairn/2', refs: [ref('m2')], scope: 'addressed' })!; const suppression = outbox.prepare({ agentName: 'cairn', sourceBranch: 'main', targetBranch: 'rollback/cairn/1', refs: [ref('m3')], scope: 'all', activationPolicy: 'explicit', suppressionIntervals: [{ fromId: 'i1', toId: 'i1' }], })!; const active = activeBatch(outbox, [ref('m4')]); - const [dispatch] = outbox.pendingDispatches('discord').filter((d) => d.key.messageId === 'm4'); - outbox.recordDispatching(dispatch); // the process dies with this request on the wire + outbox.claimDispatch('discord', (d) => d.key.messageId !== 'm4'); // dies with this request on the wire - const restarted = new DiscordAwarenessOutbox(path); + const restarted = h.reopen(); const recovered = restarted.recoverAtStartup('rollback/cairn/1'); assert.deepEqual(recovered.activated, [crashed.id]); assert.deepEqual(recovered.held, [elsewhere.id]); @@ -159,291 +160,324 @@ test('startup derives nothing from ancestry: exact-target crash completion, ever assert.equal(status.get(elsewhere.id), 'held'); assert.equal(status.get(suppression.id), 'prepared', 'left for resumePreparedDiscordSuppressions'); assert.equal(status.get(active.id), 'active'); - // The unanswered add is unknown, and as the newest op for its key it is - // retried (idempotent); the held batch's ref is not dispatched at all. - assert.deepEqual( - restarted.pendingDispatches('discord').map((d) => d.key.messageId).sort(), - ['m1', 'm4'], - ); + // The unanswered add is unknown and, still the newest for its key, retried. + assert.deepEqual(restarted.pendingDispatches('discord').map((d) => d.key.messageId).sort(), ['m1', 'm4']); + // A later restart on another branch changes nothing. + assert.deepEqual(h.reopen().recoverAtStartup('main'), { activated: [], held: [suppression.id], unknownAttempts: 0 }); +})); - // A later restart on a different branch changes nothing. - const again = new DiscordAwarenessOutbox(path).recoverAtStartup('main'); - assert.deepEqual(again, { activated: [], held: [suppression.id], unknownAttempts: 0 }); +test('a dispatch is admitted from the journal as it is now: a cancel during an in-flight request stops the rest', withJournal((outbox) => { + const batch = activeBatch(outbox, [ref('m1'), ref('m2'), ref('m3')]); + // A drain computed this list, then sent the first while holding the rest. + const stale = outbox.pendingDispatches('discord'); + assert.equal(stale.length, 3); + const first = outbox.recordDispatching(stale[0]); + assert.equal(first.length, 1); + const receipt = outbox.cancel(batch.id, 'operator'); + assert.equal(receipt.cancelled, 2); + assert.equal(receipt.inFlight, 1); + // The stale list can no longer authorize anything. + assert.deepEqual(outbox.recordDispatching(stale[1]), []); + assert.deepEqual(outbox.recordDispatching(stale[2]), []); + outbox.recordOutcome(first, 'confirmed'); + assert.equal(outbox.claimDispatch('discord'), null); })); -test('cancel stops every further send of a batch, never removes, and says what may still land', withJournal((path) => { - const outbox = new DiscordAwarenessOutbox(path); +test('cancel stops every further send of a batch, never removes, and reports what may still land', withJournal((outbox, h) => { const batch = activeBatch(outbox, [ref('m1'), ref('m2'), ref('m3'), ref('m4')]); - const dispatches = outbox.pendingDispatches('discord'); - const byId = new Map(dispatches.map((d) => [d.key.messageId, d])); - outbox.recordOutcome(outbox.recordDispatching(byId.get('m1')!), 'confirmed'); - outbox.recordOutcome(outbox.recordDispatching(byId.get('m2')!), 'unknown', { error: 'did not respond' }); - const m3Attempts = outbox.recordDispatching(byId.get('m3')!); // on the wire right now + answer(outbox, 'm1', 'confirmed'); + answer(outbox, 'm2', 'unknown', { error: 'did not respond' }); + const m3 = outbox.claimDispatch('discord', (d) => d.key.messageId !== 'm3')!; // on the wire const receipt = outbox.cancel(batch.id, 'operator'); assert.deepEqual( - { cancelled: receipt.cancelled, inFlight: receipt.inFlight, unknown: receipt.unknown, confirmed: receipt.confirmed }, - { cancelled: 1, inFlight: 1, unknown: 1, confirmed: 1 }, + { target: receipt.target, kind: receipt.kind, cancelled: receipt.cancelled, inFlight: receipt.inFlight, unknown: receipt.unknown, confirmed: receipt.confirmed, unresolvedAttempts: receipt.unresolvedAttempts }, + { target: batch.id, kind: 'batch', cancelled: 1, inFlight: 1, unknown: 1, confirmed: 1, unresolvedAttempts: 2 }, ); - // Nothing new is sent, no removal is queued, and the unknown add is not - // retried; the key with a request on the wire stays busy meanwhile. - assert.equal(outbox.pendingDispatches('discord').length, 0); - - // The in-flight request lands after the cancel: that is recorded as what - // happened, and it is still not followed by anything. - outbox.recordOutcome(m3Attempts, 'confirmed'); - const m3 = outbox.operations().find((op) => op.key.messageId === 'm3')!; - assert.equal(outbox.operationStatus(m3), 'confirmed'); + assert.equal(outbox.pendingDispatches('discord').length, 0, 'nothing new is sent, nothing is removed'); + // The in-flight request lands after the cancel: recorded, not followed up. + outbox.recordOutcome(m3.attempts, 'confirmed'); + assert.equal(outbox.operationStatus(outbox.operations().find((op) => op.key.messageId === 'm3')!), 'confirmed'); assert.equal(outbox.pendingDispatches('discord').length, 0); - - // Cancellation is durable: a restart or a branch move re-arms nothing. - const restarted = new DiscordAwarenessOutbox(path); + // Durable across a restart. + const restarted = h.reopen(); restarted.recoverAtStartup('some/other/branch'); assert.ok(restarted.batches().find((b) => b.id === batch.id)?.cancelled); assert.equal(restarted.pendingDispatches('discord').length, 0); - // Retract still reaches what may have landed: m1, m2 (unknown) and m3. - const retracted = restarted.retract(batch.id); - assert.equal(retracted.removalsQueued, 3); - assert.equal(retracted.keysWithUnresolvedAdds, 1); })); -test('cancelling a batch whose surgery has not activated it stops every mark, and its body can still resume', withJournal((path) => { - const outbox = new DiscordAwarenessOutbox(path); - const rollback = outbox.prepare({ - agentName: 'cairn', sourceBranch: 'main', targetBranch: 'rollback/cairn/1', refs: [ref('m1'), ref('m2')], scope: 'all', - })!; - const receipt = outbox.cancel(rollback.id, 'operator'); - assert.equal(receipt.cancelled, 2); +test('the cancel receipt keeps an earlier unresolved request after a later answer', withJournal((outbox) => { + const batch = activeBatch(outbox, [ref('m1'), ref('m2')]); + answer(outbox, 'm1', 'unknown', { error: 'timed out' }); + answer(outbox, 'm1', 'confirmed'); // the retry confirms + answer(outbox, 'm2', 'unknown', { error: 'timed out' }); + answer(outbox, 'm2', 'failed', { permanent: true, error: 'Unknown Message' }); + const receipt = outbox.cancel(batch.id); + assert.equal(receipt.confirmed, 1); + assert.equal(receipt.unknown, 0); + assert.equal(receipt.unresolvedAttempts, 2, 'both first attempts may still land'); +})); + +test('cancelling a batch its surgery has not activated stops every mark; its body can still resume', withJournal((outbox) => { + const rollback = outbox.prepare({ agentName: 'cairn', sourceBranch: 'main', targetBranch: 'rollback/cairn/1', refs: [ref('m1'), ref('m2')], scope: 'all' })!; + assert.equal(outbox.cancel(rollback.id, 'operator').cancelled, 2); assert.throws(() => outbox.activate(rollback.id), /cancelled before activation/); - // Not activated by startup's crash completion, and not held either. assert.deepEqual(outbox.recoverAtStartup('rollback/cairn/1'), { activated: [], held: [], unknownAttempts: 0 }); - assert.equal(outbox.pendingDispatches('discord').length, 0); const suppression = outbox.prepare({ agentName: 'cairn', sourceBranch: 'main', targetBranch: 'suppress/cairn/1', refs: [ref('m3')], scope: 'all', activationPolicy: 'explicit', suppressionIntervals: [{ fromId: 'i1', toId: 'i1' }], })!; outbox.cancel(suppression.id); - // The suppression's body still resumes on its branch; its marks never go. assert.deepEqual(outbox.preparedSuppressionsForBranch('suppress/cairn/1').map((b) => b.id), [suppression.id]); outbox.recordSuppressionComplete(suppression.id); assert.deepEqual(outbox.preparedSuppressionsForBranch('suppress/cairn/1'), []); assert.equal(outbox.pendingDispatches('discord').length, 0); })); -test('retract removes this bot\'s mark across batches, supersedes unsent adds, and discloses unresolved attempts', withJournal((path) => { - const outbox = new DiscordAwarenessOutbox(path); +test('releasing a held suppression\'s marks never certifies or forgets its unfinished body', withJournal((outbox) => { + const suppression = outbox.prepare({ + agentName: 'cairn', sourceBranch: 'main', targetBranch: 'partial', refs: [ref('m1')], scope: 'all', + activationPolicy: 'explicit', suppressionIntervals: [{ fromId: 'i1', toId: 'i2' }], + })!; + assert.deepEqual(outbox.recoverAtStartup('main').held, [suppression.id]); + assert.deepEqual(outbox.preparedSuppressionsForBranch('partial').map((b) => b.id), [suppression.id]); + const released = outbox.release(suppression.id, 'operator'); + assert.equal(released.addsQueued, 1); + assert.equal(outbox.batches()[0].status, 'active'); + assert.equal(outbox.batches()[0].suppressionComplete, undefined); + // Body recovery still owes the redactions on `partial`. + assert.deepEqual(outbox.preparedSuppressionsForBranch('partial').map((b) => b.id), [suppression.id]); +})); + +test('retract queues a removal for every selected key, whatever history says, and discloses unresolved requests', withJournal((outbox) => { const a = activeBatch(outbox, [ref('m1'), ref('m2'), ref('m3')]); const b = activeBatch(outbox, [ref('m1')]); // another batch asked for the same reaction - // m1's adds share one dispatch (same key, same action, both requested). - const m1 = outbox.pendingDispatches('discord').find((d) => d.key.messageId === 'm1')!; - assert.equal(m1.opIds.length, 2); - const m1Attempts = outbox.recordDispatching(m1); - outbox.recordOutcome(m1Attempts, 'unknown', { error: 'did not respond within 75ms' }); - // m2's add never left the host; m3's was confirmed. - const m2 = outbox.pendingDispatches('discord').find((d) => d.key.messageId === 'm2')!; - outbox.recordOutcome(outbox.recordDispatching(m2), 'not-sent'); - const m3 = outbox.pendingDispatches('discord').find((d) => d.key.messageId === 'm3')!; - outbox.recordOutcome(outbox.recordDispatching(m3), 'confirmed'); + const m1 = outbox.claimDispatch('discord', (d) => d.key.messageId !== 'm1')!; + assert.equal(m1.attempts.length, 2, "both batches' adds ride one request"); + outbox.recordOutcome(m1.attempts, 'unknown', { error: 'did not respond within 75ms' }); + answer(outbox, 'm2', 'not-sent'); + answer(outbox, 'm3', 'confirmed'); const receipt = outbox.retract(a.id, 'operator'); - // m1 and m3 had adds leave the host; m2's never did, so nothing to remove. - assert.equal(receipt.removalsQueued, 2); - assert.equal(receipt.addsSuperseded, 1, "m2's unsent add is superseded"); - assert.equal(receipt.keysWithUnresolvedAdds, 1, 'm1: a timed-out add may land after the removal'); - assert.equal(receipt.unresolvedAddAttempts, 1, "one request carried both batches' adds"); - - const dispatches = outbox.pendingDispatches('discord'); - assert.deepEqual(dispatches.map((d) => `${d.action}:${d.key.messageId}`).sort(), ['remove:m1', 'remove:m3']); - - // The removal is confirmed, and the earlier unknown add stays unresolved: - // a confirmation of a different request never resolves it. - drain(outbox, () => 'confirmed'); - const views = new Map(outbox.view().map((v) => [v.id, v])); - assert.equal(views.get(a.id)!.adds.unknown, 1); - assert.equal(views.get(a.id)!.removals.confirmed, 2); - assert.equal(views.get(b.id)!.unresolvedAttempts, 1); + assert.equal(receipt.removalsQueued, 3, 'm2 too: history does not decide whether the act is carried out'); + assert.equal(receipt.addsSuperseded, 1, "m2's due add stops"); + assert.equal(receipt.keysWithUnresolvedAdds, 1); + assert.equal(receipt.unresolvedAddAttempts, 1, 'one physical request carried both adds'); + assert.equal(receipt.keysWithLegacyUncertainty, 0); + assert.deepEqual(outbox.pendingDispatches('discord').map((d) => `${d.action}:${d.key.messageId}`).sort(), + ['remove:m1', 'remove:m2', 'remove:m3']); + for (const id of ['m1', 'm2', 'm3']) answer(outbox, id, 'confirmed'); + const views = new Map(batches(outbox).map((v) => [v.id, v])); + assert.equal(views.get(a.id)!.removals.confirmed, 3); + assert.equal(views.get(b.id)!.unresolvedAttempts, 1, 'the unknown add stays unresolved history'); + const [retractView] = retracts(outbox); + assert.equal(retractView.target, a.id); + assert.equal(retractView.removals.confirmed, 3); })); -test('a retry confirming an unknown add never erases the unresolved first attempt', withJournal((path) => { - const outbox = new DiscordAwarenessOutbox(path); +test('a newer opposite request stops the retries of a retryably failed one, in both directions', withJournal((outbox) => { const batch = activeBatch(outbox, [ref('m1')]); - const first = outbox.pendingDispatches('discord')[0]; - outbox.recordOutcome(outbox.recordDispatching(first), 'unknown'); - // Still the newest op for its key: retried (idempotent), as attempt 2. - const retry = outbox.pendingDispatches('discord')[0]; - assert.deepEqual(retry.opIds, first.opIds); - const retryAttempts = outbox.recordDispatching(retry); - assert.equal(retryAttempts[0].attempt, 2); - outbox.recordOutcome(retryAttempts, 'confirmed'); - assert.equal(outbox.pendingDispatches('discord').length, 0); - assert.equal(outbox.view()[0].unresolvedAttempts, 1, 'attempt 1 is still unknown'); - + answer(outbox, 'm1', 'failed', { error: 'Discord 503' }); // retryable const receipt = outbox.retract(batch.id); - assert.equal(receipt.keysWithUnresolvedAdds, 1); - assert.equal(receipt.unresolvedAddAttempts, 1); + assert.equal(receipt.addsSuperseded, 1); + for (let i = 0; i < 3; i++) { + const claimed = outbox.claimDispatch('discord')!; + assert.equal(claimed.dispatch.action, 'remove'); + outbox.recordOutcome(claimed.attempts, 'failed', { error: 'Discord 503' }); + } + const add = outbox.operations().find((op) => op.action === 'add')!; + assert.equal(add.attempts.length, 1, 'its history stays'); + assert.equal(add.cancelled?.reason, 'superseded'); + // Mirror: a new add supersedes the retryably failing remove. + activeBatch(outbox, [ref('m1')]); + const next = outbox.claimDispatch('discord')!; + assert.equal(next.dispatch.action, 'add'); + assert.equal(outbox.operations().find((op) => op.action === 'remove')!.cancelled?.reason, 'superseded'); })); -test('a new request reaches Discord even when an earlier one got the same answer', withJournal((path) => { - const outbox = new DiscordAwarenessOutbox(path); - const first = activeBatch(outbox, [ref('m1')]); - drain(outbox, () => 'confirmed'); - outbox.retract(first.id); - drain(outbox, () => 'confirmed'); - // Someone else's tool may have reacted again since; a fresh, explicit - // placement is sent rather than skipped on the strength of old receipts. - activeBatch(outbox, [ref('m1')]); - assert.deepEqual(outbox.pendingDispatches('discord').map((d) => d.action), ['add']); +test('a retract request can be cancelled by the id its receipt returned, even after a restart', withJournal((outbox, h) => { + const batch = activeBatch(outbox, [ref('m1'), ref('m2'), ref('m3')]); + for (const id of ['m1', 'm2', 'm3']) answer(outbox, id, 'confirmed'); + const { requestId } = outbox.retract(batch.id); + const onWire = outbox.claimDispatch('discord')!; + const restarted = h.reopen(); + // The retract is listed, so its id survives the original receipt. + assert.equal(retracts(restarted)[0].id, requestId); + const receipt = outbox.cancel(requestId, 'operator'); + assert.deepEqual( + { kind: receipt.kind, cancelled: receipt.cancelled, inFlight: receipt.inFlight }, + { kind: 'retract', cancelled: 2, inFlight: 1 }, + ); + outbox.recordOutcome(onWire.attempts, 'confirmed'); + assert.equal(outbox.claimDispatch('discord'), null); + assert.ok(retracts(outbox)[0].cancelled); })); -test('retract supersedes an add that never left the host instead of churning the reaction', withJournal((path) => { - const outbox = new DiscordAwarenessOutbox(path); - const batch = activeBatch(outbox, [ref('m2'), ref('m3')]); - // m3's add was sent and confirmed; m2's has not been sent yet. - const m3 = outbox.pendingDispatches('discord').find((d) => d.key.messageId === 'm3')!; - outbox.recordOutcome(outbox.recordDispatching(m3), 'confirmed'); +test('a retry confirming an unknown add never erases the unresolved first attempt', withJournal((outbox) => { + const batch = activeBatch(outbox, [ref('m1')]); + const first = answer(outbox, 'm1', 'unknown'); + const retry = outbox.claimDispatch('discord')!; + assert.deepEqual(retry.dispatch.opIds, first.dispatch.opIds); + assert.equal(retry.attempts[0].attempt, 2); + outbox.recordOutcome(retry.attempts, 'confirmed'); + assert.equal(batches(outbox)[0].unresolvedAttempts, 1); const receipt = outbox.retract(batch.id); - assert.equal(receipt.addsSuperseded, 1); - assert.equal(receipt.removalsQueued, 1, 'nothing was ever sent for m2, so there is nothing to remove'); - assert.deepEqual(outbox.pendingDispatches('discord').map((d) => `${d.action}:${d.key.messageId}`), ['remove:m3']); - const m2 = outbox.operations().find((op) => op.key.messageId === 'm2')!; - assert.equal(m2.cancelled?.reason, 'superseded'); + assert.equal(receipt.unresolvedAddAttempts, 1); })); -test('release queues a held batch\'s recorded operations, explicitly', withJournal((path) => { - const outbox = new DiscordAwarenessOutbox(path); - const batch = outbox.prepare({ - agentName: 'cairn', sourceBranch: 'main', targetBranch: 'rollback/cairn/9', refs: [ref('m1'), ref('m2')], scope: 'addressed', - })!; +test('release queues a held batch\'s recorded operations, explicitly', withJournal((outbox) => { + const batch = outbox.prepare({ agentName: 'cairn', sourceBranch: 'main', targetBranch: 'rollback/cairn/9', refs: [ref('m1'), ref('m2')], scope: 'addressed' })!; outbox.recoverAtStartup('main'); assert.equal(outbox.batches()[0].status, 'held'); assert.equal(outbox.pendingDispatches('discord').length, 0); const receipt = outbox.release(batch.id, 'operator'); assert.deepEqual({ adds: receipt.addsQueued, removals: receipt.removalsQueued }, { adds: 2, removals: 0 }); - assert.equal(outbox.batches()[0].status, 'active'); assert.equal(outbox.pendingDispatches('discord').length, 2); assert.throws(() => outbox.release(batch.id), /not held/); })); -test('outside error text is bounded before it is journaled', withJournal((path) => { +test('settleActivation says what the journal will do when activation cannot be recorded', withJournal((outbox) => { + const ok = outbox.prepare({ agentName: 'cairn', sourceBranch: 'main', targetBranch: 'b1', refs: [ref('m1')], scope: 'all' })!; + assert.deepEqual(outbox.settleActivation(ok.id), { status: 'queued', queued: 1 }); + const cancelled = outbox.prepare({ agentName: 'cairn', sourceBranch: 'main', targetBranch: 'b2', refs: [ref('m2')], scope: 'all' })!; + outbox.cancel(cancelled.id); + const settled = outbox.settleActivation(cancelled.id); + assert.equal(settled.status, 'not-scheduled'); + assert.equal(outbox.batches().find((b) => b.id === cancelled.id)?.status, 'discarded'); +})); + +test('outside error text is bounded before it is journaled', withJournal((outbox) => { assert.equal(boundDiscordAwarenessText('short'), 'short'); const huge = `${'x'.repeat(300)}${'界'.repeat(500_000)}😀tail`; const bounded = boundDiscordAwarenessText(huge); assert.ok(bounded.length <= 500, `bounded to ${bounded.length}`); assert.match(bounded, /chars omitted/); assert.ok(!/[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(? { - const outbox = new DiscordAwarenessOutbox(path); - const batch = outbox.prepare({ - agentName: 'cairn', sourceBranch: 'main', targetBranch: 'rollback/cairn/1', refs: [ref('m1'), ref('m2')], scope: 'all', - })!; - outbox.activate(batch.id); - const committed = readFileSync(path, 'utf8'); - // Simulate an activation whose append was cut off before its commit. - const lines = committed.trimEnd().split('\n'); - const activation = lines.filter((line) => line.includes('"txn"')); - assert.ok(activation.length >= 3, 'activation is a transaction'); - writeFileSync(path, lines.filter((line) => !line.includes('"txn"') && !line.includes('"t":"commit"')).join('\n') + '\n'); - appendFileSync(path, activation.slice(0, -1).join('\n') + '\n' + '{"t":"requested","at":1,"op'); - const reread = new DiscordAwarenessOutbox(path); - assert.equal(reread.batches()[0].status, 'prepared', 'the activation never committed'); - assert.equal(reread.pendingDispatches('discord').length, 0); - assert.equal(reread.operations().length, 0); +test('a corrupt journal entry stops the reader rather than being skipped', withJournal((_outbox, h) => { + h.store.appendJson(DISCORD_AWARENESS_RECORD_TYPE, { records: 'nope' }); + assert.throws(() => h.reopen().batches(), /Corrupt Discord awareness journal entry/); })); -test('an append after a torn tail repairs it instead of corrupting the journal', withJournal((path) => { - const outbox = new DiscordAwarenessOutbox(path); - const batch = activeBatch(outbox, [ref('m1'), ref('m2')]); - // A single-record append cut off mid-line (the process died writing it). - appendFileSync(path, '{"t":"dispatching","at":1,"opId":"x","att'); - const reread = new DiscordAwarenessOutbox(path); - assert.equal(reread.pendingDispatches('discord').length, 2, 'the torn record never applied'); - // The next append must not share the torn line. - const dispatch = reread.pendingDispatches('discord')[0]; - reread.recordOutcome(reread.recordDispatching(dispatch), 'confirmed'); - const again = new DiscordAwarenessOutbox(path); - assert.equal(again.pendingDispatches('discord').length, 1); - assert.ok(!readFileSync(path, 'utf8').includes('"att{'), 'the fragment was truncated away'); - - // A complete record that only lost its newline is kept, then terminated. - const lines = readFileSync(path, 'utf8'); - writeFileSync(path, lines.trimEnd()); - const kept = new DiscordAwarenessOutbox(path); - assert.equal(kept.pendingDispatches('discord').length, 1); - kept.cancel(batch.id); - assert.equal(new DiscordAwarenessOutbox(path).pendingDispatches('discord').length, 0); - assert.equal(new DiscordAwarenessOutbox(path).batches()[0].cancelled !== undefined, true); +test('checkpoints bound replay and reproduce the reduced state', withJournal((_outbox, h) => { + const outbox = new DiscordAwarenessOutbox(h.store, { checkpointEvery: 8 }); + const batch = activeBatch(outbox, Array.from({ length: 5 }, (_, i) => ref(`m${i}`))); + for (let round = 0; round < 3; round++) { + for (let i = 0; i < 5; i++) { + const claimed = outbox.claimDispatch('discord', (d) => d.key.messageId !== `m${i}`); + if (claimed) outbox.recordOutcome(claimed.attempts, round === 2 ? 'confirmed' : 'failed', { error: 'Discord 503' }); + } + } + outbox.retract(batch.id); + assert.ok(h.store.getRecordIdsByType(`${DISCORD_AWARENESS_RECORD_TYPE}/checkpoint`).length >= 2, 'checkpoints were written'); + const reread = new DiscordAwarenessOutbox(h.store, { checkpointEvery: 8 }); + assert.deepEqual(reread.view(), outbox.view()); + assert.deepEqual(reread.operations(), outbox.operations()); + assert.equal(batches(reread).find((v) => v.id === batch.id)!.adds.confirmed, 5); + assert.equal(retracts(reread).length, 1); })); -test('v2 import keeps what the old ledger recorded, holds pending work, and happens once', withJournal((path, dir) => { - const legacyPath = join(dir, 'recovery', 'discord-awareness-outbox.json'); - const entry = (messageId: string, fields: Record) => ({ ...ref(messageId), attempts: 1, lastAction: 'add', ...fields }); +test('v2 import keeps what the old ledger recorded as evidence, holds pending work, and happens once', withJournal((_outbox, h) => { + const entry = (messageId: string, fields: Record) => ({ ...ref(messageId), lastAction: 'add', ...fields }); const legacy = { version: 2, batches: [ { // The household's shape: an active target-branch batch, some marks - // applied, the rest marked permanent-failure by hand after shutdown. + // applied, the rest marked permanent-failure by hand after shutdown, + // one request interrupted on the wire. id: 'dd34b043', status: 'active', agentName: 'resident', sourceBranch: 'main', targetBranch: 'rollback/resident/1791358473307', emoji: '💤', createdAt: 1791358473400, activationPolicy: 'target-branch', refs: [ - entry('a1', { desired: true, markerPresent: true, deliveryStatus: 'applied' }), - entry('a2', { desired: true, markerPresent: true, deliveryStatus: 'applied' }), - entry('p1', { desired: true, markerPresent: false, deliveryStatus: 'permanent-failure', lastError: 'Cannot send request: connection to "discord" is closed' }), + entry('a1', { desired: true, markerPresent: true, deliveryStatus: 'applied', attempts: 1 }), + entry('n1', { desired: true, markerPresent: false, deliveryStatus: 'permanent-failure', attempts: 1, lastError: 'Cannot send request: connection to "discord" is closed' }), + entry('u1', { desired: true, markerPresent: false, deliveryStatus: 'permanent-failure', attempts: 1, lastError: 'Connection to MCPL server "discord" closed while awaiting response for tools/call (id=4)' }), + entry('g1', { desired: true, markerPresent: false, deliveryStatus: 'permanent-failure', attempts: 1, lastError: 'Unknown Message' }), + entry('g3', { desired: true, markerPresent: false, deliveryStatus: 'permanent-failure', attempts: 3, lastError: 'Unknown Message' }), ], }, { - // An interrupted drain that nobody edited: its pending work is held. + // June's probe: a pending entry whose one attempt's outcome is unknown. id: 'pending-batch', status: 'active', agentName: 'resident', sourceBranch: 'main', targetBranch: 'rollback/resident/2', emoji: '💤', createdAt: 1791358473500, - refs: [ - entry('q1', { desired: true, markerPresent: false, deliveryStatus: 'pending', lastError: 'Cannot send request' }), - entry('q2', { desired: true, markerPresent: true, deliveryStatus: 'applied' }), - ], + refs: [entry('p1', { desired: true, markerPresent: false, deliveryStatus: 'pending', attempts: 1, lastError: 'did not respond within 10000ms' })], }, ], }; - mkdirSync(join(dir, 'recovery'), { recursive: true }); - writeFileSync(legacyPath, JSON.stringify(legacy)); - - const outbox = new DiscordAwarenessOutbox(path); - // Reading imports in memory only; nothing is dispatched from an import. - assert.equal(outbox.pendingDispatches('discord').length, 0); - const views = new Map(outbox.view().map((v) => [v.id, v])); - assert.equal(views.get('dd34b043')!.adds.confirmed, 2); - assert.equal(views.get('dd34b043')!.adds.failed, 1); + mkdirSync(join(h.dir, 'recovery'), { recursive: true }); + writeFileSync(h.legacyPath, JSON.stringify(legacy)); + + const outbox = h.reopen(); + assert.equal(outbox.pendingDispatches('discord').length, 0, 'nothing imported is dispatched'); + assert.equal(outbox.operations().length, 0, 'no synthesized attempts'); + assert.equal(existsSync(h.legacyPath), false); + assert.equal(existsSync(`${h.legacyPath}.migrated-v2`), true); + const views = new Map(batches(outbox).map((v) => [v.id, v])); + assert.deepEqual(views.get('dd34b043')!.legacy, { + entries: 5, + lastAddConfirmed: 1, + lastRemoveConfirmed: 0, + // u1: its one outcome unrecorded; g3: two earlier attempts unrecorded. + outcomesUnrecorded: 3, + }); assert.equal(views.get('pending-batch')!.status, 'held'); assert.equal(views.get('pending-batch')!.held!.releaseActions, 1); - // The first write persists the import (with its hash) and retires the file. + // Retract carries out the act for every ref and discloses what history leaves open. const receipt = outbox.retract('dd34b043', 'operator'); - assert.equal(receipt.removalsQueued, 3, 'every add that left the host is removed; the import cannot tell which were sent'); - assert.equal(existsSync(legacyPath), false); - assert.equal(existsSync(`${legacyPath}.migrated-v2`), true); - - // A crash between the import append and the rename re-reads the same - // file: its hash is recorded, so nothing is imported or queued twice. - writeFileSync(legacyPath, JSON.stringify(legacy)); - const again = new DiscordAwarenessOutbox(path); + assert.equal(receipt.removalsQueued, 5); + assert.equal(receipt.unresolvedAddAttempts, 0, 'no physical request of this journal'); + assert.equal(receipt.keysWithLegacyUncertainty, 2, 'u1 and g3'); + const probe = outbox.retract('pending-batch'); + assert.equal(probe.removalsQueued, 1); + assert.equal(probe.keysWithLegacyUncertainty, 1); + assert.equal(outbox.cancel('pending-batch').legacyOutcomesUnrecorded, 1); + + // A crash between the import and the rename: the hash is recorded, so the + // file is only renamed, never imported twice. + writeFileSync(h.legacyPath, JSON.stringify(legacy)); + const again = h.reopen(); assert.equal(again.batches().filter((b) => b.id === 'dd34b043').length, 1); - assert.equal(again.pendingDispatches('discord').length, 3); - again.cancel('pending-batch'); - assert.equal(existsSync(legacyPath), false, 'the next write finishes the rename'); + assert.equal(batches(again).find((v) => v.id === 'dd34b043')!.legacy!.entries, 5); + assert.equal(existsSync(h.legacyPath), false); })); +// --------------------------------------------------------------------------- +// Framework seams, with stubbed collaborators +// --------------------------------------------------------------------------- + +function storeHarness() { + const dir = mkdtempSync(join(tmpdir(), 'discord-awareness-fw-')); + const store = JsStore.openOrCreate({ path: join(dir, 'store') }); + return { + dir, + store, + outbox: new DiscordAwarenessOutbox(store), + cleanup: () => { if (!store.isClosed()) store.close(); rmSync(dir, { recursive: true, force: true }); }, + }; +} + +async function muted(fn: () => Promise): Promise { + const original = console.error; + console.error = () => {}; + try { + return await fn(); + } finally { + console.error = original; + } +} + test('framework drain classifies outcomes from what reached the server', async () => { - const dir = mkdtempSync(join(tmpdir(), 'discord-awareness-drain-')); + const h = storeHarness(); try { - const outbox = new DiscordAwarenessOutbox(join(dir, 'journal.jsonl')); - activeBatch(outbox, [ref('m1'), ref('m2'), ref('m3'), ref('m4'), ref('m5')]); + activeBatch(h.outbox, [ref('m1'), ref('m2'), ref('m3'), ref('m4'), ref('m5')]); const calls: string[] = []; let connected = true; const connection = { @@ -457,218 +491,206 @@ test('framework drain classifies outcomes from what reached the server', async ( }, }; const framework = Object.create(AgentFramework.prototype) as any; - framework.discordAwarenessOutbox = outbox; + framework.discordAwarenessOutbox = h.outbox; framework.discordAwarenessDrains = new Map(); framework.discordAwarenessDeadlineMs = 1000; framework.mcplServerRegistry = { getServer: () => connection }; - const originalError = console.error; - console.error = () => {}; - try { - await framework.drainDiscordAwarenessOutbox('discord'); - } finally { - console.error = originalError; - } - assert.deepEqual(calls, ['add_reaction:m1', 'add_reaction:m2', 'add_reaction:m3', 'add_reaction:m4', 'add_reaction:m5']); - const status = new Map(outbox.operations().map((op) => [op.key.messageId, outbox.operationStatus(op)])); + await muted(() => framework.drainDiscordAwarenessOutbox('discord')); + assert.deepEqual(calls.sort(), ['add_reaction:m1', 'add_reaction:m2', 'add_reaction:m3', 'add_reaction:m4', 'add_reaction:m5']); + const status = new Map(h.outbox.operations().map((op) => [op.key.messageId, h.outbox.operationStatus(op)])); assert.equal(status.get('m1'), 'confirmed'); assert.equal(status.get('m2'), 'failed', 'Discord said the message is gone: final'); assert.equal(status.get('m3'), 'requested', 'never written: still queued'); assert.equal(status.get('m4'), 'unknown', 'written, no answer: unknown'); - assert.equal(status.get('m5'), 'confirmed'); // A disconnected route leaves its work queued without writing anything. connected = false; - const before = readFileSync(join(dir, 'journal.jsonl'), 'utf8'); - console.error = () => {}; - try { - await framework.drainDiscordAwarenessOutbox('discord'); - } finally { - console.error = originalError; - } - assert.equal(readFileSync(join(dir, 'journal.jsonl'), 'utf8'), before); + const records = h.store.getRecordIdsByType(DISCORD_AWARENESS_RECORD_TYPE).length; + await muted(() => framework.drainDiscordAwarenessOutbox('discord')); + assert.equal(h.store.getRecordIdsByType(DISCORD_AWARENESS_RECORD_TYPE).length, records); // Back online: the unsent and the unknown (newest for their keys) go again. connected = true; calls.length = 0; - await framework.drainDiscordAwarenessOutbox('discord'); + await muted(() => framework.drainDiscordAwarenessOutbox('discord')); assert.deepEqual(calls.sort(), ['add_reaction:m3', 'add_reaction:m4']); } finally { - rmSync(dir, { recursive: true, force: true }); + h.cleanup(); } }); test('framework resumes an interrupted suppression before activating its marks', async () => { - const dir = mkdtempSync(join(tmpdir(), 'discord-awareness-resume-')); + const h = storeHarness(); try { - const outbox = new DiscordAwarenessOutbox(join(dir, 'journal.jsonl')); - const batch = outbox.prepare({ - agentName: 'cairn', - sourceBranch: 'main', - targetBranch: 'recovery/cairn/suppressed', - activationPolicy: 'explicit', - suppressionIntervals: [{ fromId: 'i1', toId: 'i2' }, { fromId: 'i3', toId: 'i3' }], - refs: [ref('m1')], - scope: 'all', + const batch = h.outbox.prepare({ + agentName: 'cairn', sourceBranch: 'main', targetBranch: 'recovery/cairn/suppressed', activationPolicy: 'explicit', + suppressionIntervals: [{ fromId: 'i1', toId: 'i2' }, { fromId: 'i3', toId: 'i3' }], refs: [ref('m1')], scope: 'all', })!; const present = new Set(['i1', 'i2']); // i3 interval committed before crash. const removals: string[] = []; const cm = { getMessage: (id: string) => present.has(id) ? { id } : null, removeMessage: (id: string) => { present.delete(id); removals.push(id); }, - removeMessages: (from: string, to: string) => { - present.delete(from); present.delete(to); removals.push(`${from}..${to}`); - }, + removeMessages: (from: string, to: string) => { present.delete(from); present.delete(to); removals.push(`${from}..${to}`); }, }; const framework = Object.create(AgentFramework.prototype) as any; - framework.discordAwarenessOutbox = outbox; + framework.discordAwarenessOutbox = h.outbox; framework.store = { currentBranch: () => ({ name: 'recovery/cairn/suppressed' }) }; framework.agents = new Map([['cairn', { getContextManager: () => cm }]]); - - await framework.resumePreparedDiscordSuppressions(); - + await muted(() => framework.resumePreparedDiscordSuppressions()); assert.deepEqual(removals, ['i1..i2']); - assert.equal(outbox.batches().find((candidate) => candidate.id === batch.id)?.status, 'active'); - assert.deepEqual(outbox.pendingDispatches('discord').map((d) => d.key.messageId), ['m1']); + assert.equal(h.outbox.batches().find((candidate) => candidate.id === batch.id)?.status, 'active'); + assert.deepEqual(h.outbox.pendingDispatches('discord').map((d) => d.key.messageId), ['m1']); } finally { - rmSync(dir, { recursive: true, force: true }); + h.cleanup(); } }); -test('message-granular host/command undo places no marks unless the command asks', async () => { - const dir = mkdtempSync(join(tmpdir(), 'discord-awareness-undo-')); +/** A framework stub whose agent context is a plain list (ids i0..), for host/command. */ +function hostCommandFramework(h: ReturnType, messages: Array<{ id: string; metadata: Record }>) { + let currentBranch = 'main'; + const removed = new Set(); + const live = () => messages.filter((m) => !removed.has(m.id)); + const contextManager = { + getAllMessages: () => live(), + getMessageCount: () => live().length, + getMessageWindow: (offset: number, limit: number) => ({ + messages: live().slice(offset, offset + limit), + startIndex: offset, + totalCount: live().length, + }), + branchAt: (_id: string, name: string) => name, + switchBranch: async (name: string) => { currentBranch = name; }, + removeMessage: (id: string) => { removed.add(id); }, + removeMessages: (from: string, to: string) => { + const ids = messages.map((m) => m.id); + for (const id of ids.slice(ids.indexOf(from), ids.indexOf(to) + 1)) removed.add(id); + }, + }; + const framework = Object.create(AgentFramework.prototype) as any; + framework.agents = new Map([['cairn', { state: { status: 'idle' }, getContextManager: () => contextManager }]]); + framework.store = { currentBranch: () => ({ name: currentBranch }), sync: () => h.store.sync() }; + framework.discordAwarenessOutbox = h.outbox; + framework.discordAwarenessEmoji = '🫥'; + framework.discordAwarenessDrains = new Map(); + framework.mcplServerRegistry = null; + framework.moduleRegistry = { getModule: () => null }; + framework.lastVisiblePreview = async () => null; + framework.activeTurnTokens = new Map(); + framework.nextTurnToken = 1; + framework.deferredMessages = []; + framework.pendingAssistantBlocks = new Map(); + framework.operatorLog = new OperatorLog(undefined); + framework.opsAlert = () => {}; + return { framework, removed, setBranch: (name: string) => { currentBranch = name; } }; +} + +test('message-granular host/command undo places no marks unless asked; the marks verb reaches the controls', async () => { + const h = storeHarness(); try { - const messages = [ + const { framework, setBranch } = hostCommandFramework(h, [ { id: 'i0', metadata: { ...ref('m0') } }, { id: 'i1', metadata: { ...ref('m1'), tags: ['chat:addressed'] } }, { id: 'i2', metadata: { ...ref('m2'), tags: ['chat:ambient'] } }, { id: 'i3', metadata: { ...ref('m3') } }, - ]; - let currentBranch = 'main'; - const contextManager = { - getAllMessages: () => messages, - getMessageCount: () => messages.length, - getMessageWindow: (offset: number, limit: number) => ({ - messages: messages.slice(offset, offset + limit), - startIndex: offset, - totalCount: messages.length, - }), - branchAt: (_id: string, name: string) => name, - switchBranch: async (name: string) => { currentBranch = name; }, - }; - const framework = Object.create(AgentFramework.prototype) as any; - framework.agents = new Map([['cairn', { state: { status: 'idle' }, getContextManager: () => contextManager }]]); - framework.store = { currentBranch: () => ({ name: currentBranch }) }; - framework.discordAwarenessOutbox = new DiscordAwarenessOutbox(join(dir, 'journal.jsonl')); - framework.discordAwarenessEmoji = '🫥'; - framework.discordAwarenessDrains = new Map(); - framework.mcplServerRegistry = null; - framework.moduleRegistry = { getModule: () => null }; - framework.lastVisiblePreview = async () => null; - framework.activeTurnTokens = new Map(); - framework.nextTurnToken = 1; - framework.deferredMessages = []; - framework.pendingAssistantBlocks = new Map(); - framework.operatorLog = new OperatorLog(undefined); - const originalError = console.error; - console.error = () => {}; - try { + ]); + await muted(async () => { const silent = await framework.handleHostCommand('discord', { command: 'undo', agentName: 'cairn', messages: 1 }); assert.equal(silent.ok, true); assert.deepEqual(silent.markers, { scope: 'none', unmarked: 1, notRemoved: 0, status: 'none', queued: 0 }); - assert.equal(framework.discordAwarenessOutbox.pendingDispatches('discord').length, 0); + assert.equal(h.outbox.pendingDispatches('discord').length, 0); - currentBranch = 'main'; - const marked = await framework.handleHostCommand('discord', { - command: 'undo', agentName: 'cairn', messages: 3, marks: 'addressed', - }); + setBranch('main'); + const marked = await framework.handleHostCommand('discord', { command: 'undo', agentName: 'cairn', messages: 3, marks: 'addressed' }); assert.equal(marked.markers.status, 'queued'); assert.equal(marked.markers.queued, 1); assert.equal(marked.markers.unmarked, 2); - const dispatches = framework.discordAwarenessOutbox.pendingDispatches('discord'); - assert.deepEqual(dispatches.map((d: { key: { messageId: string; emoji: string } }) => `${d.key.messageId}${d.key.emoji}`), ['m1🫥']); + assert.deepEqual(h.outbox.pendingDispatches('discord').map((d) => `${d.key.messageId}${d.key.emoji}`), ['m1🫥']); - // A mistyped scope is refused, never guessed. - currentBranch = 'main'; - const typo = await framework.handleHostCommand('discord', { - command: 'undo', agentName: 'cairn', messages: 1, marks: 'adressed', - }); + const typo = await framework.handleHostCommand('discord', { command: 'undo', agentName: 'cairn', messages: 1, marks: 'adressed' }); assert.equal(typo.ok, false); assert.match(typo.error, /marks must be none, addressed or all/); - // The journal's controls are reachable through host/command too. const listed = await framework.handleHostCommand('discord', { command: 'marks', action: 'list' }); assert.equal(listed.ok, true); - assert.equal(listed.awareness.length, 1); - const cancelled = await framework.handleHostCommand('discord', { - command: 'marks', action: 'cancel', batchId: listed.awareness[0].id, requesterName: 'op', - }); + const batch = listed.awareness.find((v: { kind: string }) => v.kind === 'batch'); + const cancelled = await framework.handleHostCommand('discord', { command: 'marks', action: 'cancel', target: batch.id, requesterName: 'op' }); assert.equal(cancelled.ok, true); assert.equal(cancelled.awareness.cancelled, 1); - assert.equal(framework.discordAwarenessOutbox.pendingDispatches('discord').length, 0); + assert.equal(h.outbox.pendingDispatches('discord').length, 0); const missing = await framework.handleHostCommand('discord', { command: 'marks', action: 'retract' }); assert.equal(missing.ok, false); - } finally { - console.error = originalError; - } + }); } finally { - rmSync(dir, { recursive: true, force: true }); + h.cleanup(); } }); -test('host/command hide marks only on request, through the journal, routed by the issuing server for old records', async () => { - const dir = mkdtempSync(join(tmpdir(), 'discord-awareness-hide-')); +test('turn-based host/command undo honours the marks choice for the messages its turns removed', async () => { + const h = storeHarness(); try { const messages = [ + { id: 'i0', metadata: { ...ref('m0') } }, + { id: 'i1', metadata: { ...ref('m1'), tags: ['chat:addressed'] } }, // arrived during the turn + { id: 'i2', metadata: {} }, // the agent's reply + ]; + const { framework, removed } = hostCommandFramework(h, messages); + let turns = 1; + framework.undoLastTurn = () => { + if (turns === 0) return { undone: false }; + turns--; + removed.add('i1'); + removed.add('i2'); + return { undone: true }; + }; + await muted(async () => { + const result = await framework.handleHostCommand('discord', { command: 'undo', agentName: 'cairn', turns: 2, marks: 'addressed' }); + assert.equal(result.ok, true); + assert.equal(result.undone, 1); + assert.equal(result.markers.status, 'queued'); + assert.equal(result.markers.queued, 1); + assert.deepEqual(h.outbox.pendingDispatches('discord').map((d) => d.key.messageId), ['m1']); + }); + } finally { + h.cleanup(); + } +}); + +test('host/command hide takes the store reservation and marks only on request, through the journal', async () => { + const h = storeHarness(); + try { + const { framework } = hostCommandFramework(h, [ { id: 'i0', metadata: { ...ref('m0') } }, { id: 'i1', metadata: { channelId: 'discord:g1:c1', messageId: 'm1', tags: ['chat:addressed'] } }, // pre-serverId record { id: 'i2', metadata: { ...ref('m2'), tags: ['chat:ambient'] } }, { id: 'i3', metadata: { ...ref('m3') } }, - ]; - const removed: string[] = []; - const contextManager = { - getAllMessages: () => messages.filter((m) => !removed.includes(m.id)), - removeMessage: (id: string) => { removed.push(id); }, - removeMessages: (from: string, to: string) => { - const ids = messages.map((m) => m.id); - for (const id of ids.slice(ids.indexOf(from), ids.indexOf(to) + 1)) removed.push(id); - }, - }; - const framework = Object.create(AgentFramework.prototype) as any; - framework.agents = new Map([['cairn', { state: { status: 'idle' }, getContextManager: () => contextManager }]]); - framework.store = { currentBranch: () => ({ name: 'main' }) }; - framework.discordAwarenessOutbox = new DiscordAwarenessOutbox(join(dir, 'journal.jsonl')); - framework.discordAwarenessEmoji = '💤'; - framework.discordAwarenessDrains = new Map(); - framework.mcplServerRegistry = null; - framework.lastVisiblePreview = async () => null; - framework.operatorLog = new OperatorLog(undefined); - const originalError = console.error; - console.error = () => {}; - try { + ]); + await muted(async () => { + // Another agent sharing the store is mid-turn: hide refuses, removes nothing. + framework.activeTurnTokens.set('other', 99); + const busy = await framework.handleHostCommand('discord', { command: 'hide', agentName: 'cairn', fromMessageId: 'm3' }); + assert.equal(busy.ok, false); + assert.equal(busy.code, 'agent-busy'); + framework.activeTurnTokens.delete('other'); + const silent = await framework.handleHostCommand('discord', { command: 'hide', agentName: 'cairn', fromMessageId: 'm3' }); assert.equal(silent.ok, true); assert.equal(silent.markers.status, 'none'); assert.equal(silent.markers.unmarked, 1); - assert.equal(framework.discordAwarenessOutbox.batches().length, 0); + assert.equal(h.outbox.batches().length, 0); + assert.equal(framework.surgeryHold, null, 'the reservation was released'); const marked = await framework.handleHostCommand('discord', { command: 'hide', agentName: 'cairn', fromMessageId: 'm1', toMessageId: 'm2', marks: 'addressed', }); - assert.equal(marked.ok, true); assert.equal(marked.hidden, 2); assert.equal(marked.markers.status, 'queued'); assert.equal(marked.markers.queued, 1); - assert.equal(marked.markers.unmarked, 1); - const [dispatch] = framework.discordAwarenessOutbox.pendingDispatches('discord'); - assert.deepEqual(dispatch.key, { serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'm1', emoji: '💤' }); - // Explicit and already applied: a restart on this branch changes nothing. - const restarted = new DiscordAwarenessOutbox(join(dir, 'journal.jsonl')); - assert.deepEqual(restarted.recoverAtStartup('main'), { activated: [], held: [], unknownAttempts: 0 }); - assert.equal(restarted.batches()[0].status, 'active'); - } finally { - console.error = originalError; - } + const [dispatch] = h.outbox.pendingDispatches('discord'); + assert.deepEqual(dispatch.key, { serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'm1', emoji: '🫥' }); + assert.deepEqual(new DiscordAwarenessOutbox(h.store).recoverAtStartup('main'), { activated: [], held: [], unknownAttempts: 0 }); + }); } finally { - rmSync(dir, { recursive: true, force: true }); + h.cleanup(); } }); diff --git a/test/offline-recovery-branch.test.ts b/test/offline-recovery-branch.test.ts index 8b34e9aff..0ee20e99f 100644 --- a/test/offline-recovery-branch.test.ts +++ b/test/offline-recovery-branch.test.ts @@ -7,10 +7,20 @@ import { JsStore } from '@animalabs/chronicle'; import { ContextManager } from '@animalabs/context-manager'; import { createOfflineRecoveryBranch } from '../src/recovery/offline-branch.js'; import { + DISCORD_AWARENESS_RECORD_TYPE, DiscordAwarenessOutbox, - defaultDiscordAwarenessOutboxPath, } from '../src/recovery/discord-awareness-outbox.js'; +/** Read the store's awareness journal with the host (and CLI) stopped. */ +function readJournal(storePath: string, read: (outbox: DiscordAwarenessOutbox, store: JsStore) => T): T { + const store = JsStore.openOrCreate({ path: storePath }); + try { + return read(new DiscordAwarenessOutbox(store), store); + } finally { + store.close(); + } +} + test('offline recovery branches without compiling and queues discarded Discord refs', async () => { const dir = mkdtempSync(join(tmpdir(), 'offline-recovery-')); const storePath = join(dir, 'agent.chronicle'); @@ -53,17 +63,17 @@ test('offline recovery branches without compiling and queues discarded Discord r recoveredCm.close(); recoveredStore.close(); - const batches = new DiscordAwarenessOutbox( - defaultDiscordAwarenessOutboxPath(storePath), - ).pendingDispatches('discord'); + const batches = readJournal(storePath, (outbox) => outbox.pendingDispatches('discord')); assert.equal(batches.length, 2); assert.deepEqual(batches.map((dispatch) => dispatch.key.messageId), ['m-toxic-1', 'm-toxic-2']); // The outbox is metadata-only: quarantined text must never leak to it. - const rawOutbox = await import('node:fs').then(({ readFileSync }) => - readFileSync(defaultDiscordAwarenessOutboxPath(storePath), 'utf8')); - assert.ok(!rawOutbox.includes('never echo this one')); - assert.ok(!rawOutbox.includes('or this one')); + const rawJournal = readJournal(storePath, (_outbox, store) => store.getRecordIdsByType(DISCORD_AWARENESS_RECORD_TYPE) + .map((id) => Buffer.from(store.getRecord(id)!.payload).toString('utf-8')) + .join('\n')); + assert.ok(rawJournal.length > 0); + assert.ok(!rawJournal.includes('never echo this one')); + assert.ok(!rawJournal.includes('or this one')); } finally { rmSync(dir, { recursive: true, force: true }); } @@ -180,9 +190,7 @@ test('offline recovery suppresses inclusive ranges in context order', async () = recoveredCm.close(); recoveredStore.close(); - const batches = new DiscordAwarenessOutbox( - defaultDiscordAwarenessOutboxPath(storePath), - ).pendingDispatches('discord'); + const batches = readJournal(storePath, (outbox) => outbox.pendingDispatches('discord')); assert.equal(batches.length, 3); assert.deepEqual( batches.map((dispatch) => dispatch.key.messageId), @@ -300,9 +308,11 @@ test('offline recovery is local by default; addressed marks only messages that a assert.equal(local.discordAddressable, 2); assert.equal(local.discordMarkersQueued, 0); assert.equal(local.unmarked, 2); - const outbox = new DiscordAwarenessOutbox(defaultDiscordAwarenessOutboxPath(storePath)); - assert.equal(outbox.pendingDispatches('discord').length, 0, 'nothing is queued for Discord'); - assert.equal(outbox.batches().length, 0); + assert.deepEqual(local.markers, { scope: 'none', unmarked: 2, notRemoved: 0, status: 'none', queued: 0 }); + readJournal(storePath, (outbox) => { + assert.equal(outbox.pendingDispatches('discord').length, 0, 'nothing is queued for Discord'); + assert.equal(outbox.batches().length, 0); + }); // A dry run with addressed shows exactly what would be marked. rmSync(storePath, { recursive: true, force: true }); @@ -316,12 +326,49 @@ test('offline recovery is local by default; addressed marks only messages that a }); assert.equal(marked.discordMarkersQueued, 1); assert.equal(marked.unmarked, 1); + assert.equal(marked.markers?.status, 'queued'); assert.deepEqual( - new DiscordAwarenessOutbox(defaultDiscordAwarenessOutboxPath(storePath)).pendingDispatches('discord') - .map((dispatch) => dispatch.key.messageId), + readJournal(storePath, (outbox) => outbox.pendingDispatches('discord').map((dispatch) => dispatch.key.messageId)), ['m-addressed'], ); } finally { rmSync(dir, { recursive: true, force: true }); } }); + +test('offline recovery reports a marker bookkeeping failure apart from the body, which stands', async () => { + const dir = mkdtempSync(join(tmpdir(), 'offline-recovery-settle-')); + const storePath = join(dir, 'agent.chronicle'); + try { + const store = JsStore.openOrCreate({ path: storePath }); + const cm = await ContextManager.open({ store, namespace: 'agents/cairn' }); + cm.addMessage('user', [{ type: 'text', text: 'safe' }], { serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'm-safe' }); + cm.addMessage('user', [{ type: 'text', text: 'hey' }], { serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'm-gone', tags: ['chat:addressed'] }); + cm.close(); + store.close(); + const original = DiscordAwarenessOutbox.prototype.activate; + DiscordAwarenessOutbox.prototype.activate = function () { throw new Error('injected activation failure'); }; + let result: Awaited>; + try { + result = await createOfflineRecoveryBranch({ + storePath, agentName: 'cairn', messageId: 'm-safe', marks: { scope: 'all' }, branchName: 'recovery/cairn/settle', + }); + } finally { + DiscordAwarenessOutbox.prototype.activate = original; + } + assert.equal(result.markers?.status, 'not-scheduled'); + assert.match(result.markers?.status === 'not-scheduled' ? result.markers.error : '', /injected activation failure/); + assert.equal(result.discordMarkersQueued, 0); + const reopened = JsStore.openOrCreate({ path: storePath }); + try { + assert.equal(reopened.currentBranch().name, 'recovery/cairn/settle', 'the recovery branch stands'); + const outbox = new DiscordAwarenessOutbox(reopened); + assert.equal(outbox.batches()[0].status, 'discarded'); + assert.equal(outbox.pendingDispatches('discord').length, 0); + } finally { + reopened.close(); + } + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); diff --git a/test/surgery-marks-receipt.test.ts b/test/surgery-marks-receipt.test.ts index 674d08f90..523782f29 100644 --- a/test/surgery-marks-receipt.test.ts +++ b/test/surgery-marks-receipt.test.ts @@ -70,7 +70,8 @@ describe('surgery and awareness marks', () => { let framework: AgentFramework; const cm = () => framework.getAgent('resident')!.getContextManager(); - const outbox = () => new DiscordAwarenessOutbox(join(storePath, 'recovery', 'discord-awareness-journal.jsonl')); + /** The running framework's own journal instance (the journal's one writer). */ + const outbox = () => (framework as unknown as { discordAwarenessOutbox: DiscordAwarenessOutbox }).discordAwarenessOutbox; const answered = () => jsonl(eventsPath).filter((e) => e.event === 'reaction-answered').length; const calls = () => jsonl<{ name: string; args: { messageId: string; emoji: string } }>(callsPath); @@ -347,8 +348,7 @@ describe('surgery and awareness marks', () => { assert.ok(kinds.includes('awareness-cancel') && kinds.includes('awareness-retract')); }); - it('marks chosen where no awareness journal exists are reported as not scheduled, not as unwanted', async () => { - // A caller-supplied store and no storePath: no default journal location. + it('a caller-supplied store keeps its awareness journal in the store too', async () => { framework = await AgentFramework.create({ store: JsStore.openOrCreate({ path: storePath }), membrane: new MockMembrane().asMembrane(), @@ -359,12 +359,109 @@ describe('surgery and awareness marks', () => { }); const { tail } = seed(2, [0]); const result = await framework.rollbackToMessage('resident', { messageId: tail, marks: { scope: 'addressed' } }); - assert.equal(result.messagesRemoved, 2); - assert.equal(result.markers.status, 'not-scheduled'); - assert.match(result.markers.status === 'not-scheduled' ? result.markers.error : '', /no awareness journal/); + assert.equal(result.markers.status, 'queued'); + assert.equal(result.markers.queued, 1); assert.equal(result.markers.unmarked, 1); - const local = await framework.rollbackToMessage('resident', { messageId: String(cm().getAllMessages()[0].id) }); - assert.equal(local.markers.status, 'none', 'nothing chosen is still none'); + }); + + it('cancel during delivery stops the marks not yet sent: exactly one request reaches the server', async () => { + await start(true); + const { tail } = seed(3); + const result = await framework.rollbackToMessage('resident', { messageId: tail, marks: { scope: 'all' } }); + const batchId = result.markers.status === 'queued' ? result.markers.batchId : ''; + await waitFor('the first add on the wire', () => calls().length === 1); + const receipt = framework.cancelDiscordAwareness(batchId); + assert.deepEqual({ cancelled: receipt.cancelled, inFlight: receipt.inFlight }, { cancelled: 2, inFlight: 1 }); + writeFileSync(holdPath, '1'); + await waitFor('the held add answered and recorded', () => + outbox().operations().some((op) => op.attempts[0]?.outcome === 'confirmed')); + await framework.syncDiscordAwarenessMarkers(); + await new Promise((resolve) => setTimeout(resolve, 100)); + assert.equal(calls().length, 1, 'the cancelled marks were never sent'); + }); + + it('a failed rollback keeps its batch when the source cannot be confirmed, and says so', async () => { + await start(false); + const { tail } = seed(2); + const live = cm(); + const realSwitch = live.switchBranch.bind(live); + // The switch lands, then fails; the restore does nothing: the store is + // left on the fork, so the batch is the unfinished change's record. + live.switchBranch = async (name: string) => { await realSwitch(name); throw new Error('injected switch failure'); }; + const realRestore = (framework as any).restoreSourceBranch; + (framework as any).restoreSourceBranch = async () => 'source not restored (injected)'; + try { + await assert.rejects( + framework.rollbackToMessage('resident', { messageId: tail, marks: { scope: 'all' } }), + /awareness batch .* kept, since the source branch could not be confirmed/, + ); + } finally { + live.switchBranch = realSwitch; + (framework as any).restoreSourceBranch = realRestore; + } + const [batch] = outbox().batches(); + assert.equal(batch.status, 'prepared', 'not retired while the store sits on the fork'); + }); + + it('a failed suppression retires its batch only after the source is back, and reports a failed retire', async () => { + await start(false); + const { removed } = seed(3); + const live = cm(); + const realFork = live.fork.bind(live); + live.fork = async () => { throw new Error('injected fork failure'); }; + const discard = DiscordAwarenessOutbox.prototype.discard; + DiscordAwarenessOutbox.prototype.discard = function () { throw new Error('injected retire failure'); }; + try { + await assert.rejects( + framework.suppressMessages('resident', { messageIds: [removed[0]], marks: { scope: 'all' } }), + /could not be retired \(injected retire failure\)/, + ); + } finally { + live.fork = realFork; + DiscordAwarenessOutbox.prototype.discard = discard; + } + assert.equal(outbox().batches()[0].status, 'prepared'); + // With retirement working, the same failure retires it (source confirmed). + live.fork = async () => { throw new Error('injected fork failure'); }; + try { + await assert.rejects(framework.suppressMessages('resident', { messageIds: [removed[1]], marks: { scope: 'all' } })); + } finally { + live.fork = realFork; + } + assert.equal(outbox().batches().at(-1)!.status, 'discarded'); + }); + + it('an interrupted suppression: held at startup, its marks released, then its body resumed on its branch', async () => { + await start(false); + const { removed } = seed(3); + const live = cm(); + const sourceBranch = live.currentBranch().name; + // An interrupted suppression: the fork exists, nothing was redacted yet. + await live.fork('partial'); + const [target] = live.getAllMessages().filter((m) => String(m.id) === removed[1]); + const batch = outbox().prepare({ + agentName: 'resident', sourceBranch, targetBranch: 'partial', + refs: [{ serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'amb-1' }], scope: 'all', + activationPolicy: 'explicit', suppressionIntervals: [{ fromId: String(target.id), toId: String(target.id) }], + })!; + await live.switchBranch(sourceBranch); + await framework.stop(); + + await start(false); // on the source branch: the batch is held + assert.equal(outbox().batches()[0].status, 'held'); + const released = framework.releaseDiscordAwareness(batch.id); + assert.equal(released.addsQueued, 1); + await framework.stop(); + + const store = JsStore.openOrCreate({ path: storePath }); + store.switchBranch('partial'); + store.close(); + await start(false); // on the target: the body is completed, the marks untouched + assert.ok(!cm().getAllMessages().some((m) => String(m.id) === removed[1]), 'the saved redaction landed'); + const [after] = outbox().batches(); + assert.equal(after.suppressionComplete, true); + assert.equal(after.status, 'active'); + assert.equal(outbox().operations().filter((op) => op.action === 'add').length, 1, 'only the release queued marks'); }); it('a surgery that removes nothing addressable schedules no marks even when marks are chosen', async () => { From c0d2d902da4458821082693446041d329e84694a Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Wed, 7 Oct 2026 10:01:42 -0400 Subject: [PATCH 0035/1112] fix(awareness): a newer opposite intent retires in-flight and unknown requests' retries; drains stop with their route June #45033 (probe /tmp/june-1016-799-opposite-probe.ts), two lifecycle gaps at 799a16b: - an add on the wire when a retract arrived was not superseded (only `requested` ops were), so its later retryable failure put it back ahead of the remove; - an unknown add superseded by nothing (unknown ops were left alone), so cancelling the retract made it the newest live op again and revived its retries. requestRecords now retires the future retries of every opposite-action op that has not ended (requested, on the wire, or unknown): op-cancelled 'superseded'. Attempts stay as history; unresolved ones are still disclosed (keysWithUnresolvedAdds); the all-ops in-flight fence still makes the key wait for the request on the wire; a confirmed late answer still reads as confirmed. Cancelling the newer request stops it without reviving anything. Shutdown churn (found re-reading stop()): after closeAll, a running drain claimed every remaining op once, writing a durable dispatching record and a not-sent outcome for each (one store sync apiece) while the host stopped. The drain now stops claiming when its connection is gone (work stays queued; reconnect starts a new drain), and stop() awaits running drains after closing MCPL so the in-flight outcome lands before the store closes. Tests: in-flight add -> retract -> retryable failure -> next claim is the remove; unknown add -> retract -> cancel retract -> nothing due; the mirror (remove in flight, new add, retryable failure -> add); all across a restart; a drain whose route goes away writes only the in-flight attempt. Awareness set + record-journal 78/78. --- src/framework.ts | 8 +++ src/recovery/discord-awareness-outbox.ts | 16 +++--- test/discord-awareness-outbox.test.ts | 64 +++++++++++++++++++++++- 3 files changed, 80 insertions(+), 8 deletions(-) diff --git a/src/framework.ts b/src/framework.ts index fb58eea51..fe44344bf 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -2028,6 +2028,10 @@ export class AgentFramework { shutdownPromises.push(this.mcplServerRegistry.closeAll()); } await Promise.all(shutdownPromises); + // An awareness drain still recording the outcome of the request that was + // on the wire when its connection closed finishes before the store does; + // it claims nothing more once the route is gone. + await Promise.allSettled([...this.discordAwarenessDrains.values()]); // Streams may queue storage repairs while being cancelled above. Retry // after their teardown, while the store is still open, before final sync. @@ -13715,6 +13719,10 @@ export class AgentFramework { `${dispatch.opIds.join(',')}\0${dispatch.action}`; let announced = false; while (true) { + // A route that went away (shutdown, or a disconnect that will + // reconnect) leaves the rest queued: claiming would only write + // attempts that cannot be sent. + if (!connection.isConnected) break; // One dispatch at a time, chosen and written ahead in one step from // the journal as it is now: a cancel or retract made while the // previous request was on the wire takes effect here. diff --git a/src/recovery/discord-awareness-outbox.ts b/src/recovery/discord-awareness-outbox.ts index f7e6592d7..167ce380e 100644 --- a/src/recovery/discord-awareness-outbox.ts +++ b/src/recovery/discord-awareness-outbox.ts @@ -1100,11 +1100,12 @@ export class DiscordAwarenessOutbox { } /** - * Records for new requests. A request supersedes the future of every - * request for the same key with the opposite action that is still due - * (never sent, refused before writing, or failed retryably): the newest - * explicit intent wins. Their attempts stay as history, and a request on - * the wire still settles before anything else for that key is sent. + * Records for new requests. A request retires the future retries of every + * request for the same key with the opposite action that has not ended + * (due, on the wire, or unknown): the newest explicit intent wins, and + * cancelling it later never revives what it superseded. Their attempts + * stay as history (an unresolved one is still disclosed), and a request + * on the wire still settles before anything else for that key is sent. */ private requestRecords( state: JournalState, @@ -1123,8 +1124,9 @@ export class DiscordAwarenessOutbox { emoji: request.emoji ?? emoji ?? DEFAULT_DISCORD_AWARENESS_EMOJI, }; for (const op of this.opsForKey(state, key)) { - if (op.action === request.action || superseded.has(op.opId)) continue; - if (this.opStatus(op) !== 'requested') continue; + if (op.action === request.action || superseded.has(op.opId) || op.cancelled) continue; + const status = this.opStatus(op); + if (status !== 'requested' && status !== 'dispatching' && status !== 'unknown') continue; records.push({ t: 'op-cancelled', at, opId: op.opId, reason: 'superseded' }); superseded.add(op.opId); } diff --git a/test/discord-awareness-outbox.test.ts b/test/discord-awareness-outbox.test.ts index c3b4eb532..d0d46fab6 100644 --- a/test/discord-awareness-outbox.test.ts +++ b/test/discord-awareness-outbox.test.ts @@ -261,7 +261,7 @@ test('retract queues a removal for every selected key, whatever history says, an const receipt = outbox.retract(a.id, 'operator'); assert.equal(receipt.removalsQueued, 3, 'm2 too: history does not decide whether the act is carried out'); - assert.equal(receipt.addsSuperseded, 1, "m2's due add stops"); + assert.equal(receipt.addsSuperseded, 3, "m2's due add stops, and m1's unknown ones are no longer retried"); assert.equal(receipt.keysWithUnresolvedAdds, 1); assert.equal(receipt.unresolvedAddAttempts, 1, 'one physical request carried both adds'); assert.equal(receipt.keysWithLegacyUncertainty, 0); @@ -296,6 +296,39 @@ test('a newer opposite request stops the retries of a retryably failed one, in b assert.equal(outbox.operations().find((op) => op.action === 'remove')!.cancelled?.reason, 'superseded'); })); +test('a newer opposite intent retires the old request\'s retries even when it was on the wire or unknown', withJournal((outbox, h) => { + // In flight when the retract arrives, then a retryable failure: remove next. + const a = activeBatch(outbox, [ref('m1')]); + const onWire = outbox.claimDispatch('discord')!; + const retract = outbox.retract(a.id); + assert.equal(retract.addsSuperseded, 1); + assert.equal(retract.keysWithUnresolvedAdds, 1, 'the request on the wire is disclosed'); + assert.equal(outbox.claimDispatch('discord'), null, 'the key waits for the request on the wire'); + outbox.recordOutcome(onWire.attempts, 'failed', { error: 'Discord 503' }); + assert.equal(outbox.claimDispatch('discord', () => false)!.dispatch.action, 'remove'); + // Unknown, then retract, then the retract cancelled: nothing is revived. + const b = activeBatch(outbox, [ref('m2')]); + answer(outbox, 'm2', 'unknown'); + const { requestId } = outbox.retract(b.id); + outbox.cancel(requestId); + assert.ok(!outbox.pendingDispatches('discord').some((d) => d.key.messageId === 'm2')); + // Mirror: a remove in flight when a new add arrives, then a retryable failure. + const c = activeBatch(outbox, [ref('m3')]); + answer(outbox, 'm3', 'confirmed'); + outbox.retract(c.id); + const removeOnWire = outbox.claimDispatch('discord', (d) => d.key.messageId !== 'm3')!; + activeBatch(outbox, [ref('m3')]); + outbox.recordOutcome(removeOnWire.attempts, 'failed', { error: 'Discord 503' }); + const next = outbox.claimDispatch('discord', (d) => d.key.messageId !== 'm3')!; + assert.equal(next.dispatch.action, 'add'); + outbox.recordOutcome(next.attempts, 'confirmed'); + // All of it holds across a restart. + const restarted = h.reopen(); + restarted.recoverAtStartup('rollback/cairn/1'); + const due = restarted.pendingDispatches('discord').map((d) => `${d.action}:${d.key.messageId}`); + assert.ok(!due.includes('add:m1') && !due.includes('add:m2') && !due.includes('remove:m3'), due.join(',')); +})); + test('a retract request can be cancelled by the id its receipt returned, even after a restart', withJournal((outbox, h) => { const batch = activeBatch(outbox, [ref('m1'), ref('m2'), ref('m3')]); for (const id of ['m1', 'm2', 'm3']) answer(outbox, id, 'confirmed'); @@ -519,6 +552,35 @@ test('framework drain classifies outcomes from what reached the server', async ( } }); +test('a drain whose route goes away stops claiming: the rest stays queued, with no attempts written', async () => { + const h = storeHarness(); + try { + activeBatch(h.outbox, [ref('m1'), ref('m2'), ref('m3')]); + let connected = true; + const calls: string[] = []; + const connection = { + get isConnected() { return connected; }, + sendToolsCallWithDeadline: async (_name: string, args: Record) => { + calls.push(String(args.messageId)); + connected = false; // the host is shutting down + throw new McplRequestError('Connection to MCPL server "discord" closed while awaiting response', 'no-response'); + }, + }; + const framework = Object.create(AgentFramework.prototype) as any; + framework.discordAwarenessOutbox = h.outbox; + framework.discordAwarenessDrains = new Map(); + framework.discordAwarenessDeadlineMs = 1000; + framework.mcplServerRegistry = { getServer: () => connection }; + await muted(() => framework.drainDiscordAwarenessOutbox('discord')); + assert.equal(calls.length, 1); + const attempts = h.outbox.operations().reduce((sum, op) => sum + op.attempts.length, 0); + assert.equal(attempts, 1, 'only the request that was on the wire'); + assert.equal(h.outbox.pendingDispatches('discord').length, 3, 'its unknown and the two unsent stay due'); + } finally { + h.cleanup(); + } +}); + test('framework resumes an interrupted suppression before activating its marks', async () => { const h = storeHarness(); try { From d44826bdf6faa86b7a3636770dbfe0a268d7f3aa Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Wed, 7 Oct 2026 10:26:05 -0400 Subject: [PATCH 0036/1112] fix(awareness): authorization order decides opposite requests; imported active is not body proof; startup and offline body barriers; reserved turn undo June #45105 (probe /tmp/june-1016-799-prepared-retract-probe.ts): prepare -> retract -> activate let activation's adds cancel the later retract's remove. Activation completes the surgery's earlier choice; it is not a new one. Every op now carries `authorization`, the journal position (state.position, one per applied record, checkpointed) of the act it carries out: a batch's prepare for its activation's adds (activate and startup crash completion), the retract record for its removals, the release record for a release's requests. requestRecords takes the authorization (default: a new act, later than everything): an op with an opposite op authorized after it, in any state, even cancelled, is recorded superseded at birth (cancelling the later act never revives it); otherwise it retires older-authorized opposite ops not yet ended. retract('all') now covers every key of every batch not discarded (prepared, held, active). activate()/settleActivation report the adds actually queued. A release stays a new act (wins over a retract made while held). The body obligation is untouched: a suppression whose marks were retracted before activation still records suppressionComplete on activation. June #45108 (probe /tmp/june-1016-799-legacy-body-probe.ts): importLegacy emitted `activated` for v2 active batches, which certified a suppression body the old JSONL/Chronicle window could have lost. It now emits `legacy-activated` (marks history only); the resume selector picks the imported explicit suppression on its exact target, resumes intervals idempotently, and records suppression-complete after the sync. June #45172: - recoverAtStartup's crash-completion activations append with afterCommittedState (sync first; a failed sync appends nothing), as the store may be caller-supplied and already open. - offline body failure: restore the source, then retire the batch (discard, after-sync) once the source is confirmed; if not confirmed, keep it prepared and say so; a failed retire is reported. The error composes failure; restore failure; retire clause, with the original as cause. June room-203 #45217 / Agnes #45253 split: turn-based host/command undo takes reserveStoreForSurgery around the whole act (before snapshot, the undoLastTurn loop, after snapshot, marks scheduling), refusing agent-busy while any agent sharing the store is mid-turn; released in finally. Tests: authorization order (prepare->retract->activate; overlapping prepared batches; retract('all') over prepared; cancelled retract does not revive; later choice wins; release wins over retract-while-held; body completes with retracted marks; all across restart); startup sync-first and nothing on failed sync; imported active suppression not certified (outbox) and resumed on real startup (receipt test, idempotent second startup); offline failure retire/keep; turn undo refused while another agent is mid-turn. Awareness set + record-journal: 83/83. --- src/framework.ts | 82 +++++++----- src/recovery/discord-awareness-outbox.ts | 152 ++++++++++++++++++----- src/recovery/offline-branch.ts | 37 +++++- test/discord-awareness-outbox.test.ts | 113 +++++++++++++++++ test/offline-recovery-branch.test.ts | 59 +++++++++ test/surgery-marks-receipt.test.ts | 41 +++++- 6 files changed, 411 insertions(+), 73 deletions(-) diff --git a/src/framework.ts b/src/framework.ts index fe44344bf..66ce75541 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -5422,44 +5422,62 @@ export class AgentFramework { const turnMarks: DiscordAwarenessMarks = params.marks === 'addressed' || params.marks === 'all' ? { scope: params.marks } : 'none'; - // What the undone turns remove from this agent's context is whatever is - // in it now and not after: addressing metadata only, never content. - const before = this.addressingSnapshot(agentName); - let undone = 0; + // Each undone turn switches the whole store: hold it, like every surgery, + // for the whole act, so no agent sharing it can start a turn between the + // address snapshots and the switches they bracket. + let release: () => void; try { - for (let i = 0; i < requested; i++) { - const r = this.undoLastTurn(agentName, hostCommandRequester(serverId, params)); - if (!r.undone) break; - undone++; - } + release = this.reserveStoreForSurgery('undo turns', agentName); } catch (error) { - // e.g. "Cannot undo while agent is streaming" — report what happened, - // including any turns already undone before the failure. - const msg = error instanceof Error ? error.message : String(error); - if (undone === 0) return { ok: false, error: msg }; - console.error(`[host-command] undo partially failed after ${undone}/${requested}: ${msg}`); - } - - console.error( - `[host-command] undo agent=${agentName} requested=${requested} undone=${undone}` + - ` by=${params.requesterName ?? params.requesterId ?? 'unknown'} (server=${serverId})`, - ); - - if (undone === 0) { return { - ok: true, - undone: 0, - requested, - markers: { scope: marksScope(turnMarks) as SurgeryMarkerReceipt['scope'], unmarked: 0, notRemoved: 0, status: 'none', queued: 0 }, - lastVisible: null, + ok: false, + error: error instanceof Error ? error.message : String(error), + ...(error instanceof OperatorActionError ? { code: error.code } : {}), }; } + let undone = 0; + let markers: SurgeryMarkerReceipt; + try { + // What the undone turns remove from this agent's context is whatever is + // in it now and not after: addressing metadata only, never content. + const before = this.addressingSnapshot(agentName); + try { + for (let i = 0; i < requested; i++) { + const r = this.undoLastTurn(agentName, hostCommandRequester(serverId, params)); + if (!r.undone) break; + undone++; + } + } catch (error) { + // e.g. "Cannot undo while agent is streaming" — report what happened, + // including any turns already undone before the failure. + const msg = error instanceof Error ? error.message : String(error); + if (undone === 0) return { ok: false, error: msg }; + console.error(`[host-command] undo partially failed after ${undone}/${requested}: ${msg}`); + } + + console.error( + `[host-command] undo agent=${agentName} requested=${requested} undone=${undone}` + + ` by=${params.requesterName ?? params.requesterId ?? 'unknown'} (server=${serverId})`, + ); + + if (undone === 0) { + return { + ok: true, + undone: 0, + requested, + markers: { scope: marksScope(turnMarks) as SurgeryMarkerReceipt['scope'], unmarked: 0, notRemoved: 0, status: 'none', queued: 0 }, + lastVisible: null, + }; + } - // Marks are a one-shot choice about the messages these turns removed; - // the branch move itself never adds or removes any. - const after = new Set(this.addressingSnapshot(agentName).keys()); - const removed = [...before].filter(([id]) => !after.has(id)).map(([, carrier]) => carrier); - const markers = this.scheduleAppliedMarks('undo', agentName, serverId, removed, turnMarks); + // Marks are a one-shot choice about the messages these turns removed; + // the branch move itself never adds or removes any. + const after = new Set(this.addressingSnapshot(agentName).keys()); + const removed = [...before].filter(([id]) => !after.has(id)).map(([, carrier]) => carrier); + markers = this.scheduleAppliedMarks('undo', agentName, serverId, removed, turnMarks); + } finally { + release(); + } return { ok: true, undone, requested, markers, lastVisible: await this.lastVisiblePreview(agentName) }; } diff --git a/src/recovery/discord-awareness-outbox.ts b/src/recovery/discord-awareness-outbox.ts index 167ce380e..96c1e661f 100644 --- a/src/recovery/discord-awareness-outbox.ts +++ b/src/recovery/discord-awareness-outbox.ts @@ -146,6 +146,12 @@ export interface DiscordAwarenessBatch extends DiscordAwarenessBatchRecord { /** A suppression whose body is recorded complete (by its activation, or by * a later startup's resume). */ suppressionComplete?: boolean; + /** Journal position of the surgery's marks choice (its prepare): the + * authorization of the adds its activation requests. */ + authorization?: number; + /** Journal position of the operator's release: the authorization of the + * requests the release queues. */ + releaseAuthorization?: number; held?: { reason: string; at: number; releaseActions: DiscordAwarenessReleaseAction[] }; cancelled?: { at: number; by?: string }; released?: { at: number; by?: string }; @@ -185,6 +191,13 @@ export interface DiscordAwarenessOp { key: DiscordAwarenessRef & { emoji: string }; action: DiscordAwarenessAction; cause: DiscordAwarenessOpCause; + /** + * Journal position of the act that authorized this request: a surgery's + * marks choice (its prepare, even when activation comes later), a retract, + * or a release. Between opposite requests for one reaction, the later + * authorization wins, whenever each request was created. + */ + authorization: number; requestedAt: number; attempts: DiscordAwarenessAttempt[]; cancelled?: { at: number; reason: 'batch-cancelled' | 'request-cancelled' | 'superseded' | 'imported' }; @@ -400,6 +413,9 @@ function isAddressed(message: DiscordMessageMetadataCarrier): boolean { type JournalRecord = | { t: 'batch'; at: number; batch: DiscordAwarenessBatchRecord } | { t: 'activated'; at: number; batchId: string } + /** An imported ledger's `active`: a historical fact about its marks, never + * a certificate that a suppression's body completed. */ + | { t: 'legacy-activated'; at: number; batchId: string } | { t: 'discarded'; at: number; batchId: string } | { t: 'held'; at: number; batchId: string; reason: string; releaseActions: DiscordAwarenessReleaseAction[] } | { t: 'released'; at: number; batchId: string; by?: string } @@ -441,6 +457,7 @@ interface JournalSnapshot { legacy: DiscordAwarenessLegacyEvidence[]; retracts?: RetractRequest[]; importedSources: string[]; + position?: number; } export const DISCORD_AWARENESS_RECORD_TYPE = 'af:discord-awareness'; @@ -456,6 +473,8 @@ interface JournalState { /** Retract requests, by request id. */ retracts: Map; importedSources: Set; + /** Records applied so far: each record's journal position. */ + position: number; } interface RetractRequest { @@ -464,6 +483,8 @@ interface RetractRequest { at: number; by?: string; cancelled?: { at: number; by?: string }; + /** Journal position of the retract: its removals' authorization. */ + authorization: number; } /** @@ -536,7 +557,10 @@ export class DiscordAwarenessOutbox { /** * Activate a prepared batch after its body change landed: one `add` request - * per ref, durable before this returns. Returns how many were queued. + * per ref, durable before this returns. Activation completes the surgery's + * earlier choice; it is not a new one, so an add for a reaction that a + * later act (a retract, or another batch's request) already decided is + * recorded superseded and never sent. Returns how many were queued. */ activate(batchId: string): number { const state = this.load(); @@ -552,15 +576,16 @@ export class DiscordAwarenessOutbox { } const at = Date.now(); const records: JournalRecord[] = [{ t: 'activated', at, batchId }]; - records.push(...this.requestRecords(state, batch.refs.map((ref) => ({ ...ref, action: 'add' as const })), batch.emoji, { + const requested = this.requestRecords(state, batch.refs.map((ref) => ({ ...ref, action: 'add' as const })), batch.emoji, { kind: 'batch', batchId, - }, at)); + }, at, batch.authorization); + records.push(...requested); // The activation asserts the body change just committed (the switch, or // the last redaction): sync that first, so a crash can never leave marks // active for a change the store lost. this.append(records, { afterCommittedState: true, durable: true }); - return batch.refs.length; + return liveRequests(requested); } /** @@ -580,14 +605,19 @@ export class DiscordAwarenessOutbox { | { status: 'unresolved'; error: string } { let detail: string; try { - this.activate(batchId); - return { status: 'queued', queued: this.load().batches.get(batchId)?.refs.length ?? 0 }; + return { status: 'queued', queued: this.activate(batchId) }; } catch (error) { detail = boundDiscordAwarenessText(error instanceof Error ? error.message : String(error)); } try { - const current = this.load().batches.get(batchId); - if (current?.status === 'active') return { status: 'queued', queued: current.refs.length }; + const state = this.load(); + const current = state.batches.get(batchId); + if (current?.status === 'active') { + // The activation landed but its barrier failed: report what it queued. + const queued = [...state.ops.values()] + .filter((op) => op.cause.kind === 'batch' && op.cause.batchId === batchId && !op.cancelled).length; + return { status: 'queued', queued }; + } if (this.discard(batchId)) return { status: 'not-scheduled', error: detail }; } catch (error) { detail += `; retiring it also failed: ${boundDiscordAwarenessText(error instanceof Error ? error.message : String(error))}`; @@ -682,6 +712,7 @@ export class DiscordAwarenessOutbox { batch.emoji, { kind: 'batch', batchId: batch.id }, at, + batch.authorization, )); activated.push(batch.id); } else { @@ -697,7 +728,11 @@ export class DiscordAwarenessOutbox { held.push(batch.id); } } - if (records.length > 0) this.append(records, { durable: true }); + // A crash-completion activation asserts the target's state like any + // activation: the store's current state is synced before it is recorded + // (the store may be caller-supplied and already open), and a failed sync + // records nothing. + if (records.length > 0) this.append(records, { afterCommittedState: activated.length > 0, durable: true }); return { activated, held, unknownAttempts }; } @@ -790,20 +825,27 @@ export class DiscordAwarenessOutbox { /** * Remove this bot's mark (through the configured route) from a batch's refs, - * or from every key any batch requested or an imported ledger recorded an - * add for (`all`). A reaction is one per (route, message, emoji), whichever - * batches asked for it, so this acts across batches. A removal is queued for - * every selected key, whatever history says: history can't establish what - * is on Discord now, and removing an absent reaction does nothing. Add - * requests not yet sent for those keys are superseded. The receipt reports - * history only: requests whose outcome is unresolved, and imported history - * that leaves outcomes unrecorded, any of which may land or have landed. + * or (`all`) from every key of every batch its own surgery did not retire, + * whether prepared, held or active, every key an add was requested for, and + * every key an imported ledger recorded. A reaction is one per (route, + * message, emoji), whichever batches asked for it, so this acts across + * batches. A removal is queued for every selected key, whatever history + * says: history can't establish what is on Discord now, and removing an + * absent reaction does nothing. It is the latest authorization for those + * keys: adds not yet ended are superseded, and adds an earlier choice + * requests later (a prepared batch's activation) are recorded superseded. + * The receipt reports history only: requests whose outcome is unresolved, + * and imported history that leaves outcomes unrecorded, any of which may + * land or have landed. */ retract(target: string | 'all', by?: string): DiscordAwarenessRetractReceipt { const state = this.load(); let candidateKeys: Array; if (target === 'all') { candidateKeys = uniqueKeys([ + ...[...state.batches.values()] + .filter((batch) => batch.status !== 'discarded') + .flatMap((batch) => batch.refs.map((ref) => ({ ...ref, emoji: batch.emoji }))), ...[...state.ops.values()].filter((op) => op.action === 'add').map((op) => op.key), ...[...state.legacy.values()].flat().map((evidence) => evidence.key), ]); @@ -1100,12 +1142,19 @@ export class DiscordAwarenessOutbox { } /** - * Records for new requests. A request retires the future retries of every - * request for the same key with the opposite action that has not ended - * (due, on the wire, or unknown): the newest explicit intent wins, and - * cancelling it later never revives what it superseded. Their attempts - * stay as history (an unresolved one is still disclosed), and a request - * on the wire still settles before anything else for that key is sent. + * Records for new requests, ordered by authorization: between opposite + * requests for one key, the later authorization wins, whenever each + * request was created. `authorization` is the journal position of the act + * the requests carry out (a prepared batch's choice, when its activation + * comes later); left out, they are a new act, later than everything + * recorded (a retract, a release). + * - Each older-authorized opposite request that has not ended (due, on + * the wire, or unknown) has its future retries retired. Its attempts + * stay as history (an unresolved one is still disclosed), and a request + * on the wire still settles before anything else for that key is sent. + * - A request with an opposite request authorized after it, in any state, + * even cancelled, is recorded superseded and never sent: cancelling the + * later act stops it without reviving what it decided against. */ private requestRecords( state: JournalState, @@ -1113,6 +1162,7 @@ export class DiscordAwarenessOutbox { emoji: string | undefined, cause: DiscordAwarenessOpCause, at: number, + authorization: number = Number.POSITIVE_INFINITY, ): JournalRecord[] { const records: JournalRecord[] = []; const superseded = new Set(); @@ -1123,14 +1173,20 @@ export class DiscordAwarenessOutbox { messageId: request.messageId, emoji: request.emoji ?? emoji ?? DEFAULT_DISCORD_AWARENESS_EMOJI, }; - for (const op of this.opsForKey(state, key)) { - if (op.action === request.action || superseded.has(op.opId) || op.cancelled) continue; + const opposite = this.opsForKey(state, key).filter((op) => op.action !== request.action); + const opId = randomUUID(); + records.push({ t: 'requested', at, opId, key, action: request.action, cause }); + if (opposite.some((op) => op.authorization > authorization)) { + records.push({ t: 'op-cancelled', at, opId, reason: 'superseded' }); + continue; + } + for (const op of opposite) { + if (superseded.has(op.opId) || op.cancelled || op.authorization >= authorization) continue; const status = this.opStatus(op); if (status !== 'requested' && status !== 'dispatching' && status !== 'unknown') continue; records.push({ t: 'op-cancelled', at, opId: op.opId, reason: 'superseded' }); superseded.add(op.opId); } - records.push({ t: 'requested', at, opId: randomUUID(), key, action: request.action, cause }); } return records; } @@ -1225,7 +1281,13 @@ export class DiscordAwarenessOutbox { // --------------------------------------------------------------------------- function emptyState(): JournalState { - return { batches: new Map(), ops: new Map(), legacy: new Map(), retracts: new Map(), importedSources: new Set() }; + return { batches: new Map(), ops: new Map(), legacy: new Map(), retracts: new Map(), importedSources: new Set(), position: 0 }; +} + +/** Requests in these records that are not recorded superseded with them. */ +function liveRequests(records: JournalRecord[]): number { + const cancelled = new Set(records.filter((record) => record.t === 'op-cancelled').map((record) => (record as { opId: string }).opId)); + return records.filter((record) => record.t === 'requested' && !cancelled.has(record.opId)).length; } function snapshotState(state: JournalState): JournalSnapshot { @@ -1235,6 +1297,7 @@ function snapshotState(state: JournalState): JournalSnapshot { legacy: [...state.legacy.values()].flat().map((evidence) => structuredClone(evidence)), retracts: [...state.retracts.values()].map((retract) => structuredClone(retract)), importedSources: [...state.importedSources], + position: state.position, }; } @@ -1245,6 +1308,7 @@ function restoreState(snapshot: JournalSnapshot): JournalState { legacy: new Map(), retracts: new Map((snapshot.retracts ?? []).map((retract) => [retract.requestId, structuredClone(retract)])), importedSources: new Set(snapshot.importedSources), + position: snapshot.position ?? 0, }; for (const evidence of snapshot.legacy ?? []) addLegacyEvidence(state, structuredClone(evidence)); return state; @@ -1257,13 +1321,15 @@ function addLegacyEvidence(state: JournalState, evidence: DiscordAwarenessLegacy else state.legacy.set(key, [evidence]); } -/** Apply one entry's records, in order, to the reduced state. */ +/** Apply one entry's records, in order, to the reduced state. Each record + * takes the next journal position. */ function applyRecords(state: JournalState, records: JournalRecord[]): void { for (const record of records) { + const position = ++state.position; switch (record.t) { case 'batch': if (!state.batches.has(record.batch.id)) { - state.batches.set(record.batch.id, { ...structuredClone(record.batch), status: 'prepared' }); + state.batches.set(record.batch.id, { ...structuredClone(record.batch), status: 'prepared', authorization: position }); } break; case 'activated': { @@ -1271,12 +1337,17 @@ function applyRecords(state: JournalState, records: JournalRecord[]): void { if (batch && batch.status !== 'discarded') { batch.status = 'active'; // Only a surgery (or its resume) activates a suppression, after its - // last redaction: activation records the body complete. Release - // does not. + // last redaction synced: activation records the body complete. + // Release does not, and neither does an imported `active`. if (batch.suppressionIntervals?.length) batch.suppressionComplete = true; } break; } + case 'legacy-activated': { + const batch = state.batches.get(record.batchId); + if (batch && batch.status !== 'discarded') batch.status = 'active'; + break; + } case 'discarded': { const batch = state.batches.get(record.batchId); if (batch && batch.status === 'prepared') batch.status = 'discarded'; @@ -1296,6 +1367,7 @@ function applyRecords(state: JournalState, records: JournalRecord[]): void { batch.status = 'active'; delete batch.held; batch.released = { at: record.at, ...(record.by ? { by: record.by } : {}) }; + batch.releaseAuthorization = position; } break; } @@ -1317,6 +1389,7 @@ function applyRecords(state: JournalState, records: JournalRecord[]): void { key: { ...record.key }, action: record.action, cause: { ...record.cause }, + authorization: authorizationOf(state, record.cause) ?? position, requestedAt: record.at, attempts: [], }); @@ -1364,6 +1437,7 @@ function applyRecords(state: JournalState, records: JournalRecord[]): void { target: record.target, at: record.at, ...(record.by ? { by: record.by } : {}), + authorization: position, }); } break; @@ -1383,6 +1457,14 @@ function applyRecords(state: JournalState, records: JournalRecord[]): void { } } +/** The journal position of the act a request carries out. */ +function authorizationOf(state: JournalState, cause: DiscordAwarenessOpCause): number | undefined { + if (cause.kind === 'batch' && cause.batchId) return state.batches.get(cause.batchId)?.authorization; + if (cause.kind === 'release' && cause.batchId) return state.batches.get(cause.batchId)?.releaseAuthorization; + if (cause.kind === 'retract' && cause.requestId) return state.retracts.get(cause.requestId)?.authorization; + return undefined; +} + interface LegacyEntry extends DiscordAwarenessRef { desired?: boolean; markerPresent?: boolean; @@ -1448,9 +1530,11 @@ function importLegacy(parsed: unknown, path: string, sha256: string): JournalRec ...(raw.suppressionIntervals?.length ? { suppressionIntervals: raw.suppressionIntervals } : {}), }, }); - // An active batch was activated by its surgery, after its body (an - // earlier release activated suppressions only after the last redaction). - if (legacyActive) records.push({ t: 'activated', at: raw.createdAt, batchId: raw.id }); + // Its marks were active. That is history, not proof of a suppression's + // body: the old ledger could record activation before the redactions + // were synced, so a startup on the target still verifies and resumes + // the intervals before recording the body complete. + if (legacyActive) records.push({ t: 'legacy-activated', at: raw.createdAt, batchId: raw.id }); const releaseActions: DiscordAwarenessReleaseAction[] = []; for (const entry of raw.refs) { const key = { serverId: entry.serverId, channelId: entry.channelId, messageId: entry.messageId, emoji: raw.emoji }; diff --git a/src/recovery/offline-branch.ts b/src/recovery/offline-branch.ts index 26c467e49..c00a5a50d 100644 --- a/src/recovery/offline-branch.ts +++ b/src/recovery/offline-branch.ts @@ -4,6 +4,7 @@ import { ContextManager } from '@animalabs/context-manager'; import { DEFAULT_DISCORD_AWARENESS_EMOJI, DiscordAwarenessOutbox, + boundDiscordAwarenessText, defaultDiscordAwarenessOutboxPath, extractDiscordAwarenessRefs, selectDiscordAwarenessRefs, @@ -233,13 +234,37 @@ export async function createOfflineRecoveryBranch( } } catch (error) { // A partially suppressed branch is not safe to boot into. Preserve it - // for diagnosis, but leave Chronicle on the untouched source branch and - // keep the explicit batch prepared: it is the unfinished body's record - // (startup holds its marks, and resumes the body only on its branch). - if (store.currentBranch().name === createdBranch) { - await contextManager.switchBranch(sourceBranch); + // for diagnosis, but put Chronicle back on the untouched source branch. + // Once the source is confirmed, the batch is retired (after the + // restored state is synced): its surgery never happened. If it can't + // be confirmed, the batch stays prepared as the unfinished body's + // record (startup holds its marks, and resumes the body only on its + // branch). The error says which. + const failure = error instanceof Error ? error.message : String(error); + let restore = ''; + try { + if (store.currentBranch().name !== sourceBranch) await contextManager.switchBranch(sourceBranch); + } catch (restoreError) { + restore = `; restoring ${sourceBranch} failed: ${boundDiscordAwarenessText( + restoreError instanceof Error ? restoreError.message : String(restoreError), + )}`; } - throw error; + let retire = ''; + if (batch) { + if (store.currentBranch().name !== sourceBranch) { + retire = `; awareness batch ${batch.id} kept, since the source branch could not be confirmed`; + } else { + try { + outbox.discard(batch.id); + } catch (discardError) { + retire = `; awareness batch ${batch.id} could not be retired (${boundDiscordAwarenessText( + discardError instanceof Error ? discardError.message : String(discardError), + )}); startup will hold it`; + } + } + } + if (!restore && !retire) throw error; + throw new Error(`${failure}${restore}${retire}`, { cause: error }); } // The body change landed. Marker bookkeeping is reported apart from it diff --git a/test/discord-awareness-outbox.test.ts b/test/discord-awareness-outbox.test.ts index d0d46fab6..e90dcb183 100644 --- a/test/discord-awareness-outbox.test.ts +++ b/test/discord-awareness-outbox.test.ts @@ -329,6 +329,92 @@ test('a newer opposite intent retires the old request\'s retries even when it wa assert.ok(!due.includes('add:m1') && !due.includes('add:m2') && !due.includes('remove:m3'), due.join(',')); })); +test('authorization order: a retract after a surgery\'s choice wins over that choice\'s later activation', withJournal((outbox, h) => { + const due = (o: DiscordAwarenessOutbox) => o.pendingDispatches('discord').map((d) => `${d.action}:${d.key.messageId}`).sort(); + const prepare = (target: string, refs: DiscordAwarenessRef[], extra: Record = {}) => outbox.prepare({ + agentName: 'cairn', sourceBranch: 'main', targetBranch: target, refs, scope: 'all', ...extra, + })!; + + // prepare -> retract -> activate: the remove goes, the add never does. + const a = prepare('t/a', [ref('m1')]); + outbox.retract(a.id); + assert.equal(outbox.activate(a.id), 0, 'nothing queued'); + assert.deepEqual(outbox.settleActivation(a.id), { status: 'queued', queued: 0 }); + assert.deepEqual(due(outbox), ['remove:m1']); + + // Overlapping prepared batches: a retract of one decides the shared key + // for the other, which was chosen earlier; its other key is unaffected. + const b = prepare('t/b', [ref('m2')]); + const c = prepare('t/c', [ref('m2'), ref('m3')]); + outbox.retract(b.id); + assert.equal(outbox.activate(c.id), 1); + assert.equal(outbox.activate(b.id), 0); + assert.deepEqual(due(outbox), ['add:m3', 'remove:m1', 'remove:m2']); + + // retract('all') covers batches not yet activated. + const d = prepare('t/d', [ref('m4')]); + const all = outbox.retract('all'); + assert.ok(all.removalsQueued >= 4); + assert.equal(outbox.activate(d.id), 0); + assert.ok(!due(outbox).includes('add:m4')); + + // Cancelling the retract stops it without reviving the choice it overrode. + const e = prepare('t/e', [ref('m5')]); + const { requestId } = outbox.retract(e.id); + outbox.cancel(requestId); + assert.equal(outbox.activate(e.id), 0); + assert.ok(!due(outbox).some((d) => d.endsWith(':m5')), 'neither the add nor the cancelled remove'); + + // A later choice wins over an earlier retract. + const f = prepare('t/f', [ref('m1')]); + assert.equal(outbox.activate(f.id), 1); + assert.ok(due(outbox).includes('add:m1') && !due(outbox).includes('remove:m1')); + + // A release is a new act: it wins over a retract made while the batch was held. + const g = prepare('t/g', [ref('m6')]); + h.reopen().recoverAtStartup('elsewhere'); // holds g (and nothing else is prepared) + const held = h.reopen(); + held.retract(g.id); + assert.equal(held.release(g.id).addsQueued, 1); + assert.ok(due(held).includes('add:m6') && !due(held).includes('remove:m6')); + + // The body's obligation is not the marks': a suppression whose marks were + // retracted before activation still records its body complete. + const sup = held.prepare({ + agentName: 'cairn', sourceBranch: 'main', targetBranch: 't/s', refs: [ref('m7')], scope: 'all', + activationPolicy: 'explicit', suppressionIntervals: [{ fromId: 'i1', toId: 'i1' }], + })!; + held.retract(sup.id); + assert.equal(held.activate(sup.id), 0); + const settled = held.batches().find((x) => x.id === sup.id)!; + assert.equal(settled.status, 'active'); + assert.equal(settled.suppressionComplete, true); + assert.deepEqual(held.preparedSuppressionsForBranch('t/s'), []); + + // Everything above holds across a restart. + assert.deepEqual(due(h.reopen()), due(held)); +})); + +test('startup records a crash-completion activation only after syncing the store, and nothing when the sync fails', withJournal((outbox, h) => { + const batch = outbox.prepare({ agentName: 'cairn', sourceBranch: 'main', targetBranch: 'rollback/cairn/1', refs: [ref('m1')], scope: 'all' })!; + const restarted = h.reopen(); + const sync = h.store.sync.bind(h.store); + const calls: string[] = []; + const append = h.store.appendJson.bind(h.store); + (h.store as any).appendJson = (type: string, payload: unknown) => { calls.push('append'); return append(type, payload); }; + (h.store as any).sync = () => { calls.push('sync'); throw new Error('injected sync failure'); }; + try { + assert.throws(() => restarted.recoverAtStartup('rollback/cairn/1'), /injected sync failure/); + } finally { + (h.store as any).sync = sync; + (h.store as any).appendJson = append; + } + assert.deepEqual(calls, ['sync'], 'the sync came first, and nothing was appended'); + assert.equal(h.reopen().batches().find((b) => b.id === batch.id)?.status, 'prepared'); + // With a working store, the next startup completes it. + assert.deepEqual(h.reopen().recoverAtStartup('rollback/cairn/1').activated, [batch.id]); +})); + test('a retract request can be cancelled by the id its receipt returned, even after a restart', withJournal((outbox, h) => { const batch = activeBatch(outbox, [ref('m1'), ref('m2'), ref('m3')]); for (const id of ['m1', 'm2', 'm3']) answer(outbox, id, 'confirmed'); @@ -482,6 +568,24 @@ test('v2 import keeps what the old ledger recorded as evidence, holds pending wo assert.equal(existsSync(h.legacyPath), false); })); +test('an imported active suppression is not proof its body completed: startup still verifies its intervals', withJournal((_outbox, h) => { + mkdirSync(join(h.dir, 'recovery'), { recursive: true }); + writeFileSync(h.legacyPath, JSON.stringify({ + version: 2, + batches: [{ + id: 'legacy-sup', status: 'active', agentName: 'cairn', sourceBranch: 'main', targetBranch: 'partial', + emoji: '💤', createdAt: 1, activationPolicy: 'explicit', suppressionIntervals: [{ fromId: 'i1', toId: 'i1' }], + refs: [{ ...ref('m1'), desired: true, markerPresent: true, deliveryStatus: 'applied', attempts: 1, lastAction: 'add' }], + }], + })); + const outbox = h.reopen(); + const [batch] = outbox.batches(); + assert.equal(batch.status, 'active', 'its marks were active: history'); + assert.equal(batch.suppressionComplete, undefined, 'its body is not certified by the old ledger'); + assert.deepEqual(outbox.preparedSuppressionsForBranch('partial').map((b) => b.id), ['legacy-sup']); + assert.equal(outbox.pendingDispatches('discord').length, 0); +})); + // --------------------------------------------------------------------------- // Framework seams, with stubbed collaborators // --------------------------------------------------------------------------- @@ -706,7 +810,16 @@ test('turn-based host/command undo honours the marks choice for the messages its return { undone: true }; }; await muted(async () => { + // Another agent sharing the store is mid-turn: refused, nothing undone. + framework.activeTurnTokens.set('other', 99); + const busy = await framework.handleHostCommand('discord', { command: 'undo', agentName: 'cairn', turns: 2, marks: 'addressed' }); + assert.equal(busy.ok, false); + assert.equal(busy.code, 'agent-busy'); + assert.equal(turns, 1); + framework.activeTurnTokens.delete('other'); + const result = await framework.handleHostCommand('discord', { command: 'undo', agentName: 'cairn', turns: 2, marks: 'addressed' }); + assert.equal(framework.surgeryHold, null, 'the reservation was released'); assert.equal(result.ok, true); assert.equal(result.undone, 1); assert.equal(result.markers.status, 'queued'); diff --git a/test/offline-recovery-branch.test.ts b/test/offline-recovery-branch.test.ts index 0ee20e99f..4fcaa8e7b 100644 --- a/test/offline-recovery-branch.test.ts +++ b/test/offline-recovery-branch.test.ts @@ -372,3 +372,62 @@ test('offline recovery reports a marker bookkeeping failure apart from the body, rmSync(dir, { recursive: true, force: true }); } }); + +test('a failed offline suppression retires its batch once the source is confirmed, and keeps it when it is not', async () => { + const dir = mkdtempSync(join(tmpdir(), 'offline-recovery-fail-')); + const storePath = join(dir, 'agent.chronicle'); + const seed = async () => { + rmSync(storePath, { recursive: true, force: true }); + const store = JsStore.openOrCreate({ path: storePath }); + const cm = await ContextManager.open({ store, namespace: 'agents/cairn' }); + cm.addMessage('user', [{ type: 'text', text: 'keep' }], { serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'm-keep' }); + cm.addMessage('user', [{ type: 'text', text: 'hide me' }], { serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'm-a' }); + cm.addMessage('user', [{ type: 'text', text: 'anchor' }], { serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'm-current' }); + cm.close(); + store.close(); + }; + const inspect = () => { + const store = JsStore.openOrCreate({ path: storePath }); + try { + const outbox = new DiscordAwarenessOutbox(store); + return { branch: store.currentBranch().name, statuses: outbox.batches().map((b) => b.status), due: outbox.pendingDispatches().length }; + } finally { + store.close(); + } + }; + const removeMessage = ContextManager.prototype.removeMessage; + const switchBranch = ContextManager.prototype.switchBranch; + try { + ContextManager.prototype.removeMessage = function () { throw new Error('injected redaction failure'); }; + + // The source comes back: the batch is retired; nothing will ever resume or mark. + await seed(); + await assert.rejects( + createOfflineRecoveryBranch({ + storePath, agentName: 'cairn', messageId: 'm-current', suppressMessageIds: ['m-a'], + marks: { scope: 'all' }, branchName: 'recovery/cairn/fail', + }), + (error: Error) => /injected redaction failure/.test(error.message) && !/kept/.test(error.message), + ); + assert.deepEqual(inspect(), { branch: 'main', statuses: ['discarded'], due: 0 }); + + // The source can't be confirmed: the batch stays as the unfinished body's record. + await seed(); + ContextManager.prototype.switchBranch = async function (this: ContextManager, name: string) { + if (name === 'main') throw new Error('injected restore failure'); + return switchBranch.call(this, name); + }; + await assert.rejects( + createOfflineRecoveryBranch({ + storePath, agentName: 'cairn', messageId: 'm-current', suppressMessageIds: ['m-a'], + marks: { scope: 'all' }, branchName: 'recovery/cairn/fail2', + }), + /injected redaction failure; restoring main failed: injected restore failure; awareness batch .* kept, since the source branch could not be confirmed/, + ); + assert.deepEqual(inspect(), { branch: 'recovery/cairn/fail2', statuses: ['prepared'], due: 0 }); + } finally { + ContextManager.prototype.removeMessage = removeMessage; + ContextManager.prototype.switchBranch = switchBranch; + rmSync(dir, { recursive: true, force: true }); + } +}); diff --git a/test/surgery-marks-receipt.test.ts b/test/surgery-marks-receipt.test.ts index 523782f29..14448a8cd 100644 --- a/test/surgery-marks-receipt.test.ts +++ b/test/surgery-marks-receipt.test.ts @@ -21,7 +21,7 @@ import { describe, it, beforeEach, afterEach } from 'node:test'; import assert from 'node:assert/strict'; import { fileURLToPath } from 'node:url'; -import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { JsStore } from '@animalabs/chronicle'; @@ -464,6 +464,45 @@ describe('surgery and awareness marks', () => { assert.equal(outbox().operations().filter((op) => op.action === 'add').length, 1, 'only the release queued marks'); }); + it('an imported active suppression is verified on its target at startup: the old ledger does not prove its body', async () => { + await start(false); + const { removed } = seed(3); + const live = cm(); + const sourceBranch = live.currentBranch().name; + // The old ledger's window: its batch says active, but the fork still + // carries the message its suppression was to remove. + await live.fork('partial'); + const [target] = live.getAllMessages().filter((m) => String(m.id) === removed[1]); + await framework.stop(); + mkdirSync(join(storePath, 'recovery'), { recursive: true }); + writeFileSync(join(storePath, 'recovery', 'discord-awareness-outbox.json'), JSON.stringify({ + version: 2, + batches: [{ + id: 'legacy-sup', status: 'active', agentName: 'resident', sourceBranch, targetBranch: 'partial', + emoji: '💤', createdAt: 1, activationPolicy: 'explicit', + suppressionIntervals: [{ fromId: String(target.id), toId: String(target.id) }], + refs: [{ + serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'amb-1', + desired: true, markerPresent: true, deliveryStatus: 'applied', attempts: 1, lastAction: 'add', + }], + }], + })); + + await start(false); // on 'partial' + assert.equal(cm().currentBranch().name, 'partial'); + assert.ok(!cm().getAllMessages().some((m) => String(m.id) === removed[1]), 'the interval was resumed'); + const [batch] = outbox().batches(); + assert.equal(batch.suppressionComplete, true); + assert.equal(batch.status, 'active'); + assert.equal(outbox().operations().length, 0, 'its marks stay history: nothing is sent'); + + // Idempotent: a later startup on the target changes nothing. + await framework.stop(); + await start(false); + assert.equal(outbox().batches()[0].suppressionComplete, true); + assert.equal(outbox().operations().length, 0); + }); + it('a surgery that removes nothing addressable schedules no marks even when marks are chosen', async () => { await start(false); const keep = String(cm().addMessage('Operator', [{ type: 'text', text: 'kept' }], {})); From c19d5e6c01be22a32e444835ec0178bb5d5115a7 Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Wed, 7 Oct 2026 10:31:25 -0400 Subject: [PATCH 0037/1112] docs(awareness): authorization order, reserved turn undo, imported active is not body proof README live-surgery section and the three changelog fragments describe d44826b's semantics: between opposite requests for one reaction the later authorization wins (a surgery's adds carry its choice's moment; release is a new act); retract('all') covers prepared/held/active batches and cancelling it never revives what it overrode; host/command turn undo holds the store like hide; an imported active suppression is verified and resumed rather than taken as body proof. --- README.md | 39 ++++++++++++------- changelog.d/surgery-marks-controls.added.md | 9 +++-- changelog.d/surgery-marks-one-shot.changed.md | 10 +++-- changelog.d/surgery-marks-opt-in.breaking.md | 10 +++-- 4 files changed, 42 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 1cba189ae..464882dec 100644 --- a/README.md +++ b/README.md @@ -368,12 +368,16 @@ const result = await framework.rollbackToMessage('cairn', { from the journal as it is at that moment, so a cancel takes effect even while an earlier request is on the wire. A request written and never answered is recorded as `unknown`, and a later confirmation of a different - attempt never resolves it. A record that certifies a body change (an - activation, a completed suppression, a retirement) is written only after - that change is synced. A batch whose surgery was interrupted before its - branch switch was recorded is held at startup until an operator releases - it; an interrupted suppression's redactions are resumed whenever its branch - is active at startup, whatever its marks' state. + attempt never resolves it. Between opposite requests for one reaction, + the later *authorization* wins, whenever each request was created: a + surgery's adds carry the moment of its marks choice, even when activation + comes later, so a retract made in between prevails; a release is a new + act. A record that certifies a body change (an activation, a completed + suppression, a retirement) is written only after that change is synced. + A batch whose surgery was interrupted before its branch switch was + recorded is held at startup until an operator releases it; an interrupted + suppression's redactions are resumed whenever its branch is active at + startup, whatever its marks' state. - **Operator controls,** each recorded in the operator log: - `listDiscordAwareness()` lists batches and retract requests; - `cancelDiscordAwareness(batch | retractRequest)` stops all further sends @@ -382,16 +386,19 @@ const result = await framework.rollbackToMessage('cairn', { including earlier attempts later answered, any of which may still land; - `retractDiscordAwareness(batch | 'all')` queues removal of this bot's reaction, through each address's configured MCPL route, for every message - of the batch (or every message any batch or imported ledger recorded an - add for), whatever history says: removing an absent reaction does - nothing. It supersedes adds not yet sent, and its receipt discloses - earlier requests whose outcome is unknown and imported history that - leaves outcomes unrecorded; + of the batch (or of every batch not retired by its own surgery, prepared, + held or active, and every message an imported ledger recorded), whatever + history says: removing an absent reaction does nothing. As the latest + authorization for those messages, it stops adds that haven't ended and + any an earlier choice would request later; cancelling it never revives + them. Its receipt discloses earlier requests whose outcome is unknown and + imported history that leaves outcomes unrecorded; - `releaseDiscordAwareness(batch)` queues a held batch. - **Over `host/command`** (servers granted `allowHostCommands`): `undo` (by messages or by turns) and `hide` take `marks: 'none' | 'addressed' | 'all'` - (default `none`; any other value is refused) and return `markers`. `hide` - holds the store like every surgery. The `marks` verb takes `action: 'list' + (default `none`; any other value is refused) and return `markers`. Each + holds the store like every surgery, so it is refused while any agent + sharing the store is mid-turn. The `marks` verb takes `action: 'list' | 'cancel' | 'retract' | 'release'` with a `target` (a batch id; for cancel also a retract request id; for retract also `all`). @@ -399,8 +406,10 @@ A pre-journal awareness ledger (`discord-awareness-outbox.json` under the store, or `discordAwarenessOutboxPath`) is imported once on first use and the file renamed `.migrated-v2`. What it recorded about each message is kept as evidence (attempt count, last action, status and error, and how many attempt -outcomes it leaves unrecorded), never as a claim about what is on Discord. -Its undelivered work is held for an explicit release. +outcomes it leaves unrecorded), never as a claim about what is on Discord, +and an imported `active` suppression is not taken as proof that its body +completed: startup verifies and resumes its intervals on its branch. Its +undelivered work is held for an explicit release. ## Observability diff --git a/changelog.d/surgery-marks-controls.added.md b/changelog.d/surgery-marks-controls.added.md index 28be5e31f..175fb7539 100644 --- a/changelog.d/surgery-marks-controls.added.md +++ b/changelog.d/surgery-marks-controls.added.md @@ -10,9 +10,12 @@ unknown, including earlier attempts later answered, which may still land. - `retractDiscordAwareness(batch | 'all')` queues removal of this bot's reaction, through each message's configured MCPL route, for every - selected message, whatever history says. It supersedes adds not yet - sent, and its receipt discloses requests whose outcome is unknown and - imported history that leaves outcomes unrecorded. + selected message, whatever history says. As the latest authorization + for those messages, it stops adds that haven't ended and any an earlier + choice would request later (a prepared surgery's activation), and + cancelling it never revives them. Its receipt discloses requests whose + outcome is unknown and imported history that leaves outcomes + unrecorded. - `releaseDiscordAwareness(batch)` queues a held batch. Each control is recorded in the operator log. The same controls are diff --git a/changelog.d/surgery-marks-one-shot.changed.md b/changelog.d/surgery-marks-one-shot.changed.md index 043a702d6..942aedaea 100644 --- a/changelog.d/surgery-marks-one-shot.changed.md +++ b/changelog.d/surgery-marks-one-shot.changed.md @@ -4,10 +4,12 @@ did. Each request is written ahead of its dispatch, and a dispatch is admitted from the journal as it is at that moment. A request that went out and got no answer is recorded as `unknown`, and a later confirmation of a - different attempt never resolves it. A batch whose surgery was interrupted - before its branch switch was recorded is held at startup until an operator - releases it. An interrupted suppression's redactions are resumed whenever - its branch is active at startup, independent of what happened to its marks. + different attempt never resolves it. Between opposite requests for one + reaction, the later authorization wins, whenever each was created. A batch + whose surgery was interrupted before its branch switch was recorded is held + at startup until an operator releases it. An interrupted suppression's + redactions are resumed whenever its branch is active at startup, + independent of what happened to its marks. - A journal record that certifies a body change (a marks activation, a completed suppression, a retired batch) is now written only after that change is synced to the store. Previously a crash could keep marks active diff --git a/changelog.d/surgery-marks-opt-in.breaking.md b/changelog.d/surgery-marks-opt-in.breaking.md index 0b403f5ff..a0648b270 100644 --- a/changelog.d/surgery-marks-opt-in.breaking.md +++ b/changelog.d/surgery-marks-opt-in.breaking.md @@ -21,8 +21,9 @@ reaction deadline (`discordAwarenessDeadlineMs`). Marks already on Discord are left alone; nothing is removed automatically. - Receipts (`markers`) now also carry `scope`, `unmarked` and `notRemoved`. - - `host/command` `hide` now holds the store like every surgery, so it is - refused (`agent-busy`) while any agent sharing the store is mid-turn. + - `host/command` `hide` and turn-based `undo` now hold the store like + every surgery, so they are refused (`agent-busy`) while any agent + sharing the store is mid-turn. - No sibling version is required: a host that ignores the new option simply places no marks. The connectome-host and discord-mcpl companions add the choice to their surfaces. @@ -36,5 +37,6 @@ `discordAwarenessOutboxPath` now only names a previous JSON ledger to import. Such a file (by default `/recovery/discord-awareness-outbox.json`) is imported once on first use and renamed `.migrated-v2`. What it recorded - is kept as evidence, never as synthesized attempts, and its undelivered - work is held for an explicit release. + is kept as evidence, never as synthesized attempts or as proof that a + suppression's body completed, and its undelivered work is held for an + explicit release. From 85481af8f5c6a89414b1842641cd9f8db4250db5 Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Wed, 7 Oct 2026 10:45:29 -0400 Subject: [PATCH 0038/1112] fix(awareness): legacy outcomes come from the old writer's lastError, not its reconciliation state; cancel discloses released removals June #45470 (probe /tmp/june-1016-799-legacy-status-probe.ts, with the actual 64c480b writer): v2 deliveryStatus is the old writer's reconciliation state (setDesired rewrites it from the cached markerPresent without any request), not the last attempt's outcome. An unknown add reconciled onto the source read as applied -> lastAddConfirmed:1, outcomesUnrecorded:0; a confirmed add reconciled onto the source read as pending -> its attempt unrecorded. The 64c480b writer recorded a success by deleting lastError (recordSuccess) and a failure by setting it (recordFailure); nothing else touches it. Evidence now carries lastOutcome derived only from that: confirmed (no error after an attempt), not-sent (pre-write refusal), refused (Discord's definitive answer), unrecorded (anything else). outcomesUnrecorded = earlier attempts + the last when unrecorded. The old status is kept as oldDeliveryStatus, documented as reconciliation state; the view's lastAdd/RemoveConfirmed count lastOutcome confirmed. Held release actions still follow the old writer's pending work (that is what its reconciliation state legitimately says). Same probe: cancel(batch).unresolvedAttempts filtered adds only, so a released removal's unknown attempt went undisclosed while the view showed it. Cancel now counts every request the batch made (adds and released removals). Test: entries exactly as 64c480b leaves them (unknown add then source reconcile; confirmed add then source reconcile) give lastAddConfirmed 1 and outcomesUnrecorded 1; the pending removal is held; release, unknown then confirmed removal, cancel -> unresolvedAttempts 1. Outbox 30/30. --- src/recovery/discord-awareness-outbox.ts | 56 +++++++++++++++++------- test/discord-awareness-outbox.test.ts | 37 ++++++++++++++++ 2 files changed, 76 insertions(+), 17 deletions(-) diff --git a/src/recovery/discord-awareness-outbox.ts b/src/recovery/discord-awareness-outbox.ts index 96c1e661f..21ab972bd 100644 --- a/src/recovery/discord-awareness-outbox.ts +++ b/src/recovery/discord-awareness-outbox.ts @@ -265,15 +265,30 @@ export interface DiscordAwarenessLegacyEvidence { /** Attempts the old ledger counted (it kept only the last one's details). */ attempts: number; lastAction: DiscordAwarenessAction; - /** The old ledger's status after the last attempt. */ - lastStatus: 'pending' | 'applied' | 'permanent-failure'; + /** + * The last attempt's outcome, as far as the old writer's fields establish + * it: it recorded a success by clearing `lastError` and a failure by + * setting it, and nothing else touched that field. `confirmed`: no error + * after an attempt. `not-sent`: AF refused before writing the request. + * `refused`: Discord answered that the message or channel is gone or + * inaccessible. `unrecorded`: any other error, so the request may have + * landed. + */ + lastOutcome: 'confirmed' | 'not-sent' | 'refused' | 'unrecorded'; + /** + * The old writer's reconciliation state at shutdown (`deliveryStatus`): + * whether its cached belief about the marker matched what it then wanted. + * Branch reconciliation rewrote it without any request, so it says + * nothing about any attempt's outcome. + */ + oldDeliveryStatus: 'pending' | 'applied' | 'permanent-failure'; lastError?: string; /** What the last error was: AF's refusal before writing a request, a * Discord answer (message/channel gone or inaccessible), or anything else. */ lastErrorKind?: 'pre-write-refusal' | 'discord-answer' | 'other'; /** Recorded attempts whose outcome the old ledger does not establish: - * every attempt before the last, and the last unless it was applied, - * refused before writing, or answered by Discord. */ + * every attempt before the last (only the last one's details were kept), + * and the last when its outcome is `unrecorded`. */ outcomesUnrecorded: number; } @@ -761,8 +776,10 @@ export class DiscordAwarenessOutbox { inFlight: 0, unknown: 0, confirmed: 0, + // Every request the batch itself made: its adds, and any removals its + // release queued. unresolvedAttempts: unresolvedDispatches( - [...state.ops.values()].filter((op) => op.cause.batchId === batchId && op.action === 'add'), + [...state.ops.values()].filter((op) => op.cause.batchId === batchId), (op, attempt) => this.attemptUnresolved(op, attempt), ), legacyOutcomesUnrecorded: [...state.legacy.values()].flat() @@ -1086,8 +1103,8 @@ export class DiscordAwarenessOutbox { ? { legacy: { entries: evidence.length, - lastAddConfirmed: evidence.filter((entry) => entry.lastAction === 'add' && entry.lastStatus === 'applied').length, - lastRemoveConfirmed: evidence.filter((entry) => entry.lastAction === 'remove' && entry.lastStatus === 'applied').length, + lastAddConfirmed: evidence.filter((entry) => entry.lastAction === 'add' && entry.lastOutcome === 'confirmed').length, + lastRemoveConfirmed: evidence.filter((entry) => entry.lastAction === 'remove' && entry.lastOutcome === 'confirmed').length, outcomesUnrecorded: evidence.reduce((sum, entry) => sum + entry.outcomesUnrecorded, 0), }, } @@ -1571,12 +1588,18 @@ function importLegacy(parsed: unknown, path: string, sha256: string): JournalRec return records; } -/** The facts a v2 entry recorded about its attempts, and what they leave open. */ +/** + * The facts a v2 entry recorded about its attempts, and what they leave open. + * Only `lastError` speaks to the last attempt's outcome (the old writer + * cleared it on success and set it on failure); `deliveryStatus` was its + * reconciliation state, rewritten by branch switches without any request, + * and is kept as that and nothing more. + */ function legacyEvidence( batchId: string, key: DiscordAwarenessRef & { emoji: string }, entry: LegacyEntry, - status: 'pending' | 'applied' | 'permanent-failure', + oldDeliveryStatus: 'pending' | 'applied' | 'permanent-failure', ): DiscordAwarenessLegacyEvidence { const attempts = entry.attempts!; const lastError = entry.lastError; @@ -1584,21 +1607,20 @@ function legacyEvidence( : PRE_WRITE_REFUSAL.test(lastError) ? 'pre-write-refusal' : isPermanentDiscordReactionFailure(lastError) ? 'discord-answer' : 'other'; - // The last attempt's outcome is established when the ledger saw it - // applied, or its error proves it was refused before writing or answered - // by Discord. Earlier attempts' details were never kept. - const lastEstablished = status === 'applied' - || lastErrorKind === 'pre-write-refusal' - || (status === 'permanent-failure' && lastErrorKind === 'discord-answer'); + const lastOutcome = lastErrorKind === undefined ? 'confirmed' + : lastErrorKind === 'pre-write-refusal' ? 'not-sent' + : lastErrorKind === 'discord-answer' ? 'refused' + : 'unrecorded'; return { batchId, key: { ...key }, attempts, lastAction: entry.lastAction!, - lastStatus: status, + lastOutcome, + oldDeliveryStatus, ...(lastError !== undefined ? { lastError: boundDiscordAwarenessText(lastError) } : {}), ...(lastErrorKind ? { lastErrorKind } : {}), - outcomesUnrecorded: (attempts - 1) + (lastEstablished ? 0 : 1), + outcomesUnrecorded: (attempts - 1) + (lastOutcome === 'unrecorded' ? 1 : 0), }; } diff --git a/test/discord-awareness-outbox.test.ts b/test/discord-awareness-outbox.test.ts index e90dcb183..dba3803e1 100644 --- a/test/discord-awareness-outbox.test.ts +++ b/test/discord-awareness-outbox.test.ts @@ -568,6 +568,43 @@ test('v2 import keeps what the old ledger recorded as evidence, holds pending wo assert.equal(existsSync(h.legacyPath), false); })); +test('legacy evidence reads outcomes from the old writer\'s lastError, never from its reconciliation state', withJournal((_outbox, h) => { + // Entries exactly as the 64c480b writer leaves them (recordSuccess clears + // lastError, recordFailure sets it; setDesired rewrites deliveryStatus + // from the cached markerPresent without any request). + const legacy = { + version: 2, + batches: [{ + id: 'reconciled', status: 'active', agentName: 'resident', sourceBranch: 'main', targetBranch: 'rollback/resident/9', + emoji: '💤', createdAt: 1, + refs: [ + // An unknown add, then a switch to the source: "applied" only because + // the cache (false) matched desired (false). The add may have landed. + { ...ref('u1'), desired: false, markerPresent: false, deliveryStatus: 'applied', attempts: 1, lastAction: 'add', + lastError: 'MCPL server "discord" did not respond within 10000ms' }, + // A confirmed add, then a switch to the source: "pending" is the + // removal the old writer wanted; the add really succeeded. + { ...ref('c1'), desired: false, markerPresent: true, deliveryStatus: 'pending', attempts: 1, lastAction: 'add' }, + ], + }], + }; + mkdirSync(join(h.dir, 'recovery'), { recursive: true }); + writeFileSync(h.legacyPath, JSON.stringify(legacy)); + const outbox = h.reopen(); + const [view] = batches(outbox); + assert.deepEqual(view.legacy, { entries: 2, lastAddConfirmed: 1, lastRemoveConfirmed: 0, outcomesUnrecorded: 1 }); + assert.equal(view.status, 'held', "the old writer's pending removal is held for release"); + + // The released removal's own requests are disclosed by cancel, like adds. + assert.equal(outbox.release('reconciled').removalsQueued, 1); + answer(outbox, 'c1', 'unknown'); + answer(outbox, 'c1', 'confirmed'); + const receipt = outbox.cancel('reconciled'); + assert.equal(receipt.unresolvedAttempts, 1, 'the unknown removal attempt may still land'); + assert.equal(receipt.confirmed, 1); + assert.equal(receipt.legacyOutcomesUnrecorded, 1); +})); + test('an imported active suppression is not proof its body completed: startup still verifies its intervals', withJournal((_outbox, h) => { mkdirSync(join(h.dir, 'recovery'), { recursive: true }); writeFileSync(h.legacyPath, JSON.stringify({ From 9baaeb17f7035eab5843da37f4a8b9b41e8c83d8 Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Wed, 7 Oct 2026 10:52:52 -0400 Subject: [PATCH 0039/1112] fix(awareness): a repeated cancel writes nothing and still reports truthful history June #45522: cancel() and cancelRetract() returned early with zero counters once the group was cancelled, so a confirmation landing after the first cancel read as confirmed:0 on a repeat while the view showed it. The history projection (confirmed; in flight; unknown, including attempts left without an outcome by a previous process) is now computed for every call, whatever each request's cancel state, in one helper (stopRequests); only the stopping records (and `cancelled`, newly stopped requests) depend on the group not being cancelled already, and a repeat writes nothing. Test: batch with one add on the wire and one due -> cancel (cancelled 1, inFlight 1) -> the in-flight add confirms -> repeat cancel (cancelled 0, inFlight 0, confirmed 1, unresolvedAttempts 0, no new journal records); the same for a retract request. --- src/recovery/discord-awareness-outbox.ts | 63 ++++++++++++++---------- test/discord-awareness-outbox.test.ts | 27 ++++++++++ 2 files changed, 65 insertions(+), 25 deletions(-) diff --git a/src/recovery/discord-awareness-outbox.ts b/src/recovery/discord-awareness-outbox.ts index 21ab972bd..98768be62 100644 --- a/src/recovery/discord-awareness-outbox.ts +++ b/src/recovery/discord-awareness-outbox.ts @@ -786,24 +786,49 @@ export class DiscordAwarenessOutbox { .filter((evidence) => evidence.batchId === batchId) .reduce((sum, evidence) => sum + evidence.outcomesUnrecorded, 0), }; - if (batch.cancelled) return receipt; const at = Date.now(); const records: JournalRecord[] = [{ t: 'cancelled', at, batchId, ...(by ? { by } : {}) }]; - for (const op of state.ops.values()) { - if (op.cause.batchId !== batchId) continue; - const status = this.opStatus(op); - if (status === 'confirmed') { + const ops = [...state.ops.values()].filter((op) => op.cause.batchId === batchId); + records.push(...this.stopRequests(ops, receipt, at, 'batch-cancelled', !!batch.cancelled)); + // A repeated cancel writes nothing; its receipt is still today's history. + if (!batch.cancelled) this.append(records, { durable: true }); + return receipt; + } + + /** + * Count what history says about these requests into a cancel receipt, and + * return the records that stop the ones still able to go out (none when + * the group was already cancelled). History is counted whatever each + * request's cancel state: an attempt on the wire or unanswered may land + * after any cancel, and a confirmation is a fact about Discord. + */ + private stopRequests( + ops: DiscordAwarenessOp[], + receipt: DiscordAwarenessCancelReceipt, + at: number, + reason: 'batch-cancelled' | 'request-cancelled', + alreadyCancelled: boolean, + ): JournalRecord[] { + const records: JournalRecord[] = []; + for (const op of ops) { + const last = op.attempts.at(-1); + if (last?.outcome === 'confirmed') { receipt.confirmed++; continue; } + if (last && last.outcome === undefined) { + if (this.inFlight.has(op.opId)) receipt.inFlight++; + else receipt.unknown++; + } else if (last?.outcome === 'unknown') { + receipt.unknown++; + } + if (alreadyCancelled || op.cancelled) continue; + const status = this.opStatus(op); + if (status !== 'requested' && status !== 'dispatching' && status !== 'unknown') continue; if (status === 'requested') receipt.cancelled++; - else if (status === 'dispatching') receipt.inFlight++; - else if (status === 'unknown') receipt.unknown++; - else continue; // failed or already cancelled - records.push({ t: 'op-cancelled', at, opId: op.opId, reason: 'batch-cancelled' }); + records.push({ t: 'op-cancelled', at, opId: op.opId, reason }); } - this.append(records, { durable: true }); - return receipt; + return records; } /** Stop a retract's removals that are still due; nothing sent is undone. */ @@ -821,22 +846,10 @@ export class DiscordAwarenessOutbox { unresolvedAttempts: unresolvedDispatches(ops, (op, attempt) => this.attemptUnresolved(op, attempt)), legacyOutcomesUnrecorded: 0, }; - if (retract.cancelled) return receipt; const at = Date.now(); const records: JournalRecord[] = [{ t: 'request-cancelled', at, requestId, ...(by ? { by } : {}) }]; - for (const op of ops) { - const status = this.opStatus(op); - if (status === 'confirmed') { - receipt.confirmed++; - continue; - } - if (status === 'requested') receipt.cancelled++; - else if (status === 'dispatching') receipt.inFlight++; - else if (status === 'unknown') receipt.unknown++; - else continue; - records.push({ t: 'op-cancelled', at, opId: op.opId, reason: 'request-cancelled' }); - } - this.append(records, { durable: true }); + records.push(...this.stopRequests(ops, receipt, at, 'request-cancelled', !!retract.cancelled)); + if (!retract.cancelled) this.append(records, { durable: true }); return receipt; } diff --git a/test/discord-awareness-outbox.test.ts b/test/discord-awareness-outbox.test.ts index dba3803e1..a4f93015a 100644 --- a/test/discord-awareness-outbox.test.ts +++ b/test/discord-awareness-outbox.test.ts @@ -218,6 +218,33 @@ test('the cancel receipt keeps an earlier unresolved request after a later answe assert.equal(receipt.unresolvedAttempts, 2, 'both first attempts may still land'); })); +test('a repeated cancel writes nothing, and its receipt is still the truthful history', withJournal((outbox, h) => { + const batch = activeBatch(outbox, [ref('m1'), ref('m2')]); + const onWire = outbox.claimDispatch('discord', (d) => d.key.messageId !== 'm1')!; + const first = outbox.cancel(batch.id); + assert.deepEqual({ cancelled: first.cancelled, inFlight: first.inFlight, confirmed: first.confirmed }, { cancelled: 1, inFlight: 1, confirmed: 0 }); + outbox.recordOutcome(onWire.attempts, 'confirmed'); + const records = h.store.getRecordIdsByType(DISCORD_AWARENESS_RECORD_TYPE).length; + const again = outbox.cancel(batch.id); + assert.deepEqual( + { cancelled: again.cancelled, inFlight: again.inFlight, unknown: again.unknown, confirmed: again.confirmed, unresolvedAttempts: again.unresolvedAttempts }, + { cancelled: 0, inFlight: 0, unknown: 0, confirmed: 1, unresolvedAttempts: 0 }, + ); + assert.equal(h.store.getRecordIdsByType(DISCORD_AWARENESS_RECORD_TYPE).length, records, 'nothing new written'); + + // The same for a retract request. + const other = activeBatch(outbox, [ref('m3'), ref('m4')]); + answer(outbox, 'm3', 'confirmed'); + answer(outbox, 'm4', 'confirmed'); + const { requestId } = outbox.retract(other.id); + const removal = outbox.claimDispatch('discord', (d) => d.key.messageId !== 'm3')!; + const stopped = outbox.cancel(requestId); + assert.deepEqual({ cancelled: stopped.cancelled, inFlight: stopped.inFlight }, { cancelled: 1, inFlight: 1 }); + outbox.recordOutcome(removal.attempts, 'confirmed'); + const repeated = outbox.cancel(requestId); + assert.deepEqual({ cancelled: repeated.cancelled, inFlight: repeated.inFlight, confirmed: repeated.confirmed }, { cancelled: 0, inFlight: 0, confirmed: 1 }); +})); + test('cancelling a batch its surgery has not activated stops every mark; its body can still resume', withJournal((outbox) => { const rollback = outbox.prepare({ agentName: 'cairn', sourceBranch: 'main', targetBranch: 'rollback/cairn/1', refs: [ref('m1'), ref('m2')], scope: 'all' })!; assert.equal(outbox.cancel(rollback.id, 'operator').cancelled, 2); From 511bd52503b952d9c2ea265ed1100f3ee1c9c00b Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Wed, 7 Oct 2026 10:54:08 -0400 Subject: [PATCH 0040/1112] docs(awareness): legacy evidence names the outcome its error field establishes and labels the old status as reconciliation state --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 464882dec..2c6c1cd09 100644 --- a/README.md +++ b/README.md @@ -405,8 +405,10 @@ const result = await framework.rollbackToMessage('cairn', { A pre-journal awareness ledger (`discord-awareness-outbox.json` under the store, or `discordAwarenessOutboxPath`) is imported once on first use and the file renamed `.migrated-v2`. What it recorded about each message is kept as -evidence (attempt count, last action, status and error, and how many attempt -outcomes it leaves unrecorded), never as a claim about what is on Discord, +evidence (attempt count; the last action and the outcome its error field +establishes; the old writer's delivery status, labelled as the +reconciliation state it was; and how many attempt outcomes it leaves +unrecorded), never as a claim about what is on Discord, and an imported `active` suppression is not taken as proof that its body completed: startup verifies and resumes its intervals on its branch. Its undelivered work is held for an explicit release. From 7623cf2b75805c5ab69f2ad916be66d076b9d3bc Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Wed, 7 Oct 2026 11:59:59 -0400 Subject: [PATCH 0041/1112] fix(awareness): refuse a known target-branch collision before preparing intent; offline branch creation settles like any body failure June #46104 (probe /tmp/june-1016-511-offline-name-probe.ts): the offline recovery called contextManager.branchAt() outside its body try/catch, after prepare. With branchName naming an existing branch it threw "Branch already exists" and left a prepared target-branch batch; starting that pre-existing branch later crash-completed it and queued an add for a message still in context. - offline: refuse an existing target name before anything is prepared (also in a dry run, which would fail the same way), and create the branch inside the body try, so a creation failure restores/confirms the source and retires the batch (or keeps it, saying so) like a failed switch or redaction. - live rollbackToMessage/suppressMessages: the same pre-check (OperatorActionError 'invalid'), before reserving or preparing; their branch creation was already inside the failure handling. Tests: offline recovery onto an existing branch with marks chosen is refused and journals nothing; an injected branchAt failure retires the batch on a confirmed source; live rollback and suppression onto an existing branch are refused with no batch. Awareness set + record-journal 87/87. --- src/framework.ts | 8 +++++ src/recovery/offline-branch.ts | 13 ++++++-- test/discord-awareness-outbox.test.ts | 2 +- test/offline-recovery-branch.test.ts | 44 +++++++++++++++++++++++++++ test/surgery-marks-receipt.test.ts | 15 +++++++++ 5 files changed, 78 insertions(+), 4 deletions(-) diff --git a/src/framework.ts b/src/framework.ts index 66ce75541..19dfd9840 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -6182,6 +6182,11 @@ export class AgentFramework { if (targetBranch === sourceBranch) { throw new OperatorActionError('invalid', 'Rollback branch name must differ from the active branch'); } + // A known collision is refused before any intent is prepared: a batch + // whose target names an existing branch would be armed by starting it. + if (this.store.listBranches().some((branch) => branch.name === targetBranch)) { + throw new OperatorActionError('invalid', `Rollback branch ${targetBranch} already exists`); + } // Gate + reserve the whole store (see reserveStoreForSurgery); held // until the switch has landed and its marker scheduling has been @@ -6328,6 +6333,9 @@ export class AgentFramework { if (targetBranch === sourceBranch) { throw new OperatorActionError('invalid', 'Suppression branch name must differ from the active branch'); } + if (this.store.listBranches().some((branch) => branch.name === targetBranch)) { + throw new OperatorActionError('invalid', `Suppression branch ${targetBranch} already exists`); + } // Gate + reserve the whole store (see reserveStoreForSurgery); held // until the fork is fully redacted and its marker scheduling has been // recorded or reported, or the source is restored. Remote delivery is diff --git a/src/recovery/offline-branch.ts b/src/recovery/offline-branch.ts index c00a5a50d..b4a1ae5f4 100644 --- a/src/recovery/offline-branch.ts +++ b/src/recovery/offline-branch.ts @@ -181,6 +181,11 @@ export async function createOfflineRecoveryBranch( if (targetBranch === sourceBranch) { throw new Error('Recovery branch name must differ from the active source branch'); } + // A known collision is refused before anything is prepared: a batch + // whose target names an existing branch would be armed by starting it. + if (store.listBranches().some((branch) => branch.name === targetBranch)) { + throw new Error(`Recovery branch ${targetBranch} already exists; choose another --branch-name`); + } const result: OfflineRecoveryBranchResult = { dryRun: options.dryRun === true, @@ -221,10 +226,12 @@ export async function createOfflineRecoveryBranch( })), }); - // branchAt uses the target message's origin sequence. No message content is - // compiled or submitted to Membrane during this operation. - const createdBranch = contextManager.branchAt(target.id, targetBranch); try { + // branchAt uses the target message's origin sequence. No message + // content is compiled or submitted to Membrane during this operation. + // Creating the branch is part of the body: a failure here settles the + // batch like a failed switch or redaction. + const createdBranch = contextManager.branchAt(target.id, targetBranch); await contextManager.switchBranch(createdBranch); // Work from newest to oldest so earlier redactions never perturb the // live positions used by later range endpoints. diff --git a/test/discord-awareness-outbox.test.ts b/test/discord-awareness-outbox.test.ts index a4f93015a..f1a2b5677 100644 --- a/test/discord-awareness-outbox.test.ts +++ b/test/discord-awareness-outbox.test.ts @@ -799,7 +799,7 @@ function hostCommandFramework(h: ReturnType, messages: Arra }; const framework = Object.create(AgentFramework.prototype) as any; framework.agents = new Map([['cairn', { state: { status: 'idle' }, getContextManager: () => contextManager }]]); - framework.store = { currentBranch: () => ({ name: currentBranch }), sync: () => h.store.sync() }; + framework.store = { currentBranch: () => ({ name: currentBranch }), listBranches: () => [], sync: () => h.store.sync() }; framework.discordAwarenessOutbox = h.outbox; framework.discordAwarenessEmoji = '🫥'; framework.discordAwarenessDrains = new Map(); diff --git a/test/offline-recovery-branch.test.ts b/test/offline-recovery-branch.test.ts index 4fcaa8e7b..ee9b6e9fb 100644 --- a/test/offline-recovery-branch.test.ts +++ b/test/offline-recovery-branch.test.ts @@ -431,3 +431,47 @@ test('a failed offline suppression retires its batch once the source is confirme rmSync(dir, { recursive: true, force: true }); } }); + +test('an offline recovery onto an existing branch is refused before any intent, and a branch-creation failure settles like any body failure', async () => { + const dir = mkdtempSync(join(tmpdir(), 'offline-recovery-name-')); + const storePath = join(dir, 'agent.chronicle'); + try { + const store = JsStore.openOrCreate({ path: storePath }); + const cm = await ContextManager.open({ store, namespace: 'agents/cairn' }); + cm.addMessage('user', [{ type: 'text', text: 'safe' }], { serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'm-safe' }); + const last = cm.addMessage('user', [{ type: 'text', text: 'later' }], { + serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'm-later', tags: ['chat:addressed'], + }); + cm.branchAt(last, 'taken'); // an existing branch that still has both messages + cm.close(); + store.close(); + const inspect = () => { + const reopened = JsStore.openOrCreate({ path: storePath }); + try { + return { branch: reopened.currentBranch().name, statuses: new DiscordAwarenessOutbox(reopened).batches().map((b) => b.status) }; + } finally { + reopened.close(); + } + }; + + await assert.rejects( + createOfflineRecoveryBranch({ storePath, agentName: 'cairn', messageId: 'm-safe', marks: { scope: 'all' }, branchName: 'taken' }), + /Recovery branch taken already exists/, + ); + assert.deepEqual(inspect(), { branch: 'main', statuses: [] }, 'nothing prepared, so starting taken arms nothing'); + + const branchAt = ContextManager.prototype.branchAt; + ContextManager.prototype.branchAt = function () { throw new Error('injected branch failure'); }; + try { + await assert.rejects( + createOfflineRecoveryBranch({ storePath, agentName: 'cairn', messageId: 'm-safe', marks: { scope: 'all' }, branchName: 'fresh' }), + (error: Error) => /injected branch failure/.test(error.message) && !/kept/.test(error.message), + ); + } finally { + ContextManager.prototype.branchAt = branchAt; + } + assert.deepEqual(inspect(), { branch: 'main', statuses: ['discarded'] }); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); diff --git a/test/surgery-marks-receipt.test.ts b/test/surgery-marks-receipt.test.ts index 14448a8cd..2f35dfa14 100644 --- a/test/surgery-marks-receipt.test.ts +++ b/test/surgery-marks-receipt.test.ts @@ -503,6 +503,21 @@ describe('surgery and awareness marks', () => { assert.equal(outbox().operations().length, 0); }); + it('a rollback or suppression onto an existing branch is refused before any intent is prepared', async () => { + await start(false); + const { tail, removed } = seed(2); + cm().branchAt(cm().getAllMessages().at(-1)!.id, 'taken'); + await assert.rejects( + framework.rollbackToMessage('resident', { messageId: tail, branchName: 'taken', marks: { scope: 'all' } }), + /Rollback branch taken already exists/, + ); + await assert.rejects( + framework.suppressMessages('resident', { messageIds: [removed[0]], branchName: 'taken', marks: { scope: 'all' } }), + /Suppression branch taken already exists/, + ); + assert.equal(outbox().batches().length, 0); + }); + it('a surgery that removes nothing addressable schedules no marks even when marks are chosen', async () => { await start(false); const keep = String(cm().addMessage('Operator', [{ type: 'text', text: 'kept' }], {})); From dc2928ce421eb748fe645490b05e6bc06ff16893 Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Wed, 7 Oct 2026 12:01:50 -0400 Subject: [PATCH 0042/1112] fix(recover): name the CLI's actual --branch flag in the collision error --- src/recovery/offline-branch.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/recovery/offline-branch.ts b/src/recovery/offline-branch.ts index b4a1ae5f4..ef0541e42 100644 --- a/src/recovery/offline-branch.ts +++ b/src/recovery/offline-branch.ts @@ -184,7 +184,7 @@ export async function createOfflineRecoveryBranch( // A known collision is refused before anything is prepared: a batch // whose target names an existing branch would be armed by starting it. if (store.listBranches().some((branch) => branch.name === targetBranch)) { - throw new Error(`Recovery branch ${targetBranch} already exists; choose another --branch-name`); + throw new Error(`Recovery branch ${targetBranch} already exists; choose another --branch`); } const result: OfflineRecoveryBranchResult = { From e9099e5d9d3acc0710acce8712208025b71a5bd8 Mon Sep 17 00:00:00 2001 From: Basil-1369 Date: Wed, 7 Oct 2026 12:17:53 -0400 Subject: [PATCH 0043/1112] fix(drafts): held envelopes keep hybrid state, receipts read draft state, drafts keep exact runs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Answers Greptile's first review of #247 (findings 1, 3, 4, 5; 2 is answered in the review thread as intended behavior) plus the two `{{unsent}}` cases Felix-1368 and Basil-1369 found beside finding 1, which Petra-1012 and Tessa-974 agreed belong to the same repair (room-203 #45831, #45845). - Hybrid envelopes beside a send (finding 1). holdSilencedProse now moves the router's state exactly as deliverHybridProse does, withholding only publication: `>>>skip_reply` clears the pin and keeps the rest private, across segments and later rounds, until a destination is named; a named destination that resolves becomes the sticky target again (clearing privacy alone would have sent a later unsilenced round's prose to the locus, which nobody named), and an unresolved or malformed one leaves no destination ('failed'). ` !` still asks to continue. The mid-turn hybrid hold is now chained on turnSpeechChain behind earlier rounds' envelopes, because their state transitions run there; a synchronous hold could read the state before a slow earlier delivery had applied it. - `{{unsent}}` beside a send. `>>>skip_reply {{unsent}}` still sets the latest bounce aside (a send suppresses speech, not that operation). A held target envelope holds the message it would have published, `{{unsent}}` expanded under the re-bounce rule, now one helper (unsentWords) shared with bounceProse: the whole authored text kept, inherited risk (with chained sourceDraftId evidence) when the latest bounce is unconfirmed or in flight; a bare `{{unsent}}` mints no copy and the bounce stays the one draft, named in the turn's receipt. Before, the draft held the literal token, which a resend would have published. bounceProse's stale comment (describing the rejected strip-the-token remedy) is replaced by the helper's doc. - Turn-end receipt (finding 3). Each of the turn's drafts is re-read from the store and reported by its state at turn end: still held ("held as drafts … (not sent …)", wording unchanged), delivered by the resident's resend (listed in the header where it landed, so the header no longer says "nothing" over it; a hybrid `{{unsent}}` delivery already listed as plain speech gets a note instead of a second entry), unconfirmed (riskText), or dismissed. - Exact runs (finding 4). prose-segments.ts has one walker: splitProseRuns returns runs exactly as written (whitespace-only runs dropped) and splitProseSegments is those runs trimmed. Routing still publishes trimmed segments (deliverProse needs `>>` at the start, and routed speech is unchanged); every silenced hold takes the raw runs, so Draft.text is the resident's words byte for byte, as its doc and `read` claim. - Refused-call boundaries (finding 5). The walker breaks at `tool_attempt` and `tool_notice`, so the text-only path's segment split actually happens on this branch alone. This is Greta-1013's hunk from fix/xml-tool-call-notices (a05a2e3); she agreed #247 owns it and will drop hers on rebase (via Felix, room-240 #45593). A turn-ending [text, tool_attempt] still yields just the text. - Finding 2 (XML think privacy) is not changed: same_round_think_text_policy is native-round-only by upstream design (think tool description; test "legacy/no-roundContent behavior remains unchanged" routes that prose publicly), so in XML mode a send holds it like any other speech. The doc comment and changelog now say so, and a test pins it. Tests: 7 end-to-end (exact runs and resend bytes; hybrid privacy across segments/rounds and the restored pin, plus a live >>>skip_reply binding a later silenced round; the {{unsent}} cases including inherited risk; the receipt's held/resent/dismissed/unconfirmed states; XML think beside a send; locus and hybrid refused-attempt splits) and 2 unit tests for the walker. On 6c5f054 the six behavior tests fail for exactly these defects (the XML-think test passes there: it pins existing behavior). Full suite 1381/1381 on this tree (node 24, concurrency 4, Petra's feat-prose-drafts node_modules: membrane 0.5.86, context-manager 0.13.0). Not tested: the chained-hold ordering under a slow earlier delivery (needs a publish that hangs while later rounds proceed); it is reasoned from the chain's ordering, not exercised. Co-Authored-By: Claude Opus 5.5 --- changelog.d/prose-drafts.added.md | 25 +++- src/framework.ts | 229 +++++++++++++++++++++--------- src/prose-segments.ts | 35 +++-- test/prose-drafts.test.ts | 162 +++++++++++++++++++++ test/prose-segments.test.ts | 23 ++- 5 files changed, 394 insertions(+), 80 deletions(-) diff --git a/changelog.d/prose-drafts.added.md b/changelog.d/prose-drafts.added.md index 45af20a6a..e128726b0 100644 --- a/changelog.d/prose-drafts.added.md +++ b/changelog.d/prose-drafts.added.md @@ -5,15 +5,29 @@ and text-only, native and XML tool modes), hybrid prose left without a destination after a failed envelope, and explicit/hybrid bounces (which replace the in-memory latest-wins clipboard; `{{unsent}}` keeps meaning the - latest bounce). Deliberate privacy is never drafted: `skip_reply`, a - same-round private `think`, silent turns and `proseRouting: disabled` - narration stay private and are counted in the receipt. + latest bounce). A draft held beside a send keeps its run exactly as + written, indentation and surrounding newlines included; routing still + publishes trimmed segments. Deliberate privacy is never drafted: + `skip_reply` (the tool, or a hybrid `>>>skip_reply` envelope, which keeps + the rest private until a destination is named), a same-round private + `think` in a native round (where that policy applies: XML tool mode routes + such prose as ordinary speech, so a send holds it as a draft), silent turns + and `proseRouting: disabled` narration stay private and are counted in the + receipt. +- In hybrid mode a send withholds publication, not what an envelope does: a + held `>>>` envelope moves the routing state as live routing would (a + destination it names becomes the sticky target), `>>>skip_reply {{unsent}}` + still sets the latest bounce aside, and the draft holds the message the + envelope would have published, with `{{unsent}}` expanded under the + re-bounce rule (a bare `{{unsent}}` leaves the bounce as the one draft). - Drafts live in typed Chronicle records (`RecordJournal`) that do not follow branch switches: undo, rollback and checkout never change them, and they survive restarts. Nothing is ever sent from them on its own. - A private notice names held drafts at the next tool boundary when the live stream presents mid-turn messages, and the turn-end `[delivered]` receipt - names every draft held in the turn ("held as drafts d-…"); a draft no notice + names every draft held in the turn by its state at turn end: still held + ("held as drafts d-… (not sent …)"), delivered by the resident's resend + (listed where it landed), unconfirmed, or dismissed. A draft no notice reached (a crash, an aborted turn) is named at the next turn's start. - A draft is never published except by its resident's explicit resend (or `{{unsent}}`), and only that resident can list or act on it. Its notices go @@ -28,6 +42,9 @@ unknown (and none confirmed) needs `confirmDuplicate: true`, and a resend owns its drafts until it finishes, re-checking each before it is sent. Unused fields may be passed as null. +- Prose segments also break at XML tool mode's refused attempts and their + notices (`tool_attempt`, `tool_notice`): the words before an all-refused + call and after its notice are two messages, routed or held separately. - `ChannelRegistry.deliverSpeech` (routeSpeech with its `PublishOutcome`) and `ChannelRegistry.resolveDestination` are new; routeSpeech is unchanged. - The bounce notice now goes to the bouncing resident rather than the primary. diff --git a/src/framework.ts b/src/framework.ts index c462c1c55..2ea22b5fd 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -82,7 +82,7 @@ import { ProseStreamRouter } from './mcpl/prose-stream-router.js'; import { detectKnownToolWrapperProse } from './tool-wrapper-prose-guard.js'; import { InferenceRouter } from './mcpl/inference-router.js'; import { ChannelRegistry, type ChannelToolOrigin, type PublishDestination, type PublishOutcome } from './mcpl/channel-registry.js'; -import { ProseDraftStore, draftState, uncertainAttempt, type Draft, type DraftReason, type DraftSource, type InheritedRisk } from './prose-drafts.js'; +import { ProseDraftStore, draftState, uncertainAttempt, type Draft, type DraftReason, type DraftSource, type DraftState, type InheritedRisk } from './prose-drafts.js'; import { ConversationRouter, CONVERSATION_ROUTING_DEPRECATION_NOTICE } from './mcpl/conversation-router.js'; import { INBOUND_SOURCE_KEY, type InboundAcceptanceObserver, type InboundChannelSource, type InboundLane, type InboundSource, type InboundUnscopedSource } from './mcpl/inbound-source.js'; import { safeSlice } from './safe-slice.js'; @@ -109,7 +109,7 @@ import { scriptTimeLimits, validateCodeExecutionConfig, } from './code-execution/tool-definition.js'; -import { splitProseSegments } from './prose-segments.js'; +import { splitProseRuns } from './prose-segments.js'; import { cumulativeDelta } from './usage-accounting.js'; import { stampThinkingTokenEstimates } from './thinking-token-stamp.js'; @@ -8791,7 +8791,10 @@ export class AgentFramework { /** * Silenced plain speech: an explicit send holds it as resendable drafts; a * deliberate silence (skip_reply, a silent turn, a same-round private - * think) keeps it private — counted for the receipt, never drafted. + * think in a native round, where that policy applies) keeps it private — + * counted for the receipt, never drafted. `segments` are the runs exactly + * as written (splitProseRuns), so a draft keeps the resident's words byte + * for byte. */ private holdSilencedProse( agent: Agent, @@ -8809,24 +8812,71 @@ export class AgentFramework { this.holdProseDrafts(agent, segments, 'explicit-send', opts); return; } - // Hybrid prose may carry `>>>` envelopes: a draft holds the words, not - // the routing syntax (a resend must not publish the prefix), and the - // destination the resident wrote stays with it as a note. A - // `>>>skip_reply` envelope was private by the resident's own choice. + // Hybrid prose may carry `>>>` envelopes. A send suppresses speech, not + // what the envelopes do: they move the router's state exactly as + // deliverHybridProse's do. After `>>>skip_reply` the rest stays private + // (across segments and later rounds) until a destination is named; + // naming one sets the sticky target, or leaves none when it does not + // resolve; `>>>skip_reply {{unsent}}` sets the latest bounce aside, and + // ` !` asks to continue. Only publication is withheld: a draft holds + // what the envelope would have published (`{{unsent}}` under the + // re-bounce rule, unsentWords), not its routing syntax, and the + // destination the resident wrote stays with it as a note. let privateCount = 0; - const parts: Array<{ text: string; note?: string }> = []; + const parts: Array<{ text: string; note?: string; inheritedRisk?: InheritedRisk }> = []; + /** Latest bounces a bare `{{unsent}}` named: still the one draft of those words. */ + const kept: Draft[] = []; for (const segment of segments) { for (const envelope of splitHybridEnvelopes(segment)) { const parsed = parseHybridProsePrefix(envelope); - if (parsed.kind === 'private') privateCount++; - else if (parsed.kind === 'target') parts.push({ text: parsed.body, note: `written for >>>${parsed.target}` }); - else parts.push({ text: envelope }); + if (parsed.continueTurn) this.proseContinuations.add(agent.name); + if (parsed.kind === 'private') { + this.proseTargetPins.delete(agent.name); + this.proseHybridSuppressed.set(agent.name, 'private'); + if (parsed.body.includes('{{unsent}}')) this.dismissLatestBounce(agent); + privateCount++; + } else if (parsed.kind === 'target') { + const resolved = this.channelRegistry?.resolveProseTarget(parsed.target!); + if (resolved && !('error' in resolved)) { + this.proseTargetPins.set(agent.name, resolved.channelId); + this.proseHybridSuppressed.delete(agent.name); + } else { + this.proseTargetPins.delete(agent.name); + this.proseHybridSuppressed.set(agent.name, 'failed'); + } + const words = this.unsentWords(agent, parsed.body); + if ('draft' in words) { + if (!kept.includes(words.draft)) kept.push(words.draft); + } else { + parts.push({ + text: words.text, + note: `written for >>>${parsed.target}`, + ...(words.inheritedRisk ? { inheritedRisk: words.inheritedRisk } : {}), + }); + } + } else if (envelope.startsWith('>>>')) { + // Malformed: an attempted destination that names none. + this.proseTargetPins.delete(agent.name); + this.proseHybridSuppressed.set(agent.name, 'failed'); + parts.push({ text: envelope }); + } else if (this.proseHybridSuppressed.get(agent.name) === 'private') { + privateCount++; + } else { + parts.push({ text: envelope }); + } } } if (privateCount > 0) { this.turnProsePrivate.set(agent.name, (this.turnProsePrivate.get(agent.name) ?? 0) + privateCount); } this.holdProseDrafts(agent, parts, 'explicit-send', opts); + if (kept.length > 0) { + // The turn's receipt names them by their state at turn end; each was + // named when it bounced. + const list = this.turnDrafts.get(agent.name) ?? []; + for (const draft of kept) if (!list.some((d) => d.id === draft.id)) list.push(draft); + this.turnDrafts.set(agent.name, list); + } } /** @@ -9375,13 +9425,36 @@ export class AgentFramework { ); } const notes: string[] = []; - if (drafts.length > 0) { - const ids = drafts.slice(0, 8).map((d) => d.id).join(', ') + (drafts.length > 8 ? `, and ${drafts.length - 8} more` : ''); + // Each of the turn's drafts as it stands now: a mid-turn notice lets the + // resident resend or dismiss one before the turn ends, so "not sent" is + // said only of drafts still held. + const current = drafts.map((d) => this.proseDrafts.get(agent.name, d.id) ?? d); + const inState = (state: DraftState): Draft[] => current.filter((d) => draftState(d) === state); + const idList = (ds: Draft[]): string => + ds.slice(0, 8).map((d) => d.id).join(', ') + (ds.length > 8 ? `, and ${ds.length - 8} more` : ''); + const held = inState('held'); + if (held.length > 0) { notes.push( - `${drafts.length} plain-speech segment(s) held as draft${drafts.length === 1 ? '' : 's'} ${ids} ` + + `${held.length} plain-speech segment(s) held as draft${held.length === 1 ? '' : 's'} ${idList(held)} ` + '(not sent — drafts can resend them unchanged, or dismiss them)', ); } + for (const d of inState('unconfirmed')) { + notes.push(`draft ${d.id} is unconfirmed: ${this.riskText(d)} — check that channel before sending it again (resend needs confirmDuplicate: true)`); + } + const dismissed = inState('dismissed'); + if (dismissed.length > 0) notes.push(`draft${dismissed.length === 1 ? '' : 's'} ${idList(dismissed)} dismissed by you`); + for (const d of inState('delivered')) { + const attempt = d.attempts.find((a) => a.outcome?.status === 'delivered')!; + const how = attempt.via === 'resend' ? 'by your resend' : 'by your {{unsent}}'; + // A hybrid `{{unsent}}` envelope is already listed above, as the plain + // speech it was; a resend (or explicit mode's envelope) is not. + if (attempt.via === 'unsent-token' && seen.has(attempt.destination.channelId)) { + notes.push(`draft ${d.id} delivered ${how}`); + } else { + shown.push(`${AgentFramework.destinationText(attempt.destination)} (draft ${d.id}, ${how})`); + } + } if (notHeld) { notes.push( `${notHeld.count} plain-speech segment(s) suppressed and NOT held as drafts (${notHeld.error}) — ` + @@ -9783,11 +9856,52 @@ export class AgentFramework { /** Cap on consecutive bounce-triggered wakes (notices still append after). */ private static readonly PROSE_BOUNCE_WAKE_CAP = 2; + /** + * Words about to be held that may use `{{unsent}}` (a bounce, or an + * envelope an explicit send suppressed), under the one re-bounce rule. A + * bare `{{unsent}}` is the latest bounce itself, which stays the one draft + * of those words rather than being copied. Otherwise the whole authored + * text is kept, the token expanded to the latest bounce's words, nothing + * stripped or substituted. When that draft may already have been posted + * (unconfirmed, or in flight) the words carry its duplication risk: they + * can be resent only with confirmDuplicate, like the draft they copy. The + * evidence is the risk's own: the copied draft's uncertain attempt, else + * the risk that draft itself inherited, else (truly) that it was in flight + * when copied. + */ + private unsentWords(agent: Agent, text: string): { draft: Draft } | { text: string; inheritedRisk?: InheritedRisk } { + if (!text.includes('{{unsent}}')) return { text }; + const name = agent.name; + const previous = this.proseDrafts.latestBounce(name); + if (previous && text.trim() === '{{unsent}}') return { draft: previous }; + const expanded = text.replaceAll('{{unsent}}', previous?.text ?? ''); + if (!previous || !(draftState(previous) === 'unconfirmed' || this.draftsInFlight.has(`${name}\u0000${previous.id}`))) { + return { text: expanded }; + } + const risky = uncertainAttempt(previous); + const inheritedRisk: InheritedRisk = risky + ? { + draftId: previous.id, + destination: risky.destination, + at: risky.at, + reason: risky.outcome?.reason ?? 'its outcome was never recorded', + } + : previous.inheritedRisk + ? { + draftId: previous.id, + sourceDraftId: previous.inheritedRisk.sourceDraftId ?? previous.inheritedRisk.draftId, + ...(previous.inheritedRisk.destination ? { destination: previous.inheritedRisk.destination } : {}), + ...(previous.inheritedRisk.at !== undefined ? { at: previous.inheritedRisk.at } : {}), + reason: previous.inheritedRisk.reason, + } + : { draftId: previous.id, reason: 'it was being sent when these words were held' }; + return { text: expanded, inheritedRisk }; + } + /** * A routing envelope that could not be delivered: its words are held as a * draft (the `{{unsent}}` source) and the resident is told, privately. - * Bouncing `{{unsent}}` itself again keeps the one draft rather than - * minting a copy; a body that adds words around it holds the expanded text. + * `{{unsent}}` in it follows the re-bounce rule (unsentWords). */ private bounceProse( agent: Agent, @@ -9798,49 +9912,17 @@ export class AgentFramework { hold: { round?: number; notice: 'now' | 'later' } = { notice: 'later' }, ): void { const name = agent.name; - const previous = this.proseDrafts.latestBounce(name); - // A latest bounce that may already have been posted (unconfirmed, or in - // flight) is never copied into a fresh draft: that copy would resend - // without the confirmation its own uncertainty requires. It stays held as - // itself; only the resident's other words become a new draft. - const previousAtRisk = previous !== undefined - && (draftState(previous) === 'unconfirmed' || this.draftsInFlight.has(`${name}\u0000${previous.id}`)); + const words = this.unsentWords(agent, text); let draft: Draft | undefined; let inherited: InheritedRisk | undefined; - if (previous && text.trim() === '{{unsent}}') { - draft = previous; + if ('draft' in words) { + draft = words.draft; } else { - const usesUnsent = text.includes('{{unsent}}'); - const expanded = usesUnsent ? text.replaceAll('{{unsent}}', previous?.text ?? '') : text; - if (usesUnsent && previous && previousAtRisk) { - // The whole authored attempt is kept, words of the risky draft - // included, and it carries that draft's duplication risk: it can be - // resent only with confirmDuplicate, like the draft it copies. The - // evidence is the risk's own: the copied draft's uncertain attempt, - // else the risk that draft itself inherited, else (truly) that it - // was in flight when copied. - const risky = uncertainAttempt(previous); - inherited = risky - ? { - draftId: previous.id, - destination: risky.destination, - at: risky.at, - reason: risky.outcome?.reason ?? 'its outcome was never recorded', - } - : previous.inheritedRisk - ? { - draftId: previous.id, - sourceDraftId: previous.inheritedRisk.sourceDraftId ?? previous.inheritedRisk.draftId, - ...(previous.inheritedRisk.destination ? { destination: previous.inheritedRisk.destination } : {}), - ...(previous.inheritedRisk.at !== undefined ? { at: previous.inheritedRisk.at } : {}), - reason: previous.inheritedRisk.reason, - } - : { draftId: previous.id, reason: 'it was being sent when these words were held' }; - } + inherited = words.inheritedRisk; // The bounce notice below names the draft, so no separate held notice. draft = this.holdProseDrafts( agent, - [{ text: expanded, ...(inherited ? { inheritedRisk: inherited } : {}) }], + [{ text: words.text, ...(inherited ? { inheritedRisk: inherited } : {}) }], 'bounced', { round: hold.round, note: reason, notice: 'later' }, )[0]; @@ -10793,7 +10875,10 @@ export class AgentFramework { const presentsInjections = (event.context as { supportsInjectedMessages?: boolean }).supportsInjectedMessages ?? true; const holdOpts = { round: draftRound, notice: presentsInjections ? 'now' as const : 'later' as const }; - const roundSegments = splitProseSegments(assistantBlocks); + // Routing publishes trimmed segments; held drafts keep the + // runs exactly as written. + const roundRuns = splitProseRuns(assistantBlocks); + const roundSegments = roundRuns.map((run) => run.trim()); if (roundSegments.length > 0) { if (turnProseRouting === 'disabled') { console.error( @@ -10802,7 +10887,12 @@ export class AgentFramework { this.recordProseSuppression(agent.name, roundSegments.length); } else if (turnProseRouting === 'hybrid') { if (turnSilenced) { - this.holdSilencedProse(agent, roundSegments, silenceCause, hasSameRoundPrivateThink, holdOpts); + // In order behind earlier rounds' envelopes: the router + // state they leave is where this round's starts. + const cause = silenceCause; + turnSpeechChain = turnSpeechChain + .then(() => this.holdSilencedProse(agent, roundRuns, cause, hasSameRoundPrivateThink, holdOpts)) + .catch((err) => console.error('mid-turn hybrid prose hold failed:', err)); } else if (!hasSameRoundPrivateThink) { const locus = resolveTurnLocus(); for (const seg of roundSegments) { @@ -10831,7 +10921,7 @@ export class AgentFramework { ); // Never a silent black hole (n=8: the flying-scene reply): // held as drafts the resident can resend, and named. - this.holdSilencedProse(agent, roundSegments, silenceCause, hasSameRoundPrivateThink, holdOpts); + this.holdSilencedProse(agent, roundRuns, silenceCause, hasSameRoundPrivateThink, holdOpts); } else if (hasSameRoundPrivateThink) { console.error( `[routing] ${agent.name}: mid-turn round [${roundToolNames.join(', ')}] -> prose NOT routed (same_round_think_text_policy=private)`, @@ -11263,18 +11353,20 @@ export class AgentFramework { // they are never routed here — which is precisely how the `think` // tool (and any explicit send tool) yields a silent turn. if (speechContent.length > 0 && this.channelRegistry) { - const speechText = speechContent + const speechRun = speechContent .map((b) => (b as ContentBlock & { type: 'text' }).text) - .join('\n') - .trim(); + .join('\n'); // A text-only turn is one message — unless an all-refused XML // round (membrane `tool_attempt`, answered in-band with a // `tool_notice`) separates its prose: nothing was called, but // the words before the attempt and after its notice are two - // messages, routed as segments like a tool turn's. - const speechSegments = response.content.some((b) => (b.type as string) === 'tool_attempt') - ? splitProseSegments(response.content) - : speechText ? [speechText] : []; + // messages, routed as segments like a tool turn's. Routing + // publishes trimmed segments; held drafts keep the runs exactly + // as written. + const speechRuns = response.content.some((b) => (b.type as string) === 'tool_attempt') + ? splitProseRuns(response.content) + : speechRun.trim() ? [speechRun] : []; + const speechSegments = speechRuns.map((run) => run.trim()); const textOnlyHold = { round: draftRound + 1, notice: 'later' as const }; if (speechSegments.length > 0) { if (turnProseRouting === 'disabled') { @@ -11286,7 +11378,7 @@ export class AgentFramework { // (a fresh text-only turn starts unsilenced). Explicit // mode is exempt, as mid-turn. console.error(`[routing] ${agent.name}: text-only prose NOT routed (turn silenced: ${silenceCause})`); - this.holdSilencedProse(agent, speechSegments, silenceCause, false, textOnlyHold); + this.holdSilencedProse(agent, speechRuns, silenceCause, false, textOnlyHold); } else if (turnProseRouting === 'hybrid') { const locus = resolveTurnLocus(); console.error(`[prose] ${agent.name}: text-only turn -> hybrid prose gateway`); @@ -11355,7 +11447,10 @@ export class AgentFramework { silenceCause === 'private' || fallbackCause === 'private' ? 'private' : (silenceCause ?? fallbackCause); const trailingHold = { round: draftRound + 1, notice: 'later' as const }; - const segments = splitProseSegments(liveProseRouting ? terminalContent : response.content); + // Routing publishes trimmed segments; held drafts keep the runs + // exactly as written. + const runs = splitProseRuns(liveProseRouting ? terminalContent : response.content); + const segments = runs.map((run) => run.trim()); // Preserve in-channel ordering: everything enqueued live must // land before the trailing prose. Awaited even when silenced — @@ -11371,7 +11466,7 @@ export class AgentFramework { } } else if (turnProseRouting === 'hybrid') { if (silenced && segments.length > 0) { - this.holdSilencedProse(agent, segments, trailingCause, false, trailingHold); + this.holdSilencedProse(agent, runs, trailingCause, false, trailingHold); } else if (segments.length > 0) { const locus = resolveTurnLocus(); for (const seg of segments) { @@ -11401,7 +11496,7 @@ export class AgentFramework { `(${silenced ? `turn silenced: ${trailingCause}` : 'no trailing prose'})`, ); if (silenced && segments.length > 0) { - this.holdSilencedProse(agent, segments, trailingCause, false, trailingHold); + this.holdSilencedProse(agent, runs, trailingCause, false, trailingHold); } } else { // Reuse the locus pinned at the turn's first live-routed diff --git a/src/prose-segments.ts b/src/prose-segments.ts index 6d8f3f079..6e2b96f7e 100644 --- a/src/prose-segments.ts +++ b/src/prose-segments.ts @@ -10,26 +10,40 @@ * collapsing those distinct messages into one (item 4). This helper instead * yields each contiguous run of text — the segments a surface should deliver as * separate, ordered messages. Contiguous text blocks merge into one segment; - * `tool_use` / `tool_result` blocks are segment boundaries; empty or + * `tool_use` / `tool_result` blocks are segment boundaries, as are XML tool + * mode's `tool_attempt` / `tool_notice` (a refused attempt and the harness's + * notice end a round just as a call and its results do); empty or * whitespace-only runs are dropped. */ import type { ContentBlock } from '@animalabs/membrane'; -export function splitProseSegments(content: readonly ContentBlock[]): string[] { - const segments: string[] = []; +/** + * The runs exactly as written: contiguous text blocks joined with a newline, + * nothing trimmed. Whitespace-only runs are dropped. Held drafts keep these, + * so a draft's words (indentation included) are the resident's own, byte for + * byte. + */ +export function splitProseRuns(content: readonly ContentBlock[]): string[] { + const runs: string[] = []; let buf: string[] = []; const flush = (): void => { - const s = buf.join('\n').trim(); - if (s) segments.push(s); + const run = buf.join('\n'); + if (run.trim()) runs.push(run); buf = []; }; for (const block of content) { - if (block.type === 'text') { + const type = block.type as string; + if (type === 'text') { buf.push((block as ContentBlock & { type: 'text' }).text); - } else if (block.type === 'tool_use' || block.type === 'tool_result') { + } else if ( + type === 'tool_use' || + type === 'tool_result' || + type === 'tool_attempt' || + type === 'tool_notice' + ) { flush(); } // Other block types (thinking, redacted_thinking, image, …) are neither @@ -38,5 +52,10 @@ export function splitProseSegments(content: readonly ContentBlock[]): string[] { } flush(); - return segments; + return runs; +} + +/** The runs as speech is routed: each trimmed of surrounding whitespace. */ +export function splitProseSegments(content: readonly ContentBlock[]): string[] { + return splitProseRuns(content).map((run) => run.trim()); } diff --git a/test/prose-drafts.test.ts b/test/prose-drafts.test.ts index 663706489..3672dd7cc 100644 --- a/test/prose-drafts.test.ts +++ b/test/prose-drafts.test.ts @@ -892,3 +892,165 @@ describe('drafts resend ownership', () => { } }); }); + +describe('held drafts: exact runs, hybrid envelopes beside a send, and turn-end state', () => { + /** Draft ids in hold order, so a later round can name them. */ + const fixIds = (h: Awaited>, ids: string[]) => { + let next = 0; + (h.store() as unknown as { newId: () => string }).newId = () => ids[next++]!; + }; + + it('a held draft keeps its run exactly as written, and a resend publishes those bytes', async () => { + const h = await harness(); + try { + const authored = ' indented code\n second line\n'; + await h.turn([ + createMockResponse([text(authored), explicitSend('s1')], 'tool_use'), + createMockResponse([text('\n\nAfter the send.\n')]), + ]); + const held = h.drafts().reverse(); + assert.deepEqual(held.map((d) => d.text), [authored, '\n\nAfter the send.\n']); + await h.turn([ + createMockResponse([drafts('r1', { action: 'resend', draftIds: [held[0]!.id], destination: ROOM })], 'tool_use'), + createMockResponse([]), + ]); + assert.equal(h.publishes().at(-1)!.text, authored); + } finally { + await h.close(); + } + }); + + it('hybrid: after >>>skip_reply, prose a send holds back stays private across segments and rounds until a destination is named, which restores the sticky target', async () => { + const h = await harness({ agents: [{ name: 'scout', proseRouting: 'hybrid' }] }); + try { + await h.turn([ + createMockResponse([text('>>>skip_reply a private aside'), explicitSend('s1'), text('still private'), call('l1', 'channel_list', {})], 'tool_use'), + createMockResponse([text('private in a later round'), call('l2', 'channel_list', {})], 'tool_use'), + createMockResponse([text(`>>>${ROOM} words for the room`), call('l3', 'channel_list', {})], 'tool_use'), + createMockResponse([text('more for the room')]), + ]); + assert.deepEqual(h.publishes().map((p) => p.text), ['an explicit note elsewhere']); + assert.deepEqual(h.drafts().reverse().map((d) => [d.text, d.note]), [ + ['words for the room', `written for >>>${ROOM}`], + ['more for the room', undefined], + ]); + assert.equal((h.framework as unknown as { proseTargetPins: Map }).proseTargetPins.get('scout'), ROOM, + 'the named destination is the sticky target again, as live routing would leave it'); + assert.match(h.texts().find((t) => t.startsWith('[delivered]'))!, / · 3 plain-speech segment\(s\) kept private \(skip_reply\)$/); + + // A live `>>>skip_reply` in an earlier round binds a later silenced one. + const before = h.drafts().length; + await h.turn([ + createMockResponse([text('>>>skip_reply thinking aloud'), call('l4', 'channel_list', {})], 'tool_use'), + createMockResponse([text('still thinking'), explicitSend('s2')], 'tool_use'), + createMockResponse([]), + ]); + assert.equal(h.drafts().length, before, 'nothing private was drafted'); + // "still thinking" is the one kept private (live routing does not count + // the `>>>skip_reply` envelope itself). + assert.equal(h.texts().filter((t) => t.startsWith('[delivered]')).at(-1), '[delivered] nothing — 1 plain-speech segment(s) kept private (skip_reply)'); + } finally { + await h.close(); + } + }); + + it('hybrid: beside a send, {{unsent}} holds the whole message it would have published, a bare one keeps the bounce as the one draft, and >>>skip_reply {{unsent}} still sets it aside', async () => { + const h = await harness({ agents: [{ name: 'scout', proseRouting: 'hybrid' }] }); + try { + await h.turn([createMockResponse([text('>>>#nowhere lost words')])]); + const [bounce] = h.drafts(); + assert.equal(bounce?.reason, 'bounced'); + + await h.turn([createMockResponse([text(`>>>${ROOM} {{unsent}} and a postscript`), explicitSend('s1')], 'tool_use'), createMockResponse([])]); + const copy = h.drafts().find((d) => d.text.endsWith('a postscript'))!; + assert.deepEqual([copy.text, copy.note, copy.reason, copy.inheritedRisk], ['lost words and a postscript', `written for >>>${ROOM}`, 'explicit-send', undefined]); + + const count = h.drafts().length; + await h.turn([createMockResponse([text(`>>>${ROOM} {{unsent}}`), explicitSend('s2')], 'tool_use'), createMockResponse([])]); + assert.equal(h.drafts().length, count, 'a bare {{unsent}} mints no copy'); + assert.equal(h.texts().filter((t) => t.startsWith('[delivered]')).at(-1), + `[delivered] nothing — 1 plain-speech segment(s) held as draft ${bounce!.id} (not sent — drafts can resend them unchanged, or dismiss them)`); + + // Once the bounce may have been posted, held words copied from it carry its risk. + h.command({ op: 'publish-mode', mode: 'no-receipt' }); + await h.turn([createMockResponse([text(`>>>${ROOM} {{unsent}}`)])]); + assert.equal(draftState(h.store().get('scout', bounce!.id)!), 'unconfirmed'); + h.command({ op: 'publish-mode', mode: 'delivered' }); + await h.turn([createMockResponse([text(`>>>${ROOM} {{unsent}} again`), explicitSend('s3')], 'tool_use'), createMockResponse([])]); + const risky = h.drafts().find((d) => d.text === 'lost words again')!; + assert.equal(risky.inheritedRisk?.draftId, bounce!.id); + assert.equal(draftState(risky), 'unconfirmed'); + + await h.turn([createMockResponse([text('>>>skip_reply {{unsent}}'), explicitSend('s4')], 'tool_use'), createMockResponse([])]); + assert.equal(draftState(h.store().get('scout', bounce!.id)!), 'dismissed', 'a send withholds speech, not the dismissal'); + assert.ok(h.drafts().every((d) => !d.text.includes('{{unsent}}')), 'no draft holds the literal token'); + assert.deepEqual(h.publishes().map((p) => p.text).filter((t) => t !== 'an explicit note elsewhere'), ['lost words'], + 'only the live {{unsent}} went out'); + } finally { + await h.close(); + } + }); + + it('the turn-end receipt reports each draft as it stands: resent, dismissed, unconfirmed or still held', async () => { + const h = await harness(); + try { + const ids = ['d-rcpta', 'd-rcptb', 'd-rcptc', 'd-rcptd']; + fixIds(h, ids); + await h.turn([ + createMockResponse([text('resend me'), explicitSend('s1'), text('dismiss me'), call('l1', 'channel_list', {}), text('keep me'), call('l2', 'channel_list', {})], 'tool_use'), + createMockResponse([drafts('r1', { action: 'resend', draftIds: [ids[0]], destination: ROOM })], 'tool_use'), + createMockResponse([drafts('x1', { action: 'dismiss', draftIds: [ids[1]] })], 'tool_use'), + createMockResponse([]), + ]); + assert.deepEqual(h.publishes().map((p) => p.text), ['an explicit note elsewhere', 'resend me']); + assert.equal(h.texts().find((t) => t.startsWith('[delivered]')), + `[delivered] plain speech → #room (Guild One) (${ROOM}) (draft ${ids[0]}, by your resend) · ` + + `1 plain-speech segment(s) held as draft ${ids[2]} (not sent — drafts can resend them unchanged, or dismiss them) · ` + + `draft ${ids[1]} dismissed by you`); + + h.command({ op: 'publish-mode', mode: 'no-receipt' }); + await h.turn([ + createMockResponse([text('uncertain words'), explicitSend('s2')], 'tool_use'), + createMockResponse([drafts('r2', { action: 'resend', draftIds: [ids[3]], destination: ROOM })], 'tool_use'), + createMockResponse([]), + ]); + assert.match(h.texts().filter((t) => t.startsWith('[delivered]')).at(-1)!, new RegExp( + `^\\[delivered\\] nothing — draft ${ids[3]} is unconfirmed: ${ids[3]}'s attempt to #room \\(Guild One\\) \\(${ROOM}\\) at .+ ` + + 'may already have been posted: .+ — check that channel before sending it again \\(resend needs confirmDuplicate: true\\)$')); + } finally { + await h.close(); + } + }); + + it('XML tool mode: the same-round think policy is native-only, so prose beside think and a send is held like any other', async () => { + const h = await harness({ xml: true, agents: [{ name: 'scout', sameRoundThinkTextPolicy: 'private' }] }); + try { + const round = [text('Prose beside think.'), call('t1', 'think', { content: 'hmm' }), explicitSend('s1')]; + await h.turn([ + createMockResponse(round, 'tool_use'), + { ...createMockResponse(round), toolCalls: [] }, + ]); + assert.deepEqual(h.drafts().map((d) => [d.text, d.reason]), [['Prose beside think.', 'explicit-send']]); + assert.deepEqual(h.publishes().map((p) => p.text), ['an explicit note elsewhere']); + } finally { + await h.close(); + } + }); + + for (const mode of ['locus', 'hybrid'] as const) { + it(`${mode}: prose around an all-refused XML attempt posts as two messages`, async () => { + const h = await harness({ agents: [{ name: 'scout', proseRouting: mode }] }); + try { + await h.turn([createMockResponse([ + text('Before the attempt.'), + { type: 'tool_attempt', rawXml: '' } as unknown as ContentBlock, + { type: 'tool_notice', notices: [{ ordinal: 0, kind: 'refused', message: 'boundary' }] } as unknown as ContentBlock, + text('After its notice.'), + ])]); + assert.deepEqual(h.publishes().map((p) => p.text), ['Before the attempt.', 'After its notice.']); + } finally { + await h.close(); + } + }); + } +}); diff --git a/test/prose-segments.test.ts b/test/prose-segments.test.ts index 015b1594e..98ffda842 100644 --- a/test/prose-segments.test.ts +++ b/test/prose-segments.test.ts @@ -1,7 +1,7 @@ import { test } from 'node:test'; import assert from 'node:assert/strict'; import type { ContentBlock } from '@animalabs/membrane'; -import { splitProseSegments } from '../src/prose-segments.js'; +import { splitProseRuns, splitProseSegments } from '../src/prose-segments.js'; const textBlock = (text: string): ContentBlock => ({ type: 'text', text } as ContentBlock); const toolUse = (id: string): ContentBlock => @@ -60,3 +60,24 @@ test('a plain no-tool turn yields a single segment (unchanged behaviour)', () => test('a tool-only turn (no prose) yields no segments', () => { assert.deepEqual(splitProseSegments([toolUse('t1'), toolResult('t1')]), []); }); + +test('runs keep the words exactly as written; segments are those runs trimmed for routing', () => { + const content: ContentBlock[] = [ + textBlock(' indented code\n second line\n'), + toolUse('t1'), toolResult('t1'), + textBlock('\n\nAfter the tool. '), + toolUse('t2'), toolResult('t2'), + textBlock(' \n\t'), + ]; + assert.deepEqual(splitProseRuns(content), [' indented code\n second line\n', '\n\nAfter the tool. ']); + assert.deepEqual(splitProseSegments(content), ['indented code\n second line', 'After the tool.']); +}); + +test('a refused XML attempt and its notice are boundaries too, and a turn-ending attempt leaves only its prose', () => { + const attempt = { type: 'tool_attempt', rawXml: '' } as unknown as ContentBlock; + const notice = { type: 'tool_notice', notices: [{ ordinal: 0, kind: 'refused', message: 'boundary' }] } as unknown as ContentBlock; + assert.deepEqual(splitProseSegments([textBlock('Before the attempt.'), attempt, notice, textBlock('After its notice.')]), + ['Before the attempt.', 'After its notice.']); + assert.deepEqual(splitProseSegments([textBlock('Again.'), attempt]), ['Again.']); + assert.deepEqual(splitProseRuns([textBlock('Before. '), attempt, notice, textBlock(' After.')]), ['Before. ', ' After.']); +}); From 467a48ffaa5cca23995a4f3d84a5c7d2f4b22a30 Mon Sep 17 00:00:00 2001 From: Basil-1369 Date: Wed, 7 Oct 2026 12:54:35 -0400 Subject: [PATCH 0044/1112] fix(drafts): a held-draft notice gives a copied draft its risk; "nothing confirmed" beside an unconfirmed draft MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Felix-1368's review of e9099e5 (room-240 #46400, #46447): - The mid-turn held-draft notice said "not sent" and offered a plain resend for every draft it named. Since e9099e5 a held hybrid envelope can expand `{{unsent}}` from a latest bounce that may already have been posted, so the new draft is `unconfirmed` (inherited risk) from the moment it is held, and a plain resend would be refused. draftsHeldNotice now splits the batch by state: held drafts keep the existing "delivers it unchanged" wording (byte-identical when nothing is at risk), and each unconfirmed one gets its riskText and the confirmDuplicate resend it needs. Found by Felix's probe through a full native turn. - The turn-end receipt's header said "[delivered] nothing — draft d-x is unconfirmed: … may already have been posted". With an unconfirmed draft in the receipt and nothing confirmed delivered, the header now says "nothing confirmed", so its first clause doesn't contradict the evidence. - A comment no longer says explicit mode reaches the receipt: both receipt call sites exclude explicit mode (antra's original design, bdbc7e3), so the explicit-mode "nothing" I raised in room-240 is not reachable; Felix confirmed through a real turn. Tests, two adapted from Felix's probes: - a held round waits behind a slow earlier delivery for the state its `>>>skip_reply` leaves (this exercises e9099e5's chained hold; with the hold made synchronous again it fails: "the held round waits for the earlier private transition"); - the notice for a mixed batch: a free draft offered a plain resend, a copied, possibly-posted one its risk and confirmDuplicate; - XML mode: runs around a refused attempt held with their edge whitespace and resent byte for byte. The hybrid tests now end with the completion a real membrane yields (the whole turn's content, no calls), so the turn's speech chain is settled before its receipt as in production; MockMembrane's literal empty completion skipped that await. Full suite 1384/1384 on this tree (node 24, concurrency 4, Petra's feat-prose-drafts node_modules). Co-Authored-By: Claude Opus 5.5 --- changelog.d/prose-drafts.added.md | 4 +- src/framework.ts | 27 ++++++-- test/prose-drafts.test.ts | 111 +++++++++++++++++++++++++++--- 3 files changed, 124 insertions(+), 18 deletions(-) diff --git a/changelog.d/prose-drafts.added.md b/changelog.d/prose-drafts.added.md index e128726b0..7e0a0a73c 100644 --- a/changelog.d/prose-drafts.added.md +++ b/changelog.d/prose-drafts.added.md @@ -24,7 +24,9 @@ branch switches: undo, rollback and checkout never change them, and they survive restarts. Nothing is ever sent from them on its own. - A private notice names held drafts at the next tool boundary when the live - stream presents mid-turn messages, and the turn-end `[delivered]` receipt + stream presents mid-turn messages (a draft holding words copied from one + that may already have been posted is named with that risk and the + `confirmDuplicate` its resend needs), and the turn-end `[delivered]` receipt names every draft held in the turn by its state at turn end: still held ("held as drafts d-… (not sent …)"), delivered by the resident's resend (listed where it landed), unconfirmed, or dismissed. A draft no notice diff --git a/src/framework.ts b/src/framework.ts index 2ea22b5fd..433896d59 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -8966,14 +8966,25 @@ export class AgentFramework { private draftsHeldNotice(agentName: string, held: Draft[], reason: DraftReason): string { const ids = held.map((d) => d.id); const many = held.length > 1; - const idList = ids.map((id) => `"${id}"`).join(', '); + const quoted = (ds: Draft[]): string => ds.map((d) => `"${d.id}"`).join(', '); + // Words copied from a draft that may already have been posted resend + // only with the resident's confirmation; the rest resend freely. + const free = held.filter((d) => draftState(d) !== 'unconfirmed'); + const risky = held.filter((d) => draftState(d) === 'unconfirmed'); + const resend = [ + ...(free.length > 0 + ? [`drafts(action: "resend", draftIds: [${quoted(free)}], destination: "#channel") delivers ` + + `${free.length > 1 ? 'them' : 'it'} unchanged`] + : []), + ...risky.map((d) => `${this.riskText(d)}, so check that channel before resending ${d.id} ` + + `(drafts(action: "resend", draftIds: ["${d.id}"], destination: "#channel", confirmDuplicate: true))`), + ]; // Named: residents that share one message slot (#197) share this window. return ( `[drafts] ${agentName}: not sent — ${many ? `${held.length} plain-speech segments` : 'a plain-speech segment'} ` + `held as draft${many ? 's' : ''} ${ids.join(', ')} — ${AgentFramework.DRAFT_REASON_TEXT[reason]}. ` + - `Nothing publishes drafts but your own resend, and they stay until you act: drafts(action: "resend", draftIds: [${idList}], ` + - `destination: "#channel") delivers ${many ? 'them' : 'it'} unchanged; drafts(action: "dismiss", ` + - `draftIds: [${idList}]) sets ${many ? 'them' : 'it'} aside.` + `Nothing publishes drafts but your own resend, and they stay until you act: ${resend.join('; ')}; ` + + `drafts(action: "dismiss", draftIds: [${quoted(held)}]) sets ${many ? 'them' : 'it'} aside.` ); } @@ -9439,7 +9450,8 @@ export class AgentFramework { '(not sent — drafts can resend them unchanged, or dismiss them)', ); } - for (const d of inState('unconfirmed')) { + const unconfirmed = inState('unconfirmed'); + for (const d of unconfirmed) { notes.push(`draft ${d.id} is unconfirmed: ${this.riskText(d)} — check that channel before sending it again (resend needs confirmDuplicate: true)`); } const dismissed = inState('dismissed'); @@ -9448,7 +9460,7 @@ export class AgentFramework { const attempt = d.attempts.find((a) => a.outcome?.status === 'delivered')!; const how = attempt.via === 'resend' ? 'by your resend' : 'by your {{unsent}}'; // A hybrid `{{unsent}}` envelope is already listed above, as the plain - // speech it was; a resend (or explicit mode's envelope) is not. + // speech it was; a resend is not. if (attempt.via === 'unsent-token' && seen.has(attempt.destination.channelId)) { notes.push(`draft ${d.id} delivered ${how}`); } else { @@ -9468,10 +9480,11 @@ export class AgentFramework { : `${suppressed} plain-speech segment(s) suppressed`); } const suppressedNote = notes.join(' · '); + // Nothing is confirmed delivered, but an unconfirmed draft may have been. const text = shown.length > 0 ? `[delivered] plain speech → ${shown.join(' · ')}${suppressedNote ? ` · ${suppressedNote}` : ''}` - : `[delivered] nothing — ${suppressedNote}`; + : `[delivered] nothing${unconfirmed.length > 0 ? ' confirmed' : ''} — ${suppressedNote}`; try { const mid = agent.getContextManager().addMessage( 'user', diff --git a/test/prose-drafts.test.ts b/test/prose-drafts.test.ts index 3672dd7cc..a25e60664 100644 --- a/test/prose-drafts.test.ts +++ b/test/prose-drafts.test.ts @@ -899,6 +899,13 @@ describe('held drafts: exact runs, hybrid envelopes beside a send, and turn-end let next = 0; (h.store() as unknown as { newId: () => string }).newId = () => ids[next++]!; }; + /** Tool rounds, then the completion a real membrane yields: the whole + * turn's content, with no calls of its own (so the turn's speech chain is + * settled before its receipt, as in production). */ + const rounds = (...rs: ContentBlock[][]) => [ + ...rs.map((r) => createMockResponse(r, 'tool_use')), + { ...createMockResponse(rs.flat()), toolCalls: [] }, + ]; it('a held draft keeps its run exactly as written, and a resend publishes those bytes', async () => { const h = await harness(); @@ -940,11 +947,10 @@ describe('held drafts: exact runs, hybrid envelopes beside a send, and turn-end // A live `>>>skip_reply` in an earlier round binds a later silenced one. const before = h.drafts().length; - await h.turn([ - createMockResponse([text('>>>skip_reply thinking aloud'), call('l4', 'channel_list', {})], 'tool_use'), - createMockResponse([text('still thinking'), explicitSend('s2')], 'tool_use'), - createMockResponse([]), - ]); + await h.turn(rounds( + [text('>>>skip_reply thinking aloud'), call('l4', 'channel_list', {})], + [text('still thinking'), explicitSend('s2')], + )); assert.equal(h.drafts().length, before, 'nothing private was drafted'); // "still thinking" is the one kept private (live routing does not count // the `>>>skip_reply` envelope itself). @@ -961,12 +967,12 @@ describe('held drafts: exact runs, hybrid envelopes beside a send, and turn-end const [bounce] = h.drafts(); assert.equal(bounce?.reason, 'bounced'); - await h.turn([createMockResponse([text(`>>>${ROOM} {{unsent}} and a postscript`), explicitSend('s1')], 'tool_use'), createMockResponse([])]); + await h.turn(rounds([text(`>>>${ROOM} {{unsent}} and a postscript`), explicitSend('s1')])); const copy = h.drafts().find((d) => d.text.endsWith('a postscript'))!; assert.deepEqual([copy.text, copy.note, copy.reason, copy.inheritedRisk], ['lost words and a postscript', `written for >>>${ROOM}`, 'explicit-send', undefined]); const count = h.drafts().length; - await h.turn([createMockResponse([text(`>>>${ROOM} {{unsent}}`), explicitSend('s2')], 'tool_use'), createMockResponse([])]); + await h.turn(rounds([text(`>>>${ROOM} {{unsent}}`), explicitSend('s2')])); assert.equal(h.drafts().length, count, 'a bare {{unsent}} mints no copy'); assert.equal(h.texts().filter((t) => t.startsWith('[delivered]')).at(-1), `[delivered] nothing — 1 plain-speech segment(s) held as draft ${bounce!.id} (not sent — drafts can resend them unchanged, or dismiss them)`); @@ -976,12 +982,12 @@ describe('held drafts: exact runs, hybrid envelopes beside a send, and turn-end await h.turn([createMockResponse([text(`>>>${ROOM} {{unsent}}`)])]); assert.equal(draftState(h.store().get('scout', bounce!.id)!), 'unconfirmed'); h.command({ op: 'publish-mode', mode: 'delivered' }); - await h.turn([createMockResponse([text(`>>>${ROOM} {{unsent}} again`), explicitSend('s3')], 'tool_use'), createMockResponse([])]); + await h.turn(rounds([text(`>>>${ROOM} {{unsent}} again`), explicitSend('s3')])); const risky = h.drafts().find((d) => d.text === 'lost words again')!; assert.equal(risky.inheritedRisk?.draftId, bounce!.id); assert.equal(draftState(risky), 'unconfirmed'); - await h.turn([createMockResponse([text('>>>skip_reply {{unsent}}'), explicitSend('s4')], 'tool_use'), createMockResponse([])]); + await h.turn(rounds([text('>>>skip_reply {{unsent}}'), explicitSend('s4')])); assert.equal(draftState(h.store().get('scout', bounce!.id)!), 'dismissed', 'a send withholds speech, not the dismissal'); assert.ok(h.drafts().every((d) => !d.text.includes('{{unsent}}')), 'no draft holds the literal token'); assert.deepEqual(h.publishes().map((p) => p.text).filter((t) => t !== 'an explicit note elsewhere'), ['lost words'], @@ -1015,13 +1021,98 @@ describe('held drafts: exact runs, hybrid envelopes beside a send, and turn-end createMockResponse([]), ]); assert.match(h.texts().filter((t) => t.startsWith('[delivered]')).at(-1)!, new RegExp( - `^\\[delivered\\] nothing — draft ${ids[3]} is unconfirmed: ${ids[3]}'s attempt to #room \\(Guild One\\) \\(${ROOM}\\) at .+ ` + + `^\\[delivered\\] nothing confirmed — draft ${ids[3]} is unconfirmed: ${ids[3]}'s attempt to #room \\(Guild One\\) \\(${ROOM}\\) at .+ ` + 'may already have been posted: .+ — check that channel before sending it again \\(resend needs confirmDuplicate: true\\)$')); } finally { await h.close(); } }); + it('hybrid: a held round starts from the state earlier rounds leave, even while an earlier delivery is still in flight', async () => { + const h = await harness({ agents: [{ name: 'scout', proseRouting: 'hybrid' }] }); + let release: (() => void) | undefined; + try { + // Round 1's delivery blocks; its `>>>skip_reply` waits behind it on the + // speech chain. Round 2's held prose must not run ahead of it. + const registry = (h.framework as unknown as { channelRegistry: { deliverSpeech: (...args: unknown[]) => Promise } }).channelRegistry; + const deliver = registry.deliverSpeech.bind(registry); + let entered = false; + registry.deliverSpeech = async (...args: unknown[]) => { + if (args[1] === 'slow prior') { + entered = true; + await new Promise((resolve) => { release = resolve; }); + } + return deliver(...args); + }; + const running = h.turn(rounds( + [text(`>>>${ROOM} slow prior\n>>>skip_reply private aside`), call('l1', 'channel_list', {})], + [text('later private prose'), explicitSend('s1')], + )); + await h.until(() => entered && (h.membrane.lastStream?.receivedToolResults.length ?? 0) >= 2, 'both rounds while the delivery is blocked'); + assert.deepEqual(h.drafts(), [], 'the held round waits for the earlier private transition'); + release!(); + await running; + assert.deepEqual(h.drafts(), [], 'the later prose stayed private'); + assert.deepEqual(h.publishes().map((p) => p.text).sort(), ['an explicit note elsewhere', 'slow prior']); + assert.match(h.texts().filter((t) => t.startsWith('[delivered]')).at(-1)!, /1 plain-speech segment\(s\) kept private/); + } finally { + release?.(); + await h.close(); + } + }); + + it('hybrid: the held-draft notice gives a copied, possibly-posted draft its risk and the confirmation it needs', async () => { + const h = await harness({ agents: [{ name: 'scout', proseRouting: 'hybrid' }] }); + try { + const original = h.store().hold('scout', [{ text: 'possibly sent', source }], 'bounced').held[0]!; + h.store().beginAttempt('scout', original.id, dest(), 'unsent-token', false); // its outcome never recorded + await h.turn(rounds([text(`plain words\n>>>${ROOM} {{unsent}} and a new line`), explicitSend('s1')])); + const free = h.drafts().find((d) => d.text === 'plain words')!; + const copy = h.drafts().find((d) => d.text === 'possibly sent and a new line')!; + assert.equal(draftState(free), 'held'); + assert.equal(draftState(copy), 'unconfirmed'); + const notice = h.messages().find((m) => (m.metadata as { kind?: string; draftIds?: string[] } | undefined)?.kind === 'prose-drafts' + && (m.metadata as { draftIds?: string[] }).draftIds?.includes(copy.id))!; + const words = notice.content.map((b) => (b as { text?: string }).text ?? '').join(''); + assert.ok(words.includes(`drafts(action: "resend", draftIds: ["${free.id}"], destination: "#channel") delivers it unchanged`), words); + assert.ok(words.includes(`${copy.id} includes the words of ${original.id}, and ${original.id}'s attempt to #room (Guild One) (${ROOM})`), words); + assert.ok(words.includes(`so check that channel before resending ${copy.id} (drafts(action: "resend", draftIds: ["${copy.id}"], destination: "#channel", confirmDuplicate: true))`), words); + assert.ok(!words.includes(`draftIds: ["${free.id}", "${copy.id}"], destination`), 'no plain resend offered for the copy'); + } finally { + await h.close(); + } + }); + + it('XML tool mode: runs around a refused attempt are held with their edge whitespace and resent exactly', async () => { + const h = await harness({ xml: true }); + try { + const before = ' before\n'; + const after = '\n after '; + const round = [text(before), explicitSend('s1')]; + await h.turn([ + createMockResponse(round, 'tool_use'), + { + ...createMockResponse([ + ...round, + { type: 'tool_attempt', rawXml: '' } as unknown as ContentBlock, + { type: 'tool_notice', notices: [] } as unknown as ContentBlock, + text(after), + ]), + toolCalls: [], + }, + ]); + const held = h.drafts().reverse(); + assert.deepEqual(held.map((d) => d.text), [before, after]); + await h.turn([ + createMockResponse([drafts('r1', { action: 'resend', draftIds: held.map((d) => d.id), destination: ROOM })], 'tool_use'), + createMockResponse([]), + ]); + assert.deepEqual(h.publishes().slice(-2).map((p) => p.text), [before, after]); + } finally { + await h.close(); + } + }); + it('XML tool mode: the same-round think policy is native-only, so prose beside think and a send is held like any other', async () => { const h = await harness({ xml: true, agents: [{ name: 'scout', sameRoundThinkTextPolicy: 'private' }] }); try { From a14e25a0f62754a33b6845c57f13b65a4be0eeea Mon Sep 17 00:00:00 2001 From: Basil-1369 Date: Wed, 7 Oct 2026 13:14:29 -0400 Subject: [PATCH 0045/1112] fix(drafts): "not sent" only over words not at risk; exact-text claims say what holds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Felix-1368's review of 467a48f (room-240 #46701): - draftsHeldNotice opened "[drafts] name: not sent —" even when its body now says a copied draft's words may already have been posted. A batch holding any such draft now opens with the held drafts alone ("[drafts] name: 2 plain-speech segments held as drafts …"); a batch with nothing at risk keeps the old prefix. Same reason as 467a48f's "nothing confirmed". - Only speech an explicit send suppressed is held from raw runs (e9099e5); bounces and holds with no destination still come from routing, which reads trimmed segments. Draft.text's doc said "byte for byte" and the drafts tool's `read` said "exactly as written" of every draft. The doc now says which is which, and `read` labels the text "exactly as a resend publishes it", which is true of every draft. (e9099e5's message said every silenced hold made Draft.text byte for byte "as its doc and read claim"; that held for send-suppressed drafts only.) Test: the mixed-batch notice test asserts the prefix; the three-segment test's `read` label assertion follows the new wording. Full suite 1384/1384 on this tree (tsc, then node --test --test-force-exit --test-concurrency=4 dist/test/*.test.js; node 24, Petra's feat-prose-drafts node_modules). git diff --check clean. Co-Authored-By: Claude Opus 5.5 --- src/framework.ts | 8 +++++--- src/prose-drafts.ts | 5 ++++- test/prose-drafts.test.ts | 3 ++- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/framework.ts b/src/framework.ts index 433896d59..9263fb408 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -8980,8 +8980,10 @@ export class AgentFramework { `(drafts(action: "resend", draftIds: ["${d.id}"], destination: "#channel", confirmDuplicate: true))`), ]; // Named: residents that share one message slot (#197) share this window. + // "Not sent" only when no copied words may already have been posted. return ( - `[drafts] ${agentName}: not sent — ${many ? `${held.length} plain-speech segments` : 'a plain-speech segment'} ` + + `[drafts] ${agentName}: ${risky.length > 0 ? '' : 'not sent — '}` + + `${many ? `${held.length} plain-speech segments` : 'a plain-speech segment'} ` + `held as draft${many ? 's' : ''} ${ids.join(', ')} — ${AgentFramework.DRAFT_REASON_TEXT[reason]}. ` + `Nothing publishes drafts but your own resend, and they stay until you act: ${resend.join('; ')}; ` + `drafts(action: "dismiss", draftIds: [${quoted(held)}]) sets ${many ? 'them' : 'it'} aside.` @@ -9135,7 +9137,7 @@ export class AgentFramework { private async handleDraftsTool(agentName: string, rawInput: unknown): Promise { const input = (rawInput && typeof rawInput === 'object' && !Array.isArray(rawInput) ? rawInput : {}) as Record; const refuse = (error: string): ToolResult => ({ success: false, error, isError: true }); - // Plain text, not a JSON-quoted string: draft text must read exactly as written. + // Plain text, not a JSON-quoted string: draft text must read exactly as held. const ok = (text: string): ToolResult => ({ success: true, data: [{ type: 'text', text }] }); // null and omission mean the same thing: the field is not in use. const present = (v: unknown): boolean => v !== undefined && v !== null; @@ -9212,7 +9214,7 @@ export class AgentFramework { return ` attempt ${this.draftTime(a.at)} → ${AgentFramework.destinationText(a.destination)}: ${status}` + (a.confirmedDuplicate ? ' (resent knowing it might duplicate)' : ''); }), - `Text (${d.text.length} chars, exactly as written):`, + `Text (${d.text.length} chars, exactly as a resend publishes it):`, d.text, ]; return lines.join('\n'); diff --git a/src/prose-drafts.ts b/src/prose-drafts.ts index 2735fd689..7001db0fc 100644 --- a/src/prose-drafts.ts +++ b/src/prose-drafts.ts @@ -105,7 +105,10 @@ export interface InheritedRisk { export interface Draft { id: string; agent: string; - /** The held words, byte for byte. */ + /** The held words, byte for byte as a resend publishes them: a run an + * explicit send suppressed exactly as written; a bounce or a hold with no + * destination, its envelope as routing read it (surrounding whitespace + * trimmed). */ text: string; reason: DraftReason; /** Words copied in from a draft that may already have been posted. */ diff --git a/test/prose-drafts.test.ts b/test/prose-drafts.test.ts index a25e60664..bfea62ed2 100644 --- a/test/prose-drafts.test.ts +++ b/test/prose-drafts.test.ts @@ -352,7 +352,7 @@ describe('held prose drafts, end to end', () => { ]); const [list, read, resend] = h.toolResults(); assert.match(list!, /3 open drafts, newest first/); - assert.ok(read!.endsWith(`exactly as written):\n${segments[1]}`), `read shows the text in full: ${read}`); + assert.ok(read!.endsWith(`exactly as a resend publishes it):\n${segments[1]}`), `read shows the text in full: ${read}`); assert.match(resend!, new RegExp(`${ids[0]}: delivered to #room \\(Guild One\\) \\(${ROOM}\\), message posted-2`)); const sent = h.publishes().slice(1); assert.deepEqual(sent.map((p) => [p.channelId, p.text]), segments.map((s) => [ROOM, s]), 'verbatim, in order'); @@ -1074,6 +1074,7 @@ describe('held drafts: exact runs, hybrid envelopes beside a send, and turn-end const notice = h.messages().find((m) => (m.metadata as { kind?: string; draftIds?: string[] } | undefined)?.kind === 'prose-drafts' && (m.metadata as { draftIds?: string[] }).draftIds?.includes(copy.id))!; const words = notice.content.map((b) => (b as { text?: string }).text ?? '').join(''); + assert.ok(words.startsWith(`[drafts] scout: 2 plain-speech segments held as drafts ${free.id}, ${copy.id} — `), `no "not sent" over words that may be out: ${words}`); assert.ok(words.includes(`drafts(action: "resend", draftIds: ["${free.id}"], destination: "#channel") delivers it unchanged`), words); assert.ok(words.includes(`${copy.id} includes the words of ${original.id}, and ${original.id}'s attempt to #room (Guild One) (${ROOM})`), words); assert.ok(words.includes(`so check that channel before resending ${copy.id} (drafts(action: "resend", draftIds: ["${copy.id}"], destination: "#channel", confirmDuplicate: true))`), words); From 6c927dee34d3f913b20ab7592e87b93c480bcf8f Mon Sep 17 00:00:00 2001 From: Petra-1012 Date: Wed, 7 Oct 2026 15:12:41 -0400 Subject: [PATCH 0046/1112] feat(mcpl): stamp each inbound item's delivered-body and stored-copy digests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The ingestion stamp the delivery-receipt lane and the source-header work share (room-220 #46752–#47210; isolated as its own foundation at Tessa-974's request, #47586, so receipts need not wait on outgoing routing). mcpl/inbound-source.ts gains the one implementation: canonicalJson (keys sorted at every level, undefined dropped) and sourceBodyDigest(blocks) = SHA-256 hex of canonicalJson([blocks]) — the one-element framing an undecorated, unsharded stored copy has always hashed to, so stamped and older copies of the same body share a version (Hugo-1004 #46835). Both lanes stamp, outside the frozen admission envelope (they describe the delivery — a materialization, a correction — not the admission): - metadata.sourceBodyDigest: the body as delivered, after MCPL conversion and before any host decoration (on the push lane, the closed-channel invitation) or storage sharding; - metadata.storedBodyDigest: the same function over exactly the blocks handed to addMessage, decorations included (Oscar-1005 #47210). A copy that no longer hashes to it was changed after delivery — CM's editMessage keeps metadata — so a consumer can tell an intact copy from an edited one. A silent heartbeat stores nothing and gets neither field. Neither proves later presence or completeness. Tests (inbound-source.test.ts): a channels/incoming item's two digests agree and equal the legacy unsharded hash, sit outside the envelope, and an edit through the context manager leaves the stored digest unmatched; a DM push into a closed channel stores the invitation, so its stored digest covers it while the delivered digest is the adapter body alone. Co-Authored-By: Claude Opus 5.5 --- changelog.d/ingestion-body-digests.added.md | 13 +++++++ src/framework.ts | 14 +++++++- src/mcpl/inbound-source.ts | 39 +++++++++++++++++++++ test/inbound-source.test.ts | 38 ++++++++++++++++++++ 4 files changed, 103 insertions(+), 1 deletion(-) create mode 100644 changelog.d/ingestion-body-digests.added.md diff --git a/changelog.d/ingestion-body-digests.added.md b/changelog.d/ingestion-body-digests.added.md new file mode 100644 index 000000000..287bc8f7e --- /dev/null +++ b/changelog.d/ingestion-body-digests.added.md @@ -0,0 +1,13 @@ +- **Delivered-body digests on every inbound item.** Both MCPL lanes stamp two + fields at ingestion, outside the frozen source envelope: + - `metadata.sourceBodyDigest` is the version identity of the body as + delivered, taken after MCPL conversion and before any host decoration or + storage sharding. It is SHA-256 of `canonicalJson([blocks])`, the framing + an undecorated, unsharded stored copy already hashes to. + - `metadata.storedBodyDigest` uses the same function over exactly the + blocks handed to storage, decorations included. + + A copy that no longer hashes to its `storedBodyDigest` was changed after it + arrived (`editMessage` keeps metadata). `canonicalJson` and + `sourceBodyDigest` are exported from `mcpl/inbound-source.ts` as the one + implementation consumers share. diff --git a/src/framework.ts b/src/framework.ts index 9263fb408..395c224d8 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -84,7 +84,7 @@ import { InferenceRouter } from './mcpl/inference-router.js'; import { ChannelRegistry, type ChannelToolOrigin, type PublishDestination, type PublishOutcome } from './mcpl/channel-registry.js'; import { ProseDraftStore, draftState, uncertainAttempt, type Draft, type DraftReason, type DraftSource, type DraftState, type InheritedRisk } from './prose-drafts.js'; import { ConversationRouter, CONVERSATION_ROUTING_DEPRECATION_NOTICE } from './mcpl/conversation-router.js'; -import { INBOUND_SOURCE_KEY, type InboundAcceptanceObserver, type InboundChannelSource, type InboundLane, type InboundSource, type InboundUnscopedSource } from './mcpl/inbound-source.js'; +import { INBOUND_SOURCE_KEY, sourceBodyDigest, type InboundAcceptanceObserver, type InboundChannelSource, type InboundLane, type InboundSource, type InboundUnscopedSource } from './mcpl/inbound-source.js'; import { safeSlice } from './safe-slice.js'; import type { WorkspaceModule } from './modules/workspace/index.js'; import { @@ -7214,6 +7214,12 @@ export class AgentFramework { this.noteInboundAccepted(source); } metadata[INBOUND_SOURCE_KEY] = source; + // The delivered body's version identity, and the stored copy's own + // digest (mcpl/inbound-source.ts sourceBodyDigest; room-220 + // #46752–#47210). This lane stores the body as delivered, so the two + // agree until a decoration or a later edit sets them apart. + metadata.sourceBodyDigest = sourceBodyDigest(event.content); + metadata.storedBodyDigest = sourceBodyDigest(event.content); // Per-channel conversation routing: messages go to the channel's fork // agent (spawned from the template on first qualifying message), never @@ -8381,6 +8387,9 @@ export class AgentFramework { metadata[INBOUND_SOURCE_KEY] = source; const content = [...event.content]; + // The delivered body's version identity, before any host decoration + // (the closed-channel invitation below). + if (content.length > 0) metadata.sourceBodyDigest = sourceBodyDigest(content); if (triggerChannel) { const origin = (event.origin ?? {}) as Record; const invitation = this.buildClosedChannelInvitation({ @@ -8411,6 +8420,9 @@ export class AgentFramework { event.origin?.source === 'heartbeat' && event.origin?.silent === true && content.length === 0; + // The stored copy's own digest, over exactly what is stored (decorations + // included, before storage shards it). A silent heartbeat stores nothing. + if (!silentHeartbeat) metadata.storedBodyDigest = sourceBodyDigest(content); const placement: CoalescingPlacement = { agent: '' }; if (!silentHeartbeat) { diff --git a/src/mcpl/inbound-source.ts b/src/mcpl/inbound-source.ts index 29c2e65e0..0812c45f4 100644 --- a/src/mcpl/inbound-source.ts +++ b/src/mcpl/inbound-source.ts @@ -21,6 +21,8 @@ * key: `eventId` is present only when the producer supplied one. */ +import { createHash } from 'node:crypto'; + /** The MCPL admission lane that accepted an item. Its contract decides what * an `eventId` is worth: `push/event` deduplicates by eventId, and RFC-006 * coalesced admission (`coalesced: true`) guarantees stable retries and @@ -152,3 +154,40 @@ export function conversationKey(source: InboundSource): string | undefined { if (source.kind === 'surface') return `surface\u0000${source.surface}`; return undefined; } + +/** + * JSON with object keys sorted at every level and `undefined` values dropped, + * so a value hashes alike however its keys were ordered (content read back + * from the store has its keys in a different order than it was written). + */ +export function canonicalJson(value: unknown): string { + if (Array.isArray(value)) return `[${value.map(canonicalJson).join(',')}]`; + if (value && typeof value === 'object') { + const entries = Object.entries(value as Record) + .filter(([, v]) => v !== undefined) + .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)); + return `{${entries.map(([k, v]) => `${JSON.stringify(k)}:${canonicalJson(v)}`).join(',')}}`; + } + return JSON.stringify(value) ?? 'null'; +} + +/** + * The version identity of a delivered body (agreed with the receipts lane, + * room-220 #46752–#47210): SHA-256 hex of `canonicalJson([blocks])` over the + * ContentBlock[] the framework stores for the body — after MCPL conversion, + * before any host decoration is added and before storage shards it. The + * one-element array keeps the framing an undecorated, unsharded stored copy + * has always hashed to, so a stamped copy and an older copy of the same body + * share a version. + * + * Ingestion stamps it as `metadata.sourceBodyDigest`, and the same function + * over exactly the blocks handed to storage (decorations included) as + * `metadata.storedBodyDigest`, on both MCPL lanes. Both live outside the + * frozen admission envelope: they describe the body actually delivered (a + * materialization, a correction), not the admission. Neither proves later + * presence or completeness; a copy that no longer hashes to its + * storedBodyDigest was changed after delivery (editMessage keeps metadata). + */ +export function sourceBodyDigest(blocks: readonly unknown[]): string { + return createHash('sha256').update(canonicalJson([blocks])).digest('hex'); +} diff --git a/test/inbound-source.test.ts b/test/inbound-source.test.ts index 5516268dc..de9efada8 100644 --- a/test/inbound-source.test.ts +++ b/test/inbound-source.test.ts @@ -27,6 +27,8 @@ import { type ToolResult, } from '../src/index.js'; import { MockMembrane } from './helpers/mock-membrane.js'; +import { sourceBodyDigest, canonicalJson } from '../src/mcpl/inbound-source.js'; +import { createHash } from 'node:crypto'; import { fixture, eventually, TS } from './helpers/coalescing-fixture.js'; const FIXTURE = join(import.meta.dirname, 'fixtures/speech-route-mcpl-server.mjs'); @@ -160,6 +162,42 @@ describe('inbound source envelope', () => { assert.equal(observed.length, 1); }); + it('stamps the delivered body\'s digest and the stored copy\'s digest at ingestion', async () => { + command({ op: 'incoming', channelId: ROOM, messageId: 'm-d1', mode: 'ambient', text: 'digest me' }); + await waitFor(() => !!storedWith((m) => m.messageId === 'm-d1'), 'message stored'); + const message = storedWith((m) => m.messageId === 'm-d1')!; + const meta = message.metadata as Record; + // The unsharded framing receipts already hash: canonicalJson([blocks]). + const expected = createHash('sha256').update(canonicalJson([[{ type: 'text', text: 'digest me' }]])).digest('hex'); + assert.equal(meta.sourceBodyDigest, expected); + assert.equal(sourceBodyDigest([{ text: 'digest me', type: 'text' }]), expected, 'key order does not matter'); + // This lane stores the body as delivered: the stored copy hashes the same. + assert.equal(meta.storedBodyDigest, sourceBodyDigest(message.content)); + assert.equal(meta.storedBodyDigest, meta.sourceBodyDigest); + // Outside the frozen admission envelope. + assert.equal((readInboundSource(message.metadata) as unknown as Record).sourceBodyDigest, undefined); + + // An edit through the supported path keeps the metadata, so only the + // stored digest can reveal that the copy no longer holds the delivery. + const cm = framework.getAgent('scout')!.getContextManager(); + cm.editMessage(message.id, [{ type: 'text', text: 'edited later' }]); + const edited = cm.getAllMessages().find((m) => m.id === message.id)!; + assert.equal((edited.metadata as Record).storedBodyDigest, meta.storedBodyDigest); + assert.notEqual(sourceBodyDigest(edited.content), meta.storedBodyDigest); + }); + + it('on the push lane, the delivered digest excludes host decoration and the stored digest covers it', async () => { + command({ op: 'dm', eventId: 'ev-dig', authorId: '134', authorName: 'antra', rawChannelId: RAW_DM, text: 'psst' }); + await waitFor(() => !!storedWith((m) => m.eventId === 'ev-dig'), 'dm stored'); + const message = storedWith((m) => m.eventId === 'ev-dig')!; + const meta = message.metadata as Record; + assert.equal(meta.sourceBodyDigest, sourceBodyDigest([{ type: 'text', text: 'psst' }]), 'the adapter body alone'); + assert.equal(meta.storedBodyDigest, sourceBodyDigest(message.content), 'exactly what was stored'); + // The DM's channel is closed, so the host appends its invitation. + assert.equal(message.content.length, 2); + assert.notEqual(meta.storedBodyDigest, meta.sourceBodyDigest, 'the closed-channel invitation decorates the stored copy'); + }); + it('keeps a stored envelope when the channel is renamed later', async () => { command({ op: 'incoming', channelId: ROOM, messageId: 'm-2', mode: 'ambient', text: 'before rename' }); await waitFor(() => !!storedWith((m) => m.messageId === 'm-2'), 'first message stored'); From 711b400a9255d1edc90d23734807a9a7f2330bdf Mon Sep 17 00:00:00 2001 From: Petra-1012 Date: Wed, 7 Oct 2026 16:28:02 -0400 Subject: [PATCH 0047/1112] fix(mcpl): take the stored-copy digest at each storage site, over the blocks stored there MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tessa-974's recheck (room-220 #48282): on channels/incoming, storedBodyDigest was taken at ingestion, but the main storage path can still append the closed-channel invitation afterwards — when a message accepted while its channel was open is processed after the channel closed. The untouched stored copy (header + body + invitation) then failed its own witness, which the receipts lane would read as an edit. The stored digest is now taken where each path stores, over exactly the blocks it hands to storage, through one helper (withStoredDigest): - channels/incoming, main path: after the invitation is appended; - channels/incoming, an RFC-006 correction delivered to a conversation fork; - channels/incoming, conversation-router delivery; - push/event: at its addMessage (a silent heartbeat stores nothing). sourceBodyDigest stays at ingestion, from the undecorated delivered body. Deferred storage keeps the content and metadata it was given, so a deferred copy's witness still matches. Test (inbound-source.test.ts), reproducing Tessa's discriminator: an incoming message is accepted and held while its channel is open, the channel is closed, then the message is processed — it is stored with the invitation and its storedBodyDigest equals the hash of its stored blocks, while its sourceBodyDigest is the delivered body alone. Fails before this change. Co-Authored-By: Claude Opus 5.5 --- src/framework.ts | 37 +++++++++++++++++++++++++------------ test/inbound-source.test.ts | 25 +++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 12 deletions(-) diff --git a/src/framework.ts b/src/framework.ts index 395c224d8..20acf6a1d 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -7214,12 +7214,11 @@ export class AgentFramework { this.noteInboundAccepted(source); } metadata[INBOUND_SOURCE_KEY] = source; - // The delivered body's version identity, and the stored copy's own - // digest (mcpl/inbound-source.ts sourceBodyDigest; room-220 - // #46752–#47210). This lane stores the body as delivered, so the two - // agree until a decoration or a later edit sets them apart. + // The delivered body's version identity (mcpl/inbound-source.ts + // sourceBodyDigest; room-220 #46752–#47210), from the body as delivered, + // before any host decoration. The stored copy's own digest is taken at + // each storage site below, over exactly the blocks stored there. metadata.sourceBodyDigest = sourceBodyDigest(event.content); - metadata.storedBodyDigest = sourceBodyDigest(event.content); // Per-channel conversation routing: messages go to the channel's fork // agent (spawned from the template on first qualifying message), never @@ -7232,7 +7231,7 @@ export class AgentFramework { if (!target) return undefined; if (this.conversationRouter && this.conversationAgentHomes.has(target.name)) { metadata.triggered = event.triggerInference ?? false; - const id = target.getContextManager().addMessage('user', event.content, metadata); + const id = target.getContextManager().addMessage('user', event.content, AgentFramework.withStoredDigest(metadata, event.content)); // A correction may target an older engagement. Refresh only the // binding whose agent actually received the message. if (this.conversationRouter.getBinding(event.channelId)?.agentName === target.name) { @@ -7301,7 +7300,7 @@ export class AgentFramework { } const placement: CoalescingPlacement = { agent: '' }; - const id = this.addMessage('user', incomingContent, metadata, { placement, bypassDeferralFor: event.assemblingFor }); + const id = this.addMessage('user', incomingContent, AgentFramework.withStoredDigest(metadata, incomingContent), { placement, bypassDeferralFor: event.assemblingFor }); this.emitTrace({ type: 'message:added', messageId: id, source: 'mcpl:channel-incoming' }); if (event.triggerInference && !divert) { @@ -7416,7 +7415,7 @@ export class AgentFramework { const trigger = decision.trigger && event.triggerInference !== false; messageMetadata.triggered = trigger; - const id = agent.getContextManager().addMessage('user', event.content, messageMetadata); + const id = agent.getContextManager().addMessage('user', event.content, AgentFramework.withStoredDigest(messageMetadata, event.content)); router.touch(event.channelId); this.emitTrace({ type: 'message:added', messageId: id, source: 'mcpl:channel-incoming' }); @@ -7793,6 +7792,23 @@ export class AgentFramework { }; } + /** + * An inbound item's metadata with its stored copy's own digest + * (`storedBodyDigest`): the delivered-body digest's function over exactly + * the blocks handed to storage here — every decoration included, before + * storage shards them. Taken at each storage site, never earlier, because + * a path can still decorate the body after ingestion stamped it (the + * closed-channel invitation; room-220 #48282). An edit through + * editMessage keeps metadata but not this hash. + */ + private static withStoredDigest( + metadata: Record, + content: readonly ContentBlock[], + ): Record { + metadata.storedBodyDigest = sourceBodyDigest(content); + return metadata; + } + /** Source envelope of a channels/incoming event (lane facts as given). */ private channelEventSource( event: { @@ -8420,13 +8436,10 @@ export class AgentFramework { event.origin?.source === 'heartbeat' && event.origin?.silent === true && content.length === 0; - // The stored copy's own digest, over exactly what is stored (decorations - // included, before storage shards it). A silent heartbeat stores nothing. - if (!silentHeartbeat) metadata.storedBodyDigest = sourceBodyDigest(content); const placement: CoalescingPlacement = { agent: '' }; if (!silentHeartbeat) { - const id = this.addMessage('user', content, metadata, { placement, bypassDeferralFor: event.assemblingFor }); + const id = this.addMessage('user', content, AgentFramework.withStoredDigest(metadata, content), { placement, bypassDeferralFor: event.assemblingFor }); this.emitTrace({ type: 'message:added', messageId: id, source: 'mcpl:push-event' }); } else { console.error(`[heartbeat] ${event.serverId}: accepted silent scheduled wake ${event.eventId}`); diff --git a/test/inbound-source.test.ts b/test/inbound-source.test.ts index de9efada8..31e37ab95 100644 --- a/test/inbound-source.test.ts +++ b/test/inbound-source.test.ts @@ -186,6 +186,31 @@ describe('inbound source envelope', () => { assert.notEqual(sourceBodyDigest(edited.content), meta.storedBodyDigest); }); + it('a message accepted while its channel was open but stored after it closed: the stored digest covers the invitation', async () => { + // Tessa-974's discriminator (room-220 #48282): the invitation is + // appended after ingestion stamped the delivered digest, at storage + // time, when the channel has closed in between. + const registry = (framework as unknown as { + channelRegistry: { handleChannelToolCall(name: string, input: unknown, caller: unknown): Promise; isChannelOpen(id: string): boolean }; + }).channelRegistry; + const fw = framework as unknown as { pushEvent(e: { type: string }): void }; + const held: Array<{ type: string }> = []; + const push = fw.pushEvent.bind(framework); + fw.pushEvent = (e) => { if (e.type === 'mcpl:channel-incoming') held.push(e); else push(e); }; + command({ op: 'incoming', channelId: ROOM, messageId: 'm-closed', mode: 'addressed', text: 'hello while open' }); + await waitFor(() => held.length > 0, 'incoming accepted and held'); + await registry.handleChannelToolCall('channel_close', { channelId: ROOM }, { kind: 'agent', agentName: 'scout' }); + assert.equal(registry.isChannelOpen(ROOM), false); + fw.pushEvent = push; + for (const e of held) push(e); + await waitFor(() => !!storedWith((m) => m.messageId === 'm-closed'), 'message stored'); + const message = storedWith((m) => m.messageId === 'm-closed')!; + const meta = message.metadata as Record; + assert.equal(meta.channelInvitation, true, 'stored with the closed-channel invitation'); + assert.equal(meta.storedBodyDigest, sourceBodyDigest(message.content), 'the untouched copy matches its own witness'); + assert.equal(meta.sourceBodyDigest, sourceBodyDigest([{ type: 'text', text: 'hello while open' }]), 'the delivered body alone'); + }); + it('on the push lane, the delivered digest excludes host decoration and the stored digest covers it', async () => { command({ op: 'dm', eventId: 'ev-dig', authorId: '134', authorName: 'antra', rawChannelId: RAW_DM, text: 'psst' }); await waitFor(() => !!storedWith((m) => m.eventId === 'ev-dig'), 'dm stored'); From 5b34745c8b811e9480a9210faeeb27761e146b69 Mon Sep 17 00:00:00 2001 From: Ansel-1606 Date: Wed, 7 Oct 2026 16:28:11 -0400 Subject: [PATCH 0048/1112] fix(awareness): a hide or turn undo records its batch after the change is synced, and reports what the journal holds Greptile review of #250 (P1 "Marks return after failed scheduling", P1 "Hide marks can outlive removal"); both reproduced at dc2928c. scheduleAppliedMarks prepared its batch after the change was already applied (an in-place hide's redaction, a turn undo's switch), with only the after-append durability sync. A kill between the batch append and that sync could keep the batch (appends reach the OS at once) and lose the buffered removal; a hide keeps its branch, so startup activated the batch and marked messages the agent still sees. And when prepare() threw after its record reached the store (a failed post-append sync), the receipt said `not-scheduled` ("nothing was recorded") while the batch sat prepared for the current branch, and startup delivered it. The outbox now owns that protocol: settleApplied() builds the batch for exactly the change's branch, appends it with afterCommittedState (sync before append, the barrier activate/discard already use), and settles it like any prepared batch (activate, else retire, else unresolved). If the write fails, it reads the journal back: a batch that is not there is `not-scheduled`; one that is there is settled and the receipt follows that; if the read-back fails too, `unresolved` with the batch id. A journal that can't be read at all writes nothing (`not-scheduled`). The framework maps every settlement onto the receipt and ops alert through one path (settledMarkerReceipt) shared with rollback/suppress; its wording no longer says "batch retired" for a batch never recorded. The receipt docs (operator-log.ts, README) and describeMarkers say what not-scheduled and unresolved now cover. DiscordAwarenessPrepareInput and DiscordAwarenessSettlement are exported. Tests (test/discord-awareness-outbox.test.ts), through host/command hide over a store whose journal seams can be observed and faulted: - the removal is synced before the batch is appended; - a failed barrier sync before the append: not-scheduled, nothing recorded, startup activates nothing; - a failed sync right after the append: the batch is settled (queued) and the receipt names it; - every sync from the append on fails: unresolved, the receipt names the retained prepared batch, and startup may activate it, as it says; - the read-back after the failed write fails: unresolved, with the id. All five fail at dc2928c on their status/ordering assertions. --- README.md | 8 +- changelog.d/surgery-marks-one-shot.changed.md | 10 +- src/framework.ts | 77 +++++--- src/index.ts | 2 + src/operator-log.ts | 9 +- src/recovery/discord-awareness-outbox.ts | 145 +++++++++++---- test/discord-awareness-outbox.test.ts | 171 ++++++++++++++++++ 7 files changed, 346 insertions(+), 76 deletions(-) diff --git a/README.md b/README.md index 2c6c1cd09..68b6548bb 100644 --- a/README.md +++ b/README.md @@ -349,8 +349,9 @@ const result = await framework.rollbackToMessage('cairn', { accepted them. `result.markers` says whether they were scheduled (`queued` with a count and batch id), not chosen or not in scope (`none`), not to be delivered because recording the batch failed after the body change and it - was retired (`not-scheduled`), or `unresolved` (neither activation nor - retirement could be recorded; it names a batch that may still be + was retired or never recorded (`not-scheduled`), or `unresolved` (neither + activation nor retirement could be recorded, or the journal could not be + read back to say whether the batch was; it names a batch that may still be delivered). Every receipt also counts the removed messages left unmarked. Marker bookkeeping never fails or undoes an applied rollback or suppression. @@ -373,7 +374,8 @@ const result = await framework.rollbackToMessage('cairn', { surgery's adds carry the moment of its marks choice, even when activation comes later, so a retract made in between prevails; a release is a new act. A record that certifies a body change (an activation, a completed - suppression, a retirement) is written only after that change is synced. + suppression, a retirement, or the batch a hide or turn undo records after + its change) is written only after that change is synced. A batch whose surgery was interrupted before its branch switch was recorded is held at startup until an operator releases it; an interrupted suppression's redactions are resumed whenever its branch is active at diff --git a/changelog.d/surgery-marks-one-shot.changed.md b/changelog.d/surgery-marks-one-shot.changed.md index 942aedaea..44b4a2f37 100644 --- a/changelog.d/surgery-marks-one-shot.changed.md +++ b/changelog.d/surgery-marks-one-shot.changed.md @@ -11,9 +11,13 @@ redactions are resumed whenever its branch is active at startup, independent of what happened to its marks. - A journal record that certifies a body change (a marks activation, a - completed suppression, a retired batch) is now written only after that - change is synced to the store. Previously a crash could keep marks active - for a rollback the store had lost. + completed suppression, a retired batch, or the batch a `hide` or turn + `undo` records after its change) is now written only after that change is + synced to the store. Previously a crash could keep marks active for a + rollback the store had lost. If writing a `hide` or turn `undo` batch + fails, its receipt follows what the journal then holds: a batch that + reached the store is activated, retired or reported `unresolved`, never + reported `not-scheduled` while it could still be delivered. - Awareness delivery no longer holds MCPL traffic. Previously every MCPL data plane waited at startup, at reconnect, at a tools list change and after each surgery until every queued mark had been attempted, which at Discord's pace diff --git a/src/framework.ts b/src/framework.ts index 19dfd9840..04a399fe4 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -416,6 +416,7 @@ import { type DiscordAwarenessReleaseReceipt, type DiscordAwarenessRetractReceipt, type DiscordAwarenessScope, + type DiscordAwarenessSettlement, } from './recovery/discord-awareness-outbox.js'; import { OperatorActionError, @@ -737,7 +738,7 @@ function describeMarksChoice(marks: DiscordAwarenessMarks | undefined): Record ({ ...interval })) } - : {}), - }; + prepare(input: DiscordAwarenessPrepareInput): DiscordAwarenessBatch | null { + const batch = batchRecord(input); + if (!batch) return null; // Durable before the surgery's switch, so startup can crash-complete it. this.append([{ t: 'batch', at: batch.createdAt, batch }], { durable: true }); return { ...structuredClone(batch), status: 'prepared' }; } + /** + * Marks for a body change already applied on `branch` (an in-place hide, a + * turn undo): a batch for exactly that branch, recorded and activated in + * one call, with what the journal will do (as settleActivation reports + * it). Its record asserts the change before it, so it is written only + * after that change is synced: a crash can never keep the batch and lose + * the change. A batch left prepared by a crash between its two records is + * for exactly this branch, which startup completes. If writing it fails, + * the journal is read back, so the result follows what it holds: a batch + * that is not there is `not-scheduled` (nothing was recorded); one that is + * there is settled like any prepared batch (activated, else retired, else + * `unresolved`); and if the journal can't be read back, `unresolved`, since + * the batch may be there. Returns null when there is nothing to mark. + * Never throws. + */ + settleApplied( + input: Omit + & { branch: string }, + ): { batchId: string; marks: number; settled: DiscordAwarenessSettlement } | null { + const { branch, ...rest } = input; + const batch = batchRecord({ ...rest, sourceBranch: branch, targetBranch: branch, activationPolicy: 'explicit' }); + if (!batch) return null; + const settle = (settled: DiscordAwarenessSettlement) => ({ batchId: batch.id, marks: batch.refs.length, settled }); + try { + this.load(); + } catch (error) { + // Nothing is written to a journal that can't be read. + return settle({ status: 'not-scheduled', error: `the journal could not be read: ${errorText(error)}` }); + } + try { + this.append([{ t: 'batch', at: batch.createdAt, batch }], { afterCommittedState: true, durable: true }); + } catch (error) { + const detail = `recording the batch failed: ${errorText(error)}`; + let landed: boolean; + try { + landed = this.load().batches.has(batch.id); + } catch (readError) { + return settle({ status: 'unresolved', error: `${detail}; reading the journal back also failed: ${errorText(readError)}` }); + } + if (!landed) return settle({ status: 'not-scheduled', error: detail }); + // It reached the store: carry out the choice it records, like any + // prepared batch, and report what that settles. + const settled = this.settleActivation(batch.id); + return settle(settled.status === 'queued' ? settled : { ...settled, error: `${detail}; ${settled.error}` }); + } + return settle(this.settleActivation(batch.id)); + } + /** * Activate a prepared batch after its body change landed: one `add` request * per ref, durable before this returns. Activation completes the surgery's @@ -614,10 +663,7 @@ export class DiscordAwarenessOutbox { * the batch may activate it. * Never throws. */ - settleActivation(batchId: string): - | { status: 'queued'; queued: number } - | { status: 'not-scheduled'; error: string } - | { status: 'unresolved'; error: string } { + settleActivation(batchId: string): DiscordAwarenessSettlement { let detail: string; try { return { status: 'queued', queued: this.activate(batchId) }; @@ -1637,6 +1683,33 @@ function legacyEvidence( }; } +/** A new batch's record, or null when it carries neither marks nor a + * suppression journal. */ +function batchRecord(input: DiscordAwarenessPrepareInput): DiscordAwarenessBatchRecord | null { + const refs = dedupeRefs(input.refs); + if (refs.length === 0 && !input.suppressionIntervals?.length) return null; + return { + id: randomUUID(), + agentName: input.agentName, + sourceBranch: input.sourceBranch, + targetBranch: input.targetBranch, + emoji: input.emoji ?? DEFAULT_DISCORD_AWARENESS_EMOJI, + createdAt: Date.now(), + scope: input.scope, + refs, + ...(input.unmarked ? { unmarked: input.unmarked } : {}), + ...(input.notRemoved ? { notRemoved: input.notRemoved } : {}), + activationPolicy: input.activationPolicy ?? 'target-branch', + ...(input.suppressionIntervals?.length + ? { suppressionIntervals: input.suppressionIntervals.map((interval) => ({ ...interval })) } + : {}), + }; +} + +function errorText(error: unknown): string { + return boundDiscordAwarenessText(error instanceof Error ? error.message : String(error)); +} + // --------------------------------------------------------------------------- // Keys // --------------------------------------------------------------------------- diff --git a/test/discord-awareness-outbox.test.ts b/test/discord-awareness-outbox.test.ts index f1a2b5677..34a46265b 100644 --- a/test/discord-awareness-outbox.test.ts +++ b/test/discord-awareness-outbox.test.ts @@ -933,3 +933,174 @@ test('host/command hide takes the store reservation and marks only on request, t h.cleanup(); } }); + +const JOURNAL_APPEND = `append ${DISCORD_AWARENESS_RECORD_TYPE}`; +/** The call that just happened was a journal entry's append. */ +const rightAfterAppend = (calls: string[]) => calls.at(-1) === JOURNAL_APPEND; +/** A journal entry has been appended (since the seams were made). */ +const afterAnAppend = (calls: string[]) => calls.includes(JOURNAL_APPEND); + +/** + * The harness store with its journal seams observed and faultable. `fail` + * makes calls at a seam throw while `matches` holds for the calls before + * them (up to `times` of them): `before` the real call, so nothing reaches + * the store, or `after` it, so it did. + */ +function journalSeams(store: JsStore) { + type Seam = 'sync' | 'append' | 'list'; + const calls: string[] = []; + const faults: Array<{ seam: Seam; when: 'before' | 'after'; matches: (calls: string[]) => boolean; times: number }> = []; + const through = (seam: Seam, label: string, call: () => T): T => { + const fault = faults.find((candidate) => candidate.seam === seam && candidate.times > 0 && candidate.matches(calls)); + calls.push(label); + if (fault) fault.times--; + if (fault?.when === 'before') throw new Error(`injected ${seam} failure`); + const result = call(); + if (fault?.when === 'after') throw new Error(`injected ${seam} failure`); + return result; + }; + const proxy = new Proxy(store, { + get(target, prop) { + if (prop === 'sync') return () => through('sync', 'sync', () => target.sync()); + if (prop === 'appendJson') { + return (type: string, payload: unknown) => through('append', `append ${type}`, () => target.appendJson(type, payload)); + } + if (prop === 'getRecordIdsByType') { + return (type: string) => through('list', `list ${type}`, () => target.getRecordIdsByType(type)); + } + const value = Reflect.get(target, prop); + return typeof value === 'function' ? value.bind(target) : value; + }, + }); + return { + store: proxy as JsStore, + calls, + fail: (seam: Seam, when: 'before' | 'after', matches: (calls: string[]) => boolean, times = 1) => { + faults.push({ seam, when, matches, times }); + }, + }; +} + +/** + * A hide with marks over a journal whose store faults as `arrange` says. + * The removal shows in the calls as `remove`. + */ +async function hideOverFaultyJournal(arrange: (seams: ReturnType) => void) { + const h = storeHarness(); + const seams = journalSeams(h.store); + const outbox = new DiscordAwarenessOutbox(seams.store); + outbox.batches(); // loaded before the hide, as a running framework's is + const { framework, removed } = hostCommandFramework(h, [ + { id: 'i0', metadata: { ...ref('m0') } }, + { id: 'i1', metadata: { ...ref('m1'), tags: ['chat:addressed'] } }, + ]); + framework.discordAwarenessOutbox = outbox; + framework.agents.get('cairn').getContextManager().removeMessage = (id: string) => { + seams.calls.push('remove'); + removed.add(id); + }; + const alerts: string[] = []; + framework.opsAlert = (kind: string) => { alerts.push(kind); }; + arrange(seams); + const result: any = await muted(() => framework.handleHostCommand('discord', { + command: 'hide', agentName: 'cairn', fromMessageId: 'm1', marks: 'all', + })); + return { h, seams, result, alerts }; +} + +test('a hide records its batch only after the removal is synced: the batch can never outlive the change', async () => { + const { h, seams, result } = await hideOverFaultyJournal(() => {}); + try { + assert.equal(result.markers.status, 'queued'); + const removal = seams.calls.indexOf('remove'); + const batchAppend = seams.calls.indexOf(JOURNAL_APPEND, removal); + assert.ok(removal >= 0 && batchAppend > removal, `calls: ${seams.calls.join(', ')}`); + assert.ok( + seams.calls.slice(removal, batchAppend).includes('sync'), + `the removal is synced before the batch is appended: ${seams.calls.join(', ')}`, + ); + } finally { + h.cleanup(); + } +}); + +test('an applied change\'s batch that failed before reaching the store is not scheduled, and nothing delivers it', async () => { + // The barrier sync between the removal and the batch's append fails: + // nothing is written. + const { h, result, alerts } = await hideOverFaultyJournal((seams) => { + seams.fail('sync', 'before', (calls) => calls.includes('remove') && !afterAnAppend(calls)); + }); + try { + assert.equal(result.ok, true, 'the hide stands'); + assert.equal(result.markers.status, 'not-scheduled'); + const fresh = new DiscordAwarenessOutbox(h.store); + assert.equal(fresh.batches().length, 0, 'nothing was recorded'); + assert.deepEqual(fresh.recoverAtStartup('main'), { activated: [], held: [], unknownAttempts: 0 }); + assert.equal(fresh.pendingDispatches('discord').length, 0); + assert.match(result.markers.error, /recording the batch failed: injected sync failure/); + assert.deepEqual(alerts, ['discord-awareness-not-scheduled']); + } finally { + h.cleanup(); + } +}); + +test('an applied change\'s batch that reached the store before its barrier failed is settled, and the receipt follows it', async () => { + // The append lands and its durability sync fails: the batch is in the + // journal, so the operator's choice is carried out (here, activated) + // rather than reported as never scheduled. + const { h, result, alerts } = await hideOverFaultyJournal((seams) => { + seams.fail('sync', 'before', rightAfterAppend); + }); + try { + assert.equal(result.markers.status, 'queued', 'not a not-scheduled receipt over a batch that would be delivered'); + assert.equal(result.markers.queued, 1); + const fresh = new DiscordAwarenessOutbox(h.store); + const [batch] = fresh.batches(); + assert.equal(batch.id, result.markers.batchId); + assert.equal(batch.status, 'active'); + assert.deepEqual(fresh.pendingDispatches('discord').map((d) => d.key.messageId), ['m1']); + assert.deepEqual(fresh.recoverAtStartup('main'), { activated: [], held: [], unknownAttempts: 0 }); + assert.deepEqual(alerts, []); + } finally { + h.cleanup(); + } +}); + +test('an applied change\'s batch that landed but can be neither activated nor retired is unresolved, with its id', async () => { + // The append lands, and every sync from its own on fails: the batch is + // in the journal, and activation and retirement each write nothing. + const { h, result, alerts } = await hideOverFaultyJournal((seams) => { + seams.fail('sync', 'before', afterAnAppend, Number.POSITIVE_INFINITY); + }); + try { + assert.equal(result.markers.status, 'unresolved'); + const fresh = new DiscordAwarenessOutbox(h.store); + const [batch] = fresh.batches(); + assert.equal(batch.id, result.markers.batchId, 'the receipt names the retained batch'); + assert.equal(batch.status, 'prepared'); + // As the receipt says, a startup that reads it may deliver it. + assert.deepEqual(fresh.recoverAtStartup('main').activated, [batch.id]); + assert.match(result.markers.error, /recording the batch failed.*retiring it also failed/); + assert.deepEqual(alerts, ['discord-awareness-unresolved']); + } finally { + h.cleanup(); + } +}); + +test('an applied change\'s batch whose journal cannot be read back after a failed write is unresolved, with its id', async () => { + // The append lands, its durability sync fails, and the read-back fails. + const { h, result, alerts } = await hideOverFaultyJournal((seams) => { + seams.fail('sync', 'before', rightAfterAppend); + seams.fail('list', 'before', afterAnAppend); + }); + try { + assert.equal(result.markers.status, 'unresolved'); + const [batch] = new DiscordAwarenessOutbox(h.store).batches(); + assert.equal(batch.id, result.markers.batchId, 'the receipt names the batch that may be delivered'); + assert.equal(batch.status, 'prepared'); + assert.match(result.markers.error, /reading the journal back also failed: injected list failure/); + assert.deepEqual(alerts, ['discord-awareness-unresolved']); + } finally { + h.cleanup(); + } +}); From 9d17ad67b82b974026f734b2ca687d157d4d06ba Mon Sep 17 00:00:00 2001 From: Ansel-1606 Date: Wed, 7 Oct 2026 16:37:01 -0400 Subject: [PATCH 0049/1112] fix(mcpl): a reconnect holds control traffic as well as data until the new grant is established MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Greptile review of #250 (P1 "First reconnect loses channel registration"), plus the same-root case Mira-1605 identified in review. A reconnect resets the server's grant (a fresh initialize is a fresh §5.3 policy epoch), but McplServerConnection.attemptReconnect paused only the data plane, and #250's reconnect listener then called readyControlPlane() before re-running registerMcplServerFeatures. So control traffic from the fresh transport reached admission against the empty grant: a channels/register sent right after initialize (as chat bridges do) was refused with -32002 ("initial policy not yet established") and never replayed, leaving the route's incoming messages unknown channels until the server registered again. Two paths led there: - a stub whose first connect failed and which reconnects while startup is still staging other servers: its control plane was still closed until the listener's new readyControlPlane() opened it (the regression); - any connection that was already open: its control plane was never closed at the boundary (pre-existing; the old barrier code paused data only, too). attemptReconnect now closes both planes with the grant reset, as initial connect holds them, and the framework's reconnect listener no longer opens the control plane early: it re-establishes the grant (policy responses are never event-buffered, so holding control cannot deadlock the exchange) and then opens the planes through readyMcplPlanes, as it already did, also when the exchange fails. Quiesce still pauses only the data plane (pauseDataPlane's doc now says that is its use). Tests: - test/mcpl-reconnect-registration.test.ts (new, with fixture reregistering-mcpl-server.mjs): the real framework, a server that registers its channel on notifications/initialized and exits once on demand; after an ordinary reconnect the fresh process's registration must be answered without error. At dc2928c it is refused -32002; with only the listener change, or only the connection change, it is still refused; both together pass. - test/mcpl-reconnect-events.test.ts: after a reconnect, a control event is held until ready(), then released (fails at dc2928c). Changelog: mcpl-reconnect-registration.fixed.md. --- .../mcpl-reconnect-registration.fixed.md | 9 ++ src/framework.ts | 13 ++- src/mcpl/server-connection.ts | 18 ++-- test/fixtures/reregistering-mcpl-server.mjs | 68 +++++++++++++ test/mcpl-reconnect-events.test.ts | 21 ++++ test/mcpl-reconnect-registration.test.ts | 96 +++++++++++++++++++ 6 files changed, 211 insertions(+), 14 deletions(-) create mode 100644 changelog.d/mcpl-reconnect-registration.fixed.md create mode 100644 test/fixtures/reregistering-mcpl-server.mjs create mode 100644 test/mcpl-reconnect-registration.test.ts diff --git a/changelog.d/mcpl-reconnect-registration.fixed.md b/changelog.d/mcpl-reconnect-registration.fixed.md new file mode 100644 index 000000000..b52983a18 --- /dev/null +++ b/changelog.d/mcpl-reconnect-registration.fixed.md @@ -0,0 +1,9 @@ +- An MCPL server that reconnects now has its channel registration accepted. + A reconnect resets the server's grant, and its traffic used to reach + admission before the host's new §5.3 policy Request was answered: a + `channels/register` sent right after initialize, as chat bridges do, was + refused (-32002) and never replayed, so the route's incoming messages + stayed unknown channels until the server registered again. A reconnect + now holds control traffic as well as data until the new grant is + established, as initial connect already did, both for a connection that + was open and for one whose first connect had failed. diff --git a/src/framework.ts b/src/framework.ts index 04a399fe4..60d80bbf6 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -14559,13 +14559,12 @@ export class AgentFramework { // preserved across the transient close are resumed by idempotent // registration. Then refresh tools (server may have different tools). connection.on('reconnect', (info?: { attempts?: number }) => { - // The connection paused its data plane before wiring the fresh - // transport. Let control traffic through for registration, then - // re-establish the grant: the §5.3 policy Request must be answered - // before this connection's data plane opens, otherwise the first - // post-reconnect events land while the grant is still empty and are - // rejected fail-closed instead of delivered. - connection.readyControlPlane(); + // The connection closed both planes before wiring the fresh transport, + // as at initial connect. Re-establish the grant first: the §5.3 policy + // Request (whose response is never buffered) must be answered before + // any plane opens, otherwise the first post-reconnect traffic, control + // included (the server's channels/register), lands while the grant is + // still empty and is rejected fail-closed instead of delivered. void (async () => { try { const config = this.mcplServerConfigs.get(connection.id); diff --git a/src/mcpl/server-connection.ts b/src/mcpl/server-connection.ts index 273f7b61a..8dab9ac99 100644 --- a/src/mcpl/server-connection.ts +++ b/src/mcpl/server-connection.ts @@ -354,8 +354,8 @@ export class McplServerConnection extends EventEmitter { /** * Close the data plane synchronously while leaving an already-released - * control plane live. Used before reconnect adoption, so no data event can - * arrive before the server's grant is re-established, and by quiesce. + * control plane live: quiesce holds wakes this way while operator traffic + * (host/command) still flows. A reconnect closes both planes instead. */ pauseDataPlane(): void { this.dataPlaneReady = false; @@ -953,11 +953,15 @@ export class McplServerConnection extends EventEmitter { this.hostCapabilities, ); - // Close the data plane before the new transport can emit. The reconnect - // lifecycle event bypasses buffering; its framework listener releases - // the control traffic needed to re-establish the server's grant while - // data stays held until the grant is in place. - this.pauseDataPlane(); + // Close both planes before the new transport can emit, as at initial + // connect: its traffic is admitted against the grant, which this + // boundary resets, so control traffic too (a server's channels/register + // right after initialize) must wait for the new grant rather than be + // refused against the empty one. The reconnect lifecycle event bypasses + // buffering; its framework listener re-establishes the grant (§5.3 + // policy responses are never buffered) and then opens the planes. + this.controlPlaneReady = false; + this.dataPlaneReady = false; this.resetPolicyForTransportBoundary(); this.transport = transport; this.capabilities = capabilities; diff --git a/test/fixtures/reregistering-mcpl-server.mjs b/test/fixtures/reregistering-mcpl-server.mjs new file mode 100644 index 000000000..c18e3639f --- /dev/null +++ b/test/fixtures/reregistering-mcpl-server.mjs @@ -0,0 +1,68 @@ +// MCPL server for test/mcpl-reconnect-registration.test.ts. +// +// Like a chat bridge, it registers its channel as soon as the host sends +// notifications/initialized, before the host's §5.3 policy Request is +// answered, and appends the host's answer to that registration to +// EVENTS_PATH. When CRASH_PATH appears it removes the file and exits, once, +// so the host's reconnect starts a fresh process that registers again. +import { appendFileSync, existsSync, unlinkSync } from 'node:fs'; + +const eventsPath = process.env.EVENTS_PATH; +const crashPath = process.env.CRASH_PATH; + +const send = (message) => process.stdout.write(JSON.stringify(message) + '\n'); +const event = (name, extra = {}) => { + if (eventsPath) appendFileSync(eventsPath, JSON.stringify({ event: name, pid: process.pid, ...extra }) + '\n'); +}; + +let buf = ''; +process.stdin.on('data', (chunk) => { + buf += chunk.toString('utf8'); + let newline; + while ((newline = buf.indexOf('\n')) >= 0) { + const line = buf.slice(0, newline); + buf = buf.slice(newline + 1); + if (!line.trim()) continue; + const message = JSON.parse(line); + if (message.method === 'initialize') { + send({ + jsonrpc: '2.0', + id: message.id, + result: { + capabilities: { + experimental: { + mcpl: { + version: '0.5', + pushEvents: true, + channels: { register: true, incoming: true }, + featureSets: { chat: { description: 'chat', uses: ['pushEvents'] } }, + }, + }, + }, + }, + }); + } else if (message.method === 'notifications/initialized') { + send({ + jsonrpc: '2.0', + id: 100, + method: 'channels/register', + params: { channels: [{ id: 'chat:room-1', type: 'chat', label: 'room-1', direction: 'bidirectional' }] }, + }); + } else if (message.method === 'featureSets/update') { + if (message.id !== undefined && message.id !== null) send({ jsonrpc: '2.0', id: message.id, result: { accepted: true } }); + } else if (message.method === 'tools/list') { + send({ jsonrpc: '2.0', id: message.id, result: { tools: [] } }); + } else if (message.id === 100 && message.method === undefined) { + event('register-answered', { error: message.error ?? null }); + } + } +}); + +const timer = setInterval(() => { + if (crashPath && existsSync(crashPath)) { + unlinkSync(crashPath); + event('crashing'); + process.exit(9); + } +}, 5); +timer.unref(); diff --git a/test/mcpl-reconnect-events.test.ts b/test/mcpl-reconnect-events.test.ts index 2c12d7407..5a58b2abc 100644 --- a/test/mcpl-reconnect-events.test.ts +++ b/test/mcpl-reconnect-events.test.ts @@ -170,4 +170,25 @@ describe('McplServerConnection — reconnect lifecycle events', () => { grant: [], }); }); + + it('holds a reconnected transport\'s control traffic, not only its data, until the host opens the planes', async () => { + writeFileSync(FLAG_FILE, ''); + connection = await McplServerConnection.connect(makeConfig(), HOST_CAPS); + const controlEvents: string[] = []; + connection.on('tools-list-changed', () => controlEvents.push('tools-list-changed')); + let reconnected = false; + connection.on('reconnect', () => { reconnected = true; }); + connection.ready(); + connection.emit('tools-list-changed'); + assert.deepStrictEqual(controlEvents, ['tools-list-changed'], 'an open control plane delivers'); + + connection.sendToolsList().catch(() => {}); + await waitFor(() => reconnected, 5000, 'reconnect after crash'); + // What the fresh transport sends is admitted against a grant the host + // has not re-established yet: it waits, as at initial connect. + connection.emit('tools-list-changed'); + assert.deepStrictEqual(controlEvents, ['tools-list-changed'], 'control traffic is held after the boundary'); + connection.ready(); + assert.deepStrictEqual(controlEvents, ['tools-list-changed', 'tools-list-changed'], 'and released when the planes open'); + }); }); diff --git a/test/mcpl-reconnect-registration.test.ts b/test/mcpl-reconnect-registration.test.ts new file mode 100644 index 000000000..d194978bf --- /dev/null +++ b/test/mcpl-reconnect-registration.test.ts @@ -0,0 +1,96 @@ +/** + * A reconnected MCPL server's control traffic waits for its new grant. + * + * A transport boundary resets the server's grant: a fresh initialize is a + * fresh §5.3 policy epoch. A chat bridge registers its channels right after + * initialize, before the host's policy Request has been answered. If that + * registration reaches admission while the grant is still empty, it is + * refused (-32002) and never replayed, and the route's incoming messages are + * then unknown channels until the server happens to register again. Initial + * connect holds both planes until the grant is established; a reconnect must + * hold them too, both for a connection that was already open (an ordinary + * disconnect) and for a stub that had never opened (a first reconnect while + * startup is still staging other servers). + */ +import { describe, it, beforeEach, afterEach } from 'node:test'; +import assert from 'node:assert/strict'; +import { fileURLToPath } from 'node:url'; +import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { AgentFramework } from '../src/index.js'; +import { MockMembrane } from './helpers/mock-membrane.js'; + +const FIXTURE = fileURLToPath(new URL('./fixtures/reregistering-mcpl-server.mjs', import.meta.url)); + +interface ServerEvent { + event: string; + pid: number; + error?: { code: number; message: string } | null; +} + +function serverEvents(path: string): ServerEvent[] { + if (!existsSync(path)) return []; + return readFileSync(path, 'utf8').split('\n').filter(Boolean).map((line) => JSON.parse(line) as ServerEvent); +} + +async function waitFor(description: string, predicate: () => boolean, timeoutMs = 10_000): Promise { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + if (predicate()) return; + await new Promise((resolve) => setTimeout(resolve, 10)); + } + assert.fail(`timed out waiting for ${description}`); +} + +describe('MCPL reconnect and channel registration', () => { + let dir: string; + let framework: AgentFramework | undefined; + + beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), 'mcpl-reconnect-registration-')); + }); + + afterEach(async () => { + await framework?.stop(); + framework = undefined; + rmSync(dir, { recursive: true, force: true }); + }); + + it('a server that registers right after initialize is registered again after an ordinary reconnect', async () => { + const eventsPath = join(dir, 'events.jsonl'); + const crashPath = join(dir, 'crash'); + framework = await AgentFramework.create({ + storePath: join(dir, 'store'), + membrane: new MockMembrane().asMembrane(), + agents: [{ name: 'resident', model: 'test-model', systemPrompt: 'You are a resident.' }], + modules: [], + maintenanceIntervalMs: 0, + mcplServers: [{ + id: 'chat', + command: process.execPath, + args: [FIXTURE], + env: { EVENTS_PATH: eventsPath, CRASH_PATH: crashPath }, + enabledFeatureSets: ['chat'], + reconnect: true, + reconnectIntervalMs: 50, + }], + }); + const answers = () => serverEvents(eventsPath).filter((e) => e.event === 'register-answered'); + + await waitFor('the first registration answer', () => answers().length === 1); + assert.equal(answers()[0].error, null, 'initial connect admits the registration against the established grant'); + + // The server process dies; the host reconnects to a fresh one, which + // registers again before the host's new policy Request is answered. + writeFileSync(crashPath, ''); + await waitFor('the registration answer after the reconnect', () => answers().length === 2); + const [first, second] = answers(); + assert.notEqual(second.pid, first.pid, 'a fresh server process registered'); + assert.equal( + second.error, + null, + `the reconnected server's registration waits for its new grant instead of being refused: ${JSON.stringify(second.error)}`, + ); + }); +}); From 25891f07cd0160fc8171168a7afbb21dba657b27 Mon Sep 17 00:00:00 2001 From: Ansel-1606 Date: Wed, 7 Oct 2026 16:44:53 -0400 Subject: [PATCH 0050/1112] fix(recover): the CLI records its awareness controls and recoveries in the operator log Greptile review of #250 (P2 "Offline controls skip the log"), with the extension agreed in room-245 (#48091, #48115): the CLI's own recovery. The live surfaces record every operator mutation in /operator-actions.jsonl (who asked, from where, what it did, or the error), and the changelog promises that each awareness control is recorded. agent-framework-recover --awareness cancel|retract|release called the outbox directly and only printed the receipt, so the operator history lost every offline control. The CLI's recovery, which #250 lets publish marks (--marks), was not recorded either, so nothing said who chose those marks. The CLI now appends, best-effort (OperatorLog.append never throws, and the OS account lookup falls back to {via: 'cli'}): - for cancel/retract/release: kind awareness-, agent '*', params {target} or {batchId} as the live wrapper logs them, and the receipt or the error; - for a recovery: kind 'recovery', the agent, params (anchor, any suppressions, the marks choice as 'none' or {scope}), and the result (branches, messages removed and suppressed, discordRefs, markers), or the error when it is refused or fails. The requester is {via: 'cli', name: }, as the web UI records its basic-auth user. A dry run and a list record nothing. The log path is the host's default under the store. Tests: test/recover-cli-operator-log.test.ts spawns the compiled CLI: list records nothing; cancel and retract record their receipts; a refused release records its error; a dry run records nothing; a recovery records its marks choice and the receipt it printed; a refused recovery (existing branch) records its error. Docs: the CLI help, README's offline recovery section, and the controls changelog fragment. --- README.md | 5 +- changelog.d/surgery-marks-controls.added.md | 5 + src/recovery/recover-cli.ts | 125 +++++++++++++++---- test/recover-cli-operator-log.test.ts | 131 ++++++++++++++++++++ 4 files changed, 243 insertions(+), 23 deletions(-) create mode 100644 test/recover-cli-operator-log.test.ts diff --git a/README.md b/README.md index 68b6548bb..311852747 100644 --- a/README.md +++ b/README.md @@ -173,7 +173,10 @@ awareness journal kept in the store itself, and the marks are delivered when the host and that agent's `discord-mcpl` bot next connect (see [Live surgery and Discord awareness marks](#live-surgery-and-discord-awareness-marks)). If the branch is made but the marks can't be recorded, the output's `markers` -says so and the branch stands. +says so and the branch stands. Like a live surgery, the recovery (with its +marks choice and receipt) is recorded in the store's operator log, +`operator-actions.jsonl`, with the OS account that ran it; so is each +`--awareness` cancel, retract and release. A dry run records nothing. Portal and other non-Discord records are ignored. When the safe point is an assistant/tool-side record rather than a Discord diff --git a/changelog.d/surgery-marks-controls.added.md b/changelog.d/surgery-marks-controls.added.md index 175fb7539..45c6cc924 100644 --- a/changelog.d/surgery-marks-controls.added.md +++ b/changelog.d/surgery-marks-controls.added.md @@ -22,3 +22,8 @@ available as the `host/command` `marks` verb (`action: list | cancel | retract | release`, `target`) and, with the host stopped, as `agent-framework-recover --awareness list|cancel|retract|release`. +- `agent-framework-recover` records its acts in the store's operator log + (`/operator-actions.jsonl`), done or refused, with the OS account + that ran it: each `--awareness` cancel, retract and release, and each + recovery with its marks choice and receipt (kind `recovery`). A dry run or + a list records nothing. diff --git a/src/recovery/recover-cli.ts b/src/recovery/recover-cli.ts index 9191cea2a..9735e3267 100644 --- a/src/recovery/recover-cli.ts +++ b/src/recovery/recover-cli.ts @@ -1,5 +1,6 @@ #!/usr/bin/env node +import { userInfo } from 'node:os'; import { JsStore } from '@animalabs/chronicle'; import { createOfflineRecoveryBranch } from './offline-branch.js'; import { @@ -7,6 +8,12 @@ import { defaultDiscordAwarenessOutboxPath, type DiscordAwarenessMarks, } from './discord-awareness-outbox.js'; +import { + OperatorLog, + defaultOperatorLogPath, + type OperatorLogInput, + type OperatorRequester, +} from '../operator-log.js'; interface CliOptions { storePath?: string; @@ -68,6 +75,9 @@ Awareness journal (host stopped): --awareness release Queue a held batch's recorded operations Queued work is delivered when the host next connects to the Discord server. +Each recovery and each cancel, retract or release (done or refused; not a +dry run or a list) is recorded in /operator-actions.jsonl, the host's +operator log, with the OS account that ran it. The agent host must be stopped before running this command.`; } @@ -138,6 +148,28 @@ function parseArgs(args: string[]): CliOptions { return options; } +/** Who ran the command, as far as the CLI can say: the OS account. */ +function cliRequester(): OperatorRequester { + try { + return { via: 'cli', name: userInfo().username }; + } catch { + return { via: 'cli' }; + } +} + +/** + * Record an act in the store's operator log (`/operator-actions.jsonl`, + * where the host records its own), whether it was done or refused, like the + * live surfaces' acts. Best-effort: a logging failure never fails the act. + */ +function recordOperatorAction(storePath: string, entry: OperatorLogInput): void { + new OperatorLog(defaultOperatorLogPath(storePath)).append(entry); +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + function awareness(options: CliOptions): void { // The journal lives in the store; with the host stopped, this process is // its only writer. @@ -147,14 +179,28 @@ function awareness(options: CliOptions): void { legacyPath: options.outboxPath ?? defaultDiscordAwarenessOutboxPath(options.storePath!), }); const { verb, target } = options.awareness!; + if (verb === 'list') { + console.log(JSON.stringify({ store: options.storePath, batches: outbox.view() }, null, 2)); + return; + } const by = 'agent-framework-recover'; - let result: unknown; - switch (verb) { - case 'list': result = { store: options.storePath, batches: outbox.view() }; break; - case 'cancel': result = outbox.cancel(target!, by); break; - case 'retract': result = outbox.retract(target!, by); break; - case 'release': result = outbox.release(target!, by); break; + // Logged as the live controls log theirs. + const logBase: OperatorLogInput = { + kind: `awareness-${verb}`, + agent: '*', + requester: cliRequester(), + params: verb === 'release' ? { batchId: target } : { target }, + }; + let result: object; + try { + result = verb === 'cancel' ? outbox.cancel(target!, by) + : verb === 'retract' ? outbox.retract(target!, by) + : outbox.release(target!, by); + } catch (error) { + recordOperatorAction(options.storePath!, { ...logBase, error: errorMessage(error) }); + throw error; } + recordOperatorAction(options.storePath!, { ...logBase, result: { ...result } }); console.log(JSON.stringify(result, null, 2)); } finally { store.close(); @@ -181,22 +227,57 @@ async function main(): Promise { // A CLI choice and its application are the same moment (the host is // stopped), so the scope is evaluated now rather than frozen to refs. const marks: DiscordAwarenessMarks = options.marks === 'none' ? 'none' : { scope: options.marks }; - const result = await createOfflineRecoveryBranch({ - storePath: options.storePath, - agentName: options.agentName, - messageId: options.messageId, - contextId: options.contextId, - messages: options.messages, - suppressMessageIds: options.suppressMessageIds, - suppressRanges: options.suppressRanges, - branchName: options.branchName, - namespace: options.namespace, - discordServerId: options.discordServerId, - outboxPath: options.outboxPath, - emoji: options.emoji, - marks, - dryRun: options.dryRun, - }); + // A real recovery is recorded in the operator log, done or failed, with + // its marks choice and receipt, as a live rollback or suppression is. A + // dry run changes nothing and records nothing. + const logBase: OperatorLogInput = { + kind: 'recovery', + agent: options.agentName, + requester: cliRequester(), + params: { + ...(options.messageId !== undefined ? { messageId: options.messageId } : {}), + ...(options.contextId !== undefined ? { contextId: options.contextId } : {}), + ...(options.messages !== undefined ? { messages: options.messages } : {}), + ...(options.suppressMessageIds.length > 0 ? { suppressMessageIds: options.suppressMessageIds } : {}), + ...(options.suppressRanges.length > 0 ? { suppressRanges: options.suppressRanges } : {}), + marks: marks === 'none' ? 'none' : { scope: marks.scope }, + }, + }; + let result: Awaited>; + try { + result = await createOfflineRecoveryBranch({ + storePath: options.storePath, + agentName: options.agentName, + messageId: options.messageId, + contextId: options.contextId, + messages: options.messages, + suppressMessageIds: options.suppressMessageIds, + suppressRanges: options.suppressRanges, + branchName: options.branchName, + namespace: options.namespace, + discordServerId: options.discordServerId, + outboxPath: options.outboxPath, + emoji: options.emoji, + marks, + dryRun: options.dryRun, + }); + } catch (error) { + if (!options.dryRun) recordOperatorAction(options.storePath, { ...logBase, error: errorMessage(error) }); + throw error; + } + if (!result.dryRun) { + recordOperatorAction(options.storePath, { + ...logBase, + result: { + sourceBranch: result.sourceBranch, + targetBranch: result.targetBranch, + messagesRemoved: result.messagesRemoved, + messagesSuppressed: result.messagesSuppressed, + discordRefs: result.discordAddressable, + ...(result.markers ? { markers: result.markers } : {}), + }, + }); + } console.log(JSON.stringify({ ...result, diff --git a/test/recover-cli-operator-log.test.ts b/test/recover-cli-operator-log.test.ts new file mode 100644 index 000000000..1c0202e37 --- /dev/null +++ b/test/recover-cli-operator-log.test.ts @@ -0,0 +1,131 @@ +/** + * `agent-framework-recover` records its acts in the store's operator log. + * + * The live surfaces record every operator mutation (who asked, from where, + * what it did) in `/operator-actions.jsonl`. The CLI acts on the same + * store while the host is stopped: an offline recovery, which can choose to + * publish awareness marks, and the awareness controls (cancel, retract, + * release). Each is recorded the same way, done or refused, so the history + * says who chose to publish or stop marks whichever surface they used. A dry + * run and a list change nothing and record nothing. + */ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { existsSync, mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { JsStore } from '@animalabs/chronicle'; +import { ContextManager } from '@animalabs/context-manager'; +import { DiscordAwarenessOutbox } from '../src/recovery/discord-awareness-outbox.js'; +import type { OperatorLogEntry } from '../src/operator-log.js'; + +const CLI = fileURLToPath(new URL('../src/recovery/recover-cli.js', import.meta.url)); + +function recover(storePath: string, ...args: string[]) { + const run = spawnSync(process.execPath, [CLI, '--store', storePath, ...args], { encoding: 'utf8' }); + return { status: run.status, stdout: run.stdout, stderr: run.stderr }; +} + +function operatorLog(storePath: string): OperatorLogEntry[] { + const path = join(storePath, 'operator-actions.jsonl'); + if (!existsSync(path)) return []; + return readFileSync(path, 'utf8').split('\n').filter(Boolean).map((line) => JSON.parse(line) as OperatorLogEntry); +} + +const ref = (messageId: string) => ({ serverId: 'discord', channelId: 'discord:g1:c1', messageId }); + +test('the offline awareness controls record each act in the operator log, done or refused', () => { + const dir = mkdtempSync(join(tmpdir(), 'recover-cli-log-')); + const storePath = join(dir, 'store'); + try { + const store = JsStore.openOrCreate({ path: storePath }); + const outbox = new DiscordAwarenessOutbox(store); + const batch = outbox.prepare({ + agentName: 'cairn', sourceBranch: 'main', targetBranch: 'rollback/cairn/1', refs: [ref('m1'), ref('m2')], scope: 'all', + })!; + outbox.activate(batch.id); + store.close(); + + const listed = recover(storePath, '--awareness', 'list'); + assert.equal(listed.status, 0, listed.stderr); + assert.deepEqual(operatorLog(storePath), [], 'a list changes nothing and records nothing'); + + const cancelled = recover(storePath, '--awareness', 'cancel', batch.id); + assert.equal(cancelled.status, 0, cancelled.stderr); + const receipt = JSON.parse(cancelled.stdout) as Record; + assert.equal(receipt.cancelled, 2); + + const retracted = recover(storePath, '--awareness', 'retract', 'all'); + assert.equal(retracted.status, 0, retracted.stderr); + + const refused = recover(storePath, '--awareness', 'release', batch.id); + assert.equal(refused.status, 1, 'releasing a batch that is not held is refused'); + + const entries = operatorLog(storePath); + assert.deepEqual(entries.map((entry) => entry.kind), ['awareness-cancel', 'awareness-retract', 'awareness-release']); + for (const entry of entries) { + assert.equal(entry.agent, '*'); + assert.equal(entry.requester?.via, 'cli'); + assert.ok(entry.at); + } + assert.deepEqual(entries[0].params, { target: batch.id }); + assert.deepEqual(entries[0].result, receipt); + assert.deepEqual(entries[1].params, { target: 'all' }); + assert.equal(entries[1].result?.removalsQueued, 2); + assert.deepEqual(entries[2].params, { batchId: batch.id }); + assert.equal(entries[2].result, undefined); + assert.match(entries[2].error ?? '', /is active, not held/); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test('an offline recovery records its marks choice and receipt in the operator log; a dry run records nothing', async () => { + const dir = mkdtempSync(join(tmpdir(), 'recover-cli-log-')); + const storePath = join(dir, 'store'); + try { + const store = JsStore.openOrCreate({ path: storePath }); + const cm = await ContextManager.open({ store, namespace: 'agents/cairn' }); + cm.addMessage('user', [{ type: 'text', text: 'safe' }], { ...ref('m-safe') }); + cm.addMessage('user', [{ type: 'text', text: 'one' }], { ...ref('m-1'), tags: ['chat:addressed'] }); + cm.addMessage('user', [{ type: 'text', text: 'two' }], { ...ref('m-2'), tags: ['chat:ambient'] }); + cm.close(); + store.close(); + + const dry = recover(storePath, '--agent', 'cairn', '--message-id', 'm-safe', '--marks', 'addressed', '--dry-run'); + assert.equal(dry.status, 0, dry.stderr); + assert.deepEqual(operatorLog(storePath), [], 'a dry run changes nothing and records nothing'); + + const done = recover(storePath, '--agent', 'cairn', '--message-id', 'm-safe', '--marks', 'addressed', '--branch', 'recovery/cairn/1'); + assert.equal(done.status, 0, done.stderr); + const output = JSON.parse(done.stdout) as { markers: Record }; + + const again = recover(storePath, '--agent', 'cairn', '--message-id', 'm-safe', '--branch', 'main'); + assert.equal(again.status, 1, 'a recovery onto an existing branch is refused'); + + const [recorded, refused, ...rest] = operatorLog(storePath); + assert.equal(rest.length, 0); + assert.equal(recorded.kind, 'recovery'); + assert.equal(recorded.agent, 'cairn'); + assert.equal(recorded.requester?.via, 'cli'); + assert.deepEqual(recorded.params, { messageId: 'm-safe', marks: { scope: 'addressed' } }); + assert.deepEqual(recorded.result, { + sourceBranch: 'main', + targetBranch: 'recovery/cairn/1', + messagesRemoved: 2, + messagesSuppressed: 0, + discordRefs: 2, + markers: output.markers, + }); + assert.equal((recorded.result?.markers as { status: string; queued: number }).status, 'queued'); + assert.equal((recorded.result?.markers as { status: string; queued: number }).queued, 1); + + assert.equal(refused.kind, 'recovery'); + assert.deepEqual(refused.params, { messageId: 'm-safe', marks: 'none' }); + assert.match(refused.error ?? '', /already exists/); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); From 75d7eba10d1de0e9f3d7df5419b24946a128e996 Mon Sep 17 00:00:00 2001 From: Ansel-1606 Date: Wed, 7 Oct 2026 16:46:27 -0400 Subject: [PATCH 0051/1112] fix(awareness): a turn undo records its marks choice and receipt in the operator log Greptile review of #250 (P2 "Turn marks aren't logged"). Turn-based host/command undo accepts a marks choice and schedules marks for the messages its turns removed, but its only operator-log entries came from undoLastTurn(), one per turn, with the turn and branches. So nothing recorded who chose to publish (or not to publish) marks, while hide and rollback (message-granular undo) record their choice and receipt. After scheduling, the command now records one entry: kind 'undo-turns', the requester, params {turns: requested, marks: none | {scope}}, and result {undone, markers}. It is written whenever at least one turn was undone, `marks: none` included (agreed with Mira-1605: the local-only choice is worth recording, as hide and rollback do); a command that undid nothing changed nothing and records nothing. The per-turn undo-turn entries are unchanged. Test (test/discord-awareness-outbox.test.ts): a marked turn undo, a local one, and one with nothing left to undo log exactly two entries, each with the receipt the command returned. --- changelog.d/surgery-marks-controls.added.md | 4 ++ src/framework.ts | 10 ++++ test/discord-awareness-outbox.test.ts | 53 +++++++++++++++++++++ 3 files changed, 67 insertions(+) diff --git a/changelog.d/surgery-marks-controls.added.md b/changelog.d/surgery-marks-controls.added.md index 45c6cc924..51484e58b 100644 --- a/changelog.d/surgery-marks-controls.added.md +++ b/changelog.d/surgery-marks-controls.added.md @@ -22,6 +22,10 @@ available as the `host/command` `marks` verb (`action: list | cancel | retract | release`, `target`) and, with the host stopped, as `agent-framework-recover --awareness list|cancel|retract|release`. +- A turn-based `host/command` `undo` that undoes a turn records one + `undo-turns` entry in the operator log: the requested turns, the marks + choice (`none` included) and the receipt, beside the per-turn `undo-turn` + entries. - `agent-framework-recover` records its acts in the store's operator log (`/operator-actions.jsonl`), done or refused, with the OS account that ran it: each `--awareness` cancel, retract and release, and each diff --git a/src/framework.ts b/src/framework.ts index 60d80bbf6..da8c24419 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -5476,6 +5476,16 @@ export class AgentFramework { const after = new Set(this.addressingSnapshot(agentName).keys()); const removed = [...before].filter(([id]) => !after.has(id)).map(([, carrier]) => carrier); markers = this.scheduleAppliedMarks('undo', agentName, serverId, removed, turnMarks); + // Each turn is logged by undoLastTurn; the command's marks choice + // (`none` included) and what it scheduled are logged here, as hide and + // rollback log theirs. + this.recordOperatorAction({ + kind: 'undo-turns', + agent: agentName, + requester: hostCommandRequester(serverId, params), + params: { turns: requested, marks: describeMarksChoice(turnMarks) }, + result: { undone, markers }, + }); } finally { release(); } diff --git a/test/discord-awareness-outbox.test.ts b/test/discord-awareness-outbox.test.ts index 34a46265b..6329b2c77 100644 --- a/test/discord-awareness-outbox.test.ts +++ b/test/discord-awareness-outbox.test.ts @@ -895,6 +895,59 @@ test('turn-based host/command undo honours the marks choice for the messages its } }); +test('a turn undo records its marks choice and what it scheduled in the operator log, also for marks none', async () => { + const h = storeHarness(); + try { + const messages = [ + { id: 'i0', metadata: { ...ref('m0') } }, + { id: 'i1', metadata: { ...ref('m1'), tags: ['chat:addressed'] } }, + { id: 'i2', metadata: { ...ref('m2') } }, + ]; + const { framework, removed } = hostCommandFramework(h, messages); + // Each stubbed turn undo removes the next of these from the context. + let undoable = ['i1', 'i2']; + framework.undoLastTurn = () => { + const next = undoable[0]; + if (!next) return { undone: false }; + undoable = undoable.slice(1); + removed.add(next); + return { undone: true }; + }; + const logged: Array> = []; + framework.recordOperatorAction = (entry: Record) => { logged.push(entry); return entry; }; + let marked: any; + let local: any; + await muted(async () => { + marked = await framework.handleHostCommand('discord', { + command: 'undo', agentName: 'cairn', turns: 1, marks: 'addressed', requesterName: 'op', + }); + local = await framework.handleHostCommand('discord', { command: 'undo', agentName: 'cairn', turns: 3 }); + const nothing = await framework.handleHostCommand('discord', { command: 'undo', agentName: 'cairn', turns: 1, marks: 'all' }); + assert.equal(nothing.undone, 0); + }); + assert.equal(marked.markers.status, 'queued'); + assert.equal(local.markers.status, 'none'); + assert.deepEqual(logged, [ + { + kind: 'undo-turns', + agent: 'cairn', + requester: { via: 'host-command:discord', name: 'op' }, + params: { turns: 1, marks: { scope: 'addressed' } }, + result: { undone: 1, markers: marked.markers }, + }, + { + kind: 'undo-turns', + agent: 'cairn', + requester: { via: 'host-command:discord' }, + params: { turns: 3, marks: 'none' }, + result: { undone: 1, markers: local.markers }, + }, + ], 'one entry per command that undid a turn, with the receipt it returned; none when nothing was undone'); + } finally { + h.cleanup(); + } +}); + test('host/command hide takes the store reservation and marks only on request, through the journal', async () => { const h = storeHarness(); try { From 538b136cfd040a7b5fe9e6e8b0b19a12ddb2b243 Mon Sep 17 00:00:00 2001 From: Ansel-1606 Date: Wed, 7 Oct 2026 16:52:17 -0400 Subject: [PATCH 0052/1112] fix(awareness): stop checkpointing the awareness journal; replay every entry Greptile review of #250 (P2 "Checkpoints repeatedly copy all history"). Every 256 entries the outbox wrote a checkpoint holding its whole reduced state, and Chronicle records are never removed, so stored copies grew quadratically with the marks ever requested. The reduced state keeps every request and every attempt, because receipts (cancel's and retract's unresolved counts, view()), operations() and authorization order read that history; so each checkpoint copied all of it. Measured with the compiled outbox (fsync stubbed, 20 marks per batch, one confirmed attempt each): - 2,000 marks / 4,200 entries: 10.8 MB with checkpoints, 1.65 MB without; opening took 0.14 s and 0.11 s; - 8,000 marks / 16,800 entries: 149.6 MB vs 6.6 MB; opening took 0.70 s with checkpoints and 1.43 s without; - 20,000 marks: 918.5 MB with checkpoints (the run without them was not completed). Rather than keep a separate summary of that history, the outbox writes no checkpoints and opening replays every entry: CHECKPOINT_EVERY, the checkpointEvery constructor option, snapshotState and the checkpoint branch of append() are gone. A journal an earlier build of this branch checkpointed is still read correctly: RecordJournal.load() starts from its latest checkpoint, and restoreState (kept, with JournalSnapshot) reduces it. RecordJournal itself is unchanged; it is the base this PR shares with #247. Agreed with Mira-1605 in room-245. Tests (test/discord-awareness-outbox.test.ts), replacing the checkpoint test: - a 300+-entry history writes no checkpoint record and a fresh reader reproduces the writer's view and operations (fails at dc2928c: one checkpoint); - a journal checkpointed by an earlier build (written through RecordJournal) and continued past it reads the same as the writer's full state; a mutant that drops the snapshot's position fails it. --- src/recovery/discord-awareness-outbox.ts | 47 +++++++------------ test/discord-awareness-outbox.test.ts | 57 +++++++++++++++++++----- 2 files changed, 61 insertions(+), 43 deletions(-) diff --git a/src/recovery/discord-awareness-outbox.ts b/src/recovery/discord-awareness-outbox.ts index d49f8fce2..8850c3df4 100644 --- a/src/recovery/discord-awareness-outbox.ts +++ b/src/recovery/discord-awareness-outbox.ts @@ -493,7 +493,17 @@ interface JournalEntryGroup { records: JournalRecord[]; } -/** The reduced state as a checkpoint stores it. */ +/** + * The reduced state as a checkpoint stores it. This journal writes no + * checkpoints. Its reduced state keeps every request and every attempt, + * because receipts, `operations()` and authorization order read that + * history, so a snapshot of it copies all of it, and copying it every N + * entries grows the store quadratically with the marks ever requested. + * Rather than keep a separate summary of that history, opening replays every + * entry. A journal an earlier build of this journal checkpointed is still + * read from its latest checkpoint, which is where RecordJournal.load() + * starts. + */ interface JournalSnapshot { batches: DiscordAwarenessBatch[]; ops: DiscordAwarenessOp[]; @@ -504,8 +514,6 @@ interface JournalSnapshot { } export const DISCORD_AWARENESS_RECORD_TYPE = 'af:discord-awareness'; -/** Replay at open is bounded by a checkpoint every this many entries. */ -const CHECKPOINT_EVERY = 256; interface JournalState { batches: Map; @@ -539,7 +547,6 @@ interface RetractRequest { export class DiscordAwarenessOutbox { private readonly journal: RecordJournal; private readonly legacyPath?: string; - private readonly checkpointEvery: number; /** Reduced state; rebuilt from the journal when null. */ private state: JournalState | null = null; /** Operations dispatched by this process and still awaiting an outcome. */ @@ -548,12 +555,10 @@ export class DiscordAwarenessOutbox { /** * @param store the Chronicle store the journal lives in. * @param opts.legacyPath a pre-journal JSON ledger to import once, if present. - * @param opts.checkpointEvery entries between checkpoints (default 256). */ - constructor(store: JsStore, opts: { legacyPath?: string; checkpointEvery?: number } = {}) { + constructor(store: JsStore, opts: { legacyPath?: string } = {}) { this.journal = new RecordJournal(store, { type: DISCORD_AWARENESS_RECORD_TYPE }); if (opts.legacyPath) this.legacyPath = opts.legacyPath; - this.checkpointEvery = Math.max(1, Math.floor(opts.checkpointEvery ?? CHECKPOINT_EVERY)); } // -- surgery side ----------------------------------------------------------- @@ -1270,8 +1275,9 @@ export class DiscordAwarenessOutbox { /** * The reduced state, rebuilt from the journal when there is none yet or an * earlier write left the journal unreconciled (RecordJournal then requires - * a reload before writing again). The first load also imports a - * pre-journal ledger, once. + * a reload before writing again): every entry is replayed, after an + * earlier build's checkpoint if there is one (see JournalSnapshot). The + * first load also imports a pre-journal ledger, once. */ private load(): JournalState { if (this.state && !this.journal.needsReconcile) return this.state; @@ -1338,17 +1344,6 @@ export class DiscordAwarenessOutbox { throw error; } applyRecords(state, records); - if (this.journal.entriesSinceCheckpoint >= this.checkpointEvery) { - try { - this.journal.checkpoint(snapshotState(state)); - } catch (error) { - // Replay just stays longer; the journal reloads before its next write. - this.state = null; - console.error( - `[discord-awareness] checkpoint failed: ${error instanceof Error ? error.message : String(error)}`, - ); - } - } } } @@ -1366,17 +1361,7 @@ function liveRequests(records: JournalRecord[]): number { return records.filter((record) => record.t === 'requested' && !cancelled.has(record.opId)).length; } -function snapshotState(state: JournalState): JournalSnapshot { - return { - batches: [...state.batches.values()].map((batch) => structuredClone(batch)), - ops: [...state.ops.values()].map((op) => structuredClone(op)), - legacy: [...state.legacy.values()].flat().map((evidence) => structuredClone(evidence)), - retracts: [...state.retracts.values()].map((retract) => structuredClone(retract)), - importedSources: [...state.importedSources], - position: state.position, - }; -} - +/** The state an earlier build's checkpoint recorded (see JournalSnapshot). */ function restoreState(snapshot: JournalSnapshot): JournalState { const state: JournalState = { batches: new Map(snapshot.batches.map((batch) => [batch.id, structuredClone(batch)])), diff --git a/test/discord-awareness-outbox.test.ts b/test/discord-awareness-outbox.test.ts index 6329b2c77..7e47bb3f0 100644 --- a/test/discord-awareness-outbox.test.ts +++ b/test/discord-awareness-outbox.test.ts @@ -29,6 +29,7 @@ import { import { AgentFramework } from '../src/framework.js'; import { McplRequestError } from '../src/mcpl/server-connection.js'; import { OperatorLog } from '../src/operator-log.js'; +import { RecordJournal } from '../src/record-journal.js'; const ref = (messageId: string, channel = 'c1'): DiscordAwarenessRef => ({ serverId: 'discord', channelId: `discord:g1:${channel}`, messageId }); @@ -511,22 +512,54 @@ test('a corrupt journal entry stops the reader rather than being skipped', withJ assert.throws(() => h.reopen().batches(), /Corrupt Discord awareness journal entry/); })); -test('checkpoints bound replay and reproduce the reduced state', withJournal((_outbox, h) => { - const outbox = new DiscordAwarenessOutbox(h.store, { checkpointEvery: 8 }); - const batch = activeBatch(outbox, Array.from({ length: 5 }, (_, i) => ref(`m${i}`))); - for (let round = 0; round < 3; round++) { - for (let i = 0; i < 5; i++) { - const claimed = outbox.claimDispatch('discord', (d) => d.key.messageId !== `m${i}`); - if (claimed) outbox.recordOutcome(claimed.attempts, round === 2 ? 'confirmed' : 'failed', { error: 'Discord 503' }); - } +test('the journal writes no checkpoints: a long history is replayed whole', withJournal((_outbox, h) => { + // Every reduction needs the whole history, so a checkpoint would copy all + // of it, and copying it every N entries grows the store quadratically. + // (Durability is not under test here: skip the per-entry fsync.) + const unsynced = new Proxy(h.store, { + get(target, prop) { + if (prop === 'sync') return () => {}; + const value = Reflect.get(target, prop); + return typeof value === 'function' ? value.bind(target) : value; + }, + }); + const outbox = new DiscordAwarenessOutbox(unsynced); + const batch = activeBatch(outbox, Array.from({ length: 150 }, (_, i) => ref(`m${i}`))); + for (let claimed = outbox.claimDispatch('discord'); claimed; claimed = outbox.claimDispatch('discord')) { + outbox.recordOutcome(claimed.attempts, 'confirmed'); } outbox.retract(batch.id); - assert.ok(h.store.getRecordIdsByType(`${DISCORD_AWARENESS_RECORD_TYPE}/checkpoint`).length >= 2, 'checkpoints were written'); - const reread = new DiscordAwarenessOutbox(h.store, { checkpointEvery: 8 }); + const entries = h.store.getRecordIdsByType(DISCORD_AWARENESS_RECORD_TYPE).length; + assert.ok(entries > 256, `${entries} entries: more than an earlier build wrote between checkpoints`); + assert.equal(h.store.getRecordIdsByType(`${DISCORD_AWARENESS_RECORD_TYPE}/checkpoint`).length, 0); + const reread = h.reopen(); + assert.deepEqual(reread.view(), outbox.view()); + assert.deepEqual(reread.operations(), outbox.operations()); +})); + +test('a journal an earlier build checkpointed is read from that checkpoint and continues the same', withJournal((outbox, h) => { + const batch = activeBatch(outbox, [ref('m1'), ref('m2'), ref('m3')]); + answer(outbox, 'm1', 'confirmed'); + answer(outbox, 'm2', 'unknown'); + // An earlier build checkpointed its reduced state here, with the journal + // position its next record would follow. + const earlier = new RecordJournal<{ records: unknown[] }, unknown>(h.store, { type: DISCORD_AWARENESS_RECORD_TYPE }); + const { entries } = earlier.load(); + earlier.checkpoint({ + batches: outbox.batches(), + ops: outbox.operations(), + legacy: [], + retracts: [], + importedSources: [], + position: entries.reduce((sum, { entry }) => sum + entry.records.length, 0), + }); + // History continues past it: a retract, whose removals' authorization is + // its position after the checkpointed ones. + outbox.retract(batch.id); + answer(outbox, 'm3', 'confirmed'); + const reread = h.reopen(); assert.deepEqual(reread.view(), outbox.view()); assert.deepEqual(reread.operations(), outbox.operations()); - assert.equal(batches(reread).find((v) => v.id === batch.id)!.adds.confirmed, 5); - assert.equal(retracts(reread).length, 1); })); test('v2 import keeps what the old ledger recorded as evidence, holds pending work, and happens once', withJournal((_outbox, h) => { From a5439e20805d3fdafe8978c63482ddbc1f8fa869 Mon Sep 17 00:00:00 2001 From: Ansel-1606 Date: Wed, 7 Oct 2026 17:13:58 -0400 Subject: [PATCH 0053/1112] fix(mcpl): no other plane opener releases a reconnect's traffic before its policy exchange settles Review of 9d17ad6 by Mira-1605 (room-245, #48850), reproduced: Greptile's first-reconnect scenario still lost its registration. A stub whose first connect failed reconnects while startup is still staging another server; its reconnect listener correctly waits for its policy exchange, but initializeMcpl's final readyMcplPlanes() opens every connection, including this one, mid-exchange, and flushes its buffered channels/register against the empty grant (-32002, "initial policy not yet established"). resume()'s readyMcplPlanes() is the same kind of caller. The framework now marks, per connection, the transport epoch whose policy exchange a reconnect has in flight (mcplPolicyExchanges, a WeakMap keyed by the connection object, so a replacement connection under the same id is unaffected). readyMcplPlanes skips a connection whose current epoch's exchange is pending; the reconnect listener clears its mark and opens the planes once the exchange settles, granted, refused or unanswered (so an MCP-only connection and host/command still come up, as at initial connect), unless a newer transport's exchange has replaced its mark, in which case that one opens them. Tests (test/mcpl-reconnect-registration.test.ts): - Mira's reproduction as a deterministic integration test: `retry` fails its first connect and holds its retry's policy answer; `stager` answers initialize only once that policy is pending, so startup's open of every plane runs mid-exchange. Its registration must stay unanswered until the policy is released, then be admitted after it. Without this commit it is answered -32002 at startup. - Unit cases on a framework stub with deferred policy answers: a global readyMcplPlanes() during the exchange opens nothing, and the exchange's settlement opens once; a superseded exchange never opens the planes of the transport that replaced it. Both fail without this commit. The fixture gains file-based coordination (fail-first, initialize-after, policy pending/release). Object.create harnesses that emit 'reconnect' initialize the new field (mcpl-reconnect-reregistration). --- .../mcpl-reconnect-registration.fixed.md | 3 +- src/framework.ts | 36 ++++-- test/fixtures/reregistering-mcpl-server.mjs | 58 ++++++++-- test/mcpl-reconnect-registration.test.ts | 108 ++++++++++++++++++ test/mcpl-reconnect-reregistration.test.ts | 1 + 5 files changed, 187 insertions(+), 19 deletions(-) diff --git a/changelog.d/mcpl-reconnect-registration.fixed.md b/changelog.d/mcpl-reconnect-registration.fixed.md index b52983a18..daaff2abc 100644 --- a/changelog.d/mcpl-reconnect-registration.fixed.md +++ b/changelog.d/mcpl-reconnect-registration.fixed.md @@ -6,4 +6,5 @@ stayed unknown channels until the server registered again. A reconnect now holds control traffic as well as data until the new grant is established, as initial connect already did, both for a connection that - was open and for one whose first connect had failed. + was open and for one whose first connect had failed, and even when + startup or a quiesce resume opens every connection meanwhile. diff --git a/src/framework.ts b/src/framework.ts index da8c24419..ab4c36cc6 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -13874,14 +13874,26 @@ export class AgentFramework { } /** - * Open MCPL connections' planes (every connection when none is given). A - * quiesced host (issue #122) opens only control planes, so data planes stay - * held for the whole maintenance window; resume() calls this again once - * the flag is cleared. Control planes always come up, since host/command - * (and with it resume) rides the control plane. + * Per connection, the transport epoch whose §5.3 policy exchange a + * reconnect has in flight. Until it settles, that transport's traffic must + * wait for its grant, so its planes stay closed whoever else opens planes + * meanwhile (startup's staged open, resume()); the reconnect opens them. + */ + private readonly mcplPolicyExchanges = new WeakMap(); + + /** + * Open MCPL connections' planes (every connection when none is given), + * except a connection whose reconnect's policy exchange is still in flight + * (see mcplPolicyExchanges). A quiesced host (issue #122) opens only + * control planes, so data planes stay held for the whole maintenance + * window; resume() calls this again once the flag is cleared. Control + * planes always come up, since host/command (and with it resume) rides the + * control plane. */ private readyMcplPlanes(connections?: McplServerConnection[]): void { for (const connection of connections ?? this.mcplServerRegistry?.getAllServers() ?? []) { + const exchange = this.mcplPolicyExchanges.get(connection); + if (exchange !== undefined && exchange === connection.transportEpoch) continue; if (this.quiesced) connection.readyControlPlane(); else connection.ready(); } @@ -14574,7 +14586,11 @@ export class AgentFramework { // Request (whose response is never buffered) must be answered before // any plane opens, otherwise the first post-reconnect traffic, control // included (the server's channels/register), lands while the grant is - // still empty and is rejected fail-closed instead of delivered. + // still empty and is rejected fail-closed instead of delivered. The + // exchange is marked in flight for this transport, so no other opener + // (startup's staged open, resume) releases the traffic early. + const epoch = connection.transportEpoch; + this.mcplPolicyExchanges.set(connection, epoch); void (async () => { try { const config = this.mcplServerConfigs.get(connection.id); @@ -14587,7 +14603,13 @@ export class AgentFramework { error instanceof Error ? error.message : error, ); } - this.readyMcplPlanes([connection]); + // Settled, granted or not (an unanswered or refused policy leaves the + // grant empty and the connection MCP-only, as at initial connect): + // open the planes, unless a newer transport's exchange has taken over. + if (this.mcplPolicyExchanges.get(connection) === epoch) { + this.mcplPolicyExchanges.delete(connection); + this.readyMcplPlanes([connection]); + } this.handleToolsListChanged(connection.id); this.emitTrace({ type: 'mcpl:server-reconnected', diff --git a/test/fixtures/reregistering-mcpl-server.mjs b/test/fixtures/reregistering-mcpl-server.mjs index c18e3639f..b73be1c60 100644 --- a/test/fixtures/reregistering-mcpl-server.mjs +++ b/test/fixtures/reregistering-mcpl-server.mjs @@ -2,17 +2,39 @@ // // Like a chat bridge, it registers its channel as soon as the host sends // notifications/initialized, before the host's §5.3 policy Request is -// answered, and appends the host's answer to that registration to -// EVENTS_PATH. When CRASH_PATH appears it removes the file and exits, once, -// so the host's reconnect starts a fresh process that registers again. -import { appendFileSync, existsSync, unlinkSync } from 'node:fs'; +// answered, and appends the host's answer to that registration (and its own +// steps) to EVENTS_PATH, tagged with ROLE. Optional coordination, each by +// file so tests order steps without timing: +// - CRASH_PATH: when it appears, remove it and exit (once), so the host +// reconnects to a fresh process; +// - FAIL_FIRST_PATH: if it is missing at initialize, create it and exit, so +// the host's first connect fails and its retry succeeds; +// - INITIALIZE_AFTER_PATH: answer initialize only once it exists; +// - POLICY_PENDING_PATH / POLICY_RELEASE_PATH: on the policy Request, create +// the first, and answer only once the second exists. +import { appendFileSync, existsSync, unlinkSync, writeFileSync } from 'node:fs'; -const eventsPath = process.env.EVENTS_PATH; -const crashPath = process.env.CRASH_PATH; +const { + EVENTS_PATH: eventsPath, + ROLE: role = 'server', + CRASH_PATH: crashPath, + FAIL_FIRST_PATH: failFirstPath, + INITIALIZE_AFTER_PATH: initializeAfterPath, + POLICY_PENDING_PATH: policyPendingPath, + POLICY_RELEASE_PATH: policyReleasePath, +} = process.env; const send = (message) => process.stdout.write(JSON.stringify(message) + '\n'); const event = (name, extra = {}) => { - if (eventsPath) appendFileSync(eventsPath, JSON.stringify({ event: name, pid: process.pid, ...extra }) + '\n'); + if (eventsPath) appendFileSync(eventsPath, JSON.stringify({ event: name, role, pid: process.pid, ...extra }) + '\n'); +}; +const once = (path, fn) => { + if (!path || existsSync(path)) return fn(); + const timer = setInterval(() => { + if (!existsSync(path)) return; + clearInterval(timer); + fn(); + }, 5); }; let buf = ''; @@ -25,7 +47,12 @@ process.stdin.on('data', (chunk) => { if (!line.trim()) continue; const message = JSON.parse(line); if (message.method === 'initialize') { - send({ + if (failFirstPath && !existsSync(failFirstPath)) { + writeFileSync(failFirstPath, '1'); + event('initial-fail'); + process.exit(9); + } + once(initializeAfterPath, () => send({ jsonrpc: '2.0', id: message.id, result: { @@ -40,16 +67,25 @@ process.stdin.on('data', (chunk) => { }, }, }, - }); + })); } else if (message.method === 'notifications/initialized') { send({ jsonrpc: '2.0', id: 100, method: 'channels/register', - params: { channels: [{ id: 'chat:room-1', type: 'chat', label: 'room-1', direction: 'bidirectional' }] }, + params: { channels: [{ id: `chat:${role}`, type: 'chat', label: role, direction: 'bidirectional' }] }, }); + event('register-sent'); } else if (message.method === 'featureSets/update') { - if (message.id !== undefined && message.id !== null) send({ jsonrpc: '2.0', id: message.id, result: { accepted: true } }); + if (message.id === undefined || message.id === null) continue; + if (policyPendingPath) { + writeFileSync(policyPendingPath, '1'); + event('policy-held'); + } + once(policyReleasePath, () => { + event('policy-answered'); + send({ jsonrpc: '2.0', id: message.id, result: { accepted: true } }); + }); } else if (message.method === 'tools/list') { send({ jsonrpc: '2.0', id: message.id, result: { tools: [] } }); } else if (message.id === 100 && message.method === undefined) { diff --git a/test/mcpl-reconnect-registration.test.ts b/test/mcpl-reconnect-registration.test.ts index d194978bf..5bc7d40cb 100644 --- a/test/mcpl-reconnect-registration.test.ts +++ b/test/mcpl-reconnect-registration.test.ts @@ -18,13 +18,16 @@ import { fileURLToPath } from 'node:url'; import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; +import { EventEmitter } from 'node:events'; import { AgentFramework } from '../src/index.js'; +import { FeatureSetManager } from '../src/mcpl/feature-set-manager.js'; import { MockMembrane } from './helpers/mock-membrane.js'; const FIXTURE = fileURLToPath(new URL('./fixtures/reregistering-mcpl-server.mjs', import.meta.url)); interface ServerEvent { event: string; + role: string; pid: number; error?: { code: number; message: string } | null; } @@ -93,4 +96,109 @@ describe('MCPL reconnect and channel registration', () => { `the reconnected server's registration waits for its new grant instead of being refused: ${JSON.stringify(second.error)}`, ); }); + + it('a first reconnect during staged startup keeps its registration held through startup\'s open of every plane', async () => { + // Mira-1605's reproduction (room-245): `retry` fails its first connect; + // its retry reconnects while `stager` is still staging, and holds its + // policy answer; `stager` answers initialize only once that policy is + // pending, so startup's final open of every connection's planes runs + // while the retry's exchange is still in flight. + const eventsPath = join(dir, 'events.jsonl'); + const pending = join(dir, 'policy-pending'); + const release = join(dir, 'policy-release'); + const server = (id: string, env: Record) => ({ + id, + command: process.execPath, + args: [FIXTURE], + env: { EVENTS_PATH: eventsPath, ROLE: id, ...env }, + enabledFeatureSets: ['chat'], + ...(id === 'retry' ? { reconnect: true, reconnectIntervalMs: 20 } : {}), + }); + framework = await AgentFramework.create({ + storePath: join(dir, 'store'), + membrane: new MockMembrane().asMembrane(), + agents: [{ name: 'resident', model: 'test-model', systemPrompt: 'You are a resident.' }], + modules: [], + maintenanceIntervalMs: 0, + mcplServers: [ + server('retry', { FAIL_FIRST_PATH: join(dir, 'failed-once'), POLICY_PENDING_PATH: pending, POLICY_RELEASE_PATH: release }), + server('stager', { INITIALIZE_AFTER_PATH: pending }), + ], + }); + const retry = () => serverEvents(eventsPath).filter((e) => e.role === 'retry'); + // Startup has opened every plane it may; the retry's policy is still held. + await new Promise((resolve) => setTimeout(resolve, 100)); + assert.ok(retry().some((e) => e.event === 'policy-held'), `events: ${JSON.stringify(retry())}`); + assert.equal( + retry().find((e) => e.event === 'register-answered'), + undefined, + `its registration is not answered while its policy exchange is in flight: ${JSON.stringify(retry())}`, + ); + + writeFileSync(release, '1'); + await waitFor('the retry\'s registration answer', () => retry().some((e) => e.event === 'register-answered')); + const steps = retry().map((e) => e.event); + const answer = retry().find((e) => e.event === 'register-answered')!; + assert.equal(answer.error, null, `admitted against the new grant: ${JSON.stringify(answer.error)}`); + assert.ok(steps.indexOf('policy-answered') < steps.indexOf('register-answered'), `steps: ${steps.join(', ')}`); + }); +}); + +describe('MCPL reconnect: a pending policy exchange holds its planes', () => { + /** A framework stub with one connection whose policy answers are deferred. */ + function harness() { + const answers: Array<(receipt: { accepted: true }) => void> = []; + let opened = 0; + const connection = Object.assign(new EventEmitter(), { + id: 'srv', + transportEpoch: 0, + capabilities: { version: '0.5', pushEvents: true, featureSets: { chat: { description: 'chat', uses: ['pushEvents'] } } }, + mcpToolsAdvertised: false, + establishGrant: () => {}, + sendFeatureSetsUpdateRequest: () => new Promise<{ accepted: true }>((resolve) => { answers.push(resolve); }), + ready: () => { opened++; }, + readyControlPlane: () => { opened++; }, + }); + const fw = Object.create(AgentFramework.prototype) as any; + fw.traceListeners = []; + fw.mcplPolicyExchanges = new WeakMap(); + fw.mcplServerConfigs = new Map([['srv', { id: 'srv', command: 'unused', enabledFeatureSets: ['chat'] }]]); + fw.featureSetManager = new FeatureSetManager(); + fw.checkpointManager = null; + fw.mcplServerRegistry = { getAllServers: () => [connection], getServer: () => connection }; + fw.handleToolsListChanged = () => {}; + fw.wireMcplEvents(connection); + return { + fw, + connection, + answers, + opened: () => opened, + reconnect: () => { connection.transportEpoch++; connection.emit('reconnect', { attempts: 1 }); }, + }; + } + + it('no other opener (startup\'s staged open, resume) releases a reconnect\'s traffic before its exchange settles', async () => { + const h = harness(); + h.reconnect(); + h.fw.readyMcplPlanes(); // startup's open of every connection, or resume() + assert.equal(h.opened(), 0, 'held while the policy exchange is in flight'); + h.answers[0]({ accepted: true }); + await new Promise((resolve) => setImmediate(resolve)); + assert.equal(h.opened(), 1, 'opened by the reconnect once its exchange settled'); + h.fw.readyMcplPlanes(); + assert.equal(h.opened(), 2, 'and no longer held'); + }); + + it('a superseded exchange never opens the planes of the transport that replaced it', async () => { + const h = harness(); + h.reconnect(); + h.reconnect(); // the first fresh transport died; a newer one is exchanging + h.answers[0]({ accepted: true }); + await new Promise((resolve) => setImmediate(resolve)); + h.fw.readyMcplPlanes(); + assert.equal(h.opened(), 0, 'the newer exchange is still in flight'); + h.answers[1]({ accepted: true }); + await new Promise((resolve) => setImmediate(resolve)); + assert.equal(h.opened(), 1); + }); }); diff --git a/test/mcpl-reconnect-reregistration.test.ts b/test/mcpl-reconnect-reregistration.test.ts index 35a1fadd6..eb3d20630 100644 --- a/test/mcpl-reconnect-reregistration.test.ts +++ b/test/mcpl-reconnect-reregistration.test.ts @@ -108,6 +108,7 @@ async function makeHarness() { fw.channelRegistry = null; fw.inferenceRouter = null; fw.eventGate = null; + fw.mcplPolicyExchanges = new WeakMap(); fw.featureSetManager = new FeatureSetManager(); fw.scopeManager = new ScopeManager(); From 485a3c75a0029fe1aab490af4fb2c13927fc17ed Mon Sep 17 00:00:00 2001 From: Ansel-1606 Date: Wed, 7 Oct 2026 17:13:58 -0400 Subject: [PATCH 0054/1112] fix(recover): refuse --dry-run with an awareness control instead of carrying the control out Review of 25891f0 by Mira-1605 (room-245, #48850), reproduced: the CLI parsed --dry-run but the --awareness path ignored it, so `--awareness cancel ID --dry-run` cancelled the batch's adds and logged it, while the new help text said a dry run changes and records nothing; a "preview" of retract or release would have published removals or held marks. A control has no preview (inspect with --awareness list), so the CLI now refuses --dry-run with --awareness before it opens or creates any store, as Mira favoured, rather than simulating controls. The help text limits --dry-run to a recovery and says each control acts at once. Test (test/recover-cli-operator-log.test.ts): cancel, retract and release with --dry-run each exit 1 with the refusal; the journal (views and operations) is unchanged and nothing is logged; against a store path that doesn't exist, nothing is created. Fails without this commit (cancel exits 0). --- changelog.d/surgery-marks-controls.added.md | 5 ++- src/recovery/recover-cli.ts | 16 ++++++-- test/recover-cli-operator-log.test.ts | 44 +++++++++++++++++++++ 3 files changed, 59 insertions(+), 6 deletions(-) diff --git a/changelog.d/surgery-marks-controls.added.md b/changelog.d/surgery-marks-controls.added.md index 51484e58b..b562b6481 100644 --- a/changelog.d/surgery-marks-controls.added.md +++ b/changelog.d/surgery-marks-controls.added.md @@ -29,5 +29,6 @@ - `agent-framework-recover` records its acts in the store's operator log (`/operator-actions.jsonl`), done or refused, with the OS account that ran it: each `--awareness` cancel, retract and release, and each - recovery with its marks choice and receipt (kind `recovery`). A dry run or - a list records nothing. + recovery with its marks choice and receipt (kind `recovery`). A list, and + a recovery's dry run, record nothing. `--dry-run` with `--awareness` is + refused: a control has no preview. diff --git a/src/recovery/recover-cli.ts b/src/recovery/recover-cli.ts index 9735e3267..4940165f8 100644 --- a/src/recovery/recover-cli.ts +++ b/src/recovery/recover-cli.ts @@ -61,7 +61,8 @@ Options: --outbox A pre-journal awareness ledger to import (default: /recovery/discord-awareness-outbox.json) --emoji Marker reaction (default: 💤) - --dry-run Inspect counts/addresses without writing + --dry-run Inspect a recovery's counts/addresses without + writing (a recovery only; refused with --awareness) --help Show this help Awareness journal (host stopped): @@ -73,11 +74,13 @@ Awareness journal (host stopped): ref's configured MCPL route) on a batch's refs, or on every ref an add was ever sent for --awareness release Queue a held batch's recorded operations + Each acts at once: inspect first with --awareness list. Queued work is delivered when the host next connects to the Discord server. -Each recovery and each cancel, retract or release (done or refused; not a -dry run or a list) is recorded in /operator-actions.jsonl, the host's -operator log, with the OS account that ran it. +Each recovery, and each cancel, retract or release, is recorded in +/operator-actions.jsonl, the host's operator log, with the OS account +that ran it, whether it was carried out or refused. A dry run and a list +record nothing. The agent host must be stopped before running this command.`; } @@ -213,6 +216,11 @@ async function main(): Promise { if (!options.storePath) { throw new Error(`--awareness needs --store\n\n${usage()}`); } + // There is no preview of a control: refuse rather than act on a command + // that asked not to change anything. + if (options.dryRun) { + throw new Error('--dry-run previews a recovery; it does not apply to --awareness (inspect with --awareness list)'); + } awareness(options); return; } diff --git a/test/recover-cli-operator-log.test.ts b/test/recover-cli-operator-log.test.ts index 1c0202e37..91c470490 100644 --- a/test/recover-cli-operator-log.test.ts +++ b/test/recover-cli-operator-log.test.ts @@ -82,6 +82,50 @@ test('the offline awareness controls record each act in the operator log, done o } }); +test('--dry-run with an awareness control is refused before the store opens: nothing is done or recorded', () => { + const dir = mkdtempSync(join(tmpdir(), 'recover-cli-log-')); + const storePath = join(dir, 'store'); + const journal = () => { + const store = JsStore.openOrCreate({ path: storePath }); + try { + const outbox = new DiscordAwarenessOutbox(store); + return { view: outbox.view(), operations: outbox.operations() }; + } finally { + store.close(); + } + }; + try { + const store = JsStore.openOrCreate({ path: storePath }); + const outbox = new DiscordAwarenessOutbox(store); + const active = outbox.prepare({ + agentName: 'cairn', sourceBranch: 'main', targetBranch: 'rollback/cairn/1', refs: [ref('m1')], scope: 'all', + })!; + outbox.activate(active.id); + const held = outbox.prepare({ + agentName: 'cairn', sourceBranch: 'main', targetBranch: 'rollback/cairn/interrupted', refs: [ref('m2')], scope: 'all', + })!; + outbox.recoverAtStartup('main'); // its switch was never recorded: held + store.close(); + const before = journal(); + const heldView = before.view.find((v) => v.id === held.id); + assert.equal(heldView?.kind === 'batch' ? heldView.status : undefined, 'held'); + + for (const control of [['cancel', active.id], ['retract', 'all'], ['release', held.id]]) { + const run = recover(storePath, '--awareness', ...control, '--dry-run'); + assert.equal(run.status, 1, `${control[0]} --dry-run is refused`); + assert.match(run.stderr, /--dry-run previews a recovery; it does not apply to --awareness/); + } + assert.deepEqual(journal(), before, 'nothing was cancelled, retracted or released'); + assert.deepEqual(operatorLog(storePath), [], 'and nothing was recorded'); + + const missing = join(dir, 'no-such-store'); + assert.equal(recover(missing, '--awareness', 'cancel', active.id, '--dry-run').status, 1); + assert.equal(existsSync(missing), false, 'refused before any store is opened or created'); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + test('an offline recovery records its marks choice and receipt in the operator log; a dry run records nothing', async () => { const dir = mkdtempSync(join(tmpdir(), 'recover-cli-log-')); const storePath = join(dir, 'store'); From d36a5ad5d37b274055c1b1e527fde1df40334c1c Mon Sep 17 00:00:00 2001 From: Petra-1759 Date: Wed, 7 Oct 2026 18:54:27 -0400 Subject: [PATCH 0055/1112] fix(mcpl): every stored push gets the host's delivered-body digest; export sourceBodyDigest from the package root Greptile's review of #252 at 711b400 (review 5448991826): P1 (discussion r4212520929). handleMcplPushEvent stamped metadata.sourceBodyDigest only when the body had blocks, a guard meant for the silent heartbeat, which stores nothing. But PushHandler maps an ordinary empty payload to [], and handleMcplPushEvent stores it unless the heartbeat's exact marker holds. So an ordinary empty push, or an addressed empty push into a closed channel (stored as the host's invitation alone), was stored with a storedBodyDigest and no host-computed sourceBodyDigest; and since push metadata starts from the adapter's origin, an origin.sourceBodyDigest survived on the stored copy as if the host had computed it. The digest is now taken for every push, an empty body included, before any decoration, written over whatever the origin carried. A silent heartbeat still stores nothing; its metadata is never kept. channels/incoming already stamped unconditionally, and every storage site on both lanes already wrote storedBodyDigest over the adapter's value. P2 (discussion r4212520946). The helpers were reachable only from src/mcpl/inbound-source.ts, and the package's exports map exposes only the root, so a package consumer could not import them and would have to copy the hashing code. sourceBodyDigest is now re-exported from src/mcpl/index.ts and the package root. canonicalJson stays out of the package API: it is the digest's serializer, and keeping serialization and framing inside sourceBodyDigest saves consumers from duplicating either, which is the drift the finding is about (a consumer hashing canonicalJson(blocks) itself gets a different digest from sourceBodyDigest(blocks)). It keeps its module export for in-repo tests that pin the framing. Agreed with Petra-1012 (room-203 #49948) and June-1758 (room-262 #49931); wording per June-1758's review (#50126). The changelog fragment said both helpers were exported as the shared implementation and that both fields were stamped at ingestion. It now names the one public function and states the algorithm, says the stored digest is taken where each path stores (711b400), that an empty body has a digest, and that adapter-supplied values are replaced. Tests (inbound-source.test.ts, over the real MCPL child): - an ordinary empty unscoped push whose origin carries forged digests is stored with sourceBodyDigest = storedBodyDigest = SHA-256 of "[[]]"; - an empty DM into a closed channel is stored as the invitation alone, its sourceBodyDigest the empty body's and its storedBodyDigest the stored blocks'. Both fail before this change: sourceBodyDigest came back as the adapter's 'adapter-value'. The test file now imports sourceBodyDigest from the package root, so the public export is typechecked. The speech-route fixture's dm and push ops accept "content" (blocks, [] included) in place of "text", as its incoming op does on the receipts branch. Co-Authored-By: Claude Opus 5.5 --- changelog.d/ingestion-body-digests.added.md | 23 ++++++++----- src/framework.ts | 6 ++-- src/index.ts | 6 ++-- src/mcpl/inbound-source.ts | 7 ++++ src/mcpl/index.ts | 5 +-- test/fixtures/speech-route-mcpl-server.mjs | 10 +++--- test/inbound-source.test.ts | 37 ++++++++++++++++++++- 7 files changed, 74 insertions(+), 20 deletions(-) diff --git a/changelog.d/ingestion-body-digests.added.md b/changelog.d/ingestion-body-digests.added.md index 287bc8f7e..ea2ac3d73 100644 --- a/changelog.d/ingestion-body-digests.added.md +++ b/changelog.d/ingestion-body-digests.added.md @@ -1,13 +1,18 @@ -- **Delivered-body digests on every inbound item.** Both MCPL lanes stamp two - fields at ingestion, outside the frozen source envelope: +- **Delivered-body digests on every stored inbound item.** Both MCPL lanes + stamp two fields, outside the frozen source envelope: - `metadata.sourceBodyDigest` is the version identity of the body as - delivered, taken after MCPL conversion and before any host decoration or - storage sharding. It is SHA-256 of `canonicalJson([blocks])`, the framing - an undecorated, unsharded stored copy already hashes to. + delivered, taken at ingestion, after MCPL conversion and before any host + decoration or storage sharding. It is the SHA-256 (hex) of `[blocks]` + as canonical JSON (keys sorted at every level, `undefined` dropped), the + framing an undecorated, unsharded stored copy already hashes to. An + empty body has one too. - `metadata.storedBodyDigest` uses the same function over exactly the - blocks handed to storage, decorations included. + blocks handed to storage, decorations included, taken where each path + stores. + Both are always the host's own: values an adapter supplies are replaced. A copy that no longer hashes to its `storedBodyDigest` was changed after it - arrived (`editMessage` keeps metadata). `canonicalJson` and - `sourceBodyDigest` are exported from `mcpl/inbound-source.ts` as the one - implementation consumers share. + arrived (`editMessage` keeps metadata). `sourceBodyDigest(blocks)` is + exported from the package root as the one implementation consumers share; + it carries the framing, so check a copy with it rather than hashing your + own serialization. diff --git a/src/framework.ts b/src/framework.ts index 20acf6a1d..b2d8ec8f4 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -8404,8 +8404,10 @@ export class AgentFramework { const content = [...event.content]; // The delivered body's version identity, before any host decoration - // (the closed-channel invitation below). - if (content.length > 0) metadata.sourceBodyDigest = sourceBodyDigest(content); + // (the closed-channel invitation below). Every push gets one, an empty + // body included, written over any value the adapter's origin carried. + // A silent heartbeat stores nothing, so its metadata is never kept. + metadata.sourceBodyDigest = sourceBodyDigest(content); if (triggerChannel) { const origin = (event.origin ?? {}) as Record; const invitation = this.buildClosedChannelInvitation({ diff --git a/src/index.ts b/src/index.ts index 9c1cc4ac4..db533b6d5 100644 --- a/src/index.ts +++ b/src/index.ts @@ -83,8 +83,10 @@ export type { GateConfig, GateOptions, GatePolicy, GatePolicyMatch, GateBehavior // MCPL channel registry (exposed for modules that need channel-level operations) export { ChannelRegistry } from './mcpl/index.js'; -// Where each accepted inbound item came from (`metadata.inboundSource`) -export { INBOUND_SOURCE_KEY, readInboundSource, conversationKey } from './mcpl/index.js'; +// Where each accepted inbound item came from (`metadata.inboundSource`), and +// the digest its delivered and stored bodies are stamped with +// (`metadata.sourceBodyDigest`, `metadata.storedBodyDigest`) +export { INBOUND_SOURCE_KEY, readInboundSource, conversationKey, sourceBodyDigest } from './mcpl/index.js'; export type { InboundSource, InboundChannelSource, diff --git a/src/mcpl/inbound-source.ts b/src/mcpl/inbound-source.ts index 0812c45f4..4f51a1860 100644 --- a/src/mcpl/inbound-source.ts +++ b/src/mcpl/inbound-source.ts @@ -159,6 +159,10 @@ export function conversationKey(source: InboundSource): string | undefined { * JSON with object keys sorted at every level and `undefined` values dropped, * so a value hashes alike however its keys were ordered (content read back * from the store has its keys in a different order than it was written). + * + * sourceBodyDigest's serializer, kept out of the package API: serialization + * and framing both stay inside sourceBodyDigest, so consumers don't + * duplicate either (`canonicalJson(blocks)` alone hashes differently). */ export function canonicalJson(value: unknown): string { if (Array.isArray(value)) return `[${value.map(canonicalJson).join(',')}]`; @@ -187,6 +191,9 @@ export function canonicalJson(value: unknown): string { * materialization, a correction), not the admission. Neither proves later * presence or completeness; a copy that no longer hashes to its * storedBodyDigest was changed after delivery (editMessage keeps metadata). + * + * Exported from the package root, so a consumer checks a copy against either + * field with this same function rather than a copy of it. */ export function sourceBodyDigest(blocks: readonly unknown[]): string { return createHash('sha256').update(canonicalJson([blocks])).digest('hex'); diff --git a/src/mcpl/index.ts b/src/mcpl/index.ts index 1b933017a..e7a54fb9b 100644 --- a/src/mcpl/index.ts +++ b/src/mcpl/index.ts @@ -157,8 +157,9 @@ export { InferenceRouter } from './inference-router.js'; // Channel registry (channel lifecycle, incoming messages, synthesized tools) export { ChannelRegistry } from './channel-registry.js'; -// Inbound source envelope: where each accepted item came from, stamped once -export { INBOUND_SOURCE_KEY, readInboundSource, conversationKey } from './inbound-source.js'; +// Inbound source envelope: where each accepted item came from, stamped once; +// and the digest of a delivered or stored body +export { INBOUND_SOURCE_KEY, readInboundSource, conversationKey, sourceBodyDigest } from './inbound-source.js'; export type { InboundSource, InboundChannelSource, diff --git a/test/fixtures/speech-route-mcpl-server.mjs b/test/fixtures/speech-route-mcpl-server.mjs index 8cebbf70c..6d6a757d5 100644 --- a/test/fixtures/speech-route-mcpl-server.mjs +++ b/test/fixtures/speech-route-mcpl-server.mjs @@ -5,8 +5,10 @@ // BOTH MCPL lanes. The TEST appends one JSON object per line to COMMAND_PATH: // {"op":"incoming","channelId":…,"messageId":…,"mode":"ambient"|"addressed", // "text":…, "threadId"?, "metadata"?, "eventId"?, "coalesce"?} -// {"op":"dm","eventId":…,"authorId":…,"authorName"?,"rawChannelId":…,"text":…} -// {"op":"push","eventId":…,"origin"?,"tags"?,"text":…,"featureSet"?} +// {"op":"dm","eventId":…,"authorId":…,"authorName"?,"rawChannelId":…,"text":…, +// "content"? (content blocks, instead of text; [] included), "origin"?} +// {"op":"push","eventId":…,"origin"?,"tags"?,"text":…,"featureSet"?, +// "content"? (content blocks, instead of text; [] included)} // {"op":"rename","channelId":…,"label":…} (channels/changed) // {"op":"publish-mode","mode":"delivered"|"not-delivered"|"no-receipt"|"error"|"hang"} // {"op":"history","channelId":…,"history":[ChannelIncomingMessage…]} @@ -90,7 +92,7 @@ function pollCommands() { ...(c.origin ?? {}), }, tags: ['chat:dm', 'chat:addressed', 'chat:private'], - payload: { content: textBlock(c.text ?? '') }, + payload: { content: c.content ?? textBlock(c.text ?? '') }, }, }); } else if (c.op === 'push') { @@ -105,7 +107,7 @@ function pollCommands() { timestamp: c.timestamp ?? new Date().toISOString(), ...(c.origin ? { origin: c.origin } : {}), ...(c.tags ? { tags: c.tags } : {}), - payload: { content: textBlock(c.text ?? '') }, + payload: { content: c.content ?? textBlock(c.text ?? '') }, }, }); } else if (c.op === 'rename') { diff --git a/test/inbound-source.test.ts b/test/inbound-source.test.ts index 31e37ab95..5d3f3dafb 100644 --- a/test/inbound-source.test.ts +++ b/test/inbound-source.test.ts @@ -16,6 +16,8 @@ import { INBOUND_SOURCE_KEY, readInboundSource, conversationKey, + // The package's public digest: consumers import it from the root. + sourceBodyDigest, type InboundSource, type Module, type ModuleContext, @@ -27,7 +29,7 @@ import { type ToolResult, } from '../src/index.js'; import { MockMembrane } from './helpers/mock-membrane.js'; -import { sourceBodyDigest, canonicalJson } from '../src/mcpl/inbound-source.js'; +import { canonicalJson } from '../src/mcpl/inbound-source.js'; import { createHash } from 'node:crypto'; import { fixture, eventually, TS } from './helpers/coalescing-fixture.js'; @@ -223,6 +225,39 @@ describe('inbound source envelope', () => { assert.notEqual(meta.storedBodyDigest, meta.sourceBodyDigest, 'the closed-channel invitation decorates the stored copy'); }); + it('an ordinary empty push is stored with the host\'s digests of its empty body, never the adapter\'s', async () => { + // Only a silent heartbeat stores nothing (test/silent-heartbeat.test.ts); + // any other push is stored, an empty one included, so it is stamped like + // any other — over whatever its origin claimed. + command({ + op: 'push', eventId: 'empty-1', content: [], + origin: { source: 'timer', sourceBodyDigest: 'adapter-value', storedBodyDigest: 'adapter-value' }, + }); + await waitFor(() => !!storedWith((m) => m.eventId === 'empty-1'), 'empty push stored'); + const message = storedWith((m) => m.eventId === 'empty-1')!; + const meta = message.metadata as Record; + assert.deepEqual(message.content, []); + // The empty body in the one-element framing: SHA-256 of `[[]]`. + const empty = createHash('sha256').update('[[]]').digest('hex'); + assert.equal(meta.sourceBodyDigest, empty, 'the empty delivered body'); + assert.equal(meta.storedBodyDigest, empty, 'stored undecorated'); + assert.equal(meta.storedBodyDigest, sourceBodyDigest(message.content), 'the untouched copy matches its own witness'); + }); + + it('an empty DM pushed into a closed channel: the delivered digest is the empty body, the stored digest covers the invitation', async () => { + command({ + op: 'dm', eventId: 'ev-empty', authorId: '134', authorName: 'antra', rawChannelId: RAW_DM, content: [], + origin: { sourceBodyDigest: 'adapter-value', storedBodyDigest: 'adapter-value' }, + }); + await waitFor(() => !!storedWith((m) => m.eventId === 'ev-empty'), 'empty dm stored'); + const message = storedWith((m) => m.eventId === 'ev-empty')!; + const meta = message.metadata as Record; + assert.equal(meta.channelInvitation, true, 'stored with the closed-channel invitation'); + assert.equal(message.content.length, 1, 'the invitation alone'); + assert.equal(meta.sourceBodyDigest, sourceBodyDigest([]), 'the empty delivered body, not the adapter\'s value'); + assert.equal(meta.storedBodyDigest, sourceBodyDigest(message.content), 'exactly what was stored, not the adapter\'s value'); + }); + it('keeps a stored envelope when the channel is renamed later', async () => { command({ op: 'incoming', channelId: ROOM, messageId: 'm-2', mode: 'ambient', text: 'before rename' }); await waitFor(() => !!storedWith((m) => m.messageId === 'm-2'), 'first message stored'); From 10380ddfae09151b4469b4697ae97fa6f5b79d98 Mon Sep 17 00:00:00 2001 From: Petra-1012 Date: Wed, 7 Oct 2026 05:45:51 -0400 Subject: [PATCH 0056/1112] feat(mcpl): type MCPL request failures by what reached the server MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A failed request to an MCPL server used to reject with a plain Error, so the only way to tell "never written" from "written and unanswered" from "answered with an error" was to match message text. Callers that act in the world need that difference as a fact: a publish or a reaction that was never sent left the world as it was, while one that was handed to the transport and never answered may have landed. (A rollback reaction drain met exactly this: its pending reactions failed at shutdown, and only their message text said whether each had been written.) McplRequestError extends Error with `outcome`: - `not-sent`: sendRequest refused before writing (connection already closed); - `error-response`: a JSON-RPC error response; `code` and `data` kept verbatim (a connector can describe a partial multi-part post there); - `no-response`: the request was handed to the transport and no answer came (per-request timeout, a deliberate close while awaiting, or an unexpected disconnect). It does not prove the request reached the server — the transport can decline or drop a write — only that the host can no longer say it didn't. Messages are byte-identical to before, and every rejection is still an Error. Exported from the mcpl index and the package root. Tests (test/mcpl-request-outcome.test.ts, a real stdio server): an error response keeps code and data; an unanswered request and a process exit while awaiting are no-response; a request on a closed connection is not-sent; the class keeps Error identity. 12/12 with the existing request-timeout tests. Reviewed by Tessa-974 (room-203 #43780), whose wording correction for no-response this carries. Shared with room-225's lane 1 (Ada-1017). Co-Authored-By: Claude Opus 5.5 --- changelog.d/mcpl-request-outcome.added.md | 10 +++ src/index.ts | 2 +- src/mcpl/index.ts | 2 +- src/mcpl/server-connection.ts | 45 +++++++++-- test/mcpl-request-outcome.test.ts | 92 +++++++++++++++++++++++ 5 files changed, 144 insertions(+), 7 deletions(-) create mode 100644 changelog.d/mcpl-request-outcome.added.md create mode 100644 test/mcpl-request-outcome.test.ts diff --git a/changelog.d/mcpl-request-outcome.added.md b/changelog.d/mcpl-request-outcome.added.md new file mode 100644 index 000000000..241ec226e --- /dev/null +++ b/changelog.d/mcpl-request-outcome.added.md @@ -0,0 +1,10 @@ +- **`McplRequestError`: why an MCPL request produced no result.** A request + to an MCPL server that fails now rejects with `McplRequestError` (exported), + whose `outcome` says which of three things happened: `not-sent` (refused + before anything was written: the connection was already closed), + `error-response` (the server answered with a JSON-RPC error; `code` and the + error's `data` are kept), or `no-response` (the request was handed to the + transport and no answer came back: a timeout, or the connection closed + while awaiting; it may or may not have reached the server). Only + `not-sent` proves the server never saw the request. Messages are unchanged, + and it is still an `Error`, so existing handling keeps working. diff --git a/src/index.ts b/src/index.ts index adfab80b2..6baa23fab 100644 --- a/src/index.ts +++ b/src/index.ts @@ -90,7 +90,7 @@ export type { McplServerConfig } from './mcpl/index.js'; // MCPL client connection (exposed so external tooling — server playtests, // health probes, protocol harnesses — can dial an MCPL server exactly the // way the framework host does, over stdio or WebSocket) -export { McplServerConnection } from './mcpl/index.js'; +export { McplServerConnection, McplRequestError } from './mcpl/index.js'; export type { McplHostCapabilities } from './mcpl/index.js'; // MCPL tool lifecycle (RFC-007) and tool classes (RFC-008): the types a host diff --git a/src/mcpl/index.ts b/src/mcpl/index.ts index d82790c67..9033077ec 100644 --- a/src/mcpl/index.ts +++ b/src/mcpl/index.ts @@ -106,7 +106,7 @@ export { } from './errors.js'; // Server connection and registry -export { McplServerConnection } from './server-connection.js'; +export { McplServerConnection, McplRequestError } from './server-connection.js'; export { McplServerRegistry, type McplCapabilityQuery } from './server-registry.js'; // Feature set management (permission layer) diff --git a/src/mcpl/server-connection.ts b/src/mcpl/server-connection.ts index f29fea22d..18f82d096 100644 --- a/src/mcpl/server-connection.ts +++ b/src/mcpl/server-connection.ts @@ -62,6 +62,34 @@ const DEFAULT_REQUEST_TIMEOUT_MS = 60_000; /** MCP protocol version used in the initialize handshake. */ const MCP_PROTOCOL_VERSION = '2024-11-05'; +/** + * Why a request to an MCPL server produced no result, as far as the host can + * know it. Messages are unchanged from plain errors; the `outcome` makes the + * difference between "never left the host" and "left, and no answer came + * back" a fact a caller can act on instead of a string to match: + * - `not-sent`: the request was refused before anything was written (the + * connection was already closed); + * - `error-response`: the server answered with a JSON-RPC error (it may have + * acted before failing; the error alone does not say); + * - `no-response`: the request was handed to the transport and no answer + * came back (timeout, or the connection closed while awaiting). It does + * not prove the request reached the server — the transport can decline or + * drop a write — only that the host can no longer say it didn't. + */ +export class McplRequestError extends Error { + constructor( + message: string, + readonly outcome: 'not-sent' | 'error-response' | 'no-response', + readonly code?: number, + /** The JSON-RPC error's `data`, verbatim, when the server supplied one + * (e.g. which parts of a multi-part send were posted). */ + readonly data?: unknown, + ) { + super(message); + this.name = 'McplRequestError'; + } +} + /** * Represents a pending JSON-RPC request awaiting a response. */ @@ -846,7 +874,7 @@ export class McplServerConnection extends EventEmitter { // Reject all pending requests for (const [id, pending] of this.pendingRequests) { if (pending.timer) clearTimeout(pending.timer); - pending.reject(new Error(`Connection to MCPL server "${this.id}" closed while awaiting response for ${pending.method} (id=${id})`)); + pending.reject(new McplRequestError(`Connection to MCPL server "${this.id}" closed while awaiting response for ${pending.method} (id=${id})`, 'no-response')); } this.pendingRequests.clear(); this.orphanedRequests.clear(); @@ -973,7 +1001,7 @@ export class McplServerConnection extends EventEmitter { options: RequestOptions = {}, ): Promise { if (this.closed) { - return Promise.reject(new Error(`Cannot send request: connection to "${this.id}" is closed`)); + return Promise.reject(new McplRequestError(`Cannot send request: connection to "${this.id}" is closed`, 'not-sent')); } const id = this.nextRequestId++; @@ -994,13 +1022,14 @@ export class McplServerConnection extends EventEmitter { this.orphanedRequests.set(id, { method, ...(method === 'push/render' ? { renderParams: { featureSet: params.featureSet, key: params.key, eventId: params.eventId } } : {}) }); if (this.orphanedRequests.size > 4096) this.orphanedRequests.delete(this.orphanedRequests.keys().next().value!); } - reject(new Error( + reject(new McplRequestError( `MCPL server "${this.id}" did not respond to ${method} (id=${id}) ` + `within ${timeoutMs}ms — the server may be hung. The response ` + `outcome is unknown: the request was abandoned locally but was not ` + `cancelled, so the tool may still have completed server-side; verify ` + `state before retrying; a blind retry of a stateful/side-effecting ` + `tool may duplicate it.`, + 'no-response', )); }, timeoutMs); // Don't hold the event loop open for the watchdog alone. @@ -1215,7 +1244,12 @@ export class McplServerConnection extends EventEmitter { if (response.error) { pending.reject( - new Error(`MCPL server "${this.id}" returned error for ${pending.method}: [${response.error.code}] ${response.error.message}`), + new McplRequestError( + `MCPL server "${this.id}" returned error for ${pending.method}: [${response.error.code}] ${response.error.message}`, + 'error-response', + typeof response.error.code === 'number' ? response.error.code : undefined, + (response.error as { data?: unknown }).data, + ), ); } else { pending.resolve(response.result); @@ -1267,8 +1301,9 @@ export class McplServerConnection extends EventEmitter { for (const [id, pending] of this.pendingRequests) { if (pending.timer) clearTimeout(pending.timer); pending.reject( - new Error( + new McplRequestError( `MCPL server "${this.id}" disconnected unexpectedly (code=${info.code ?? 'n/a'}, signal=${info.signal ?? 'n/a'}, reason=${info.reason ?? 'unknown'}) while awaiting ${pending.method} (id=${id})`, + 'no-response', ), ); } diff --git a/test/mcpl-request-outcome.test.ts b/test/mcpl-request-outcome.test.ts new file mode 100644 index 000000000..1b0f45647 --- /dev/null +++ b/test/mcpl-request-outcome.test.ts @@ -0,0 +1,92 @@ +/** + * McplRequestError: a request that produced no result says which of three + * things happened, as a fact rather than a message to parse. + * not-sent refused before anything was written (connection closed) + * error-response the server answered with a JSON-RPC error (code, data kept) + * no-response the request was handed to the transport and no answer + * came back (timeout, or the connection closed while + * awaiting): it may or may not have reached the server + * Callers that act in the world (publishing speech, placing reactions) need + * the difference: only `not-sent` proves the server never saw the request. + */ +import { test, afterEach } from 'node:test'; +import assert from 'node:assert/strict'; + +import { McplServerConnection, McplRequestError } from '../src/mcpl/server-connection.js'; +import type { McplHostCapabilities, McplServerConfig } from '../src/mcpl/types.js'; + +// Stdio server: answers initialize; tools/call 'fail' gets a JSON-RPC error +// with data, 'stuck' gets nothing, 'exit' kills the process mid-request. +const SERVER = ` +let buf = ''; +process.stdin.on('data', (c) => { + buf += c.toString('utf8'); + let i; + while ((i = buf.indexOf('\\n')) >= 0) { + const line = buf.slice(0, i); buf = buf.slice(i + 1); + if (!line.trim()) continue; + let m; try { m = JSON.parse(line); } catch { continue; } + const out = (o) => process.stdout.write(JSON.stringify({ jsonrpc: '2.0', id: m.id, ...o }) + '\\n'); + if (m.method === 'initialize') out({ result: { capabilities: {} } }); + else if (m.method === 'tools/call' && m.params.name === 'fail') { + out({ error: { code: -32001, message: 'POSTED: part 1 of 2', data: { posted: ['111'] } } }); + } else if (m.method === 'tools/call' && m.params.name === 'exit') { + process.exit(3); + } + // 'stuck': deliberately no response. + } +}); +setInterval(() => {}, 1 << 30); +`; + +const HOST_CAPS: McplHostCapabilities = { version: '0.4', pushEvents: true, contextHooks: { beforeInference: true }, featureSets: true }; +const config = (overrides?: Partial): McplServerConfig => + ({ id: 'outcome', command: process.execPath, args: ['-e', SERVER], ...overrides }); + +let connection: McplServerConnection | null = null; +afterEach(async () => { + await connection?.close(); + connection = null; +}); + +const rejectsWith = async (promise: Promise, outcome: McplRequestError['outcome']): Promise => { + let caught: unknown; + await assert.rejects(promise, (err: unknown) => { caught = err; return true; }); + assert.ok(caught instanceof McplRequestError, `expected McplRequestError, got ${String(caught)}`); + assert.equal((caught as McplRequestError).outcome, outcome); + return caught as McplRequestError; +}; + +test('an error response is error-response, with its code and data kept', async () => { + connection = await McplServerConnection.connect(config(), HOST_CAPS); + const err = await rejectsWith(connection.sendToolsCall('fail', {}), 'error-response'); + assert.equal(err.code, -32001); + assert.deepEqual(err.data, { posted: ['111'] }); + assert.match(err.message, /returned error for tools\/call: \[-32001\] POSTED: part 1 of 2/); +}); + +test('a request nobody answers is no-response, and the message is unchanged', async () => { + connection = await McplServerConnection.connect(config({ requestTimeoutMs: 200 }), HOST_CAPS); + const err = await rejectsWith(connection.sendToolsCall('stuck', {}), 'no-response'); + assert.match(err.message, /did not respond to tools\/call/); +}); + +test('a connection lost while awaiting is no-response', async () => { + connection = await McplServerConnection.connect(config({ requestTimeoutMs: 10_000 }), HOST_CAPS); + await rejectsWith(connection.sendToolsCall('exit', {}), 'no-response'); +}); + +test('a request on a closed connection is not-sent', async () => { + connection = await McplServerConnection.connect(config(), HOST_CAPS); + await connection.close(); + const err = await rejectsWith(connection.sendToolsCall('fail', {}), 'not-sent'); + assert.match(err.message, /Cannot send request: connection to "outcome" is closed/); + connection = null; +}); + +test('McplRequestError is an Error that keeps its message and name', () => { + const err = new McplRequestError('boom', 'not-sent'); + assert.ok(err instanceof Error); + assert.equal(err.name, 'McplRequestError'); + assert.equal(err.message, 'boom'); +}); From 2c9aff13402ea872572a49236906a576e7c77445 Mon Sep 17 00:00:00 2001 From: Olive-1804 Date: Thu, 8 Oct 2026 01:11:39 -0400 Subject: [PATCH 0057/1112] feat(mcp): legacy engine's protocol boundary, family selection, paginated tools/list First batch of room-284's modern-MCP client work (shelf-487; design in personal:olive-1804 #52935, revised #53069, implementation authorized #53087). Protocol family and transport are decided from configuration alone (src/mcpl/protocol-family.ts): a URL's scheme decides a network server (ws/wss = legacy MCPL, http/https = modern MCP 2026-07-28); only a stdio server has a choice, `protocol: 'legacy' | 'modern'`, defaulting to legacy so every existing stdio config opens exactly as before. No cross-family probing: the spec's discover-first probe makes real MCPL servers exit (heartbeat-mcpl), and the SDK's stdio probe launches every server twice. serverConfigProblems() adds checks only for what the modern family brought: protocol on a URL, a modern requestTimeoutMs outside 1..2^31-1, and effective MCPL-only policy on a modern server. Every legacy shape still passes. Legacy engine (McplServerConnection): - Offers and now requires MCP 2024-11-05. Another returned revision, or none, is McplProtocolVersionError kind 'mismatch'. An initialize answered -32022 is kind 'rejected', keeping the server's supported list, code and data, with a protocol: 'modern' hint when the list names modern MCP. The established revision is on connection.protocolVersion. - A version verdict is permanent: connectWithReconnect makes no retry stub, and attemptReconnect stops its loop and emits reconnect-failed with permanent: true. The framework raises the ops alert at once for it, since no fifth attempt will come, and its connect-failed trace says willRetry: false. - tools/list follows nextCursor to the complete inventory. A repeated cursor, or more than 100 pages, is an error. A single-page answer is returned exactly as before, other fields included (mcpl-capability-scoping relies on that). - A modern config is refused before anything is dialed. Carries Petra-1012's McplRequestError commit (7144970, cherry-picked as 10380dd) as the shared request-outcome contract, per Tessa (room-203 #53124). Test stubs that answered initialize without protocolVersion now send 2024-11-05: six existing files, plus mcpl-request-outcome.test.ts from the carried commit. Every probed real server family (TS SDK 1.x, Python FastMCP 1.x/2.x, SDK 2 dual-era, discord/heartbeat/slack/spend MCPL servers) already sends it. The changelog fragment names the hand-written-server consequence. Tests: mcp-protocol-family (6), mcpl-legacy-version (10). Full suite 1397/1397 (baseline 1377 + 4 carried + 16 new). --- .../legacy-mcp-protocol-boundary.breaking.md | 17 ++ src/framework.ts | 15 +- src/index.ts | 10 +- src/mcpl/index.ts | 12 +- src/mcpl/protocol-family.ts | 160 +++++++++++++++++ src/mcpl/server-connection.ts | 163 ++++++++++++++++-- src/mcpl/types.ts | 29 +++- .../awareness-barrier-mcpl-server.mjs | 1 + test/fixtures/legacy-version-server.mjs | 65 +++++++ test/host-quiesce.test.ts | 2 +- test/mcp-protocol-family.test.ts | 103 +++++++++++ test/mcpl-capability-scoping.test.ts | 2 +- test/mcpl-legacy-version.test.ts | 155 +++++++++++++++++ test/mcpl-request-outcome.test.ts | 2 +- test/mcpl-request-timeout.test.ts | 2 +- test/mcpl-runtime-lifecycle.test.ts | 4 +- test/mcpl-ws-transport.test.ts | 4 +- 17 files changed, 710 insertions(+), 36 deletions(-) create mode 100644 changelog.d/legacy-mcp-protocol-boundary.breaking.md create mode 100644 src/mcpl/protocol-family.ts create mode 100644 test/fixtures/legacy-version-server.mjs create mode 100644 test/mcp-protocol-family.test.ts create mode 100644 test/mcpl-legacy-version.test.ts diff --git a/changelog.d/legacy-mcp-protocol-boundary.breaking.md b/changelog.d/legacy-mcp-protocol-boundary.breaking.md new file mode 100644 index 000000000..92d319362 --- /dev/null +++ b/changelog.d/legacy-mcp-protocol-boundary.breaking.md @@ -0,0 +1,17 @@ +- **MCPL/MCP server authors:** the legacy engine now enforces the revision it + speaks. `McplServerConnection` offers MCP `2024-11-05` and now also requires + it back: an `initialize` result with any other `protocolVersion`, or with + none, fails as `McplProtocolVersionError` (`kind: 'mismatch'`), stating what + each side said. Every server family probed answers `2024-11-05`, but a + hand-written server that omits the field must add it. An `initialize` + answered with JSON-RPC `-32022` fails as `McplProtocolVersionError` + (`kind: 'rejected'`), keeping the server's `supported` list, code and data. + When that list names modern MCP, the message says to configure the server + with `protocol: 'modern'`. A protocol-version verdict is never retried: no + reconnect stub, and a reconnecting connection stops its backoff loop + (`reconnect-failed` carries `permanent: true` and raises the ops alert at + once). The established revision is on `connection.protocolVersion`. +- `tools/list` now follows `nextCursor` across pages, so a paginating server's + whole inventory reaches the agent; before, every page after the first was + silently dropped. A repeated cursor, or more than 100 pages, is an error + rather than a partial inventory. diff --git a/src/framework.ts b/src/framework.ts index 349bdd693..3e5188f72 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -391,6 +391,7 @@ import { EventGate, formatShadowWarning } from './gate/event-gate.js'; import { UsageTracker, type PersistedUsageState } from './usage/usage-tracker.js'; import type { SessionUsageSnapshot, UsageUpdatedEvent } from './usage/types.js'; import type { McplServerConnection } from './mcpl/server-connection.js'; +import { McplProtocolVersionError } from './mcpl/server-connection.js'; import type { McplServerConfig, McplHostCapabilities, @@ -13311,7 +13312,9 @@ export class AgentFramework { serverId: config.id, error: err.message, attempt: 0, - willRetry: config.reconnect === true, + // A protocol-version verdict is never retried (no common revision; + // only configuration can fix it). + willRetry: config.reconnect === true && !(error instanceof McplProtocolVersionError), }); } } @@ -14485,7 +14488,7 @@ export class AgentFramework { willRetry: connection.willReconnect, }); }); - connection.on('reconnect-failed', (params: { error: string; attempt: number }) => { + connection.on('reconnect-failed', (params: { error: string; attempt: number; permanent?: boolean }) => { this.emitTrace({ type: 'mcpl:server-connect-failed', serverId: connection.id, @@ -14493,11 +14496,13 @@ export class AgentFramework { attempt: params.attempt, willRetry: connection.willReconnect, }); - // The reconnect loop never gives up (backoff caps at ~300s), so - // "the server is effectively down" is an attempt-count judgment: + // The reconnect loop gives up only on a protocol-version verdict + // (`permanent`), which no later attempt can change — alert at once, + // since no fifth attempt will come. Otherwise backoff caps at ~300s, + // so "the server is effectively down" is an attempt-count judgment: // 5 failed attempts ≈ a few minutes of outage. Throttled per // (serverId, kind), so a long outage re-posts every ~15 min. - if (params.attempt >= 5) { + if (params.permanent || params.attempt >= 5) { this.opsAlert( 'mcpl-down', connection.id, diff --git a/src/index.ts b/src/index.ts index 6baa23fab..9e6b0073f 100644 --- a/src/index.ts +++ b/src/index.ts @@ -90,7 +90,15 @@ export type { McplServerConfig } from './mcpl/index.js'; // MCPL client connection (exposed so external tooling — server playtests, // health probes, protocol harnesses — can dial an MCPL server exactly the // way the framework host does, over stdio or WebSocket) -export { McplServerConnection, McplRequestError } from './mcpl/index.js'; +export { McplServerConnection, McplRequestError, McplProtocolVersionError } from './mcpl/index.js'; +export { + LEGACY_MCP_PROTOCOL_VERSION, + MODERN_MCP_PROTOCOL_VERSION, + resolveServerBinding, + serverConfigProblems, + checkServerConfig, +} from './mcpl/index.js'; +export type { McpProtocolFamily, McpTransportKind, ServerBinding } from './mcpl/index.js'; export type { McplHostCapabilities } from './mcpl/index.js'; // MCPL tool lifecycle (RFC-007) and tool classes (RFC-008): the types a host diff --git a/src/mcpl/index.ts b/src/mcpl/index.ts index 9033077ec..c3bc7ef31 100644 --- a/src/mcpl/index.ts +++ b/src/mcpl/index.ts @@ -106,7 +106,17 @@ export { } from './errors.js'; // Server connection and registry -export { McplServerConnection, McplRequestError } from './server-connection.js'; +export { McplServerConnection, McplRequestError, McplProtocolVersionError } from './server-connection.js'; +export { + LEGACY_MCP_PROTOCOL_VERSION, + MODERN_MCP_PROTOCOL_VERSION, + MAX_TIMER_MS, + MCPL_ONLY_POLICY_FIELDS, + resolveServerBinding, + serverConfigProblems, + checkServerConfig, +} from './protocol-family.js'; +export type { McpProtocolFamily, McpTransportKind, ServerBinding } from './protocol-family.js'; export { McplServerRegistry, type McplCapabilityQuery } from './server-registry.js'; // Feature set management (permission layer) diff --git a/src/mcpl/protocol-family.ts b/src/mcpl/protocol-family.ts new file mode 100644 index 000000000..5037d1205 --- /dev/null +++ b/src/mcpl/protocol-family.ts @@ -0,0 +1,160 @@ +/** + * Which MCP protocol family a configured server speaks, and over which + * transport. Both are decided from the configuration alone, before anything + * is spawned or dialed: there is no cross-family probing and no fallback. + * + * - **Legacy**: MCP 2024-11-05, with MCPL riding on it as an experimental + * capability. `McplServerConnection` speaks it, over stdio or MCPL's + * WebSocket binding. + * - **Modern**: MCP 2026-07-28, which is stateless. It has no `initialize`; + * every request carries its version and client capabilities in `_meta`, and + * server-to-client requests are replaced by in-band continuation. + * `ModernMcpConnection` speaks it, over stdio or Streamable HTTP. + * + * A URL's scheme decides a network server's family: `ws://`/`wss://` is + * legacy, `http://`/`https://` is modern, because neither family has the + * other's binding. A stdio (`command`) server can be either. It is legacy + * unless its config says `protocol: 'modern'`, so every existing stdio + * configuration opens exactly as before. + * + * Why no auto-detection: the modern spec's discover-first probe makes some + * real MCPL servers exit (heartbeat-mcpl closes on any first message that + * isn't `initialize`), the SDK's stdio probe launches the server twice, and a + * server's era is a stable fact its configuration can state. + */ +import type { McplServerConfig } from './types.js'; + +/** The one revision the legacy engine offers and accepts. */ +export const LEGACY_MCP_PROTOCOL_VERSION = '2024-11-05'; +/** The revision the modern engine pins. */ +export const MODERN_MCP_PROTOCOL_VERSION = '2026-07-28'; + +export type McpProtocolFamily = 'legacy' | 'modern'; +export type McpTransportKind = 'stdio' | 'websocket' | 'http'; + +export interface ServerBinding { + family: McpProtocolFamily; + transport: McpTransportKind; +} + +/** The largest delay a Node/Bun timer honors (2³¹−1 ms, about 24.8 days). */ +export const MAX_TIMER_MS = 2_147_483_647; + +/** + * Configuration fields that are MCPL policy: they have no meaning for a modern + * server, which cannot negotiate MCPL. Setting one there is rejected rather + * than ignored, so an operator who chose a policy learns that it cannot apply. + */ +export const MCPL_ONLY_POLICY_FIELDS = [ + 'enabledFeatureSets', + 'disabledFeatureSets', + 'enabledCapabilities', + 'disabledCapabilities', + 'scopes', + 'channelSubscription', + 'allowHostCommands', + 'toolLifecycle', + 'autofetch', + 'shouldTriggerInference', +] as const satisfies readonly (keyof McplServerConfig)[]; + +type BindingFields = Pick; + +function urlScheme(config: BindingFields): string { + let parsed: URL; + try { + parsed = new URL(config.url!); + } catch { + throw new Error(`MCP server "${config.id}": invalid url "${config.url}"`); + } + return parsed.protocol; +} + +/** + * Resolve a server's family and transport. Throws on a configuration that + * names no usable transport (for example an `ftp://` URL, or + * `transport: 'http'` with a WebSocket URL). + */ +export function resolveServerBinding(config: BindingFields): ServerBinding { + const networkKind = (): McpTransportKind => { + const scheme = urlScheme(config); + if (scheme === 'ws:' || scheme === 'wss:') return 'websocket'; + if (scheme === 'http:' || scheme === 'https:') return 'http'; + throw new Error( + `MCP server "${config.id}": url must be ws:// or wss:// (legacy MCPL) or http:// or https:// (modern MCP), got "${scheme}"`, + ); + }; + + let transport: McpTransportKind; + if (config.transport === 'stdio' || (config.transport === undefined && config.command)) { + if (!config.command) { + throw new Error(`MCP server "${config.id}": the stdio transport requires "command"`); + } + transport = 'stdio'; + } else if (config.url) { + transport = networkKind(); + if (config.transport !== undefined && config.transport !== transport) { + throw new Error( + `MCP server "${config.id}": transport "${config.transport}" does not match url "${config.url}"`, + ); + } + } else { + throw new Error(`MCP server "${config.id}": needs "command" (stdio) or "url" (WebSocket or HTTP)`); + } + + if (transport === 'stdio') return { family: config.protocol ?? 'legacy', transport }; + return { family: transport === 'http' ? 'modern' : 'legacy', transport }; +} + +/** A configured value that would take effect, as opposed to an empty default. */ +function isEffective(value: unknown): boolean { + if (value === undefined || value === null || value === false) return false; + if (Array.isArray(value)) return value.length > 0; + if (typeof value === 'object') return Object.keys(value as object).length > 0; + return true; +} + +/** + * Problems with a server configuration's protocol settings, as messages; empty + * when it is usable. Legacy configurations are checked only for what this + * design added, so every configuration that worked before still passes. + */ +export function serverConfigProblems(config: McplServerConfig): string[] { + let binding: ServerBinding; + try { + binding = resolveServerBinding(config); + } catch (error) { + return [(error as Error).message]; + } + const problems: string[] = []; + if (config.protocol !== undefined) { + if (config.protocol !== 'legacy' && config.protocol !== 'modern') { + problems.push(`MCP server "${config.id}": protocol must be 'legacy' or 'modern', got ${JSON.stringify(config.protocol)}`); + } else if (binding.transport !== 'stdio') { + problems.push( + `MCP server "${config.id}": "protocol" applies only to stdio (command) servers; the url's scheme decides a network server's family`, + ); + } + } + if (binding.family === 'modern') { + const timeout = config.requestTimeoutMs; + if (timeout !== undefined && !(Number.isInteger(timeout) && timeout >= 1 && timeout <= MAX_TIMER_MS)) { + problems.push( + `MCP server "${config.id}": requestTimeoutMs must be an integer from 1 to ${MAX_TIMER_MS} for a modern server (got ${String(timeout)}); every modern call has a deadline`, + ); + } + for (const field of MCPL_ONLY_POLICY_FIELDS) { + if (isEffective(config[field])) { + problems.push(`MCP server "${config.id}": "${field}" is MCPL policy, which a modern MCP server cannot negotiate`); + } + } + } + return problems; +} + +/** Throw one error listing every problem, or return the resolved binding. */ +export function checkServerConfig(config: McplServerConfig): ServerBinding { + const problems = serverConfigProblems(config); + if (problems.length > 0) throw new Error(problems.join('; ')); + return resolveServerBinding(config); +} diff --git a/src/mcpl/server-connection.ts b/src/mcpl/server-connection.ts index 18f82d096..0a13dcf0a 100644 --- a/src/mcpl/server-connection.ts +++ b/src/mcpl/server-connection.ts @@ -15,6 +15,11 @@ import { openTransport, type McplTransport, type TransportCloseInfo } from './tr import { maskNegotiatedCapabilities } from './capability-mask.js'; import { CapabilityGrant, expandAdvertisementShorthand } from './capability-grant.js'; import { CAPABILITY_DISABLED } from './errors.js'; +import { + LEGACY_MCP_PROTOCOL_VERSION, + MODERN_MCP_PROTOCOL_VERSION, + resolveServerBinding, +} from './protocol-family.js'; import type { McplServerConfig, @@ -59,8 +64,67 @@ const INITIALIZE_TIMEOUT_MS = 30_000; * tool work while still bounding the hang. */ const DEFAULT_REQUEST_TIMEOUT_MS = 60_000; -/** MCP protocol version used in the initialize handshake. */ -const MCP_PROTOCOL_VERSION = '2024-11-05'; +/** MCP protocol version used in the initialize handshake: the only revision + * this legacy engine offers or accepts. */ +const MCP_PROTOCOL_VERSION = LEGACY_MCP_PROTOCOL_VERSION; + +/** JSON-RPC code for an unsupported protocol version (MCP 2026-07-28 §basic, + * the code a modern-only server answers our `initialize` with). */ +const UNSUPPORTED_PROTOCOL_VERSION_CODE = -32022; + +/** Upper bound on `tools/list` pages, against a server whose cursors never + * end. Generous: real inventories fit in a handful of pages. */ +const MAX_TOOL_LIST_PAGES = 100; + +/** + * The server and this legacy engine have no protocol revision in common, so + * the connection was never established. This is a protocol-negotiation fact, + * not a request outcome. Retrying cannot change it: the engine offers one + * revision, and only configuration can fix the mismatch. A connection that + * gets this error does not schedule reconnects. + * + * - `rejected`: the server answered `initialize` with error -32022. + * `supported` is the server's own list, when it gave one. + * - `mismatch`: the server answered `initialize` with a revision other than + * 2024-11-05, or with none. + */ +export class McplProtocolVersionError extends Error { + readonly offered = MCP_PROTOCOL_VERSION; + constructor( + message: string, + readonly kind: 'rejected' | 'mismatch', + /** Revision the server answered with (`mismatch`; null when it gave none). */ + readonly returned: string | null, + /** Revisions the server says it supports (`rejected`, when stated). */ + readonly supported: readonly string[] | null, + /** The JSON-RPC error's code and data, verbatim (`rejected`). */ + readonly code?: number, + readonly data?: unknown, + ) { + super(message); + this.name = 'McplProtocolVersionError'; + } +} + +function protocolVersionRejection(id: string, error: { code: number; message: string; data?: unknown }): McplProtocolVersionError { + const data = error.data as { supported?: unknown } | undefined; + const supported = Array.isArray(data?.supported) && data.supported.every((v) => typeof v === 'string') + ? (data.supported as string[]) + : null; + const modernHint = supported?.includes(MODERN_MCP_PROTOCOL_VERSION) + ? ` It speaks modern MCP: configure it with protocol: 'modern' (stdio) or reach it over http(s).` + : ''; + return new McplProtocolVersionError( + `MCPL server "${id}" does not support MCP ${MCP_PROTOCOL_VERSION}` + + (supported ? ` (it supports: ${supported.join(', ') || 'none listed'}).` : ` (${error.message}).`) + + modernHint, + 'rejected', + null, + supported, + error.code, + error.data, + ); +} /** * Why a request to an MCPL server produced no result, as far as the host can @@ -144,6 +208,10 @@ export class McplServerConnection extends EventEmitter { * the sole source of the `tools` capability path). */ mcpToolsAdvertised = false; + /** The MCP revision established by the last successful handshake (always + * 2024-11-05 for this engine), or null before the first one. */ + protocolVersion: string | null = null; + /** Host-owned per-server authority for host/command (config * allowHostCommands, default false). No capability path exists and the * grant cannot confer it — server params never decide admin authority. */ @@ -499,11 +567,12 @@ export class McplServerConnection extends EventEmitter { config: McplServerConfig, hostCapabilities: McplHostCapabilities, ): Promise { - const { transport, capabilities, droppedCapabilities, mcpToolsAdvertised } = await McplServerConnection.handshake(config, hostCapabilities); + const { transport, capabilities, droppedCapabilities, mcpToolsAdvertised, protocolVersion } = await McplServerConnection.handshake(config, hostCapabilities); const connection = new McplServerConnection(config.id, capabilities, transport); connection.droppedCapabilities = droppedCapabilities; connection.mcpToolsAdvertised = mcpToolsAdvertised; + connection.protocolVersion = protocolVersion; connection.allowHostCommands = config.allowHostCommands === true; connection.requestTimeoutMs = config.requestTimeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS; @@ -529,7 +598,19 @@ export class McplServerConnection extends EventEmitter { private static async handshake( config: McplServerConfig, hostCapabilities: McplHostCapabilities, - ): Promise<{ transport: McplTransport; capabilities: McplCapabilities | null; droppedCapabilities: ReadonlySet; mcpToolsAdvertised: boolean }> { + ): Promise<{ transport: McplTransport; capabilities: McplCapabilities | null; droppedCapabilities: ReadonlySet; mcpToolsAdvertised: boolean; protocolVersion: string }> { + // This engine speaks the legacy family only. A configuration that + // resolves to the modern family belongs to ModernMcpConnection; refuse it + // here rather than dialing an http(s) url as a WebSocket. A configuration + // that doesn't resolve at all keeps openTransport's existing errors. + let family: string | null = null; + try { family = resolveServerBinding(config).family; } catch { /* openTransport reports it */ } + if (family === 'modern') { + throw new Error( + `MCP server "${config.id}" is configured for modern MCP (${MODERN_MCP_PROTOCOL_VERSION}); ` + + `McplServerConnection speaks the legacy family (${MCP_PROTOCOL_VERSION} + MCPL) only`, + ); + } const transport = await openTransport(config); try { @@ -547,7 +628,7 @@ export class McplServerConnection extends EventEmitter { // Await the initialize response, racing an early transport close/error and // a timeout. All three paths clean up their listeners. - const { mcpl: rawCapabilities, mcpToolsAdvertised } = await new Promise<{ mcpl: McplCapabilities | null; mcpToolsAdvertised: boolean }>((resolve, reject) => { + const { mcpl: rawCapabilities, mcpToolsAdvertised, protocolVersion } = await new Promise<{ mcpl: McplCapabilities | null; mcpToolsAdvertised: boolean; protocolVersion: string }>((resolve, reject) => { const cleanup = () => { transport.off('line', onLine); transport.off('close', onClose); @@ -564,10 +645,26 @@ export class McplServerConnection extends EventEmitter { if (msg.id !== initId) return; cleanup(); if (msg.error) { - reject(new Error(`MCPL server "${config.id}" initialize error: ${msg.error.message}`)); + reject(msg.error.code === UNSUPPORTED_PROTOCOL_VERSION_CODE + ? protocolVersionRejection(config.id, msg.error) + : new Error(`MCPL server "${config.id}" initialize error: ${msg.error.message}`)); return; } const result = msg.result as Record | undefined; + // The server's answer is the established revision. This engine + // implements exactly one, so any other answer (or none) means the + // two sides never agreed on what they're speaking. + const returned = typeof result?.protocolVersion === 'string' ? result.protocolVersion : null; + if (returned !== MCP_PROTOCOL_VERSION) { + reject(new McplProtocolVersionError( + `MCPL server "${config.id}" answered initialize with protocolVersion ` + + `${returned === null ? '(none)' : JSON.stringify(returned)}; this client's legacy engine speaks only ${MCP_PROTOCOL_VERSION}`, + 'mismatch', + returned, + null, + )); + return; + } const caps = result?.capabilities as Record | undefined; const experimental = caps?.experimental as Record | undefined; // §5.1: `tools` capability is the OUTER standard MCP member — the @@ -576,6 +673,7 @@ export class McplServerConnection extends EventEmitter { resolve({ mcpl: (experimental?.mcpl as McplCapabilities) ?? null, mcpToolsAdvertised: caps?.tools !== undefined, + protocolVersion: returned, }); }; const onClose = (info: TransportCloseInfo) => { @@ -613,7 +711,7 @@ export class McplServerConnection extends EventEmitter { console.error(`MCPL server "${config.id}" capabilities masked by host config: ${dropped.join(', ')}`); } - return { transport, capabilities: scoped, droppedCapabilities: new Set(dropped), mcpToolsAdvertised }; + return { transport, capabilities: scoped, droppedCapabilities: new Set(dropped), mcpToolsAdvertised, protocolVersion }; } catch (err) { // Never leak the child / socket if the handshake fails. await transport.close().catch(() => { /* best effort */ }); @@ -634,7 +732,9 @@ export class McplServerConnection extends EventEmitter { try { return await McplServerConnection.connect(config, hostCapabilities); } catch (error) { - if (!config.reconnect) { + // No common protocol revision is a configuration fact, not an outage: + // a retry stub would restart the server forever for nothing. + if (!config.reconnect || error instanceof McplProtocolVersionError) { throw error; } @@ -815,9 +915,38 @@ export class McplServerConnection extends EventEmitter { // Standard MCP methods // ========================================================================== - /** Send `tools/list` and return the server's tool definitions. */ - sendToolsList(): Promise<{ tools: McpToolDefinition[] }> { - return this.sendRequest('tools/list', {}) as Promise<{ tools: McpToolDefinition[] }>; + /** + * Send `tools/list` and return the server's complete tool inventory, + * following `nextCursor` across pages (2024-11-05 pagination). A server + * whose cursors don't terminate — a repeated cursor, or more than + * {@link MAX_TOOL_LIST_PAGES} pages — gets an error rather than a silently + * partial inventory. + */ + async sendToolsList(): Promise<{ tools: McpToolDefinition[] }> { + const tools: McpToolDefinition[] = []; + const seen = new Set(); + let cursor: string | undefined; + let first: Record | undefined; + for (let page = 0; page < MAX_TOOL_LIST_PAGES; page++) { + const result = await this.sendRequest('tools/list', cursor === undefined ? {} : { cursor }) as { + tools?: McpToolDefinition[]; + nextCursor?: unknown; + }; + first ??= result; + tools.push(...(result.tools ?? [])); + if (typeof result.nextCursor !== 'string' || result.nextCursor === '') { + // The first page's other fields are kept as they came, so a + // single-page answer is returned exactly as before. + const { nextCursor: _done, ...rest } = first as { nextCursor?: unknown }; + return { ...rest, tools }; + } + if (seen.has(result.nextCursor)) { + throw new Error(`MCPL server "${this.id}" repeated tools/list cursor ${JSON.stringify(result.nextCursor)}`); + } + seen.add(result.nextCursor); + cursor = result.nextCursor; + } + throw new Error(`MCPL server "${this.id}" tools/list did not finish within ${MAX_TOOL_LIST_PAGES} pages`); } /** Send `tools/call` and return the result. Optionally includes state/checkpoint for stateful tools. */ @@ -948,7 +1077,7 @@ export class McplServerConnection extends EventEmitter { const attempt = Math.max(1, this.reconnectAttempts); try { - const { transport, capabilities, droppedCapabilities, mcpToolsAdvertised } = await McplServerConnection.handshake( + const { transport, capabilities, droppedCapabilities, mcpToolsAdvertised, protocolVersion } = await McplServerConnection.handshake( this.config, this.hostCapabilities, ); @@ -963,6 +1092,7 @@ export class McplServerConnection extends EventEmitter { this.capabilities = capabilities; this.droppedCapabilities = droppedCapabilities; this.mcpToolsAdvertised = mcpToolsAdvertised; + this.protocolVersion = protocolVersion; this.allowHostCommands = this.config?.allowHostCommands === true; this.closed = false; this.nextRequestId = 1; @@ -975,8 +1105,13 @@ export class McplServerConnection extends EventEmitter { } catch (error) { console.error(`MCPL server "${this.id}" reconnect failed:`, (error as Error).message); this.reconnectAttempts = attempt + 1; - this.emit('reconnect-failed', { error: (error as Error).message, attempt }); - this.scheduleReconnect(); + // No common protocol revision: every further attempt would restart the + // server to hear the same answer. Stop retrying; the configuration is + // what has to change. Every other failure keeps the backoff loop. + const permanent = error instanceof McplProtocolVersionError; + if (permanent) this.reconnectEnabled = false; + this.emit('reconnect-failed', { error: (error as Error).message, attempt, ...(permanent ? { permanent: true } : {}) }); + if (!permanent) this.scheduleReconnect(); } } diff --git a/src/mcpl/types.ts b/src/mcpl/types.ts index 0f913ea5a..209229b01 100644 --- a/src/mcpl/types.ts +++ b/src/mcpl/types.ts @@ -243,18 +243,31 @@ export interface McplServerConfig { inheritEnv?: boolean; /** - * WebSocket URL for the network transport (`ws://` or `wss://`). Mutually - * exclusive with `command`. When set (or `transport: 'websocket'`), the host - * dials this endpoint instead of spawning a process — newline-delimited - * JSON-RPC MCPL runs over the socket identically to stdio. + * URL for a network server, used instead of `command`. Its scheme decides + * the protocol family: + * - `ws://` or `wss://` is a legacy MCPL server over MCPL's WebSocket + * binding. The host dials it and runs the same newline-delimited JSON-RPC + * as over stdio. + * - `http://` or `https://` is a modern MCP (2026-07-28) server over + * Streamable HTTP. */ url?: string; /** - * Transport selector. Defaults to `'websocket'` when a bare `url` is given - * (no `command`), `'stdio'` otherwise. Set explicitly to disambiguate. + * Transport selector. When it is absent, a `command` means stdio and a bare + * `url` means the transport its scheme names. Set it explicitly to + * disambiguate; it must agree with the url's scheme. */ - transport?: 'stdio' | 'websocket'; + transport?: 'stdio' | 'websocket' | 'http'; + + /** + * Protocol family for a stdio (`command`) server. Defaults to `'legacy'`, + * which is MCP 2024-11-05 + MCPL and today's behavior. `'modern'` speaks + * MCP 2026-07-28 through the SDK-backed modern engine. A network server's + * family comes from its url's scheme, so setting this on one is a + * configuration error. There is no probing between families. + */ + protocol?: 'legacy' | 'modern'; /** * Bearer token for WebSocket auth. Appended to `url` as a `token` query @@ -1315,6 +1328,8 @@ export interface McpToolDefinition { name: string; description?: string; inputSchema: Record; + /** Server metadata for the tool (e.g. RFC-008 `mcpl/class` hints). */ + _meta?: Record; } /** diff --git a/test/fixtures/awareness-barrier-mcpl-server.mjs b/test/fixtures/awareness-barrier-mcpl-server.mjs index a16b342b3..24156d023 100644 --- a/test/fixtures/awareness-barrier-mcpl-server.mjs +++ b/test/fixtures/awareness-barrier-mcpl-server.mjs @@ -166,6 +166,7 @@ function beginServerTraffic() { function handle(message) { if (message.method === 'initialize') { reply(message.id, { + protocolVersion: '2024-11-05', capabilities: { experimental: { mcpl: { diff --git a/test/fixtures/legacy-version-server.mjs b/test/fixtures/legacy-version-server.mjs new file mode 100644 index 000000000..33979b28e --- /dev/null +++ b/test/fixtures/legacy-version-server.mjs @@ -0,0 +1,65 @@ +// Stdio MCP server for the legacy engine's protocol-version and pagination +// tests. Usage: node legacy-version-server.mjs [startLog] +// +// Modes (how `initialize` is answered): +// ok protocolVersion 2024-11-05 +// reject error -32022 with data { supported: ['2026-07-28'], requested } +// reject-nodata error -32022 without data +// mismatch protocolVersion 2025-06-18 +// missing no protocolVersion at all +// ok-then-reject `ok` on the first start recorded in startLog, `reject` after +// +// tools/list pages: `ok` serves three pages (cursors p2, p3). Tool `cursor-loop` +// mode is selected with MODE_LIST=loop (always nextCursor "same") or +// MODE_LIST=empty (nextCursor ""). Tool `die` exits the process. +import { appendFileSync, existsSync, readFileSync } from 'node:fs'; + +let mode = process.argv[2] ?? 'ok'; +const startLog = process.argv[3]; +if (startLog) { + const priorStarts = existsSync(startLog) ? readFileSync(startLog, 'utf8').split('\n').filter(Boolean).length : 0; + appendFileSync(startLog, `start ${process.pid}\n`); + if (mode === 'ok-then-reject') mode = priorStarts === 0 ? 'ok' : 'reject'; +} +const listMode = process.env.MODE_LIST ?? 'pages'; + +const send = (m) => process.stdout.write(JSON.stringify(m) + '\n'); +const reply = (id, result) => send({ jsonrpc: '2.0', id, result }); +const error = (id, code, message, data) => send({ jsonrpc: '2.0', id, error: { code, message, ...(data === undefined ? {} : { data }) } }); + +const tool = (name) => ({ name, description: name, inputSchema: { type: 'object' } }); + +let buf = ''; +process.stdin.on('data', (chunk) => { + buf += chunk.toString('utf8'); + let i; + while ((i = buf.indexOf('\n')) >= 0) { + const line = buf.slice(0, i); + buf = buf.slice(i + 1); + if (!line.trim()) continue; + let m; + try { m = JSON.parse(line); } catch { continue; } + if (m.method === 'initialize') { + const requested = m.params?.protocolVersion; + if (mode === 'reject') error(m.id, -32022, `Unsupported protocol version: ${requested}`, { supported: ['2026-07-28'], requested }); + else if (mode === 'reject-nodata') error(m.id, -32022, `Unsupported protocol version: ${requested}`); + else if (mode === 'mismatch') reply(m.id, { protocolVersion: '2025-06-18', capabilities: { tools: {} } }); + else if (mode === 'missing') reply(m.id, { capabilities: { tools: {} } }); + else reply(m.id, { protocolVersion: '2024-11-05', capabilities: { tools: {} } }); + } else if (m.method === 'tools/list') { + const cursor = m.params?.cursor; + if (listMode === 'loop') reply(m.id, { tools: [tool(`loop-${cursor ?? 'first'}`)], nextCursor: 'same' }); + else if (listMode === 'empty') reply(m.id, { tools: [tool('only')], nextCursor: '' }); + else if (cursor === undefined) reply(m.id, { tools: [tool('a1'), tool('a2')], nextCursor: 'p2' }); + else if (cursor === 'p2') reply(m.id, { tools: [tool('b1')], nextCursor: 'p3' }); + else if (cursor === 'p3') reply(m.id, { tools: [tool('c1'), tool('die')] }); + else error(m.id, -32602, `unknown cursor ${cursor}`); + } else if (m.method === 'tools/call') { + if (m.params?.name === 'die') process.exit(3); + reply(m.id, { content: [{ type: 'text', text: 'ok' }] }); + } else if (m.id !== undefined) { + error(m.id, -32601, `Method not found: ${m.method}`); + } + } +}); +process.stdin.on('end', () => process.exit(0)); diff --git a/test/host-quiesce.test.ts b/test/host-quiesce.test.ts index cae60190e..14be7b64d 100644 --- a/test/host-quiesce.test.ts +++ b/test/host-quiesce.test.ts @@ -364,7 +364,7 @@ process.stdin.on('data', (c) => { if (!line.trim()) continue; let m; try { m = JSON.parse(line); } catch { continue; } const reply = (result) => process.stdout.write(JSON.stringify({ jsonrpc: '2.0', id: m.id, result }) + '\\n'); - if (m.method === 'initialize') reply({ capabilities: { experimental: { mcpl: { version: '0.4', pushEvents: true } } } }); + if (m.method === 'initialize') reply({ protocolVersion: '2024-11-05', capabilities: { experimental: { mcpl: { version: '0.4', pushEvents: true } } } }); else if (m.method === 'tools/list') reply({ tools: [] }); else if (m.id === 900) log('push-response', { error: m.error ?? null }); else if (m.id === 901) log('resume-response', { result: m.result ?? null, error: m.error ?? null }); diff --git a/test/mcp-protocol-family.test.ts b/test/mcp-protocol-family.test.ts new file mode 100644 index 000000000..cd53aba66 --- /dev/null +++ b/test/mcp-protocol-family.test.ts @@ -0,0 +1,103 @@ +/** + * Protocol family and transport selection (room-284 design, shelf-487). + * A URL's scheme decides a network server's family; only a stdio server has + * a choice, and it defaults to legacy. Validation adds checks only for what + * the modern family brought, so every legacy configuration still passes. + */ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; + +import { + resolveServerBinding, + serverConfigProblems, + MCPL_ONLY_POLICY_FIELDS, + MAX_TIMER_MS, +} from '../src/mcpl/protocol-family.js'; +import type { McplServerConfig } from '../src/mcpl/types.js'; + +const cfg = (c: Partial): McplServerConfig => ({ id: 's', ...c }); + +test('family and transport follow the configuration', () => { + assert.deepEqual(resolveServerBinding(cfg({ command: 'srv' })), { family: 'legacy', transport: 'stdio' }); + assert.deepEqual(resolveServerBinding(cfg({ command: 'srv', protocol: 'legacy' })), { family: 'legacy', transport: 'stdio' }); + assert.deepEqual(resolveServerBinding(cfg({ command: 'srv', protocol: 'modern' })), { family: 'modern', transport: 'stdio' }); + assert.deepEqual(resolveServerBinding(cfg({ url: 'ws://localhost:3000/mcpl' })), { family: 'legacy', transport: 'websocket' }); + assert.deepEqual(resolveServerBinding(cfg({ url: 'wss://example.com/mcpl' })), { family: 'legacy', transport: 'websocket' }); + assert.deepEqual(resolveServerBinding(cfg({ url: 'http://localhost:8080/mcp' })), { family: 'modern', transport: 'http' }); + assert.deepEqual(resolveServerBinding(cfg({ url: 'https://example.com/mcp' })), { family: 'modern', transport: 'http' }); + // Existing semantics: a command wins over a url unless transport says otherwise. + assert.deepEqual(resolveServerBinding(cfg({ command: 'srv', url: 'ws://x/mcpl' })), { family: 'legacy', transport: 'stdio' }); + assert.deepEqual( + resolveServerBinding(cfg({ command: 'srv', url: 'ws://x/mcpl', transport: 'websocket' })), + { family: 'legacy', transport: 'websocket' }, + ); + assert.deepEqual( + resolveServerBinding(cfg({ url: 'https://x/mcp', transport: 'http' })), + { family: 'modern', transport: 'http' }, + ); +}); + +test('a configuration that names no usable transport throws', () => { + assert.throws(() => resolveServerBinding(cfg({})), /needs "command"/); + assert.throws(() => resolveServerBinding(cfg({ url: 'ftp://x' })), /ws:\/\/ or wss:\/\/.*http:\/\/ or https:\/\//); + assert.throws(() => resolveServerBinding(cfg({ url: 'not a url' })), /invalid url/); + assert.throws(() => resolveServerBinding(cfg({ url: 'ws://x', transport: 'http' })), /does not match url/); + assert.throws(() => resolveServerBinding(cfg({ url: 'https://x', transport: 'websocket' })), /does not match url/); + assert.throws(() => resolveServerBinding(cfg({ url: 'ws://x', transport: 'stdio' })), /requires "command"/); +}); + +test('every legacy configuration shape passes validation unchanged', () => { + const legacy: McplServerConfig[] = [ + cfg({ command: 'srv', requestTimeoutMs: 0 }), // zero still disables the legacy watchdog + cfg({ command: 'srv', enabledFeatureSets: ['a.*'], allowHostCommands: true, channelSubscription: 'auto' }), + cfg({ url: 'wss://x/mcpl', token: 't', toolLifecycle: { observe: { tools: ['*'] } } as never, autofetch: { maxBytes: 1 } }), + cfg({ command: 'srv', protocol: 'legacy', disabledCapabilities: ['contextHooks.*'] }), + ]; + for (const c of legacy) assert.deepEqual(serverConfigProblems(c), [], JSON.stringify(c)); +}); + +test('protocol is a stdio-only choice', () => { + assert.match(serverConfigProblems(cfg({ url: 'https://x/mcp', protocol: 'modern' })).join(), /applies only to stdio/); + assert.match(serverConfigProblems(cfg({ url: 'ws://x/mcpl', protocol: 'legacy' })).join(), /applies only to stdio/); + assert.match(serverConfigProblems(cfg({ command: 'srv', protocol: 'new' as never })).join(), /must be 'legacy' or 'modern'/); +}); + +test('a modern server needs a real deadline', () => { + for (const bad of [0, -1, 1.5, Number.NaN, Number.POSITIVE_INFINITY, MAX_TIMER_MS + 1]) { + const problems = serverConfigProblems(cfg({ command: 'srv', protocol: 'modern', requestTimeoutMs: bad })); + assert.match(problems.join(), /requestTimeoutMs must be an integer from 1/, String(bad)); + } + for (const good of [undefined, 1, 60_000, MAX_TIMER_MS]) { + assert.deepEqual(serverConfigProblems(cfg({ url: 'https://x/mcp', requestTimeoutMs: good })), [], String(good)); + } +}); + +test('MCPL policy on a modern server is rejected, empty values are not', () => { + const effective: Partial = { + enabledFeatureSets: ['a'], + disabledFeatureSets: ['b'], + enabledCapabilities: ['pushEvents'], + disabledCapabilities: ['channels'], + scopes: { a: {} as never }, + channelSubscription: 'manual', + allowHostCommands: true, + toolLifecycle: { observe: {} } as never, + autofetch: { maxBytes: 10 }, + shouldTriggerInference: () => true, + }; + for (const field of MCPL_ONLY_POLICY_FIELDS) { + const problems = serverConfigProblems(cfg({ url: 'https://x/mcp', [field]: effective[field] })); + assert.equal(problems.length, 1, field); + assert.match(problems[0]!, new RegExp(`"${field}" is MCPL policy`)); + } + const empty = cfg({ + command: 'srv', + protocol: 'modern', + enabledFeatureSets: [], + disabledCapabilities: [], + scopes: {}, + allowHostCommands: false, + toolLifecycle: {} as never, + }); + assert.deepEqual(serverConfigProblems(empty), []); +}); diff --git a/test/mcpl-capability-scoping.test.ts b/test/mcpl-capability-scoping.test.ts index e0c7777ef..29f223e16 100644 --- a/test/mcpl-capability-scoping.test.ts +++ b/test/mcpl-capability-scoping.test.ts @@ -179,7 +179,7 @@ process.stdin.on('data', (c) => { if (!line.trim()) continue; let m; try { m = JSON.parse(line); } catch { continue; } if (m.method === 'initialize') { - send({ jsonrpc: '2.0', id: m.id, result: { capabilities: { experimental: { mcpl: { + send({ jsonrpc: '2.0', id: m.id, result: { protocolVersion: '2024-11-05', capabilities: { experimental: { mcpl: { version: '0.4', pushEvents: true, contextHooks: { beforeInference: true, afterInference: { blocking: true } }, diff --git a/test/mcpl-legacy-version.test.ts b/test/mcpl-legacy-version.test.ts new file mode 100644 index 000000000..1d6a171e0 --- /dev/null +++ b/test/mcpl-legacy-version.test.ts @@ -0,0 +1,155 @@ +/** + * The legacy engine's protocol boundary and inventory (room-284, shelf-487): + * - it offers and accepts exactly MCP 2024-11-05; anything else is a typed + * McplProtocolVersionError carrying what each side said; + * - such a verdict is never retried (no reconnect stub, no backoff loop); + * - tools/list follows nextCursor to the complete inventory, and refuses a + * server whose cursors don't terminate. + */ +import { test, afterEach } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtempSync, readFileSync, existsSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, dirname } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +import { McplServerConnection, McplProtocolVersionError, McplRequestError } from '../src/mcpl/server-connection.js'; +import type { McplHostCapabilities, McplServerConfig } from '../src/mcpl/types.js'; + +const here = dirname(fileURLToPath(import.meta.url)); +const SERVER = join(here, 'fixtures', 'legacy-version-server.mjs'); +const HOST_CAPS: McplHostCapabilities = { version: '0.5', pushEvents: true }; + +function config(mode: string, extra: Partial = {}, startLog?: string): McplServerConfig { + return { + id: 'legacy', + command: process.execPath, + args: [SERVER, mode, ...(startLog ? [startLog] : [])], + ...extra, + }; +} + +const cleanups: Array<() => Promise | void> = []; +afterEach(async () => { + while (cleanups.length) await cleanups.pop()!(); +}); + +async function connect(c: McplServerConfig): Promise { + const connection = await McplServerConnection.connect(c, HOST_CAPS); + cleanups.push(() => connection.close()); + connection.ready(); + return connection; +} + +test('a 2024-11-05 answer establishes the connection and records its revision', async () => { + const connection = await connect(config('ok')); + assert.equal(connection.protocolVersion, '2024-11-05'); +}); + +test('-32022 with a supported list is a typed rejection naming the modern remedy', async () => { + await assert.rejects(McplServerConnection.connect(config('reject'), HOST_CAPS), (err: unknown) => { + assert.ok(err instanceof McplProtocolVersionError); + assert.equal(err.kind, 'rejected'); + assert.equal(err.offered, '2024-11-05'); + assert.deepEqual(err.supported, ['2026-07-28']); + assert.equal(err.code, -32022); + assert.deepEqual(err.data, { supported: ['2026-07-28'], requested: '2024-11-05' }); + assert.match(err.message, /does not support MCP 2024-11-05 \(it supports: 2026-07-28\)/); + assert.match(err.message, /protocol: 'modern'/); + // A negotiation verdict, not a request outcome. + assert.ok(!(err instanceof McplRequestError)); + return true; + }); +}); + +test('-32022 without data still rejects, with the server\'s own message', async () => { + await assert.rejects(McplServerConnection.connect(config('reject-nodata'), HOST_CAPS), (err: unknown) => { + assert.ok(err instanceof McplProtocolVersionError); + assert.equal(err.kind, 'rejected'); + assert.equal(err.supported, null); + assert.match(err.message, /Unsupported protocol version: 2024-11-05/); + assert.doesNotMatch(err.message, /protocol: 'modern'/); + return true; + }); +}); + +test('another legacy revision, or none, is a mismatch rather than a silent connection', async () => { + await assert.rejects(McplServerConnection.connect(config('mismatch'), HOST_CAPS), (err: unknown) => { + assert.ok(err instanceof McplProtocolVersionError); + assert.equal(err.kind, 'mismatch'); + assert.equal(err.returned, '2025-06-18'); + assert.match(err.message, /protocolVersion "2025-06-18"; this client's legacy engine speaks only 2024-11-05/); + return true; + }); + await assert.rejects(McplServerConnection.connect(config('missing'), HOST_CAPS), (err: unknown) => { + assert.ok(err instanceof McplProtocolVersionError); + assert.equal(err.kind, 'mismatch'); + assert.equal(err.returned, null); + assert.match(err.message, /protocolVersion \(none\)/); + return true; + }); +}); + +test('a version verdict gets no reconnect stub even with reconnect: true', async () => { + await assert.rejects( + McplServerConnection.connectWithReconnect(config('reject', { reconnect: true, reconnectIntervalMs: 20 }), HOST_CAPS), + (err: unknown) => err instanceof McplProtocolVersionError, + ); +}); + +test('a version verdict on reconnect stops the backoff loop', async () => { + const dir = mkdtempSync(join(tmpdir(), 'legacy-version-')); + cleanups.push(() => rmSync(dir, { recursive: true, force: true })); + const startLog = join(dir, 'starts.log'); + const connection = await McplServerConnection.connectWithReconnect( + config('ok-then-reject', { reconnect: true, reconnectIntervalMs: 20, reconnectMaxIntervalMs: 40 }, startLog), + HOST_CAPS, + ); + cleanups.push(() => connection.close()); + connection.ready(); + const failures: Array<{ error: string; attempt: number; permanent?: boolean }> = []; + connection.on('reconnect-failed', (info) => failures.push(info)); + + // The first process answers 2024-11-05; make it exit so the loop runs. + await assert.rejects(connection.sendToolsCall('die', {})); + const deadline = Date.now() + 5_000; + while (failures.length === 0 && Date.now() < deadline) await new Promise((r) => setTimeout(r, 20)); + assert.equal(failures.length, 1, 'one failed reconnect'); + assert.equal(failures[0]!.permanent, true); + assert.match(failures[0]!.error, /does not support MCP 2024-11-05/); + assert.equal(connection.willReconnect, false); + + // No further attempts: the start count stays at 2 (original + one retry). + await new Promise((r) => setTimeout(r, 300)); + const starts = existsSync(startLog) ? readFileSync(startLog, 'utf8').split('\n').filter(Boolean).length : 0; + assert.equal(starts, 2); + assert.equal(failures.length, 1); +}); + +test('tools/list follows nextCursor to the complete inventory', async () => { + const connection = await connect(config('ok')); + const { tools } = await connection.sendToolsList(); + assert.deepEqual(tools.map((t) => t.name), ['a1', 'a2', 'b1', 'c1', 'die']); +}); + +test('an empty nextCursor ends the inventory', async () => { + const connection = await connect(config('ok', { env: { MODE_LIST: 'empty' } })); + const { tools } = await connection.sendToolsList(); + assert.deepEqual(tools.map((t) => t.name), ['only']); +}); + +test('a repeating cursor is an error, not a silently partial inventory', async () => { + const connection = await connect(config('ok', { env: { MODE_LIST: 'loop' } })); + await assert.rejects(connection.sendToolsList(), /repeated tools\/list cursor "same"/); +}); + +test('a modern configuration is refused by the legacy engine before anything is dialed', async () => { + await assert.rejects( + McplServerConnection.connect({ id: 'modern', url: 'https://example.invalid/mcp' }, HOST_CAPS), + /configured for modern MCP \(2026-07-28\); McplServerConnection speaks the legacy family/, + ); + await assert.rejects( + McplServerConnection.connect({ id: 'modern', command: process.execPath, protocol: 'modern' }, HOST_CAPS), + /configured for modern MCP/, + ); +}); diff --git a/test/mcpl-request-outcome.test.ts b/test/mcpl-request-outcome.test.ts index 1b0f45647..43821d1d1 100644 --- a/test/mcpl-request-outcome.test.ts +++ b/test/mcpl-request-outcome.test.ts @@ -27,7 +27,7 @@ process.stdin.on('data', (c) => { if (!line.trim()) continue; let m; try { m = JSON.parse(line); } catch { continue; } const out = (o) => process.stdout.write(JSON.stringify({ jsonrpc: '2.0', id: m.id, ...o }) + '\\n'); - if (m.method === 'initialize') out({ result: { capabilities: {} } }); + if (m.method === 'initialize') out({ result: { protocolVersion: '2024-11-05', capabilities: {} } }); else if (m.method === 'tools/call' && m.params.name === 'fail') { out({ error: { code: -32001, message: 'POSTED: part 1 of 2', data: { posted: ['111'] } } }); } else if (m.method === 'tools/call' && m.params.name === 'exit') { diff --git a/test/mcpl-request-timeout.test.ts b/test/mcpl-request-timeout.test.ts index 2778469e3..038274f4f 100644 --- a/test/mcpl-request-timeout.test.ts +++ b/test/mcpl-request-timeout.test.ts @@ -23,7 +23,7 @@ process.stdin.on('data', (c) => { if (!line.trim()) continue; let m; try { m = JSON.parse(line); } catch { continue; } const reply = (result) => process.stdout.write(JSON.stringify({ jsonrpc: '2.0', id: m.id, result }) + '\\n'); - if (m.method === 'initialize') reply({ capabilities: {} }); + if (m.method === 'initialize') reply({ protocolVersion: '2024-11-05', capabilities: {} }); else if (m.method === 'tools/list') reply({ tools: [{ name: 'stuck', description: 's', inputSchema: { type: 'object' } }] }); // tools/call: deliberately NO response, connection stays open. } diff --git a/test/mcpl-runtime-lifecycle.test.ts b/test/mcpl-runtime-lifecycle.test.ts index 82c48137a..56c10c52b 100644 --- a/test/mcpl-runtime-lifecycle.test.ts +++ b/test/mcpl-runtime-lifecycle.test.ts @@ -28,7 +28,7 @@ process.stdin.on('data', (c) => { if (!line.trim()) continue; let m; try { m = JSON.parse(line); } catch { continue; } const reply = (result) => process.stdout.write(JSON.stringify({ jsonrpc: '2.0', id: m.id, result }) + '\\n'); - if (m.method === 'initialize') reply({ capabilities: { experimental: { mcpl: { version: '0.4', pushEvents: true } } } }); + if (m.method === 'initialize') reply({ protocolVersion: '2024-11-05', capabilities: { experimental: { mcpl: { version: '0.4', pushEvents: true } } } }); else if (m.method === 'tools/list') reply({ tools: [{ name: 'echo', description: 'e', inputSchema: { type: 'object' } }] }); else if (m.method === 'tools/call') reply({ content: [{ type: 'text', text: String((m.params && m.params.arguments && m.params.arguments.text) || '') }] }); } @@ -139,7 +139,7 @@ test('connectMcplServer hot-connects a network-only (WebSocket) MCPL server', as let msg: { id?: unknown; method?: string; params?: { arguments?: Record } }; try { msg = JSON.parse(String(data)); } catch { return; } const reply = (result: unknown) => ws.send(JSON.stringify({ jsonrpc: '2.0', id: msg.id, result })); - if (msg.method === 'initialize') reply({ capabilities: { experimental: { mcpl: { version: '0.4', pushEvents: true } } } }); + if (msg.method === 'initialize') reply({ protocolVersion: '2024-11-05', capabilities: { experimental: { mcpl: { version: '0.4', pushEvents: true } } } }); else if (msg.method === 'tools/list') reply({ tools: [{ name: 'echo', description: 'e', inputSchema: { type: 'object' } }] }); }); }); diff --git a/test/mcpl-ws-transport.test.ts b/test/mcpl-ws-transport.test.ts index 0599910c3..7626eaf76 100644 --- a/test/mcpl-ws-transport.test.ts +++ b/test/mcpl-ws-transport.test.ts @@ -28,7 +28,7 @@ function startMockMcplWsServer(onSocket?: (ws: WebSocket) => void) { const reply = (result: unknown) => ws.send(JSON.stringify({ jsonrpc: '2.0', id: msg.id, result })); switch (msg.method) { case 'initialize': - reply({ capabilities: { experimental: { mcpl: { version: '0.4', pushEvents: true } } } }); + reply({ protocolVersion: '2024-11-05', capabilities: { experimental: { mcpl: { version: '0.4', pushEvents: true } } } }); break; case 'notifications/initialized': break; // notification, no reply @@ -133,7 +133,7 @@ process.stdin.on('data', (c) => { if (!line.trim()) continue; let m; try { m = JSON.parse(line); } catch { continue; } const reply = (result) => process.stdout.write(JSON.stringify({ jsonrpc: '2.0', id: m.id, result }) + '\\n'); - if (m.method === 'initialize') reply({ capabilities: { experimental: { mcpl: { version: '0.4', pushEvents: true } } } }); + if (m.method === 'initialize') reply({ protocolVersion: '2024-11-05', capabilities: { experimental: { mcpl: { version: '0.4', pushEvents: true } } } }); else if (m.method === 'tools/list') reply({ tools: [{ name: 'echo', description: 'e', inputSchema: { type: 'object' } }] }); else if (m.method === 'tools/call') reply({ content: [{ type: 'text', text: String((m.params && m.params.arguments && m.params.arguments.text) || '') }] }); } From 3a3104ccd7e7fd12e7185a14231b57aabbffad98 Mon Sep 17 00:00:00 2001 From: Olive-1804 Date: Thu, 8 Oct 2026 01:12:48 -0400 Subject: [PATCH 0058/1112] =?UTF-8?q?feat(mcp):=20ModernMcpConnection=20?= =?UTF-8?q?=E2=80=94=20MCP=202026-07-28=20over=20stdio=20and=20Streamable?= =?UTF-8?q?=20HTTP=20via=20SDK=202.x?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Second batch of room-284's modern-MCP client work (shelf-487). Adds the modern engine as its own connection class. Framework integration is the next batch; nothing in the framework constructs it yet. The SDK (@modelcontextprotocol/client ^2.3.1, pinned to 2026-07-28) owns the modern wire: server/discover, the per-request _meta envelope, resultType, input_required continuation, cancellation, subscriptions/listen, and the HTTP binding's headers, 401 retry and header-mismatch relist. Connectome owns: - stdio through its own spawner (SpawnerStdioTransport over StdioTransport). The child launches once per connect: a custom transport probes in place, where the SDK's own stdio transport starts a throwaway sibling. It gets the same env allowlist, inheritEnv and stderr lines as a legacy child, and cancellation by notifications/cancelled. - HTTP through the SDK's StreamableHTTPClientTransport, with token and accessProvider behind the SDK's AuthProvider seam: a cached bearer, re-resolved once on 401. - One deadline per call. requestTimeoutMs is always positive for modern (0 is rejected by config validation, since the SDK reads 0 as "time out now"), and is passed as timeout, maxTotalTimeout and an outer AbortSignal, so continuation rounds and pre-dispatch retries can't renew the wait. At the deadline the SDK requests cancellation and the error says the outcome is unknown. Nothing is ever replayed. - Failures use the McplRequestError contract, claiming only what's established. A JSON-RPC error, or an HTTP 401/403 carrying the status, is error-response. Everything after hand-off, the deadline included, is no-response. Connectome's own pre-send refusal is not-sent. A result the server returned that can't be used (failed validation, input this client can't supply, a rounds cap) is a plain Error, not an outcome. - Results kept by presence: structuredContent false/0/null are values. - Lifetime: reconnect with the legacy backoff (reconnect / intervals). A lost list-change subscription is reopened while the connection lives, followed by one inventory refresh. Each connect is a generation, so a superseded client never touches a newer one. Client capabilities are {}: no sampling, elicitation or roots. Tests (13, at our seams): stdio launch count, env allowlist, stderr, results by presence, the deadline cancelled server-side (warm-up first, since SDK 2.0.0's server ignored cancellation of request id 0 in probes), list change, crash and reconnect with relaunch count 2, no-reconnect refusal as not-sent, retrying first connect stopped by close, config errors; plus one HTTP smoke covering accessProvider refresh, required headers, deadline abort seen server-side, 401 connect failure, and auth wall as error-response 401. Stable across 3 reruns. --- package.json | 2 + src/index.ts | 2 + src/mcpl/index.ts | 2 + src/mcpl/modern-connection.ts | 455 ++++++++++++++++++++++++++++ test/fixtures/modern-mcp-server.mjs | 13 + test/fixtures/modern-tools.mjs | 73 +++++ test/mcp-modern-connection.test.ts | 186 ++++++++++++ test/mcp-modern-http.test.ts | 138 +++++++++ 8 files changed, 871 insertions(+) create mode 100644 src/mcpl/modern-connection.ts create mode 100644 test/fixtures/modern-mcp-server.mjs create mode 100644 test/fixtures/modern-tools.mjs create mode 100644 test/mcp-modern-connection.test.ts create mode 100644 test/mcp-modern-http.test.ts diff --git a/package.json b/package.json index 4bf3dbcaa..138d9d6db 100644 --- a/package.json +++ b/package.json @@ -49,11 +49,13 @@ "@animalabs/chronicle": "^0.4.0", "@animalabs/context-manager": "^0.13.0", "@animalabs/membrane": "^0.5.78", + "@modelcontextprotocol/client": "^2.3.1", "chokidar": "^4.0.3", "discord.js": "^14.25.1", "ws": "^8.18.0" }, "devDependencies": { + "@modelcontextprotocol/server": "^2.3.1", "@types/node": "^22.0.0", "@types/ws": "^8.5.13", "discord.js": "^14.25.1", diff --git a/src/index.ts b/src/index.ts index 9e6b0073f..34b61109a 100644 --- a/src/index.ts +++ b/src/index.ts @@ -91,6 +91,8 @@ export type { McplServerConfig } from './mcpl/index.js'; // health probes, protocol harnesses — can dial an MCPL server exactly the // way the framework host does, over stdio or WebSocket) export { McplServerConnection, McplRequestError, McplProtocolVersionError } from './mcpl/index.js'; +export { ModernMcpConnection } from './mcpl/index.js'; +export type { ModernToolCallResult } from './mcpl/index.js'; export { LEGACY_MCP_PROTOCOL_VERSION, MODERN_MCP_PROTOCOL_VERSION, diff --git a/src/mcpl/index.ts b/src/mcpl/index.ts index c3bc7ef31..8e09c869e 100644 --- a/src/mcpl/index.ts +++ b/src/mcpl/index.ts @@ -107,6 +107,8 @@ export { // Server connection and registry export { McplServerConnection, McplRequestError, McplProtocolVersionError } from './server-connection.js'; +export { ModernMcpConnection } from './modern-connection.js'; +export type { ModernToolCallResult } from './modern-connection.js'; export { LEGACY_MCP_PROTOCOL_VERSION, MODERN_MCP_PROTOCOL_VERSION, diff --git a/src/mcpl/modern-connection.ts b/src/mcpl/modern-connection.ts new file mode 100644 index 000000000..cce545d09 --- /dev/null +++ b/src/mcpl/modern-connection.ts @@ -0,0 +1,455 @@ +/** + * ModernMcpConnection: a server that speaks modern MCP (2026-07-28), reached + * over stdio or Streamable HTTP through the official SDK client. + * + * Responsibilities divide like this. The SDK owns the modern wire: + * `server/discover`, the per-request `_meta` envelope, `resultType`, + * continuation of `input_required` results, cancellation, list-change + * subscriptions, and the HTTP binding's headers, auth retry and + * header-mismatch handling. This class owns what Connectome decides: + * - **stdio** runs through Connectome's own spawner, so the child gets the + * same env allowlist, `inheritEnv` and stderr surfacing as a legacy server, + * and it starts once per connect. The SDK's own stdio transport starts a + * throwaway second copy for discovery. + * - **HTTP credentials** come from `token`/`accessProvider`: a cached bearer + * token, re-resolved once when the server answers 401. + * - **One deadline per tool call** (`requestTimeoutMs`, always positive) + * bounds every leg of the call. At the deadline the SDK requests + * cancellation and the outcome is unknown. Calls are never replayed. + * - **Lifetime.** On a lost transport the connection restarts with the same + * backoff settings as legacy, when `reconnect` is set. A lost list-change + * stream is reopened while the connection lives. Each connect is a + * generation, so a superseded client can never touch a newer one. + * + * Nothing here is MCPL: a modern server has no grant, no planes and no + * server→host requests. The framework uses this connection only through its + * tool paths, and the MCPL machinery never sees it. + */ +import { EventEmitter } from 'node:events'; + +import { + Client, + ProtocolError, + SdkError, + SdkErrorCode, + SdkHttpError, + StreamableHTTPClientTransport, + type AuthProvider, + type CallToolResult, + type JSONRPCMessage, + type McpSubscription, + type Transport, +} from '@modelcontextprotocol/client'; + +import { StdioTransport, type McplTransport } from './transport.js'; +import { McplRequestError } from './server-connection.js'; +import { MODERN_MCP_PROTOCOL_VERSION, checkServerConfig } from './protocol-family.js'; +import type { McplServerConfig, McpToolDefinition } from './types.js'; + +const DEFAULT_REQUEST_TIMEOUT_MS = 60_000; +/** Bound on establishing the connection (`server/discover`), like the legacy + * engine's initialize timeout. */ +const CONNECT_TIMEOUT_MS = 30_000; +/** Inventory listing has its own bound: it is host work, not a resident's call. */ +const LIST_TIMEOUT_MS = 30_000; + +/** A modern tool result as the framework consumes it. `structuredContent` is + * kept by presence: `false`, `0` and `null` are values, not absence. */ +export interface ModernToolCallResult { + content: CallToolResult['content']; + isError?: boolean; + structuredContent?: unknown; + _meta?: Record; +} + +/** + * SDK `Transport` over Connectome's stdio spawner. Framing is the same + * newline-delimited JSON-RPC as legacy. Lines that are not JSON are ignored, + * as the legacy engine ignores them. Because this is a custom transport, the + * SDK runs `server/discover` on this very process instead of starting a + * throwaway copy, and cancels with `notifications/cancelled`, which is the + * stdio binding's mechanism. + */ +class SpawnerStdioTransport implements Transport { + onclose?: () => void; + onerror?: (error: Error) => void; + onmessage?: (message: JSONRPCMessage) => void; + private line: McplTransport | null = null; + + constructor( + private readonly config: McplServerConfig, + private readonly onStderr: (line: string) => void, + ) {} + + async start(): Promise { + const line = StdioTransport.spawn(this.config); + this.line = line; + line.on('line', (text: string) => { + let message: JSONRPCMessage; + try { + message = JSON.parse(text) as JSONRPCMessage; + } catch { + return; + } + this.onmessage?.(message); + }); + line.on('stderr', (text: string) => this.onStderr(text)); + line.on('error', (error: Error) => this.onerror?.(error)); + line.on('close', () => this.onclose?.()); + } + + async send(message: JSONRPCMessage): Promise { + if (!this.line || this.line.closed) throw new Error(`MCP server "${this.config.id}" stdio transport is closed`); + this.line.writeLine(JSON.stringify(message)); + } + + async close(): Promise { + await this.line?.close(); + } +} + +/** Connectome's credential plumbing as the SDK's minimal bearer seam: a + * cached token, fetched on first use and once again after a 401. */ +function bearerAuth(config: McplServerConfig): AuthProvider | undefined { + const provider = config.accessProvider; + if (!provider && !config.token) return undefined; + let cached: string | undefined; + const resolve = async (): Promise => { + if (provider) { + try { + const fresh = await provider(); + if (fresh) return fresh; + } catch (error) { + // Same fallback as the WebSocket dial: use the static token, and let + // the server refuse a stale credential loudly. + console.error( + `[mcp] server "${config.id}": access provider failed (${error instanceof Error ? error.message : error}) — using configured fallback`, + ); + } + } + return config.token; + }; + return { + token: async () => (cached ??= await resolve()), + onUnauthorized: async () => { + cached = await resolve(); + }, + }; +} + +export class ModernMcpConnection extends EventEmitter { + readonly id: string; + readonly family = 'modern' as const; + readonly transportKind: 'stdio' | 'http'; + /** The pinned revision, once a connect has established it; null before. */ + protocolVersion: string | null = null; + + private client: Client | null = null; + private connected = false; + private generation = 0; + private closedByHost = false; + private readonly reconnectEnabled: boolean; + private readonly reconnectIntervalMs: number; + private readonly reconnectMaxIntervalMs: number; + private reconnectTimer: ReturnType | null = null; + private reconnectAttempts = 0; + private relistenTimer: ReturnType | null = null; + private relistenAttempts = 0; + + private constructor(private readonly config: McplServerConfig) { + super(); + this.id = config.id; + const binding = checkServerConfig(config); + if (binding.family !== 'modern') { + throw new Error(`MCP server "${config.id}" is not configured for modern MCP; it belongs to McplServerConnection`); + } + this.transportKind = binding.transport === 'http' ? 'http' : 'stdio'; + this.reconnectEnabled = config.reconnect === true; + this.reconnectIntervalMs = config.reconnectIntervalMs ?? 5000; + this.reconnectMaxIntervalMs = config.reconnectMaxIntervalMs ?? 300_000; + // An 'error' with no listener would throw out of an SDK callback and take + // the host down; out-of-band errors here are diagnostics, never fatal. + this.on('error', (error: Error) => { + if (this.listenerCount('error') === 1) console.error(`[mcp] ${this.id}: ${error.message}`); + }); + } + + /** The per-call deadline: `requestTimeoutMs`, validated positive. */ + get requestTimeoutMs(): number { + return this.config.requestTimeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS; + } + + get isConnected(): boolean { + return this.connected; + } + + /** Whether a lost connection will be re-established in the background. */ + get willReconnect(): boolean { + return this.reconnectEnabled && !this.closedByHost; + } + + /** + * Connect, or with `reconnect` set, return a connection that keeps + * retrying in the background when the first attempt fails (the legacy + * engine's contract). Configuration errors always throw. + */ + static async connect(config: McplServerConfig): Promise { + const connection = new ModernMcpConnection(config); + try { + await connection.open(); + } catch (error) { + if (!connection.reconnectEnabled) throw error; + console.error(`MCP server "${config.id}" initial connect failed, will retry:`, (error as Error).message); + connection.reconnectAttempts = 1; + // A macrotask, so the caller has wired its listeners (synchronously, + // right after this resolves) before the event fires. + setImmediate(() => connection.emit('connect-failed', { error: (error as Error).message, attempt: 0 })); + connection.scheduleReconnect(); + } + return connection; + } + + /** One connect generation: a fresh client and transport. */ + private async open(): Promise { + const generation = ++this.generation; + const live = () => generation === this.generation && !this.closedByHost; + + const transport: Transport = this.transportKind === 'stdio' + ? new SpawnerStdioTransport(this.config, (line) => { if (live()) this.emit('stderr', { line }); }) + : new StreamableHTTPClientTransport(new URL(this.config.url!), { authProvider: bearerAuth(this.config) }); + + const client = new Client( + { name: 'agent-framework', version: '1.0.0' }, + { + versionNegotiation: { mode: { pin: MODERN_MCP_PROTOCOL_VERSION } }, + // Nothing advertised: this client supplies no sampling, elicitation + // or roots, so a conforming server cannot require them. + capabilities: {}, + listChanged: { + tools: { + autoRefresh: false, + debounceMs: 0, + onChanged: () => { if (live()) this.emit('tools-list-changed'); }, + }, + }, + }, + ); + client.onerror = (error: Error) => { + // The SDK reports late responses to deadline-abandoned calls here + // ("unknown message ID"), among other out-of-band errors. They are + // logged, never delivered: the call's outcome was already reported. + if (live()) this.emit('error', error); + }; + client.onclose = () => { + if (live()) this.handleLost('transport closed'); + }; + + try { + await client.connect(transport, { timeout: CONNECT_TIMEOUT_MS }); + } catch (error) { + await client.close().catch(() => {}); + throw error; + } + if (!live()) { + await client.close().catch(() => {}); + return; + } + this.client = client; + this.connected = true; + this.protocolVersion = MODERN_MCP_PROTOCOL_VERSION; + this.reconnectAttempts = 0; + this.watchSubscription(client.autoOpenedSubscription, generation); + } + + /** + * Keep list-change awareness while the connection lives. The SDK opens the + * subscription at connect when the server advertises tool list changes. + * When the subscription ends without our asking, it is reopened with + * backoff, and the inventory is refreshed once it's back, since changes + * may have been missed in between. A transport loss is the reconnect + * path's job, not this one's. + */ + private watchSubscription(subscription: McpSubscription | undefined, generation: number): void { + if (!subscription) return; + void subscription.closed.then((reason) => { + if (reason === 'local' || generation !== this.generation || this.closedByHost || !this.connected) return; + this.scheduleRelisten(generation); + }); + } + + private scheduleRelisten(generation: number): void { + if (this.relistenTimer) return; + const delay = this.backoffDelay(this.relistenAttempts); + this.relistenTimer = setTimeout(() => { + this.relistenTimer = null; + void this.relisten(generation); + }, delay); + this.relistenTimer.unref?.(); + } + + private async relisten(generation: number): Promise { + const client = this.client; + if (!client || generation !== this.generation || this.closedByHost || !this.connected) return; + try { + const subscription = await client.listen({ toolsListChanged: true }, { timeout: CONNECT_TIMEOUT_MS }); + if (generation !== this.generation || this.closedByHost) { + await subscription.close().catch(() => {}); + return; + } + this.relistenAttempts = 0; + this.watchSubscription(subscription, generation); + this.emit('tools-list-changed'); + } catch (error) { + this.relistenAttempts++; + this.emit('error', new Error(`MCP server "${this.id}" could not reopen its list-change subscription: ${(error as Error).message}`)); + this.scheduleRelisten(generation); + } + } + + /** The transport went away (child exit, or a closed HTTP client). */ + private handleLost(reason: string): void { + if (!this.connected) return; + this.connected = false; + this.client = null; + this.emit('close', { reason }); + if (this.willReconnect) this.scheduleReconnect(); + } + + /** The legacy engine's backoff: doubling from the base interval, capped, + * with ±25% jitter. */ + private backoffDelay(failures: number): number { + const exponent = Math.max(0, Math.min(failures - 1, 30)); + const capped = Math.min(this.reconnectIntervalMs * 2 ** exponent, this.reconnectMaxIntervalMs); + return capped * (0.75 + Math.random() * 0.5); + } + + private scheduleReconnect(): void { + if (!this.willReconnect || this.reconnectTimer) return; + this.reconnectTimer = setTimeout(() => { + this.reconnectTimer = null; + void this.attemptReconnect(); + }, this.backoffDelay(this.reconnectAttempts)); + } + + private async attemptReconnect(): Promise { + if (!this.willReconnect || this.connected) return; + const attempt = Math.max(1, this.reconnectAttempts); + try { + await this.open(); + if (!this.connected) return; + console.error(`MCP server "${this.id}" reconnected`); + this.emit('reconnect', { attempts: attempt }); + } catch (error) { + this.reconnectAttempts = attempt + 1; + this.emit('reconnect-failed', { error: (error as Error).message, attempt }); + this.scheduleReconnect(); + } + } + + /** The server's complete tool inventory (the SDK follows every page). */ + async listTools(): Promise { + const client = this.requireClient('tools/list'); + try { + const { tools } = await client.listTools(undefined, { cacheMode: 'refresh', timeout: LIST_TIMEOUT_MS }); + return tools.map((tool) => ({ + name: tool.name, + ...(tool.description !== undefined ? { description: tool.description } : {}), + inputSchema: tool.inputSchema as Record, + ...(tool._meta !== undefined ? { _meta: tool._meta as Record } : {}), + })); + } catch (error) { + throw this.requestFailure('tools/list', error); + } + } + + /** + * Call a tool within the configured deadline. One budget covers the whole + * logical call: the SDK's per-leg timer and its `maxTotalTimeout` across + * continuation rounds, and an abort signal around any retry the SDK makes + * before dispatch (an auth refresh, or a header-mismatch relist). + */ + async callTool(name: string, args: Record): Promise { + const client = this.requireClient('tools/call'); + const timeoutMs = this.requestTimeoutMs; + try { + const result = await client.callTool( + { name, arguments: args }, + { timeout: timeoutMs, maxTotalTimeout: timeoutMs, signal: AbortSignal.timeout(timeoutMs) }, + ); + return { + content: result.content ?? [], + ...(result.isError !== undefined ? { isError: result.isError } : {}), + ...('structuredContent' in result ? { structuredContent: result.structuredContent } : {}), + ...(result._meta !== undefined ? { _meta: result._meta as Record } : {}), + }; + } catch (error) { + throw this.requestFailure(`tools/call "${name}"`, error, timeoutMs); + } + } + + /** Close for good: no reconnect, no reopened subscription. */ + async close(): Promise { + this.closedByHost = true; + if (this.reconnectTimer) clearTimeout(this.reconnectTimer); + if (this.relistenTimer) clearTimeout(this.relistenTimer); + this.reconnectTimer = null; + this.relistenTimer = null; + const client = this.client; + const wasConnected = this.connected; + this.client = null; + this.connected = false; + if (client) await client.close().catch(() => {}); + if (wasConnected) this.emit('close', { reason: 'closed by host' }); + } + + /** Connectome refuses before anything reaches the SDK: provably not sent. */ + private requireClient(method: string): Client { + if (!this.client || !this.connected) { + throw new McplRequestError(`Cannot send ${method}: connection to "${this.id}" is not established`, 'not-sent'); + } + return this.client; + } + + /** + * Map an SDK failure onto the request-outcome contract, claiming no more + * than the failure establishes: + * - `error-response`: the server answered with an error. That is a + * JSON-RPC error, or an HTTP auth refusal, which carries the status as + * its code. + * - `no-response`: everything after hand-off that proves nothing about + * delivery, including the deadline, where cancellation was requested and + * the outcome is unknown. + * A call the server answered with a result Connectome can't use (it fails + * validation, or needs input this client can't supply) is a plain error: + * the server did act, so no request outcome applies. + */ + private requestFailure(what: string, error: unknown, timeoutMs?: number): Error { + const prefix = `MCP server "${this.id}"`; + if (error instanceof ProtocolError) { + return new McplRequestError(`${prefix} returned error for ${what}: [${error.code}] ${error.message}`, 'error-response', error.code, error.data); + } + if (error instanceof SdkHttpError && (error.code === SdkErrorCode.ClientHttpAuthentication || error.code === SdkErrorCode.ClientHttpForbidden)) { + return new McplRequestError(`${prefix} refused ${what}: HTTP ${error.status} (${error.message})`, 'error-response', error.status, error.data); + } + if (error instanceof SdkError) { + switch (error.code) { + case SdkErrorCode.RequestTimeout: + return new McplRequestError( + `${prefix} did not answer ${what}${timeoutMs ? ` within ${timeoutMs}ms` : ''}. Cancellation was requested; ` + + `the outcome is unknown: the tool may still complete server-side. Verify state before retrying; ` + + `a blind retry of a side-effecting tool may duplicate it.`, + 'no-response', + ); + case SdkErrorCode.InvalidResult: + case SdkErrorCode.UnsupportedResultType: + case SdkErrorCode.InputRequiredRoundsExceeded: + case SdkErrorCode.CapabilityNotSupported: + return new Error(`${prefix} answered ${what}, but the result can't be used: ${error.message}`); + default: + return new McplRequestError(`${prefix} ${what} failed: ${error.message}`, 'no-response'); + } + } + const message = error instanceof Error ? error.message : String(error); + return new McplRequestError(`${prefix} ${what} failed: ${message}`, 'no-response'); + } +} diff --git a/test/fixtures/modern-mcp-server.mjs b/test/fixtures/modern-mcp-server.mjs new file mode 100644 index 000000000..21c324d38 --- /dev/null +++ b/test/fixtures/modern-mcp-server.mjs @@ -0,0 +1,13 @@ +// Stdio modern-MCP fixture: SDK 2.x serveStdio pinned to 2026-07-28 with +// legacy openings refused (the house services' posture). +// Usage: node modern-mcp-server.mjs [startLog] [eventsFile] +// startLog receives one line per process start, to count launches. +import { appendFileSync } from 'node:fs'; +import { serveStdio } from '@modelcontextprotocol/server/stdio'; +import { createServer } from './modern-tools.mjs'; + +const [startLog, eventsFile] = process.argv.slice(2); +if (startLog) appendFileSync(startLog, `start ${process.pid}\n`); +process.stderr.write(`modern fixture ${process.pid} up\n`); + +serveStdio(() => createServer({ eventsFile }), { legacy: 'reject' }); diff --git a/test/fixtures/modern-tools.mjs b/test/fixtures/modern-tools.mjs new file mode 100644 index 000000000..d4e28a724 --- /dev/null +++ b/test/fixtures/modern-tools.mjs @@ -0,0 +1,73 @@ +// Tools for the modern-MCP fixture servers (SDK 2.x McpServer), shared by the +// stdio fixture (modern-mcp-server.mjs) and the in-process HTTP test server. +import { appendFileSync } from 'node:fs'; +import { fromJsonSchema, McpServer } from '@modelcontextprotocol/server'; + +const obj = (properties = {}) => fromJsonSchema({ type: 'object', properties }); + +/** + * @param {{ log?: (line: string) => void, eventsFile?: string }} [opts] + * `eventsFile` (optional) receives one line per notable server-side event + * (`slow-start`, `slow-aborted`, `slow-finished`) so a test can observe what + * the server saw, independently of what the client reports. + */ +export function createServer(opts = {}) { + const note = (line) => { + opts.log?.(line); + if (opts.eventsFile) appendFileSync(opts.eventsFile, `${line}\n`); + }; + const server = new McpServer( + { name: 'modern-fixture', version: '1.0.0' }, + { capabilities: { tools: { listChanged: true } } }, + ); + server.registerTool('echo', { description: 'Echo text back', inputSchema: obj({ text: { type: 'string' } }) }, + async (args) => ({ content: [{ type: 'text', text: `echo:${args.text}` }] })); + server.registerTool('structured_only', { description: 'Only structured content', inputSchema: obj() }, + async () => ({ content: [], structuredContent: { answer: 42, ok: false, nothing: null } })); + server.registerTool('structured_and_text', { description: 'Summary text plus structured data', inputSchema: obj() }, + async () => ({ content: [{ type: 'text', text: 'Found 2 rows.' }], structuredContent: { rows: [{ id: 1 }, { id: 2 }] } })); + server.registerTool('structured_zero', { description: 'Falsy structured value', inputSchema: obj() }, + async () => ({ content: [], structuredContent: 0 })); + server.registerTool('media', { description: 'Mixed standard content', inputSchema: obj() }, + async () => ({ + content: [ + { type: 'text', text: 'before' }, + { type: 'resource_link', uri: 'file:///srv/report.pdf', name: 'report.pdf', mimeType: 'application/pdf' }, + { type: 'resource', resource: { uri: 'memo://note', mimeType: 'text/plain', text: 'embedded note text' } }, + { type: 'resource', resource: { uri: 'memo://bin', mimeType: 'application/octet-stream', blob: Buffer.from('binary!').toString('base64') } }, + { type: 'audio', data: Buffer.from('RIFF0000WAVEfmt ').toString('base64'), mimeType: 'audio/wav' }, + { type: 'image', data: 'iVBORw0KGgo=', mimeType: 'image/png' }, + ], + })); + server.registerTool('fail', { description: 'Reports a tool error', inputSchema: obj() }, + async () => ({ content: [{ type: 'text', text: 'the tool failed on purpose' }], isError: true })); + server.registerTool('slow', { description: 'Sleeps, then answers', inputSchema: obj({ ms: { type: 'number' } }) }, + async (args, ctx) => { + const ms = Number(args.ms ?? 1000); + const signal = ctx?.mcpReq?.signal; + note('slow-start'); + signal?.addEventListener('abort', () => note('slow-aborted')); + await new Promise((resolve) => setTimeout(resolve, ms)); + note('slow-finished'); + return { content: [{ type: 'text', text: `slept ${ms}` }] }; + }); + server.registerTool('env', { description: 'Reports selected environment variables', inputSchema: obj() }, + async () => ({ content: [{ type: 'text', text: JSON.stringify({ + declared: process.env.FIXTURE_DECLARED ?? null, + hostOnly: process.env.FIXTURE_HOST_ONLY ?? null, + path: Boolean(process.env.PATH), + }) }] })); + let added = 0; + server.registerTool('add_tool', { description: 'Registers another tool (list change)', inputSchema: obj() }, + async () => { + const name = `added_${++added}`; + server.registerTool(name, { description: 'added at runtime', inputSchema: obj() }, + async () => ({ content: [{ type: 'text', text: name }] })); + return { content: [{ type: 'text', text: `registered ${name}` }] }; + }); + server.registerTool('shout', { description: 'Writes a line to stderr', inputSchema: obj({ text: { type: 'string' } }) }, + async (args) => { process.stderr.write(`shout: ${args.text}\n`); return { content: [{ type: 'text', text: 'shouted' }] }; }); + server.registerTool('die', { description: 'Exits the server process', inputSchema: obj() }, + async () => { setTimeout(() => process.exit(7), 10); return { content: [{ type: 'text', text: 'bye' }] }; }); + return server; +} diff --git a/test/mcp-modern-connection.test.ts b/test/mcp-modern-connection.test.ts new file mode 100644 index 000000000..f496ec677 --- /dev/null +++ b/test/mcp-modern-connection.test.ts @@ -0,0 +1,186 @@ +/** + * The modern engine (room-284, shelf-487) over stdio, against an SDK 2.x + * fixture that refuses legacy openings, as the house services do. What + * Connectome owns here, rather than the SDK: + * - one launch per connect, through Connectome's own spawner (env allowlist, + * stderr), instead of the SDK's sibling-probe launch; + * - one deadline per call, which ends as `no-response` with cancellation + * requested, and is never replayed; + * - results kept by presence (structured `0` is a value); + * - reconnect with the legacy backoff settings, and list-change awareness; + * - the request-outcome contract on failures. + */ +import { test, afterEach } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtempSync, readFileSync, existsSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, dirname } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +import { ModernMcpConnection } from '../src/mcpl/modern-connection.js'; +import { McplRequestError } from '../src/mcpl/server-connection.js'; +import type { McplServerConfig } from '../src/mcpl/types.js'; + +const here = dirname(fileURLToPath(import.meta.url)); +const SERVER = join(here, 'fixtures', 'modern-mcp-server.mjs'); + +const cleanups: Array<() => Promise | void> = []; +afterEach(async () => { + while (cleanups.length) await cleanups.pop()!(); +}); + +function scratch(): string { + const dir = mkdtempSync(join(tmpdir(), 'modern-mcp-')); + cleanups.push(() => rmSync(dir, { recursive: true, force: true })); + return dir; +} + +const lines = (file: string) => (existsSync(file) ? readFileSync(file, 'utf8').split('\n').filter(Boolean) : []); +const until = async (cond: () => boolean, ms = 5000) => { + const deadline = Date.now() + ms; + while (!cond() && Date.now() < deadline) await new Promise((r) => setTimeout(r, 20)); + assert.ok(cond(), 'condition reached in time'); +}; + +function config(dir: string, extra: Partial = {}): McplServerConfig { + return { + id: 'modern', + command: process.execPath, + args: [SERVER, join(dir, 'starts.log'), join(dir, 'events.log')], + protocol: 'modern', + ...extra, + }; +} + +async function connect(c: McplServerConfig): Promise { + const connection = await ModernMcpConnection.connect(c); + cleanups.push(() => connection.close()); + return connection; +} + +test('connects once over Connectome\'s spawner and speaks 2026-07-28', async () => { + const dir = scratch(); + const stderr: string[] = []; + const connection = await connect(config(dir)); + connection.on('stderr', ({ line }: { line: string }) => stderr.push(line)); + assert.equal(connection.isConnected, true); + assert.equal(connection.protocolVersion, '2026-07-28'); + assert.equal(connection.transportKind, 'stdio'); + const names = (await connection.listTools()).map((t) => t.name); + assert.ok(names.includes('echo') && names.includes('structured_only'), names.join()); + const echo = await connection.callTool('echo', { text: 'hi' }); + assert.deepEqual(echo.content, [{ type: 'text', text: 'echo:hi' }]); + // In-place discovery: the SDK's sibling probe would have made this 2. + assert.equal(lines(join(dir, 'starts.log')).length, 1); + // The child's stderr surfaces as lines, as a legacy child's does. + await connection.callTool('shout', { text: 'hello' }); + await until(() => stderr.includes('shout: hello')); +}); + +test('the child gets the env allowlist plus declared env, as legacy children do', async () => { + const dir = scratch(); + process.env.FIXTURE_HOST_ONLY = 'leaked'; + cleanups.push(() => { delete process.env.FIXTURE_HOST_ONLY; }); + const connection = await connect(config(dir, { env: { FIXTURE_DECLARED: 'yes' } })); + const result = await connection.callTool('env', {}); + const seen = JSON.parse((result.content[0] as { text: string }).text); + assert.deepEqual(seen, { declared: 'yes', hostOnly: null, path: true }); +}); + +test('results are kept by presence: structured-only, falsy structured, tool errors', async () => { + const connection = await connect(config(scratch())); + const only = await connection.callTool('structured_only', {}); + assert.deepEqual(only.content, []); + assert.deepEqual(only.structuredContent, { answer: 42, ok: false, nothing: null }); + const zero = await connection.callTool('structured_zero', {}); + assert.ok('structuredContent' in zero); + assert.equal(zero.structuredContent, 0); + const plain = await connection.callTool('echo', { text: 'x' }); + assert.ok(!('structuredContent' in plain)); + const fail = await connection.callTool('fail', {}); + assert.equal(fail.isError, true); +}); + +test('a deadline requests cancellation and reports no-response, outcome unknown', async () => { + const dir = scratch(); + const connection = await connect(config(dir, { requestTimeoutMs: 300 })); + await connection.callTool('echo', { text: 'warm' }); // the slow call is not the connection's first request + const started = Date.now(); + await assert.rejects(connection.callTool('slow', { ms: 2000 }), (err: unknown) => { + assert.ok(err instanceof McplRequestError); + assert.equal(err.outcome, 'no-response'); + assert.match(err.message, /did not answer tools\/call "slow" within 300ms/); + assert.match(err.message, /Cancellation was requested; the outcome is unknown/); + return true; + }); + assert.ok(Date.now() - started < 1500, 'rejected at the deadline, not at the tool\'s end'); + // The server saw the cancellation: the request was cancelled, not merely abandoned. + await until(() => lines(join(dir, 'events.log')).includes('slow-aborted')); +}); + +test('a list change is announced, and the next listing has the new tool', async () => { + const connection = await connect(config(scratch())); + let changes = 0; + connection.on('tools-list-changed', () => changes++); + await connection.callTool('add_tool', {}); + await until(() => changes > 0); + assert.ok((await connection.listTools()).some((t) => t.name === 'added_1')); +}); + +test('a lost child reconnects with backoff, relaunching once, and lists again', async () => { + const dir = scratch(); + const connection = await connect(config(dir, { reconnect: true, reconnectIntervalMs: 50, reconnectMaxIntervalMs: 100 })); + const events: string[] = []; + connection.on('close', () => events.push('close')); + connection.on('reconnect', () => events.push('reconnect')); + await connection.callTool('die', {}); + await until(() => events.includes('reconnect')); + assert.deepEqual(events, ['close', 'reconnect']); + assert.equal(connection.isConnected, true); + assert.ok((await connection.listTools()).some((t) => t.name === 'echo')); + assert.equal(lines(join(dir, 'starts.log')).length, 2); +}); + +test('without reconnect a lost child stays down, and calls are refused before sending', async () => { + const connection = await connect(config(scratch())); + let closed = false; + connection.on('close', () => { closed = true; }); + await connection.callTool('die', {}); + await until(() => closed); + assert.equal(connection.isConnected, false); + assert.equal(connection.willReconnect, false); + await assert.rejects(connection.callTool('echo', { text: 'x' }), (err: unknown) => { + assert.ok(err instanceof McplRequestError); + assert.equal(err.outcome, 'not-sent'); + return true; + }); +}); + +test('a first connect that fails keeps retrying with reconnect, and close stops it', async () => { + const dir = scratch(); + const failures: unknown[] = []; + const connection = await connect({ + id: 'nothing', + command: process.execPath, + args: ['-e', 'process.exit(1)'], + protocol: 'modern', + reconnect: true, + reconnectIntervalMs: 30, + reconnectMaxIntervalMs: 60, + }); + connection.on('connect-failed', (e) => failures.push(e)); + connection.on('reconnect-failed', (e) => failures.push(e)); + assert.equal(connection.isConnected, false); + await until(() => failures.length >= 2); + await connection.close(); + const count = failures.length; + await new Promise((r) => setTimeout(r, 200)); + assert.equal(failures.length, count, 'no attempts after close'); + void dir; +}); + +test('configuration mistakes throw before anything is launched', async () => { + await assert.rejects(ModernMcpConnection.connect({ id: 'x', url: 'https://x.invalid/mcp', protocol: 'modern' }), /applies only to stdio/); + await assert.rejects(ModernMcpConnection.connect({ id: 'x', command: 'srv', protocol: 'modern', requestTimeoutMs: 0 }), /requestTimeoutMs must be an integer/); + await assert.rejects(ModernMcpConnection.connect({ id: 'x', command: 'srv' }), /not configured for modern MCP/); +}); diff --git a/test/mcp-modern-http.test.ts b/test/mcp-modern-http.test.ts new file mode 100644 index 000000000..106bf9b50 --- /dev/null +++ b/test/mcp-modern-http.test.ts @@ -0,0 +1,138 @@ +/** + * The modern engine over Streamable HTTP (room-284, shelf-487), against an + * in-process SDK 2.x `createMcpHandler` that refuses legacy openings. This is + * the one HTTP smoke path. It covers Connectome's own seams, and leaves the + * SDK's conformance to the SDK: + * - credentials: a cached bearer from `accessProvider`, refreshed once on 401; + * - the required headers reach the server; + * - a deadline aborts the request stream, which the server sees as cancellation; + * - an auth wall is an `error-response` carrying the HTTP status. + */ +import { test, afterEach } from 'node:test'; +import assert from 'node:assert/strict'; +import { createServer as createHttpServer, type Server } from 'node:http'; +import { Readable } from 'node:stream'; +import { join, dirname } from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; +import type { AddressInfo } from 'node:net'; + +import { createMcpHandler } from '@modelcontextprotocol/server'; +import { ModernMcpConnection } from '../src/mcpl/modern-connection.js'; +import { McplRequestError } from '../src/mcpl/server-connection.js'; + +const here = dirname(fileURLToPath(import.meta.url)); + +interface Seen { method: string; auth: string | null; version: string | null; mcpMethod: string | null; mcpName: string | null } + +async function startServer(accepted: (token: string | null) => boolean): Promise<{ url: string; seen: Seen[]; events: string[]; server: Server }> { + const { createServer } = await import(pathToFileURL(join(here, 'fixtures', 'modern-tools.mjs')).href) as { + createServer: (opts: { log: (line: string) => void }) => unknown; + }; + const seen: Seen[] = []; + const events: string[] = []; + const handler = createMcpHandler(() => createServer({ log: (line) => events.push(line) }) as never, { legacy: 'reject' }); + const server = createHttpServer(async (req, res) => { + const auth = req.headers.authorization ?? null; + seen.push({ + method: req.method ?? '', + auth, + version: (req.headers['mcp-protocol-version'] as string | undefined) ?? null, + mcpMethod: (req.headers['mcp-method'] as string | undefined) ?? null, + mcpName: (req.headers['mcp-name'] as string | undefined) ?? null, + }); + if (!accepted(auth?.replace(/^Bearer /, '') ?? null)) { + res.writeHead(401, { 'content-type': 'text/plain' }).end('unauthorized'); + return; + } + const chunks: Buffer[] = []; + for await (const chunk of req) chunks.push(chunk as Buffer); + const headers = new Headers(); + for (const [key, value] of Object.entries(req.headers)) { + if (typeof value === 'string') headers.set(key, value); + else if (Array.isArray(value)) headers.set(key, value.join(', ')); + } + // The client closing its request stream is the HTTP binding's + // cancellation: carry it into the handler as an abort. + const abort = new AbortController(); + res.on('close', () => { if (!res.writableEnded) abort.abort(); }); + const response = await handler.fetch(new Request(`http://127.0.0.1${req.url}`, { + method: req.method, + headers, + body: req.method === 'GET' || req.method === 'HEAD' ? undefined : Buffer.concat(chunks), + signal: abort.signal, + })); + res.writeHead(response.status, Object.fromEntries(response.headers)); + if (response.body) Readable.fromWeb(response.body as never).pipe(res); + else res.end(); + }); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + const { port } = server.address() as AddressInfo; + return { url: `http://127.0.0.1:${port}/mcp`, seen, events, server }; +} + +const cleanups: Array<() => Promise | void> = []; +afterEach(async () => { + while (cleanups.length) await cleanups.pop()!(); +}); + +test('HTTP: bearer from accessProvider, one refresh on 401, required headers, calls', async () => { + const { url, seen, server } = await startServer((token) => token === 'token-2'); + cleanups.push(() => new Promise((resolve) => { server.closeAllConnections?.(); server.close(() => resolve()); })); + let issued = 0; + const connection = await ModernMcpConnection.connect({ + id: 'remote', + url, + accessProvider: async () => `token-${++issued}`, + }); + cleanups.push(() => connection.close()); + + assert.equal(connection.transportKind, 'http'); + assert.equal(connection.protocolVersion, '2026-07-28'); + assert.deepEqual((await connection.callTool('echo', { text: 'over http' })).content, [{ type: 'text', text: 'echo:over http' }]); + const only = await connection.callTool('structured_only', {}); + assert.deepEqual(only.structuredContent, { answer: 42, ok: false, nothing: null }); + + assert.equal(issued, 2, 'token-1 was refused once, token-2 cached thereafter'); + const posts = seen.filter((s) => s.method === 'POST'); + assert.ok(posts.every((s) => s.version === '2026-07-28'), 'every POST names the revision'); + assert.ok(posts.every((s) => s.mcpMethod !== null), 'every POST names its method'); + assert.ok(posts.some((s) => s.mcpMethod === 'tools/call' && s.mcpName === 'echo'), 'tools/call names its tool'); + assert.ok(posts.slice(1).every((s) => s.auth === 'Bearer token-2')); +}); + +test('HTTP: a deadline aborts the request stream, and the server sees the cancellation', async () => { + const { url, events, server } = await startServer(() => true); + cleanups.push(() => new Promise((resolve) => { server.closeAllConnections?.(); server.close(() => resolve()); })); + const connection = await ModernMcpConnection.connect({ id: 'remote', url, token: 't', requestTimeoutMs: 300 }); + cleanups.push(() => connection.close()); + await assert.rejects(connection.callTool('slow', { ms: 1500 }), (err: unknown) => { + assert.ok(err instanceof McplRequestError); + assert.equal(err.outcome, 'no-response'); + assert.match(err.message, /Cancellation was requested/); + return true; + }); + const deadline = Date.now() + 3000; + while (!events.includes('slow-aborted') && Date.now() < deadline) await new Promise((r) => setTimeout(r, 20)); + assert.ok(events.includes('slow-aborted'), `server events: ${events.join(', ')}`); +}); + +test('HTTP: a credential the server keeps refusing fails the connect, naming 401', async () => { + const { url, server } = await startServer(() => false); + cleanups.push(() => new Promise((resolve) => { server.closeAllConnections?.(); server.close(() => resolve()); })); + await assert.rejects(ModernMcpConnection.connect({ id: 'remote', url, token: 'stale' }), /401/); +}); + +test('HTTP: an auth wall on a call is an error-response carrying the status', async () => { + let allow = true; + const { url, server } = await startServer(() => allow); + cleanups.push(() => new Promise((resolve) => { server.closeAllConnections?.(); server.close(() => resolve()); })); + const connection = await ModernMcpConnection.connect({ id: 'remote', url, token: 't' }); + cleanups.push(() => connection.close()); + allow = false; + await assert.rejects(connection.callTool('echo', { text: 'x' }), (err: unknown) => { + assert.ok(err instanceof McplRequestError); + assert.equal(err.outcome, 'error-response'); + assert.equal(err.code, 401); + return true; + }); +}); From 7f7e91b600e2e3b2563b8e3b99b4c7f088588246 Mon Sep 17 00:00:00 2001 From: Olive-1804 Date: Thu, 8 Oct 2026 01:38:00 -0400 Subject: [PATCH 0059/1112] =?UTF-8?q?fix(mcp):=20review=20corrections=20to?= =?UTF-8?q?=20batches=201=E2=80=932,=20outcomes=20from=20wire=20evidence?= =?UTF-8?q?=20and=20owned=20lifetimes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Theo-1805's review of 2c9aff1/3a3104c (room-284 #53300, #53316, #53318, #53323) and Petra-1012's read of the outcome mapping (room-203 #53269), reproduced against SDK 2.3.1. Request outcomes from the wire (was: inferred from error classes). ProtocolError doesn't prove a remote error: SDK 2.3.1 raises it locally for an outputSchema that won't compile (before any tools/call) and for output validation failures (after a successful result), and both were reported as error-response. Now an ObservedTransport sits between the SDK and the real transport, and a per-request AsyncLocalStorage scope records two things: whether a request of the observed method was handed off, and how the server answered the most recent one. - Never handed off → not-sent. This covers SDK pre-dispatch rejects, a pre-aborted signal, and no transport. - A JSON-RPC error on the wire → error-response with that error's code and data. - A result → a plain Error, "answered, but the result can't be used". - Handed off and unanswered → no-response. At the deadline, cancellation was requested. - HTTP 401/403 → error-response with code undefined and data.httpStatus, so JSON-RPC and HTTP codes never share `code` (Petra). Auxiliary requests inside a call, such as the SDK's header-mismatch relist, are another method, so they never overwrite the call's evidence. listTools() uses the same contract with tools/list as its method, and one LIST_TIMEOUT_MS budget now bounds the whole paginated walk (an outer signal), not each page. Lifetime: - close() owns a connect in flight. During server/discover the SDK hasn't attached its Protocol yet, so client.close() alone leaves the probe's child alive. close() now closes the opening transport, then awaits the connect's end. - List-change awareness covers a subscription the server advertises but that never opened. The SDK swallows a refused initial listen into onerror and still connects, so keepListening() schedules a reopen when capabilities.tools.listChanged is set and no subscription exists. - Reopen timers are keyed by generation. A stale failure can't take the single slot or schedule across generations; handleLost and each new generation cancel it. Legacy pagination (2c9aff1): cursors are opaque, so '' continues and only an absent nextCursor ends the inventory. A page without a tools array, or with a non-string nextCursor (null included), is an error rather than a partial or empty inventory published as complete. Tests: mcp-modern-outcomes (6; a hand-written server logs every request, and each outcome is checked against that log: 0 calls; 1 call answered; a wire error; listen refused then reopened, also across a reconnect; close() during a hanging reconnect handshake kills that launch, and no launch follows). Legacy: '' cursor continues; malformed pages rejected. HTTP auth-wall test updated for code/httpStatus. Full suite 1413/1413; the touched files repeated 3x stable. --- src/mcpl/modern-connection.ts | 420 ++++++++++++++++-------- src/mcpl/server-connection.ts | 27 +- test/fixtures/legacy-version-server.mjs | 14 +- test/fixtures/modern-raw-server.mjs | 92 ++++++ test/mcp-modern-http.test.ts | 6 +- test/mcp-modern-outcomes.test.ts | 146 ++++++++ test/mcpl-legacy-version.test.ts | 15 +- 7 files changed, 573 insertions(+), 147 deletions(-) create mode 100644 test/fixtures/modern-raw-server.mjs create mode 100644 test/mcp-modern-outcomes.test.ts diff --git a/src/mcpl/modern-connection.ts b/src/mcpl/modern-connection.ts index cce545d09..9aaeae664 100644 --- a/src/mcpl/modern-connection.ts +++ b/src/mcpl/modern-connection.ts @@ -16,20 +16,27 @@ * - **One deadline per tool call** (`requestTimeoutMs`, always positive) * bounds every leg of the call. At the deadline the SDK requests * cancellation and the outcome is unknown. Calls are never replayed. + * - **Request outcomes from the wire.** A failed call is classified by what + * crossed Connectome's transport boundary, never by the error's class. The + * SDK raises the same error classes locally (an invalid outputSchema before + * dispatch, output validation after a successful result) as it does for + * server answers. * - **Lifetime.** On a lost transport the connection restarts with the same - * backoff settings as legacy, when `reconnect` is set. A lost list-change - * stream is reopened while the connection lives. Each connect is a - * generation, so a superseded client can never touch a newer one. + * backoff settings as legacy, when `reconnect` is set. List-change + * awareness is kept while the connection lives: a subscription the server + * advertises but that didn't open, or that ended, is reopened. Each connect + * is a generation, so a superseded client never touches a newer one. + * `close()` owns a connect in flight too. * * Nothing here is MCPL: a modern server has no grant, no planes and no * server→host requests. The framework uses this connection only through its * tool paths, and the MCPL machinery never sees it. */ +import { AsyncLocalStorage } from 'node:async_hooks'; import { EventEmitter } from 'node:events'; import { Client, - ProtocolError, SdkError, SdkErrorCode, SdkHttpError, @@ -38,7 +45,9 @@ import { type CallToolResult, type JSONRPCMessage, type McpSubscription, + type MessageExtraInfo, type Transport, + type TransportSendOptions, } from '@modelcontextprotocol/client'; import { StdioTransport, type McplTransport } from './transport.js'; @@ -62,6 +71,102 @@ export interface ModernToolCallResult { _meta?: Record; } +/** + * What crossed Connectome's transport boundary for one logical request: were + * any of its requests handed to the transport, and how did the server answer + * the most recent one? A logical call can take several legs: a + * header-mismatch retry, or the rounds of an `input_required` continuation. + */ +interface WireRecord { + readonly method: string; + handedOff: boolean; + currentId?: string | number; + answer: 'none' | 'result' | 'error'; + error?: { code: number; message: string; data?: unknown }; +} + +/** The logical request a send belongs to. The SDK hands each request to the + * transport inside the caller's async chain, so the scope reaches `send`. */ +const wireScope = new AsyncLocalStorage(); + +/** + * The boundary between the SDK and the actual transport. It records, for the + * logical request in scope, each request of the observed method that it + * hands on, and the answer that comes back for it. Everything else passes + * through untouched: the SDK sees the inner transport's own members. + */ +class ObservedTransport implements Transport { + onclose?: () => void; + onerror?: (error: Error) => void; + onmessage?: (message: T, extra?: MessageExtraInfo) => void; + private readonly awaiting = new Map(); + + constructor(private readonly inner: Transport) {} + + get sessionId(): string | undefined { + return this.inner.sessionId; + } + + get hasPerRequestStream(): boolean | undefined { + return this.inner.hasPerRequestStream; + } + + setProtocolVersion(version: string): void { + this.inner.setProtocolVersion?.(version); + } + + setSupportedProtocolVersions(versions: string[]): void { + this.inner.setSupportedProtocolVersions?.(versions); + } + + async start(): Promise { + this.inner.onmessage = (message, extra) => { + this.observe(message); + this.onmessage?.(message, extra); + }; + this.inner.onerror = (error) => this.onerror?.(error); + this.inner.onclose = () => this.onclose?.(); + await this.inner.start(); + } + + async send(message: JSONRPCMessage, options?: TransportSendOptions): Promise { + const record = wireScope.getStore(); + if (record && 'method' in message && 'id' in message && message.method === record.method) { + // Handed off from here on: past this point nothing proves it unsent. + record.handedOff = true; + record.currentId = message.id; + record.answer = 'none'; + record.error = undefined; + this.awaiting.set(message.id, record); + } + await this.inner.send(message, options); + } + + async close(): Promise { + await this.inner.close(); + } + + /** Drop a finished request's entries (unanswered legs included). */ + forget(record: WireRecord): void { + for (const [id, entry] of this.awaiting) if (entry === record) this.awaiting.delete(id); + } + + private observe(message: JSONRPCMessage): void { + if (!('id' in message) || (!('result' in message) && !('error' in message))) return; + const record = this.awaiting.get(message.id as string | number); + if (!record) return; + this.awaiting.delete(message.id as string | number); + // A superseded leg's late answer says nothing about the current one. + if (record.currentId !== message.id) return; + if ('error' in message) { + record.answer = 'error'; + record.error = message.error as WireRecord['error']; + } else { + record.answer = 'result'; + } + } +} + /** * SDK `Transport` over Connectome's stdio spawner. Framing is the same * newline-delimited JSON-RPC as legacy. Lines that are not JSON are ignored, @@ -73,7 +178,7 @@ export interface ModernToolCallResult { class SpawnerStdioTransport implements Transport { onclose?: () => void; onerror?: (error: Error) => void; - onmessage?: (message: JSONRPCMessage) => void; + onmessage?: (message: T, extra?: MessageExtraInfo) => void; private line: McplTransport | null = null; constructor( @@ -137,6 +242,13 @@ function bearerAuth(config: McplServerConfig): AuthProvider | undefined { }; } +/** One connect generation's client and the boundary it talks through. */ +interface Session { + readonly generation: number; + readonly client: Client; + readonly wire: ObservedTransport; +} + export class ModernMcpConnection extends EventEmitter { readonly id: string; readonly family = 'modern' as const; @@ -144,8 +256,9 @@ export class ModernMcpConnection extends EventEmitter { /** The pinned revision, once a connect has established it; null before. */ protocolVersion: string | null = null; - private client: Client | null = null; - private connected = false; + private session: Session | null = null; + /** A connect in flight, owned so `close()` can end it. */ + private opening: { session: Session; done: Promise } | null = null; private generation = 0; private closedByHost = false; private readonly reconnectEnabled: boolean; @@ -153,7 +266,8 @@ export class ModernMcpConnection extends EventEmitter { private readonly reconnectMaxIntervalMs: number; private reconnectTimer: ReturnType | null = null; private reconnectAttempts = 0; - private relistenTimer: ReturnType | null = null; + /** At most one pending reopen, and only ever for the current generation. */ + private relisten: { generation: number; timer: ReturnType } | null = null; private relistenAttempts = 0; private constructor(private readonly config: McplServerConfig) { @@ -180,7 +294,7 @@ export class ModernMcpConnection extends EventEmitter { } get isConnected(): boolean { - return this.connected; + return this.session !== null; } /** Whether a lost connection will be re-established in the background. */ @@ -209,14 +323,18 @@ export class ModernMcpConnection extends EventEmitter { return connection; } - /** One connect generation: a fresh client and transport. */ + private isCurrent(generation: number): boolean { + return generation === this.generation && !this.closedByHost; + } + + /** One connect generation: a fresh client over a fresh transport. */ private async open(): Promise { const generation = ++this.generation; - const live = () => generation === this.generation && !this.closedByHost; - - const transport: Transport = this.transportKind === 'stdio' - ? new SpawnerStdioTransport(this.config, (line) => { if (live()) this.emit('stderr', { line }); }) + this.cancelRelisten(); + const inner: Transport = this.transportKind === 'stdio' + ? new SpawnerStdioTransport(this.config, (line) => { if (this.isCurrent(generation)) this.emit('stderr', { line }); }) : new StreamableHTTPClientTransport(new URL(this.config.url!), { authProvider: bearerAuth(this.config) }); + const wire = new ObservedTransport(inner); const client = new Client( { name: 'agent-framework', version: '1.0.0' }, @@ -229,77 +347,98 @@ export class ModernMcpConnection extends EventEmitter { tools: { autoRefresh: false, debounceMs: 0, - onChanged: () => { if (live()) this.emit('tools-list-changed'); }, + onChanged: () => { if (this.isCurrent(generation)) this.emit('tools-list-changed'); }, }, }, }, ); client.onerror = (error: Error) => { // The SDK reports late responses to deadline-abandoned calls here - // ("unknown message ID"), among other out-of-band errors. They are - // logged, never delivered: the call's outcome was already reported. - if (live()) this.emit('error', error); + // ("unknown message ID"), and a list-change subscription it could not + // open at connect, among other out-of-band errors. They are logged, + // never delivered: the call's outcome was already reported. + if (this.isCurrent(generation)) this.emit('error', error); }; client.onclose = () => { - if (live()) this.handleLost('transport closed'); + if (this.isCurrent(generation)) this.handleLost(generation, 'transport closed'); }; + const session: Session = { generation, client, wire }; + const done = client.connect(wire, { timeout: CONNECT_TIMEOUT_MS }); + this.opening = { session, done }; try { - await client.connect(transport, { timeout: CONNECT_TIMEOUT_MS }); + await done; } catch (error) { await client.close().catch(() => {}); + await wire.close().catch(() => {}); throw error; + } finally { + if (this.opening?.session === session) this.opening = null; } - if (!live()) { + if (!this.isCurrent(generation)) { + // Superseded or closed while connecting: this generation never serves. await client.close().catch(() => {}); + await wire.close().catch(() => {}); return; } - this.client = client; - this.connected = true; + this.session = session; this.protocolVersion = MODERN_MCP_PROTOCOL_VERSION; this.reconnectAttempts = 0; - this.watchSubscription(client.autoOpenedSubscription, generation); + this.keepListening(session, client.autoOpenedSubscription); } /** - * Keep list-change awareness while the connection lives. The SDK opens the - * subscription at connect when the server advertises tool list changes. - * When the subscription ends without our asking, it is reopened with - * backoff, and the inventory is refreshed once it's back, since changes - * may have been missed in between. A transport loss is the reconnect - * path's job, not this one's. + * Keep list-change awareness while this generation lives. The SDK opens the + * subscription at connect when the server advertises tool list changes. If + * that open failed (the SDK reports it through `onerror` and connects + * anyway), or the subscription later ends without our asking, it is + * reopened with backoff. Once it's back the inventory is refreshed, since + * changes may have been missed. A lost transport is the reconnect path's + * job, not this one's. */ - private watchSubscription(subscription: McpSubscription | undefined, generation: number): void { - if (!subscription) return; + private keepListening(session: Session, subscription: McpSubscription | undefined): void { + if (!session.client.getServerCapabilities()?.tools?.listChanged) return; + if (!subscription) { + this.scheduleRelisten(session.generation); + return; + } void subscription.closed.then((reason) => { - if (reason === 'local' || generation !== this.generation || this.closedByHost || !this.connected) return; - this.scheduleRelisten(generation); + if (reason === 'local' || this.session !== session || !this.isCurrent(session.generation)) return; + this.scheduleRelisten(session.generation); }); } + private cancelRelisten(): void { + if (this.relisten) clearTimeout(this.relisten.timer); + this.relisten = null; + } + private scheduleRelisten(generation: number): void { - if (this.relistenTimer) return; - const delay = this.backoffDelay(this.relistenAttempts); - this.relistenTimer = setTimeout(() => { - this.relistenTimer = null; - void this.relisten(generation); - }, delay); - this.relistenTimer.unref?.(); - } - - private async relisten(generation: number): Promise { - const client = this.client; - if (!client || generation !== this.generation || this.closedByHost || !this.connected) return; + if (!this.isCurrent(generation)) return; + if (this.relisten?.generation === generation) return; + this.cancelRelisten(); + const timer = setTimeout(() => { + if (this.relisten?.timer === timer) this.relisten = null; + void this.reopenSubscription(generation); + }, this.backoffDelay(Math.max(1, this.relistenAttempts))); + timer.unref?.(); + this.relisten = { generation, timer }; + } + + private async reopenSubscription(generation: number): Promise { + const session = this.session; + if (!session || session.generation !== generation || !this.isCurrent(generation)) return; try { - const subscription = await client.listen({ toolsListChanged: true }, { timeout: CONNECT_TIMEOUT_MS }); - if (generation !== this.generation || this.closedByHost) { + const subscription = await session.client.listen({ toolsListChanged: true }, { timeout: CONNECT_TIMEOUT_MS }); + if (this.session !== session || !this.isCurrent(generation)) { await subscription.close().catch(() => {}); return; } this.relistenAttempts = 0; - this.watchSubscription(subscription, generation); + this.keepListening(session, subscription); this.emit('tools-list-changed'); } catch (error) { + if (this.session !== session || !this.isCurrent(generation)) return; this.relistenAttempts++; this.emit('error', new Error(`MCP server "${this.id}" could not reopen its list-change subscription: ${(error as Error).message}`)); this.scheduleRelisten(generation); @@ -307,10 +446,10 @@ export class ModernMcpConnection extends EventEmitter { } /** The transport went away (child exit, or a closed HTTP client). */ - private handleLost(reason: string): void { - if (!this.connected) return; - this.connected = false; - this.client = null; + private handleLost(generation: number, reason: string): void { + if (this.session?.generation !== generation) return; + this.session = null; + this.cancelRelisten(); this.emit('close', { reason }); if (this.willReconnect) this.scheduleReconnect(); } @@ -332,34 +471,41 @@ export class ModernMcpConnection extends EventEmitter { } private async attemptReconnect(): Promise { - if (!this.willReconnect || this.connected) return; + if (!this.willReconnect || this.session) return; const attempt = Math.max(1, this.reconnectAttempts); try { await this.open(); - if (!this.connected) return; + if (!this.session) return; console.error(`MCP server "${this.id}" reconnected`); this.emit('reconnect', { attempts: attempt }); } catch (error) { + if (this.closedByHost) return; this.reconnectAttempts = attempt + 1; this.emit('reconnect-failed', { error: (error as Error).message, attempt }); this.scheduleReconnect(); } } - /** The server's complete tool inventory (the SDK follows every page). */ + /** + * The server's complete tool inventory. The SDK follows every page; one + * budget, {@link LIST_TIMEOUT_MS}, bounds the whole walk, not each page. + * Its failures use the request-outcome contract with `tools/list` as the + * observed method. A relist the SDK makes inside a tool call is auxiliary + * to that call, and leaves the call's evidence alone. + */ async listTools(): Promise { - const client = this.requireClient('tools/list'); - try { - const { tools } = await client.listTools(undefined, { cacheMode: 'refresh', timeout: LIST_TIMEOUT_MS }); - return tools.map((tool) => ({ - name: tool.name, - ...(tool.description !== undefined ? { description: tool.description } : {}), - inputSchema: tool.inputSchema as Record, - ...(tool._meta !== undefined ? { _meta: tool._meta as Record } : {}), - })); - } catch (error) { - throw this.requestFailure('tools/list', error); - } + const { tools } = await this.request('tools/list', 'tools/list', (client) => + client.listTools(undefined, { + cacheMode: 'refresh', + timeout: LIST_TIMEOUT_MS, + signal: AbortSignal.timeout(LIST_TIMEOUT_MS), + }), LIST_TIMEOUT_MS); + return tools.map((tool) => ({ + name: tool.name, + ...(tool.description !== undefined ? { description: tool.description } : {}), + inputSchema: tool.inputSchema as Record, + ...(tool._meta !== undefined ? { _meta: tool._meta as Record } : {}), + })); } /** @@ -369,87 +515,103 @@ export class ModernMcpConnection extends EventEmitter { * before dispatch (an auth refresh, or a header-mismatch relist). */ async callTool(name: string, args: Record): Promise { - const client = this.requireClient('tools/call'); const timeoutMs = this.requestTimeoutMs; - try { - const result = await client.callTool( + const result = await this.request('tools/call', `tools/call "${name}"`, (client) => + client.callTool( { name, arguments: args }, { timeout: timeoutMs, maxTotalTimeout: timeoutMs, signal: AbortSignal.timeout(timeoutMs) }, - ); - return { - content: result.content ?? [], - ...(result.isError !== undefined ? { isError: result.isError } : {}), - ...('structuredContent' in result ? { structuredContent: result.structuredContent } : {}), - ...(result._meta !== undefined ? { _meta: result._meta as Record } : {}), - }; + ), timeoutMs); + return { + content: result.content ?? [], + ...(result.isError !== undefined ? { isError: result.isError } : {}), + ...('structuredContent' in result ? { structuredContent: result.structuredContent } : {}), + ...(result._meta !== undefined ? { _meta: result._meta as Record } : {}), + }; + } + + /** Run one logical request in a wire scope, and classify its failure by + * what that scope saw. */ + private async request(method: string, what: string, run: (client: Client) => Promise, timeoutMs?: number): Promise { + const session = this.session; + if (!session) { + // Connectome refuses before anything reaches the SDK: provably not sent. + throw new McplRequestError(`Cannot send ${what}: connection to "${this.id}" is not established`, 'not-sent'); + } + const record: WireRecord = { method, handedOff: false, answer: 'none' }; + try { + return await wireScope.run(record, () => run(session.client)); } catch (error) { - throw this.requestFailure(`tools/call "${name}"`, error, timeoutMs); + throw this.requestFailure(what, record, error, timeoutMs); + } finally { + session.wire.forget(record); } } - /** Close for good: no reconnect, no reopened subscription. */ + /** Close for good: no reconnect, no reopened subscription, no connect left + * in flight. Awaits the in-flight connect's end. */ async close(): Promise { this.closedByHost = true; if (this.reconnectTimer) clearTimeout(this.reconnectTimer); - if (this.relistenTimer) clearTimeout(this.relistenTimer); this.reconnectTimer = null; - this.relistenTimer = null; - const client = this.client; - const wasConnected = this.connected; - this.client = null; - this.connected = false; - if (client) await client.close().catch(() => {}); - if (wasConnected) this.emit('close', { reason: 'closed by host' }); - } - - /** Connectome refuses before anything reaches the SDK: provably not sent. */ - private requireClient(method: string): Client { - if (!this.client || !this.connected) { - throw new McplRequestError(`Cannot send ${method}: connection to "${this.id}" is not established`, 'not-sent'); + this.cancelRelisten(); + const opening = this.opening; + if (opening) { + // Closing the transport ends the probe; the connect then rejects and + // its own cleanup runs. Wait for that, so nothing outlives close(). + await opening.session.wire.close().catch(() => {}); + await opening.session.client.close().catch(() => {}); + await opening.done.catch(() => {}); + } + const session = this.session; + this.session = null; + if (session) { + await session.client.close().catch(() => {}); + await session.wire.close().catch(() => {}); + this.emit('close', { reason: 'closed by host' }); } - return this.client; } /** - * Map an SDK failure onto the request-outcome contract, claiming no more - * than the failure establishes: - * - `error-response`: the server answered with an error. That is a - * JSON-RPC error, or an HTTP auth refusal, which carries the status as - * its code. - * - `no-response`: everything after hand-off that proves nothing about - * delivery, including the deadline, where cancellation was requested and - * the outcome is unknown. - * A call the server answered with a result Connectome can't use (it fails - * validation, or needs input this client can't supply) is a plain error: - * the server did act, so no request outcome applies. + * Map a failure onto the request-outcome contract by what crossed the + * transport boundary for this request, claiming no more than that shows: + * - nothing handed to the transport: `not-sent`, whatever the SDK raised + * (a pre-dispatch validation, a signal already aborted, no transport); + * - an HTTP 401/403 refusal: `error-response`. The server answered, at the + * HTTP layer, so the status is in `data.httpStatus` and `code` stays the + * JSON-RPC code space's (absent); + * - the most recent leg answered with a JSON-RPC error: `error-response` + * with that error's code and data; + * - the most recent leg answered with a result: the server acted, but what + * it returned can't be used (output validation, input this client can't + * supply, a rounds cap). That's a plain error, not a request outcome; + * - handed off and unanswered: `no-response`. At the deadline, cancellation + * was requested and the outcome is unknown. */ - private requestFailure(what: string, error: unknown, timeoutMs?: number): Error { + private requestFailure(what: string, record: WireRecord, error: unknown, timeoutMs?: number): Error { const prefix = `MCP server "${this.id}"`; - if (error instanceof ProtocolError) { - return new McplRequestError(`${prefix} returned error for ${what}: [${error.code}] ${error.message}`, 'error-response', error.code, error.data); - } + const message = error instanceof Error ? error.message : String(error); if (error instanceof SdkHttpError && (error.code === SdkErrorCode.ClientHttpAuthentication || error.code === SdkErrorCode.ClientHttpForbidden)) { - return new McplRequestError(`${prefix} refused ${what}: HTTP ${error.status} (${error.message})`, 'error-response', error.status, error.data); + const data = error.data && typeof error.data === 'object' ? error.data : {}; + return new McplRequestError(`${prefix} refused ${what}: HTTP ${error.status} (${message})`, 'error-response', undefined, { ...data, httpStatus: error.status }); } - if (error instanceof SdkError) { - switch (error.code) { - case SdkErrorCode.RequestTimeout: - return new McplRequestError( - `${prefix} did not answer ${what}${timeoutMs ? ` within ${timeoutMs}ms` : ''}. Cancellation was requested; ` + - `the outcome is unknown: the tool may still complete server-side. Verify state before retrying; ` + - `a blind retry of a side-effecting tool may duplicate it.`, - 'no-response', - ); - case SdkErrorCode.InvalidResult: - case SdkErrorCode.UnsupportedResultType: - case SdkErrorCode.InputRequiredRoundsExceeded: - case SdkErrorCode.CapabilityNotSupported: - return new Error(`${prefix} answered ${what}, but the result can't be used: ${error.message}`); - default: - return new McplRequestError(`${prefix} ${what} failed: ${error.message}`, 'no-response'); - } + if (!record.handedOff) { + return new McplRequestError(`${prefix} ${what} was not sent: ${message}`, 'not-sent'); + } + if (record.answer === 'error' && record.error) { + const { code, message: wireMessage, data } = record.error; + return new McplRequestError(`${prefix} returned error for ${what}: [${code}] ${wireMessage}`, 'error-response', code, data); + } + if (record.answer === 'result') { + return new Error(`${prefix} answered ${what}, but the result can't be used: ${message}`); + } + if (error instanceof SdkError && error.code === SdkErrorCode.RequestTimeout) { + return new McplRequestError( + `${prefix} did not answer ${what}${timeoutMs ? ` within ${timeoutMs}ms` : ''}. Cancellation was requested; ` + + `the outcome is unknown: the tool may still complete server-side. Verify state before retrying; ` + + `a blind retry of a side-effecting tool may duplicate it.`, + 'no-response', + ); } - const message = error instanceof Error ? error.message : String(error); return new McplRequestError(`${prefix} ${what} failed: ${message}`, 'no-response'); } } diff --git a/src/mcpl/server-connection.ts b/src/mcpl/server-connection.ts index 0a13dcf0a..66a2949c0 100644 --- a/src/mcpl/server-connection.ts +++ b/src/mcpl/server-connection.ts @@ -917,10 +917,12 @@ export class McplServerConnection extends EventEmitter { /** * Send `tools/list` and return the server's complete tool inventory, - * following `nextCursor` across pages (2024-11-05 pagination). A server - * whose cursors don't terminate — a repeated cursor, or more than - * {@link MAX_TOOL_LIST_PAGES} pages — gets an error rather than a silently - * partial inventory. + * following `nextCursor` across pages (2024-11-05 pagination). Cursors are + * opaque strings, `''` included: only an absent `nextCursor` ends the + * inventory. A page that isn't well formed (no `tools` array, a cursor + * that isn't a string), or cursors that don't terminate (a repeat, or more + * than {@link MAX_TOOL_LIST_PAGES} pages), are errors, never a partial or + * empty inventory published as complete. */ async sendToolsList(): Promise<{ tools: McpToolDefinition[] }> { const tools: McpToolDefinition[] = []; @@ -928,18 +930,23 @@ export class McplServerConnection extends EventEmitter { let cursor: string | undefined; let first: Record | undefined; for (let page = 0; page < MAX_TOOL_LIST_PAGES; page++) { - const result = await this.sendRequest('tools/list', cursor === undefined ? {} : { cursor }) as { - tools?: McpToolDefinition[]; - nextCursor?: unknown; - }; + const result = await this.sendRequest('tools/list', cursor === undefined ? {} : { cursor }) as Record | null; + const malformed = (why: string) => + new Error(`MCPL server "${this.id}" sent a malformed tools/list page ${page + 1}: ${why}`); + if (!result || typeof result !== 'object' || !Array.isArray(result.tools)) { + throw malformed('no tools array'); + } first ??= result; - tools.push(...(result.tools ?? [])); - if (typeof result.nextCursor !== 'string' || result.nextCursor === '') { + tools.push(...(result.tools as McpToolDefinition[])); + if (result.nextCursor === undefined) { // The first page's other fields are kept as they came, so a // single-page answer is returned exactly as before. const { nextCursor: _done, ...rest } = first as { nextCursor?: unknown }; return { ...rest, tools }; } + if (typeof result.nextCursor !== 'string') { + throw malformed(`nextCursor must be a string, got ${JSON.stringify(result.nextCursor)}`); + } if (seen.has(result.nextCursor)) { throw new Error(`MCPL server "${this.id}" repeated tools/list cursor ${JSON.stringify(result.nextCursor)}`); } diff --git a/test/fixtures/legacy-version-server.mjs b/test/fixtures/legacy-version-server.mjs index 33979b28e..9a84796f1 100644 --- a/test/fixtures/legacy-version-server.mjs +++ b/test/fixtures/legacy-version-server.mjs @@ -9,9 +9,10 @@ // missing no protocolVersion at all // ok-then-reject `ok` on the first start recorded in startLog, `reject` after // -// tools/list pages: `ok` serves three pages (cursors p2, p3). Tool `cursor-loop` -// mode is selected with MODE_LIST=loop (always nextCursor "same") or -// MODE_LIST=empty (nextCursor ""). Tool `die` exits the process. +// tools/list pages: by default three pages (cursors p2, p3). MODE_LIST picks +// an edge case: loop (always nextCursor "same"), empty (nextCursor "" then a +// second page), notools (no tools array), badcursor (nextCursor 5), +// nullcursor (nextCursor null). Tool `die` exits the process. import { appendFileSync, existsSync, readFileSync } from 'node:fs'; let mode = process.argv[2] ?? 'ok'; @@ -49,7 +50,12 @@ process.stdin.on('data', (chunk) => { } else if (m.method === 'tools/list') { const cursor = m.params?.cursor; if (listMode === 'loop') reply(m.id, { tools: [tool(`loop-${cursor ?? 'first'}`)], nextCursor: 'same' }); - else if (listMode === 'empty') reply(m.id, { tools: [tool('only')], nextCursor: '' }); + // '' is an opaque cursor like any other: it continues to a second page. + else if (listMode === 'empty' && cursor === undefined) reply(m.id, { tools: [tool('first')], nextCursor: '' }); + else if (listMode === 'empty' && cursor === '') reply(m.id, { tools: [tool('second')] }); + else if (listMode === 'notools') reply(m.id, {}); + else if (listMode === 'badcursor') reply(m.id, { tools: [tool('x')], nextCursor: 5 }); + else if (listMode === 'nullcursor') reply(m.id, { tools: [tool('x')], nextCursor: null }); else if (cursor === undefined) reply(m.id, { tools: [tool('a1'), tool('a2')], nextCursor: 'p2' }); else if (cursor === 'p2') reply(m.id, { tools: [tool('b1')], nextCursor: 'p3' }); else if (cursor === 'p3') reply(m.id, { tools: [tool('c1'), tool('die')] }); diff --git a/test/fixtures/modern-raw-server.mjs b/test/fixtures/modern-raw-server.mjs new file mode 100644 index 000000000..84a0e054a --- /dev/null +++ b/test/fixtures/modern-raw-server.mjs @@ -0,0 +1,92 @@ +// A hand-written modern-MCP (2026-07-28) stdio server, for the edge cases an +// SDK server won't produce on request. It logs every launch and request to +// a JSONL file, so tests can check what reached the wire against what the +// client reports. +// Usage: node modern-raw-server.mjs +// Flags, comma-separated: +// invalid-schema tools/list advertises an outputSchema that doesn't compile +// listen-fail-first the first subscriptions/listen of each launch is refused +// hang-discover-later launches after the first never answer server/discover +// Tools: op (outputSchema {value: integer}, answers with a string: invalid +// structured content), plain (text), err (a JSON-RPC error), touch (announces +// a tool list change on the open subscription), die (exits). +import { createInterface } from 'node:readline'; +import { appendFileSync, existsSync, readFileSync } from 'node:fs'; + +const [flagList = '', log] = process.argv.slice(2); +const flags = new Set(flagList.split(',').filter(Boolean)); +const prior = existsSync(log) ? readFileSync(log, 'utf8').split('\n').filter(Boolean).map((l) => JSON.parse(l)) : []; +const launch = prior.filter((x) => x.event === 'start').length + 1; +const note = (x) => appendFileSync(log, `${JSON.stringify({ ...x, launch })}\n`); +note({ event: 'start', pid: process.pid }); + +const send = (obj) => process.stdout.write(`${JSON.stringify(obj)}\n`); +const result = (id, body) => send({ jsonrpc: '2.0', id, result: { resultType: 'complete', ...body } }); +const error = (id, code, message, data) => send({ jsonrpc: '2.0', id, error: { code, message, ...(data === undefined ? {} : { data }) } }); +const SUB = 'io.modelcontextprotocol/subscriptionId'; + +let listens = 0; +let subscription = null; + +createInterface({ input: process.stdin }).on('line', (line) => { + let m; + try { m = JSON.parse(line); } catch { return; } + note({ event: m.id === undefined ? 'notification' : 'request', method: m.method, id: m.id ?? null, ...(m.method === 'tools/call' ? { tool: m.params?.name } : {}) }); + switch (m.method) { + case 'server/discover': + if (flags.has('hang-discover-later') && launch > 1) return; + return result(m.id, { + supportedVersions: ['2026-07-28'], + capabilities: { tools: { listChanged: true } }, + ttlMs: 60000, + cacheScope: 'private', + _meta: { 'io.modelcontextprotocol/serverInfo': { name: 'raw-fixture', version: '1' } }, + }); + case 'subscriptions/listen': + listens++; + if (flags.has('listen-fail-first') && listens === 1) return error(m.id, -32603, 'first listen unavailable'); + subscription = m.id; + return send({ + jsonrpc: '2.0', + method: 'notifications/subscriptions/acknowledged', + params: { notifications: { toolsListChanged: true }, _meta: { [SUB]: m.id } }, + }); + case 'tools/list': + // 2026-07-28 list results carry cache hints. + return result(m.id, { + ttlMs: 60000, + cacheScope: 'private', + tools: [ + { + name: 'op', + inputSchema: { type: 'object' }, + outputSchema: { + type: 'object', + properties: { value: { type: flags.has('invalid-schema') ? 'not-a-json-schema-type' : 'integer' } }, + required: ['value'], + }, + }, + { name: 'plain', inputSchema: { type: 'object' } }, + { name: 'err', inputSchema: { type: 'object' } }, + { name: 'touch', inputSchema: { type: 'object' } }, + { name: 'die', inputSchema: { type: 'object' } }, + ], + }); + case 'tools/call': { + const name = m.params?.name; + if (name === 'op') return result(m.id, { content: [], structuredContent: { value: 'not an integer' } }); + if (name === 'err') return error(m.id, -32603, 'boom', { why: 'on purpose' }); + if (name === 'touch') { + if (subscription !== null) { + send({ jsonrpc: '2.0', method: 'notifications/tools/list_changed', params: { _meta: { [SUB]: subscription } } }); + } + return result(m.id, { content: [{ type: 'text', text: subscription === null ? 'no subscription' : 'touched' }] }); + } + if (name === 'die') { setTimeout(() => process.exit(5), 10); return result(m.id, { content: [{ type: 'text', text: 'bye' }] }); } + return result(m.id, { content: [{ type: 'text', text: 'plain' }] }); + } + default: + if (m.id !== undefined) error(m.id, -32601, `Method not found: ${m.method}`); + } +}); +process.stdin.on('end', () => process.exit(0)); diff --git a/test/mcp-modern-http.test.ts b/test/mcp-modern-http.test.ts index 106bf9b50..235bccb72 100644 --- a/test/mcp-modern-http.test.ts +++ b/test/mcp-modern-http.test.ts @@ -122,7 +122,7 @@ test('HTTP: a credential the server keeps refusing fails the connect, naming 401 await assert.rejects(ModernMcpConnection.connect({ id: 'remote', url, token: 'stale' }), /401/); }); -test('HTTP: an auth wall on a call is an error-response carrying the status', async () => { +test('HTTP: an auth wall on a call is an error-response with the status in data', async () => { let allow = true; const { url, server } = await startServer(() => allow); cleanups.push(() => new Promise((resolve) => { server.closeAllConnections?.(); server.close(() => resolve()); })); @@ -132,7 +132,9 @@ test('HTTP: an auth wall on a call is an error-response carrying the status', as await assert.rejects(connection.callTool('echo', { text: 'x' }), (err: unknown) => { assert.ok(err instanceof McplRequestError); assert.equal(err.outcome, 'error-response'); - assert.equal(err.code, 401); + // The status is HTTP's, not a JSON-RPC code: it rides in data. + assert.equal(err.code, undefined); + assert.equal((err.data as { httpStatus?: number }).httpStatus, 401); return true; }); }); diff --git a/test/mcp-modern-outcomes.test.ts b/test/mcp-modern-outcomes.test.ts new file mode 100644 index 000000000..5c6b2dca7 --- /dev/null +++ b/test/mcp-modern-outcomes.test.ts @@ -0,0 +1,146 @@ +/** + * The modern engine's request outcomes and lifetime against what actually + * crossed the wire (room-284 review, Theo #53300 / Petra #53269). A + * hand-written server logs every request, and each case checks the reported + * outcome against that log: + * - an SDK-local rejection before dispatch is `not-sent` (0 calls on the wire); + * - a result that fails validation is an answer that can't be used (1 call, + * answered), never a claimed server error; + * - a JSON-RPC error on the wire is `error-response` with its code and data; + * - a list-change subscription refused at connect is reopened; + * - `close()` during a reconnect handshake ends the launch it was waiting on. + */ +import { test, afterEach } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtempSync, readFileSync, existsSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, dirname } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +import { ModernMcpConnection } from '../src/mcpl/modern-connection.js'; +import { McplRequestError } from '../src/mcpl/server-connection.js'; +import type { McplServerConfig } from '../src/mcpl/types.js'; + +const here = dirname(fileURLToPath(import.meta.url)); +const SERVER = join(here, 'fixtures', 'modern-raw-server.mjs'); + +interface LogEntry { event: string; method?: string; id?: unknown; tool?: string; pid?: number; launch: number } + +const cleanups: Array<() => Promise | void> = []; +afterEach(async () => { + while (cleanups.length) await cleanups.pop()!(); +}); + +function scratchLog(): string { + const dir = mkdtempSync(join(tmpdir(), 'modern-raw-')); + cleanups.push(() => rmSync(dir, { recursive: true, force: true })); + return join(dir, 'wire.jsonl'); +} + +const wire = (log: string): LogEntry[] => + existsSync(log) ? readFileSync(log, 'utf8').split('\n').filter(Boolean).map((l) => JSON.parse(l) as LogEntry) : []; +const calls = (log: string, tool: string) => wire(log).filter((e) => e.method === 'tools/call' && e.tool === tool).length; + +async function until(cond: () => boolean, what: string, ms = 5000): Promise { + const deadline = Date.now() + ms; + while (!cond() && Date.now() < deadline) await new Promise((r) => setTimeout(r, 20)); + assert.ok(cond(), `in time: ${what}`); +} + +async function connect(flags: string, log: string, extra: Partial = {}): Promise { + const connection = await ModernMcpConnection.connect({ + id: 'raw', + command: process.execPath, + args: [SERVER, flags, log], + protocol: 'modern', + ...extra, + }); + cleanups.push(() => connection.close()); + return connection; +} + +test('an outputSchema the SDK cannot compile rejects before dispatch: not-sent, nothing on the wire', async () => { + const log = scratchLog(); + const connection = await connect('invalid-schema', log); + await connection.listTools(); + await assert.rejects(connection.callTool('op', {}), (err: unknown) => { + assert.ok(err instanceof McplRequestError, String(err)); + assert.equal(err.outcome, 'not-sent'); + assert.match(err.message, /tools\/call "op" was not sent: .*invalid outputSchema/); + return true; + }); + assert.equal(calls(log, 'op'), 0); +}); + +test('a result that fails its outputSchema was answered: a plain error, not a server error', async () => { + const log = scratchLog(); + const connection = await connect('', log); + await connection.listTools(); + await assert.rejects(connection.callTool('op', {}), (err: unknown) => { + assert.ok(!(err instanceof McplRequestError), 'no request outcome is claimed'); + assert.match((err as Error).message, /answered tools\/call "op", but the result can't be used: .*output schema/); + return true; + }); + assert.equal(calls(log, 'op'), 1); +}); + +test('a JSON-RPC error on the wire is error-response with its code and data', async () => { + const log = scratchLog(); + const connection = await connect('', log); + await connection.listTools(); + await assert.rejects(connection.callTool('err', {}), (err: unknown) => { + assert.ok(err instanceof McplRequestError); + assert.equal(err.outcome, 'error-response'); + assert.equal(err.code, -32603); + assert.deepEqual(err.data, { why: 'on purpose' }); + assert.match(err.message, /returned error for tools\/call "err": \[-32603\] boom/); + return true; + }); + // The plain call on the same connection is unaffected. + assert.deepEqual((await connection.callTool('plain', {})).content, [{ type: 'text', text: 'plain' }]); +}); + +test('a subscription refused at connect is reopened, and change awareness returns', async () => { + const log = scratchLog(); + const connection = await connect('listen-fail-first', log, { reconnectIntervalMs: 30, reconnectMaxIntervalMs: 60 }); + let changes = 0; + connection.on('tools-list-changed', () => changes++); + await until(() => wire(log).filter((e) => e.method === 'subscriptions/listen').length >= 2, 'a second listen'); + // Reopening refreshes once, since changes may have been missed meanwhile. + await until(() => changes >= 1, 'the refresh after reopening'); + const before = changes; + const touched = await connection.callTool('touch', {}); + assert.deepEqual(touched.content, [{ type: 'text', text: 'touched' }]); + await until(() => changes > before, 'the announced change'); +}); + +test('awareness survives a reconnect whose first listen is refused again', async () => { + const log = scratchLog(); + const connection = await connect('listen-fail-first', log, { reconnect: true, reconnectIntervalMs: 30, reconnectMaxIntervalMs: 60 }); + let reconnected = false; + connection.on('reconnect', () => { reconnected = true; }); + await connection.callTool('die', {}); + await until(() => reconnected, 'reconnect'); + // Launch 2 refuses its first listen too; the newer generation reopens it. + await until(() => wire(log).filter((e) => e.method === 'subscriptions/listen' && e.launch === 2).length >= 2, 'a second listen on launch 2'); + let changes = 0; + connection.on('tools-list-changed', () => changes++); + await new Promise((r) => setTimeout(r, 50)); + const before = changes; + await connection.callTool('touch', {}); + await until(() => changes > before, 'the announced change on launch 2'); +}); + +test('close() during a reconnect handshake ends the launch it was waiting on, and stops', async () => { + const log = scratchLog(); + const connection = await connect('hang-discover-later', log, { reconnect: true, reconnectIntervalMs: 30, reconnectMaxIntervalMs: 60 }); + await connection.callTool('die', {}); + await until(() => wire(log).some((e) => e.launch === 2 && e.method === 'server/discover'), 'launch 2 waiting on discover'); + const second = wire(log).find((e) => e.launch === 2 && e.event === 'start')!; + await connection.close(); + const alive = (pid: number) => { try { process.kill(pid, 0); return true; } catch { return false; } }; + await until(() => !alive(second.pid!), 'the hanging launch has exited', 3000); + await new Promise((r) => setTimeout(r, 200)); + assert.equal(wire(log).filter((e) => e.event === 'start').length, 2, 'no launch after close'); + assert.equal(connection.isConnected, false); +}); diff --git a/test/mcpl-legacy-version.test.ts b/test/mcpl-legacy-version.test.ts index 1d6a171e0..d672fea31 100644 --- a/test/mcpl-legacy-version.test.ts +++ b/test/mcpl-legacy-version.test.ts @@ -132,10 +132,21 @@ test('tools/list follows nextCursor to the complete inventory', async () => { assert.deepEqual(tools.map((t) => t.name), ['a1', 'a2', 'b1', 'c1', 'die']); }); -test('an empty nextCursor ends the inventory', async () => { +test('an empty-string cursor is a cursor: only an absent nextCursor ends the inventory', async () => { const connection = await connect(config('ok', { env: { MODE_LIST: 'empty' } })); const { tools } = await connection.sendToolsList(); - assert.deepEqual(tools.map((t) => t.name), ['only']); + assert.deepEqual(tools.map((t) => t.name), ['first', 'second']); +}); + +test('a malformed page is an error, never a partial or empty inventory', async () => { + for (const [mode, pattern] of [ + ['notools', /malformed tools\/list page 1: no tools array/], + ['badcursor', /malformed tools\/list page 1: nextCursor must be a string, got 5/], + ['nullcursor', /malformed tools\/list page 1: nextCursor must be a string, got null/], + ] as const) { + const connection = await connect(config('ok', { env: { MODE_LIST: mode } })); + await assert.rejects(connection.sendToolsList(), pattern, mode); + } }); test('a repeating cursor is an error, not a silently partial inventory', async () => { From 9a2710808954930c0252fbca9bc1a7c9b8a93d7c Mon Sep 17 00:00:00 2001 From: Olive-1804 Date: Thu, 8 Oct 2026 02:21:14 -0400 Subject: [PATCH 0060/1112] fix(mcp): second review round, continuation scope, reaping, malformed frames, startup stderr MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Theo-1805's recheck of 7f7e91b (room-284 #53325, #53542, #53568, #53570). - input_required continuations keep their evidence. The SDK drives the next leg from the transport's inbound callback, outside the caller's async chain, so leg 2's send had no wire scope, and an error or a silence on leg 2 was reported as leg 1's "answered, but can't be used". Now the observer hands each matched answer to the SDK inside its record's scope (wireScope.run), so every leg's send is recorded against its own call. A superseded leg's late answer re-enters nothing. - stdio close reaps. StdioTransport.close() resolved as soon as kill() set child.killed, before the child exited, so a child handling SIGTERM outlived close(). Now it waits for the real exit. After 2 s it sends SIGKILL and waits up to 2 s more. A child that never spawned returns at once. This is a shared primitive, so the legacy engine reaps too. - Writes that race a child's exit no longer crash the host. An EPIPE on child.stdin had no listener: it is now reported as a transport 'error' when anyone listens, and otherwise ignored, since 'close' already reports the exit. A full-suite run caught this as an uncaught EPIPE in a reconnect test. - A JSON line that isn't a JSON-RPC message is a diagnostic, never a crash. `null`, an array or a bare value went to observe() as a message and threw from the stream callback. Lines are now validated with the SDK's own parseJSONRPCMessage. A malformed one goes to onerror (bounded excerpt) and is dropped, and observe() guards non-objects as well. - Startup stderr is kept. Lines emitted while connect() was still pending had no listener and were dropped. They are now held, the newest 200, and delivered when the first 'stderr' listener attaches. Tests, raw fixture flags/tools added (ignore-sigterm, nullframe, cont): - an error or a hang on the second leg is error-response with leg 2's code/data, or no-response; - three concurrent continuations (two error, one ok) never cross evidence; - four non-JSON-RPC lines → four diagnostics, and the connection stays usable; - close() returns only after a SIGTERM-ignoring child is dead; - the startup stderr banner reaches a listener attached after connect(). Outcome and connection files: 19/19. --- src/mcpl/modern-connection.ts | 64 +++++++++++++++++++++++++---- src/mcpl/transport.ts | 37 +++++++++++++---- test/fixtures/modern-raw-server.mjs | 22 +++++++++- test/mcp-modern-connection.test.ts | 4 +- test/mcp-modern-outcomes.test.ts | 63 ++++++++++++++++++++++++++++ 5 files changed, 170 insertions(+), 20 deletions(-) diff --git a/src/mcpl/modern-connection.ts b/src/mcpl/modern-connection.ts index 9aaeae664..4df032dc0 100644 --- a/src/mcpl/modern-connection.ts +++ b/src/mcpl/modern-connection.ts @@ -41,6 +41,7 @@ import { SdkErrorCode, SdkHttpError, StreamableHTTPClientTransport, + parseJSONRPCMessage, type AuthProvider, type CallToolResult, type JSONRPCMessage, @@ -61,6 +62,8 @@ const DEFAULT_REQUEST_TIMEOUT_MS = 60_000; const CONNECT_TIMEOUT_MS = 30_000; /** Inventory listing has its own bound: it is host work, not a resident's call. */ const LIST_TIMEOUT_MS = 30_000; +/** Startup stderr kept for the first listener. */ +const STDERR_BACKLOG_LINES = 200; /** A modern tool result as the framework consumes it. `structuredContent` is * kept by presence: `false`, `0` and `null` are values, not absence. */ @@ -121,8 +124,13 @@ class ObservedTransport implements Transport { async start(): Promise { this.inner.onmessage = (message, extra) => { - this.observe(message); - this.onmessage?.(message, extra); + // An answer to a request in scope is handed to the SDK inside that + // request's scope. The SDK drives an input_required continuation from + // this very callback, outside the caller's async chain, and the next + // leg's send must still be recorded against the right call. + const record = this.observe(message); + if (record) wireScope.run(record, () => this.onmessage?.(message, extra)); + else this.onmessage?.(message, extra); }; this.inner.onerror = (error) => this.onerror?.(error); this.inner.onclose = () => this.onclose?.(); @@ -151,19 +159,23 @@ class ObservedTransport implements Transport { for (const [id, entry] of this.awaiting) if (entry === record) this.awaiting.delete(id); } - private observe(message: JSONRPCMessage): void { - if (!('id' in message) || (!('result' in message) && !('error' in message))) return; + /** Record an answer to a request in scope; returns its record when it + * answers that record's current leg. */ + private observe(message: JSONRPCMessage): WireRecord | undefined { + if (typeof message !== 'object' || message === null) return undefined; + if (!('id' in message) || (!('result' in message) && !('error' in message))) return undefined; const record = this.awaiting.get(message.id as string | number); - if (!record) return; + if (!record) return undefined; this.awaiting.delete(message.id as string | number); // A superseded leg's late answer says nothing about the current one. - if (record.currentId !== message.id) return; + if (record.currentId !== message.id) return undefined; if ('error' in message) { record.answer = 'error'; record.error = message.error as WireRecord['error']; } else { record.answer = 'result'; } + return record; } } @@ -190,12 +202,26 @@ class SpawnerStdioTransport implements Transport { const line = StdioTransport.spawn(this.config); this.line = line; line.on('line', (text: string) => { - let message: JSONRPCMessage; + // A line that isn't JSON is ignored, as the legacy engine ignores it. + // A line that is JSON but not a JSON-RPC message (`null`, an array, a + // bare value) is a diagnostic. It must never reach a handler that + // assumes the shape: thrown from this callback, it would end the host. + let parsed: unknown; try { - message = JSON.parse(text) as JSONRPCMessage; + parsed = JSON.parse(text); } catch { return; } + let message: JSONRPCMessage; + try { + message = parseJSONRPCMessage(parsed); + } catch (error) { + this.onerror?.(new Error( + `MCP server "${this.config.id}" sent a malformed JSON-RPC message (${text.length > 200 ? `${text.slice(0, 200)}…` : text}): ` + + (error instanceof Error ? error.message.split('\n')[0] : String(error)), + )); + return; + } this.onmessage?.(message); }); line.on('stderr', (text: string) => this.onStderr(text)); @@ -269,6 +295,10 @@ export class ModernMcpConnection extends EventEmitter { /** At most one pending reopen, and only ever for the current generation. */ private relisten: { generation: number; timer: ReturnType } | null = null; private relistenAttempts = 0; + /** Child stderr from before anyone listens (the caller attaches only once + * connect() returns), delivered to the first listener. Bounded: the + * newest lines are kept. */ + private stderrBacklog: string[] = []; private constructor(private readonly config: McplServerConfig) { super(); @@ -286,6 +316,22 @@ export class ModernMcpConnection extends EventEmitter { this.on('error', (error: Error) => { if (this.listenerCount('error') === 1) console.error(`[mcp] ${this.id}: ${error.message}`); }); + this.on('newListener', (event: string | symbol) => { + if (event !== 'stderr' || this.stderrBacklog.length === 0) return; + const pending = this.stderrBacklog; + this.stderrBacklog = []; + // After the listener is added (newListener fires just before). + queueMicrotask(() => { for (const line of pending) this.emit('stderr', { line }); }); + }); + } + + private deliverStderr(line: string): void { + if (this.listenerCount('stderr') > 0) { + this.emit('stderr', { line }); + return; + } + this.stderrBacklog.push(line); + if (this.stderrBacklog.length > STDERR_BACKLOG_LINES) this.stderrBacklog.shift(); } /** The per-call deadline: `requestTimeoutMs`, validated positive. */ @@ -332,7 +378,7 @@ export class ModernMcpConnection extends EventEmitter { const generation = ++this.generation; this.cancelRelisten(); const inner: Transport = this.transportKind === 'stdio' - ? new SpawnerStdioTransport(this.config, (line) => { if (this.isCurrent(generation)) this.emit('stderr', { line }); }) + ? new SpawnerStdioTransport(this.config, (line) => { if (this.isCurrent(generation)) this.deliverStderr(line); }) : new StreamableHTTPClientTransport(new URL(this.config.url!), { authProvider: bearerAuth(this.config) }); const wire = new ObservedTransport(inner); diff --git a/src/mcpl/transport.ts b/src/mcpl/transport.ts index 044249cc4..d344e8949 100644 --- a/src/mcpl/transport.ts +++ b/src/mcpl/transport.ts @@ -188,6 +188,12 @@ export class StdioTransport extends McplTransport { }); this.child.on('error', (err) => this.emit('error', err)); + // A write racing the child's exit fails with EPIPE on stdin. That is the + // exit's echo, which 'close' already reports, never a reason to crash: an + // unhandled stream 'error' would end the host. + this.child.stdin?.on('error', (err) => { + if (this.listenerCount('error') > 0) this.emit('error', err); + }); this.child.on('exit', (code, signal) => this.markClosed({ code, signal, reason: 'child process exited' }), ); @@ -216,22 +222,35 @@ export class StdioTransport extends McplTransport { this.child.stdin?.write(json + '\n'); } + /** + * Close and reap: resolves once the child has actually exited, not merely + * been signalled (`child.killed` is true as soon as a signal is sent). A + * child that outlives SIGTERM by {@link STDIO_EXIT_GRACE_MS} gets SIGKILL, + * and close waits that long again, so shutdown stays bounded. + */ async close(): Promise { this.markClosed({ reason: 'closed by host' }); this.rl.close(); - if (this.child && !this.child.killed) { - this.child.kill(); - } - await new Promise((resolve) => { - if (!this.child || this.child.exitCode !== null || this.child.killed) { - resolve(); - } else { - this.child.once('exit', () => resolve()); - } + const child = this.child; + // Never started (a spawn error): there is no process to reap. + if (child.pid === undefined) return; + if (child.exitCode !== null || child.signalCode !== null) return; + const exited = new Promise((resolve) => child.once('exit', () => resolve())); + const exitWithin = (ms: number) => new Promise((resolve) => { + const timer = setTimeout(() => resolve(false), ms); + void exited.then(() => { clearTimeout(timer); resolve(true); }); }); + child.kill('SIGTERM'); + if (await exitWithin(STDIO_EXIT_GRACE_MS)) return; + console.error(`[mcpl] stdio child ${child.pid} ignored SIGTERM for ${STDIO_EXIT_GRACE_MS}ms — sending SIGKILL`); + child.kill('SIGKILL'); + await exitWithin(STDIO_EXIT_GRACE_MS); } } +/** How long a stdio child gets to exit after SIGTERM, and again after SIGKILL. */ +const STDIO_EXIT_GRACE_MS = 2_000; + // --------------------------------------------------------------------------- // WebSocket // --------------------------------------------------------------------------- diff --git a/test/fixtures/modern-raw-server.mjs b/test/fixtures/modern-raw-server.mjs index 84a0e054a..f63c1a82c 100644 --- a/test/fixtures/modern-raw-server.mjs +++ b/test/fixtures/modern-raw-server.mjs @@ -7,9 +7,13 @@ // invalid-schema tools/list advertises an outputSchema that doesn't compile // listen-fail-first the first subscriptions/listen of each launch is refused // hang-discover-later launches after the first never answer server/discover +// ignore-sigterm the process survives SIGTERM (only SIGKILL ends it) // Tools: op (outputSchema {value: integer}, answers with a string: invalid // structured content), plain (text), err (a JSON-RPC error), touch (announces -// a tool list change on the open subscription), die (exits). +// a tool list change on the open subscription), nullframe (writes JSON lines +// that aren't JSON-RPC messages, then answers), cont (a state-only +// input_required first leg; the second leg does what arguments.leg2 says: +// 'error', 'hang' or 'ok'), die (exits). import { createInterface } from 'node:readline'; import { appendFileSync, existsSync, readFileSync } from 'node:fs'; @@ -19,6 +23,7 @@ const prior = existsSync(log) ? readFileSync(log, 'utf8').split('\n').filter(Boo const launch = prior.filter((x) => x.event === 'start').length + 1; const note = (x) => appendFileSync(log, `${JSON.stringify({ ...x, launch })}\n`); note({ event: 'start', pid: process.pid }); +if (flags.has('ignore-sigterm')) process.on('SIGTERM', () => note({ event: 'sigterm-ignored' })); const send = (obj) => process.stdout.write(`${JSON.stringify(obj)}\n`); const result = (id, body) => send({ jsonrpc: '2.0', id, result: { resultType: 'complete', ...body } }); @@ -69,6 +74,8 @@ createInterface({ input: process.stdin }).on('line', (line) => { { name: 'plain', inputSchema: { type: 'object' } }, { name: 'err', inputSchema: { type: 'object' } }, { name: 'touch', inputSchema: { type: 'object' } }, + { name: 'nullframe', inputSchema: { type: 'object' } }, + { name: 'cont', inputSchema: { type: 'object' } }, { name: 'die', inputSchema: { type: 'object' } }, ], }); @@ -83,6 +90,19 @@ createInterface({ input: process.stdin }).on('line', (line) => { return result(m.id, { content: [{ type: 'text', text: subscription === null ? 'no subscription' : 'touched' }] }); } if (name === 'die') { setTimeout(() => process.exit(5), 10); return result(m.id, { content: [{ type: 'text', text: 'bye' }] }); } + if (name === 'nullframe') { + process.stdout.write('null\n[1,2]\n"just a string"\n{"no":"jsonrpc"}\n'); + return result(m.id, { content: [{ type: 'text', text: 'after the noise' }] }); + } + if (name === 'cont') { + const leg2 = m.params?.arguments?.leg2; + if (m.params?.requestState === undefined) { + return send({ jsonrpc: '2.0', id: m.id, result: { resultType: 'input_required', requestState: `state-for-${leg2}` } }); + } + if (leg2 === 'error') return error(m.id, -32603, 'second leg failed', { leg: 2, state: m.params.requestState }); + if (leg2 === 'hang') return; + return result(m.id, { content: [{ type: 'text', text: `leg 2 done (${m.params.requestState})` }] }); + } return result(m.id, { content: [{ type: 'text', text: 'plain' }] }); } default: diff --git a/test/mcp-modern-connection.test.ts b/test/mcp-modern-connection.test.ts index f496ec677..d71e2b6f5 100644 --- a/test/mcp-modern-connection.test.ts +++ b/test/mcp-modern-connection.test.ts @@ -72,7 +72,9 @@ test('connects once over Connectome\'s spawner and speaks 2026-07-28', async () assert.deepEqual(echo.content, [{ type: 'text', text: 'echo:hi' }]); // In-place discovery: the SDK's sibling probe would have made this 2. assert.equal(lines(join(dir, 'starts.log')).length, 1); - // The child's stderr surfaces as lines, as a legacy child's does. + // The child's stderr surfaces as lines, as a legacy child's does, + // startup lines included: they came before anyone could listen. + await until(() => stderr.some((l) => /^modern fixture \d+ up$/.test(l))); await connection.callTool('shout', { text: 'hello' }); await until(() => stderr.includes('shout: hello')); }); diff --git a/test/mcp-modern-outcomes.test.ts b/test/mcp-modern-outcomes.test.ts index 5c6b2dca7..0f5f45ab0 100644 --- a/test/mcp-modern-outcomes.test.ts +++ b/test/mcp-modern-outcomes.test.ts @@ -144,3 +144,66 @@ test('close() during a reconnect handshake ends the launch it was waiting on, an assert.equal(wire(log).filter((e) => e.event === 'start').length, 2, 'no launch after close'); assert.equal(connection.isConnected, false); }); + +test('an input_required continuation keeps its evidence: the second leg decides the outcome', async () => { + const log = scratchLog(); + const connection = await connect('', log, { requestTimeoutMs: 600 }); + await connection.listTools(); + await assert.rejects(connection.callTool('cont', { leg2: 'error' }), (err: unknown) => { + assert.ok(err instanceof McplRequestError, String(err)); + assert.equal(err.outcome, 'error-response'); + assert.equal(err.code, -32603); + assert.deepEqual(err.data, { leg: 2, state: 'state-for-error' }); + return true; + }); + await assert.rejects(connection.callTool('cont', { leg2: 'hang' }), (err: unknown) => { + assert.ok(err instanceof McplRequestError, String(err)); + assert.equal(err.outcome, 'no-response'); + assert.match(err.message, /Cancellation was requested/); + return true; + }); + assert.equal(calls(log, 'cont'), 4, 'two legs each'); +}); + +test('concurrent continuations never cross their evidence', async () => { + const log = scratchLog(); + const connection = await connect('', log, { requestTimeoutMs: 2000 }); + await connection.listTools(); + const [failing, passing, failingToo] = await Promise.allSettled([ + connection.callTool('cont', { leg2: 'error' }), + connection.callTool('cont', { leg2: 'ok' }), + connection.callTool('cont', { leg2: 'error' }), + ]); + for (const settled of [failing, failingToo]) { + assert.equal(settled.status, 'rejected'); + const err = (settled as PromiseRejectedResult).reason; + assert.ok(err instanceof McplRequestError); + assert.equal(err.outcome, 'error-response'); + assert.deepEqual(err.data, { leg: 2, state: 'state-for-error' }); + } + assert.equal(passing.status, 'fulfilled'); + assert.deepEqual((passing as PromiseFulfilledResult<{ content: unknown }>).value.content, [{ type: 'text', text: 'leg 2 done (state-for-ok)' }]); +}); + +test('a JSON line that is not a JSON-RPC message is a diagnostic, never a crash', async () => { + const log = scratchLog(); + const connection = await connect('', log); + const diagnostics: string[] = []; + connection.on('error', (e: Error) => diagnostics.push(e.message)); + await connection.listTools(); + assert.deepEqual((await connection.callTool('nullframe', {})).content, [{ type: 'text', text: 'after the noise' }]); + await until(() => diagnostics.length >= 4, 'four diagnostics'); + assert.ok(diagnostics.every((d) => /sent a malformed JSON-RPC message/.test(d)), diagnostics.join(' | ')); + assert.equal(connection.isConnected, true); + assert.deepEqual((await connection.callTool('plain', {})).content, [{ type: 'text', text: 'plain' }]); +}); + +test('close() reaps: a child that ignores SIGTERM is killed before close resolves', async () => { + const log = scratchLog(); + const connection = await connect('ignore-sigterm', log); + const pid = wire(log).find((e) => e.event === 'start')!.pid!; + await connection.close(); + const alive = (p: number) => { try { process.kill(p, 0); return true; } catch { return false; } }; + assert.equal(alive(pid), false, 'dead when close() resolved'); + assert.ok(wire(log).some((e) => e.event === 'sigterm-ignored'), 'it really did ignore SIGTERM'); +}); From f9954a5cc628728a2653761b9776a1bd2e48ca25 Mon Sep 17 00:00:00 2001 From: Olive-1804 Date: Thu, 8 Oct 2026 02:21:45 -0400 Subject: [PATCH 0061/1112] feat(mcp): modern servers in the framework, standard result reading, structured results Third batch of room-284's modern-MCP client work (shelf-487): the framework integration. Connecting. connectMcplServerInternal validates first: serverConfigProblems for any config whose binding resolves, so a modern config with MCPL policy, a deadline of 0, or protocol on a URL is an error before any routing is registered. A config naming no usable transport keeps the transport's own error. Duplicate ids are refused across both engines. A modern binding goes to ModernMcpConnection and is kept in its own map, apart from the MCPL registry, so quiesce's plane pauses, awareness gates and policy never see it. Fatal startup paths and stop() close modern connections too, including any connect still in flight. Events (wireModernEvents). stderr, errors, connect and reconnect failures (with the ops alert at attempt 5) and close get the MCPL traces. A list change, and the relist after a reconnect, take handleToolsListChanged, the legacy refresh path, so any wake goes through ordinary admission. No awareness gate: it orders MCPL data-plane traffic, which this family doesn't have. A closed connection without reconnect is dropped, as the MCPL registry drops one. Tools. refreshMcplTools lists modern servers under the same prefix, tool policy and RFC-008 class hints. A failed listing leaves the server unlisted (unknown) and traces it. Dispatch. Both the model path and the direct path (executeToolCall, ModuleContext.callTool) route modern calls, sharing the not-found and policy refusals with MCPL. Both read results through mcpToolResult: - A standard peer (an MCP-only legacy server, or a modern server) gets normalizeStandardToolResult (src/mcpl/tool-result-normalize.ts): - text stays text, and inline images stay native; - audio and blobs are saved to the workspace `tool-results/`, with a bounded stub saying where; - a resource_link is shown and never fetched; embedded text is shown; - structuredContent is kept by presence on the new ToolResult.structured, and shown to the model as JSON only when the content has no text. - MCPL peers keep the RFC-005 reading (autofetch only for them now), gaining `structured` and the JSON view for empty content. - The direct path now fails on isError. It used to report every answer as success, so tts-relay's edit check read a failed edit as success. Scripts. A result with structured content reaches the script as one JSON object, { content, structuredContent, isError }, with text content as the text itself, under the same 5 MB cap. Other results keep the history string contract. Status. listMcplServers entries gain family, protocolVersion and transport. Host lifetime evidence (Tessa #53152, Agnes/Kit #53184, Theo #53178), in test/mcp-modern-framework.test.ts against a real framework: - a modern call in flight when quiesce begins completes, and the drain succeeds: the server finished, nothing was aborted, and the result reached the turn; - a list change while quiesced refreshes the inventory, and its wake parks (gatedRequests) with no inference until resume, then runs; - a modern child lost while quiesced is relaunched and relisted with no turn and no new wake. Also covered: listing and status, the direct path (structured by presence including 0, tool error as a failure, media stubs), the model path, the script object, a config error at connect, and the standard reading for an MCP-only legacy peer. Plus 10 unit tests for the normalizer. Docs: README "Modern MCP servers" (family selection table, what differs), requestTimeoutMs doc for modern, and changelog fragments (modern-mcp-client.added, mcp-standard-tool-results.changed). Receipts (Lethe, Node 24.19.0): every test file run standalone gives 1441/1441 (133 files 1375 plus mcpl-coalescing 66). The full-suite runner twice reported fewer tests than exist (1420, 1425) with 0 failures, because one file's later subtests went missing from the report; per-file runs are complete. mcpl-coalescing hung once at R16 under 6-way parallel load and once right after that run was killed. It then passed twice standalone (66/66) and R16 alone 3x; baseline 02adf27 passed R16 once. Not attributed either way. --- README.md | 36 ++ .../mcp-standard-tool-results.changed.md | 24 ++ changelog.d/modern-mcp-client.added.md | 28 ++ src/framework.ts | 345 +++++++++++++++--- src/mcpl/tool-result-normalize.ts | 153 ++++++++ src/mcpl/types.ts | 5 + src/types/events.ts | 8 + test/mcp-modern-framework.test.ts | 258 +++++++++++++ test/mcp-tool-result-normalize.test.ts | 111 ++++++ test/mcpl-list-status.test.ts | 4 + 10 files changed, 916 insertions(+), 56 deletions(-) create mode 100644 changelog.d/mcp-standard-tool-results.changed.md create mode 100644 changelog.d/modern-mcp-client.added.md create mode 100644 src/mcpl/tool-result-normalize.ts create mode 100644 test/mcp-modern-framework.test.ts create mode 100644 test/mcp-tool-result-normalize.test.ts diff --git a/README.md b/README.md index 7e4075c09..545279646 100644 --- a/README.md +++ b/README.md @@ -266,6 +266,42 @@ A pattern that matches no tool is reported once, as a console line and an A pattern that could name a server's tools is not judged until that server has listed them, so late connects and reconnects don't cause early reports. +#### Modern MCP servers (2026-07-28) + +The same `mcplServers` list also takes servers that speak modern MCP +(revision `2026-07-28`), reached through the official SDK client. The +configuration chooses the family, and nothing is probed: + +| Configuration | Family | Transport | +|---|---|---| +| `url: 'ws://…'` / `'wss://…'` | MCPL (MCP `2024-11-05`) | WebSocket | +| `url: 'http://…'` / `'https://…'` | modern MCP | Streamable HTTP (`token` / `accessProvider` as bearer) | +| `command` | MCPL | stdio | +| `command` + `protocol: 'modern'` | modern MCP | stdio | + +```typescript +{ + mcplServers: [ + { id: 'docs', url: 'https://mcp.example.com/mcp', accessProvider: getToken }, + { id: 'search', command: 'search-mcp', protocol: 'modern', reconnect: true }, + ], +} +``` + +A modern server's tools get the same prefix, `enabledTools`/`disabledTools`, +class overrides and dispatch as an MCPL server's, and `listMcplServers()` +shows its `family`, `protocolVersion` and `transport`. Its config differs in +two ways: +- It has no MCPL surface (grant, push, channels, inference requests, feature + sets), so setting MCPL-only fields on one is a configuration error. +- `requestTimeoutMs` is one deadline per tool call and must be 1 to 2^31−1; + 0 is refused, where MCPL reads it as no watchdog. + +At the deadline cancellation is requested, the outcome is reported as +unknown, and the call is never retried. An MCPL server that refuses +`2024-11-05` with `-32022` fails with `McplProtocolVersionError`, which names +the fix, and is not retried. + #### Feature sets `enabledFeatureSets` and `disabledFeatureSets` select which of a server's diff --git a/changelog.d/mcp-standard-tool-results.changed.md b/changelog.d/mcp-standard-tool-results.changed.md new file mode 100644 index 000000000..ae9cad54a --- /dev/null +++ b/changelog.d/mcp-standard-tool-results.changed.md @@ -0,0 +1,24 @@ +- MCP tool results from servers that aren't MCPL peers (MCP-only legacy + servers and modern servers) are now read by their content types: + - text stays text, and inline images stay native; + - audio and binary resources are saved to the workspace (`tool-results/`), + with a bounded stub saying where; + - a `resource_link` is shown as a reference and never fetched; + - an embedded text resource shows its text. + + Before, any non-text block turned the whole result into raw JSON. + `structuredContent` is now kept by presence on the new + `ToolResult.structured`, so `false`, `0` and `null` are values. The model + sees it as JSON only when the content carries no text; a structured-only + result used to reach the model as an empty string. Scripts receive + `{ content, structuredContent, isError }` as one JSON object whenever a + result has structured content. MCPL peers keep their RFC-005 reading and + gain `structured`. +- The direct tool path (`executeToolCall`, `ModuleContext.callTool`) now + treats an MCP tool error (`isError: true`) as a failure (`success: false` + with the error text), as the model path does. Before, every answer counted + as success. Its `data` now has the model path's shape too: text joined into + a string, or an array when an image is present. +- Closing a stdio MCP/MCPL server now waits for the child to exit. A child + still alive 2 s after SIGTERM gets SIGKILL. Before, close returned as soon + as the signal was sent. diff --git a/changelog.d/modern-mcp-client.added.md b/changelog.d/modern-mcp-client.added.md new file mode 100644 index 000000000..285d8bc13 --- /dev/null +++ b/changelog.d/modern-mcp-client.added.md @@ -0,0 +1,28 @@ +- Modern MCP servers (revision `2026-07-28`) can now be configured beside + MCPL ones, through the official SDK 2.x client. A server's URL decides its + family: `ws://`/`wss://` stays MCPL (MCP `2024-11-05`), and + `http://`/`https://` is modern MCP over Streamable HTTP. A stdio server + stays legacy unless its config sets `protocol: 'modern'`. A modern server's + tools use the same prefix, tool policy, RFC-008 class hints and dispatch + paths as any MCPL server's. It has no MCPL surface: no grant, push, + channels, inference requests, quiesce plane or awareness gate. MCPL-only + policy on one is a configuration error, and so is a modern + `requestTimeoutMs` outside 1..2^31−1, since 0 means "time out now" to the + SDK. Details: + - **stdio:** runs through the framework's own spawner (env allowlist, + `inheritEnv`, stderr lines including startup), launched once per connect. + - **HTTP credentials:** come from `token`/`accessProvider`, as a cached + bearer refreshed once on 401. + - **Deadline:** each call gets one `requestTimeoutMs` deadline across every + leg. At the deadline cancellation is requested, the outcome is reported + as unknown, and nothing is replayed. + - **Failures:** reported as `McplRequestError`, with the outcome taken from + what actually crossed the transport: `not-sent`, `error-response` (HTTP + 401/403 with `data.httpStatus`) or `no-response`. A server answer that + can't be used is a plain error. + - **Lifetime:** reconnects use the legacy backoff settings. A tool-list + change subscription is kept open, reopened if refused or lost, and its + changes go through ordinary admission, so they park under quiesce like + any wake. `close()` and `stop()` end any connect in flight. + - **Status:** `listMcplServers()` entries gain `family`, `protocolVersion` + and `transport`. diff --git a/src/framework.ts b/src/framework.ts index 3e5188f72..14f5cbe23 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -392,6 +392,9 @@ import { UsageTracker, type PersistedUsageState } from './usage/usage-tracker.js import type { SessionUsageSnapshot, UsageUpdatedEvent } from './usage/types.js'; import type { McplServerConnection } from './mcpl/server-connection.js'; import { McplProtocolVersionError } from './mcpl/server-connection.js'; +import { ModernMcpConnection } from './mcpl/modern-connection.js'; +import { resolveServerBinding, serverConfigProblems } from './mcpl/protocol-family.js'; +import { normalizeStandardToolResult } from './mcpl/tool-result-normalize.js'; import type { McplServerConfig, McplHostCapabilities, @@ -1372,6 +1375,13 @@ export class AgentFramework { private toolPatternKnownPrefixes: Map = new Map(); /** Maps serverId → McplServerConfig for prefix lookup. */ private mcplServerConfigs: Map = new Map(); + /** Modern-MCP (2026-07-28) servers, by id; read through modernServers(). + * They are kept apart from the MCPL registry on purpose: they have no + * grant, planes or server→host traffic, so the MCPL machinery (quiesce's + * plane pauses, awareness gates, policy) never sees them. They take part + * only through the tool paths, and their list changes reach agents + * through ordinary admission. */ + private modernMcpConnections: Map | undefined; /** Host capabilities advertised during the MCP handshake — stored so servers * can be connected at runtime (connectMcplServer) after initialization. */ private mcplHostCapabilities: McplHostCapabilities | null = null; @@ -2024,6 +2034,7 @@ export class AgentFramework { if (this.mcplServerRegistry) { shutdownPromises.push(this.mcplServerRegistry.closeAll()); } + shutdownPromises.push(this.closeModernMcpConnections()); await Promise.all(shutdownPromises); // Streams may queue storage repairs while being cancelled above. Retry @@ -11188,8 +11199,9 @@ export class AgentFramework { if (!this.mcplServerRegistry) { return { success: false, error: 'MCPL not initialized', isError: true }; } - const server = this.mcplServerRegistry.getServer(config.id); - if (!server) { + const modern = this.modernServers().get(config.id) ?? null; + const server = modern ? null : this.mcplServerRegistry.getServer(config.id); + if (!modern && !server) { return { success: false, error: `MCPL server ${config.id} not found`, isError: true }; } const prefix = config.toolPrefix ?? `mcpl--${config.id}`; @@ -11201,12 +11213,13 @@ export class AgentFramework { isError: true, }; } + const args = (call.input && typeof call.input === 'object') ? call.input as Record : {}; try { - const result = await server.sendToolsCall(toolName, call.input as Record); - return { - success: true, - data: result.content, - }; + // The same reading as the model path, so a tool error is a failure + // here too (this path used to report every answer as success). + if (modern) return await this.mcpToolResult(await modern.callTool(toolName, args), call.id, false); + const result = await server!.sendToolsCall(toolName, args); + return await this.mcpToolResult(result, call.id, server!.capabilities !== null); } catch (err) { return { success: false, @@ -11827,10 +11840,14 @@ export class AgentFramework { // stream while the script still runs and wedge the tool round. script.endTurn = true; } - if (result.isError) { + // An MCP structured result reaches the script whole, as one JSON object, + // so code can use structuredContent directly and doesn't have to parse + // it back out of text. + const structured = Object.prototype.hasOwnProperty.call(result, 'structured'); + if (result.isError && !structured) { return `Error: ${result.error ?? 'tool call failed'}`; } - if (result.data === undefined) return ''; + if (result.data === undefined && !structured) return ''; // RFC-005 "eagerly lazy": a script receiving a result is the strongest // available signal that the bytes are wanted (scripts process, they do // not browse) — materialize every referenced payload now, so the stub @@ -11849,6 +11866,21 @@ export class AgentFramework { // Scripts get (nearly) full data — the script environment IS the spill: // filtering big results in code is the whole point. 5MB protocol safety // cap only; no context-cap truncation, no file spill. + if (structured) { + // Inside the object, text content is the text itself; arrays get the + // history rendering (images as placeholders). The same 5MB protocol + // safety cap applies, over the whole object. + const content = result.isError + ? (result.error ?? '') + : typeof result.data === 'string' + ? result.data + : result.data === undefined ? '' : toolResultDataToHistoryString(result.data, 5_000_000); + return truncateForHistory(JSON.stringify({ + content, + structuredContent: result.structured, + isError: result.isError === true, + }), 5_000_000); + } return toolResultDataToHistoryString(result.data, 5_000_000); } @@ -13298,6 +13330,7 @@ export class AgentFramework { } catch (error) { if (error instanceof DiscordAwarenessAccountingError) { await this.mcplServerRegistry.closeAll(); + await this.closeModernMcpConnections(); throw error; } // Fail-open: log and continue with remaining servers @@ -13329,6 +13362,7 @@ export class AgentFramework { this.completeMcplDataPlaneGate(startupBarrier); } catch (error) { await this.mcplServerRegistry.closeAll(); + await this.closeModernMcpConnections(); throw error; } @@ -13336,6 +13370,20 @@ export class AgentFramework { await this.refreshMcplTools(); } + /** The modern connections, created on first use, so a framework assembled + * field by field (as unit tests do) needs no setup for them. */ + private modernServers(): Map { + return (this.modernMcpConnections ??= new Map()); + } + + /** Close every modern-MCP connection, including any connect still in + * flight, so none outlives a failed startup or stop(). */ + private async closeModernMcpConnections(): Promise { + const modern = [...this.modernServers().values()]; + this.modernServers().clear(); + await Promise.all(modern.map((connection) => connection.close())); + } + /** Reconcile the durable ledger with Chronicle, then deliver every server's work. */ async syncDiscordAwarenessMarkers(_onlyServerId?: string): Promise { if (!this.discordAwarenessOutbox) return; @@ -13693,12 +13741,36 @@ export class AgentFramework { throw new Error('MCPL subsystem is not initialized'); } + // Configuration mistakes the family brought (protocol on a URL, a modern + // deadline of 0, MCPL policy on a modern server) fail here, before any + // routing is registered. A configuration that names no usable transport + // keeps the transport's own error, as before. + let family: 'legacy' | 'modern' | null = null; + try { family = resolveServerBinding(config).family; } catch { /* the transport reports it */ } + if (family !== null) { + const problems = serverConfigProblems(config); + if (problems.length > 0) { + throw new Error(`MCP server "${config.id}" configuration: ${problems.join('; ')}`); + } + } + if (this.modernServers().has(config.id) || this.mcplServerRegistry.getServer(config.id)) { + throw new Error(`MCPL server "${config.id}" is already registered`); + } + // Register prefix + config for tool dispatch routing (idempotent with // the pre-registration pass in initializeMcpl). const prefix = config.toolPrefix ?? `mcpl--${config.id}`; this.mcplPrefixMap.set(prefix, config.id); this.mcplServerConfigs.set(config.id, config); + if (family === 'modern') { + const modern = await ModernMcpConnection.connect(config); + this.modernServers().set(config.id, modern); + this.wireModernEvents(modern); + this.emitTrace({ type: 'module:added', moduleName: `mcpl:${config.id}` }); + return; + } + // Record per-server channel subscription policy before the server // registers channels — handleRegister fires during the handshake. if (this.channelRegistry) { @@ -14124,6 +14196,11 @@ export class AgentFramework { const config = this.mcplServerConfigs.get(id); const oldToolNames = new Set(this.mcplTools.map(t => t.name)); + const modern = this.modernServers().get(id); + if (modern) { + this.modernServers().delete(id); + await modern.close(); + } await this.mcplServerRegistry.removeServer(id); this.channelRegistry?.removeServer(id); @@ -14168,6 +14245,13 @@ export class AgentFramework { id: string; connected: boolean; retrying: boolean; + /** Which engine serves it: `legacy` (MCP 2024-11-05 + MCPL) or `modern` + * (MCP 2026-07-28), as its configuration selects. */ + family: 'legacy' | 'modern'; + /** The MCP revision its last handshake established; null until one has. */ + protocolVersion: string | null; + /** How it is reached. */ + transport: 'stdio' | 'websocket' | 'http'; toolPrefix: string; toolCount: number; /** True only after the §5.3 policy receipt activated the grant. */ @@ -14209,6 +14293,8 @@ export class AgentFramework { }> { const result: Array<{ id: string; connected: boolean; retrying: boolean; + family: 'legacy' | 'modern'; protocolVersion: string | null; + transport: 'stdio' | 'websocket' | 'http'; toolPrefix: string; toolCount: number; policyEstablished: boolean; effectiveGrant: string[]; maskedCapabilities: string[]; deniedCapabilities: string[]; allowHostCommands: boolean; @@ -14223,8 +14309,15 @@ export class AgentFramework { }> = []; for (const [id, config] of this.mcplServerConfigs) { const prefix = config.toolPrefix ?? `mcpl--${id}`; - const connection = this.mcplServerRegistry?.getServer(id) ?? null; - const connected = connection?.isConnected ?? false; + const modern = this.modernServers().get(id) ?? null; + const connection = modern ? null : this.mcplServerRegistry?.getServer(id) ?? null; + const connected = modern?.isConnected ?? connection?.isConnected ?? false; + let binding: { family: 'legacy' | 'modern'; transport: 'stdio' | 'websocket' | 'http' }; + try { + binding = resolveServerBinding(config); + } catch { + binding = { family: 'legacy', transport: config.url ? 'websocket' : 'stdio' }; + } // Attribute through the dispatch resolver: a bare prefix match would // also count a nested prefix's tools (`foo` vs `foo--bar`) as ours. const ownTools = (this.mcplTools ?? []).flatMap((t) => { @@ -14234,7 +14327,10 @@ export class AgentFramework { result.push({ id, connected, - retrying: !connected && (connection?.willReconnect ?? false), + retrying: !connected && (modern?.willReconnect ?? connection?.willReconnect ?? false), + family: binding.family, + protocolVersion: modern?.protocolVersion ?? connection?.protocolVersion ?? null, + transport: binding.transport, toolPrefix: prefix, toolCount: ownTools.length, policyEstablished: connection?.policyEstablished ?? false, @@ -14568,6 +14664,63 @@ export class AgentFramework { }); } + /** + * Wire a modern-MCP connection's events. There's less here than for MCPL + * on purpose: no push, inference, channels or policy exist on this family. + * A list change, including the relist after a reconnect, takes the legacy + * engine's refresh path (`handleToolsListChanged`), so any wake it causes + * goes through ordinary admission: it parks under quiesce or a hold like + * every other wake. It installs no awareness gate, since that gate orders + * MCPL data-plane traffic, which this family doesn't carry. + */ + private wireModernEvents(connection: ModernMcpConnection): void { + connection.on('stderr', (params: { line: string }) => { + this.emitTrace({ type: 'mcpl:server-stderr', serverId: connection.id, line: params.line }); + }); + connection.on('tools-list-changed', () => { + this.handleToolsListChanged(connection.id); + }); + connection.on('reconnect', (info?: { attempts?: number }) => { + this.emitTrace({ type: 'mcpl:server-reconnected', serverId: connection.id, attempts: info?.attempts ?? 0 }); + this.emitTrace({ type: 'module:added', moduleName: `mcpl:${connection.id}` }); + this.handleToolsListChanged(connection.id); + }); + const failed = (params: { error: string; attempt: number }) => { + this.emitTrace({ + type: 'mcpl:server-connect-failed', + serverId: connection.id, + error: params.error, + attempt: params.attempt, + willRetry: connection.willReconnect, + }); + if (params.attempt >= 5) { + this.opsAlert('mcpl-down', connection.id, `MCP server unreachable (attempt ${params.attempt}): ${params.error}`); + } + }; + connection.on('connect-failed', failed); + connection.on('reconnect-failed', failed); + connection.on('error', (err: Error) => { + this.emitTrace({ type: 'mcpl:server-error', serverId: connection.id, error: err.message }); + }); + connection.on('close', () => { + // A closed provider's class hints are stale (RFC-008), as for MCPL. + this.dropMcplToolClasses(connection.id); + this.emitTrace({ + type: 'mcpl:server-closed', + serverId: connection.id, + code: null, + signal: null, + willReconnect: connection.willReconnect, + }); + this.emitTrace({ type: 'module:removed', moduleName: `mcpl:${connection.id}` }); + // Without reconnect a closed server is gone, as the MCPL registry + // drops one; its configuration stays listed as disconnected. + if (!connection.willReconnect && this.modernServers().get(connection.id) === connection) { + this.modernServers().delete(connection.id); + } + }); + } + /** * Discover tools from all connected MCPL servers and cache them. * Tools are namespaced as `{toolPrefix}--{toolName}` per server config. @@ -14625,6 +14778,37 @@ export class AgentFramework { } } + // Modern servers: the same namespacing, tool policy and RFC-008 class + // hints. Feature sets are MCPL and don't apply. A server whose listing + // fails stays unlisted (unknown), never "listed nothing". + for (const server of this.modernServers().values()) { + if (!server.isConnected) continue; + const config = this.mcplServerConfigs.get(server.id); + const prefix = config?.toolPrefix ?? `mcpl--${server.id}`; + try { + const listed = await server.listTools(); + listedServers.add(server.id); + for (const tool of listed) { + if (!isToolAllowed(tool.name, config)) continue; + const namespacedName = `${prefix}--${tool.name}`; + toolServers.set(namespacedName, server.id); + const declaredClasses = parseDeclaredClasses(tool._meta, namespacedName); + if (declaredClasses) toolClasses.set(namespacedName, declaredClasses); + tools.push({ + name: namespacedName, + description: tool.description ?? '', + inputSchema: tool.inputSchema as import('./types/index.js').ToolDefinition['inputSchema'], + }); + } + } catch (error) { + this.emitTrace({ + type: 'mcpl:server-error', + serverId: server.id, + error: `tools/list failed: ${error instanceof Error ? error.message : String(error)}`, + }); + } + } + this.mcplTools = tools; this.mcplToolFeatureSets = toolFeatureSets; this.mcplToolClasses = toolClasses; @@ -14776,9 +14960,10 @@ export class AgentFramework { */ private dispatchMcplToolCall(agentName: string, call: ToolCall, serverId: string, prefix: string): void { const toolName = call.name.slice(prefix.length + 2); // strip "{prefix}--" - const server = this.mcplServerRegistry!.getServer(serverId); + const modern = this.modernServers().get(serverId) ?? null; + const server = modern ? null : this.mcplServerRegistry!.getServer(serverId); - if (!server) { + if (!modern && !server) { // RFC-007: refused before executing — no lifecycle events. this.toolLifecycleEmitter?.refuse(agentName, call.id); this.pushEvent({ @@ -14813,6 +14998,36 @@ export class AgentFramework { const startTime = Date.now(); const args = (call.input && typeof call.input === 'object') ? call.input as Record : {}; + const failDispatch = (error: unknown) => { + const err = error instanceof Error ? error : new Error(String(error)); + this.emitTrace({ type: 'tool:failed', module: `mcpl:${serverId}`, tool: toolName, callId: call.id, error: err.message, stack: err.stack }); + // RFC-007: transport error, closed connection or timeout — no result. + this.toolLifecycleEmitter?.markDispatchFailure(agentName, call.id); + this.pushEvent({ + type: 'tool-result', + callId: call.id, + agentName, + moduleName: `mcpl:${serverId}`, + result: { success: false, error: err.message, isError: true }, + }); + }; + + if (modern) { + // Modern MCP: no state exchange, no channels, no RFC-005 references. + modern.callTool(toolName, args) + .then((result) => this.mcpToolResult(result, call.id, false)) + .then((result) => { + this.emitTrace({ type: 'tool:completed', module: `mcpl:${serverId}`, tool: toolName, callId: call.id, durationMs: Date.now() - startTime }); + this.pushEvent({ type: 'tool-result', callId: call.id, agentName, moduleName: `mcpl:${serverId}`, result }); + }) + .catch(failDispatch); + return; + } + const legacy = server!; + // An MCPL peer's results get the RFC-005 reading; an MCP-only peer's + // the standard one, the same as a modern server's. + const mcplPeer = legacy.capabilities !== null; + // Build state params for stateful tools (Step 8). // A tools/list attribution selects that exact stateful set. Untagged tools // retain the host-managed or unique server-managed fallback, but never guess @@ -14860,15 +15075,18 @@ export class AgentFramework { } } - server.sendToolsCall(toolName, args, stateParams) + legacy.sendToolsCall(toolName, args, stateParams) .then(async (result) => { // RFC-005: register references and eagerly fetch the small ones - // before serialization, so their stubs carry a saved path. - await this.autoFetchReferences(result.content, serverId, true) - .catch((e) => console.error('[rfc005] autofetch error:', (e as Error).message)); - return result; + // before serialization, so their stubs carry a saved path. Only an + // MCPL peer's blocks are references; an MCP-only peer's are standard. + if (mcplPeer) { + await this.autoFetchReferences(result.content, serverId, true) + .catch((e) => console.error('[rfc005] autofetch error:', (e as Error).message)); + } + return { result, toolResult: await this.mcpToolResult(result, call.id, mcplPeer) }; }) - .then((result) => { + .then(({ result, toolResult }) => { const durationMs = Date.now() - startTime; this.emitTrace({ type: 'tool:completed', module: `mcpl:${serverId}`, tool: toolName, callId: call.id, durationMs }); @@ -14937,50 +15155,65 @@ export class AgentFramework { } } - // Convert MCP tool result to framework ToolResult. - // When the result contains non-text blocks (e.g. images from an MCP - // tool like zulip-mcp's fetch_attachment), pass the full content array - // through so toMembraneToolResult can preserve image blocks natively. - // Text-only results still collapse to a joined string for backward - // compatibility with callers that expect data to be string-ish. - const textContent = result.content - ?.filter((c) => c.type === 'text' && c.text) - .map((c) => c.text!) - .join('\n'); - const hasNonText = result.content?.some((c) => c.type !== 'text'); - const data = result.isError - ? undefined - : hasNonText - ? result.content - : (textContent || undefined); - this.pushEvent({ type: 'tool-result', callId: call.id, agentName, moduleName: `mcpl:${serverId}`, - result: { - success: !result.isError, - data, - error: result.isError ? (textContent || 'Tool call failed') : undefined, - isError: result.isError ?? false, - }, + result: toolResult, }); }) - .catch((error) => { - const err = error instanceof Error ? error : new Error(String(error)); - this.emitTrace({ type: 'tool:failed', module: `mcpl:${serverId}`, tool: toolName, callId: call.id, error: err.message, stack: err.stack }); - // RFC-007: transport error, closed connection or timeout — no result. - this.toolLifecycleEmitter?.markDispatchFailure(agentName, call.id); + .catch(failDispatch); + } - this.pushEvent({ - type: 'tool-result', - callId: call.id, - agentName, - moduleName: `mcpl:${serverId}`, - result: { success: false, error: err.message, isError: true }, - }); - }); + /** + * The framework's ToolResult for an MCP tool result, on both dispatch + * paths (model and direct). A standard peer, meaning an MCP-only legacy + * server or any modern server, gets the standard reading + * (tool-result-normalize.ts): typed content blocks, workspace-saved + * payloads, and `structured` kept by presence. An MCPL peer keeps the + * RFC-005 reading. Its content array passes through when it holds + * non-text blocks, so references and images render natively. It gets + * `structured` too, with the JSON view when its content is empty. + */ + private async mcpToolResult( + result: { content?: unknown; isError?: boolean; structuredContent?: unknown }, + callId: string, + mcplPeer: boolean, + ): Promise { + const hasStructured = Object.prototype.hasOwnProperty.call(result, 'structuredContent'); + if (!mcplPeer) { + const label = `${new Date().toISOString().slice(0, 10)}-${callId}`.replace(/[^A-Za-z0-9._-]/g, '_').slice(0, 80); + const workspace = this.getWorkspaceModule(); + const mount = workspace ? this.firstWritableMountName(workspace) : null; + const save = workspace && mount + ? async (fileName: string, bytes: Buffer, mimeType: string) => { + const path = `${mount}/tool-results/${fileName}`; + const written = await workspace.writeBinary(path, bytes, mimeType); + return written.success ? path : null; + } + : null; + return normalizeStandardToolResult(result, label, save); + } + const blocks = (Array.isArray(result.content) ? result.content : []) as Array<{ type?: string; text?: string }>; + // When the result contains non-text blocks (RFC-005 references, images), + // pass the full content array through so toMembraneToolResult can + // preserve them natively. Text-only results collapse to a joined string + // for callers that expect data to be string-ish. + let textContent = blocks + .filter((c) => c.type === 'text' && c.text) + .map((c) => c.text!) + .join('\n'); + if (blocks.length === 0 && hasStructured) textContent = JSON.stringify(result.structuredContent) ?? 'null'; + const hasNonText = blocks.some((c) => c.type !== 'text'); + const structured = hasStructured ? { structured: result.structuredContent } : {}; + return { + success: !result.isError, + data: result.isError ? undefined : hasNonText ? blocks : (textContent || undefined), + error: result.isError ? (textContent || 'Tool call failed') : undefined, + isError: result.isError ?? false, + ...structured, + }; } /** diff --git a/src/mcpl/tool-result-normalize.ts b/src/mcpl/tool-result-normalize.ts new file mode 100644 index 000000000..4b82ed85a --- /dev/null +++ b/src/mcpl/tool-result-normalize.ts @@ -0,0 +1,153 @@ +/** + * What a standard MCP tool result becomes for the framework. Standard means + * the server did not negotiate MCPL: a legacy MCP-only peer or a modern + * server. MCPL peers keep their RFC-005 reading (flat `resource` and + * uri-form media as references), which is not this module's job. + * + * An MCP result has two views, and each consumer gets its own: + * - **The content array is the server's model-facing view.** Text stays text + * and inline images stay native. Anything the provider format can't carry + * (audio, binary blobs) is saved to the workspace, and a bounded stub says + * where it went: a stub tells the reader where the payload is; it never + * replaces it silently. A `resource_link` is shown as a reference and never + * fetched. An embedded resource shows its text, or is saved like a blob. + * - **`structuredContent` is the machine-readable view.** It is kept by + * presence on `ToolResult.structured`, for scripts and modules. The model + * sees it rendered as JSON only when the content carries no text, because + * otherwise the server already chose what the model reads. + * + * Before this, a structured-only result reached the model as an empty string, + * and a mixed result became one JSON string with its image as base64 text + * (room-284 probe P1c). + */ +import type { ToolResult } from '../types/index.js'; + +/** Saves a payload under a host-chosen name; resolves to its workspace path, + * or null when there is nowhere writable to put it. */ +export type SaveToolPayload = (fileName: string, bytes: Buffer, mimeType: string) => Promise; + +/** Stub fields are bounded independently of anything the server sends. */ +const STUB_FIELD_CHARS = 200; + +type Block = { type: 'text'; text: string } | { type: 'image'; data: string; mimeType: string }; + +function bounded(value: unknown): string { + const text = typeof value === 'string' ? value : JSON.stringify(value) ?? ''; + return text.length > STUB_FIELD_CHARS ? `${text.slice(0, STUB_FIELD_CHARS)}…` : text; +} + +function decodedSize(base64: string): number { + const padding = base64.endsWith('==') ? 2 : base64.endsWith('=') ? 1 : 0; + return Math.max(0, Math.floor((base64.length * 3) / 4) - padding); +} + +function humanSize(bytes: number): string { + if (bytes < 1024) return `${bytes} B`; + if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`; + return `${(bytes / (1024 * 1024)).toFixed(1)} MB`; +} + +function extensionFor(mimeType: string): string { + const subtype = mimeType.split('/')[1]?.split(';')[0]?.trim().toLowerCase() ?? ''; + const known: Record = { 'octet-stream': 'bin', 'mpeg': 'mp3', 'plain': 'txt', 'x-wav': 'wav' }; + const ext = known[subtype] ?? subtype.replace(/[^a-z0-9]/g, ''); + return ext.length > 0 && ext.length <= 10 ? ext : 'bin'; +} + +export interface StandardToolResult { + content?: unknown; + isError?: boolean; + structuredContent?: unknown; +} + +/** + * Normalize a standard MCP tool result into the framework's `ToolResult`. + * `label` names saved payloads (one per call; payloads get an index suffix). + * Without `save`, payloads are described but not saved, and the stub says so. + */ +export async function normalizeStandardToolResult( + result: StandardToolResult, + label: string, + save: SaveToolPayload | null, +): Promise { + const raw = Array.isArray(result.content) ? result.content : []; + const blocks: Block[] = []; + let saved = 0; + + const payloadStub = async (kind: string, base64: string, mimeType: string, origin?: string): Promise => { + const size = humanSize(decodedSize(base64)); + const what = `[${kind}${origin ? ` ${bounded(origin)}` : ''}: ${bounded(mimeType)}, ${size}`; + if (!save) return `${what}. Not shown: the model can't take this type, and no workspace is mounted to save it]`; + const path = await save(`${label}-${++saved}.${extensionFor(mimeType)}`, Buffer.from(base64, 'base64'), mimeType) + .catch(() => null); + return path + ? `${what}, saved to workspace file ${path}]` + : `${what}. Not shown: the model can't take this type, and saving it to the workspace failed]`; + }; + + for (const entry of raw) { + const block = (entry && typeof entry === 'object' ? entry : {}) as Record; + switch (block.type) { + case 'text': + if (typeof block.text === 'string') blocks.push({ type: 'text', text: block.text }); + break; + case 'image': + if (typeof block.data === 'string' && typeof block.mimeType === 'string') { + blocks.push({ type: 'image', data: block.data, mimeType: block.mimeType }); + } else { + blocks.push({ type: 'text', text: `[image without inline data, not fetched${typeof block.uri === 'string' ? `: ${bounded(block.uri)}` : ''}]` }); + } + break; + case 'audio': + if (typeof block.data === 'string' && typeof block.mimeType === 'string') { + blocks.push({ type: 'text', text: await payloadStub('audio', block.data, block.mimeType) }); + } else { + blocks.push({ type: 'text', text: `[audio without inline data, not fetched${typeof block.uri === 'string' ? `: ${bounded(block.uri)}` : ''}]` }); + } + break; + case 'resource_link': { + const name = typeof block.name === 'string' ? block.name : undefined; + const mime = typeof block.mimeType === 'string' ? `, ${bounded(block.mimeType)}` : ''; + blocks.push({ type: 'text', text: `[resource link${name ? ` "${bounded(name)}"` : ''}: ${bounded(block.uri)}${mime}; not fetched]` }); + break; + } + case 'resource': { + const resource = (block.resource && typeof block.resource === 'object' ? block.resource : {}) as Record; + const uri = typeof resource.uri === 'string' ? resource.uri : undefined; + const mime = typeof resource.mimeType === 'string' ? resource.mimeType : undefined; + if (typeof resource.text === 'string') { + blocks.push({ type: 'text', text: `[resource ${bounded(uri ?? '(no uri)')}${mime ? `, ${bounded(mime)}` : ''}]\n${resource.text}` }); + } else if (typeof resource.blob === 'string') { + blocks.push({ type: 'text', text: await payloadStub('resource', resource.blob, mime ?? 'application/octet-stream', uri) }); + } else { + blocks.push({ type: 'text', text: `[resource ${bounded(uri ?? '(no uri)')} with no contents]` }); + } + break; + } + default: + // Unknown to this client: shown as data rather than dropped. + blocks.push({ type: 'text', text: JSON.stringify(entry) ?? '' }); + } + } + + const hasStructured = Object.prototype.hasOwnProperty.call(result, 'structuredContent'); + const hasText = blocks.some((b) => b.type === 'text' && b.text.trim().length > 0); + if (hasStructured && !hasText) { + blocks.push({ type: 'text', text: JSON.stringify(result.structuredContent) ?? 'null' }); + } + + const text = blocks.filter((b): b is Extract => b.type === 'text').map((b) => b.text).join('\n'); + const structured = hasStructured ? { structured: result.structuredContent } : {}; + if (result.isError) { + return { success: false, error: text || 'Tool call failed', isError: true, ...structured }; + } + const hasImage = blocks.some((b) => b.type === 'image'); + return { + success: true, + // Text-only results keep the joined-string convention callers expect. + // An image keeps the array, so the model sees it natively. + data: hasImage ? blocks : (text || undefined), + isError: false, + ...structured, + }; +} diff --git a/src/mcpl/types.ts b/src/mcpl/types.ts index 209229b01..3daf4e49e 100644 --- a/src/mcpl/types.ts +++ b/src/mcpl/types.ts @@ -388,6 +388,11 @@ export interface McplServerConfig { * agent turn forever; with the timeout the pending request rejects with a * descriptive error, which the framework surfaces as a normal isError * tool_result. Set 0 to disable. Default: 60000 (60 seconds). + * + * For a modern server it is one deadline per tool call, across every leg + * of the call, and must be an integer from 1 to 2^31−1. It can't be + * disabled: the SDK reads 0 as "time out now". At the deadline + * cancellation is requested and the outcome is unknown. */ requestTimeoutMs?: number; diff --git a/src/types/events.ts b/src/types/events.ts index b5e8afb8e..36037c65c 100644 --- a/src/types/events.ts +++ b/src/types/events.ts @@ -188,6 +188,14 @@ export interface ToolResult { error?: string; /** Whether this was an error (for LLM) */ isError?: boolean; + /** + * An MCP tool's `structuredContent`: its machine-readable result, beside + * `data`, which is the content view. Present exactly when the server sent + * one, so `false`, `0` and `null` are values here, not absence. Programs + * read it (scripts get it in their result object); the model sees content, + * and sees this rendered as JSON only when the content carries no text. + */ + structured?: unknown; /** * When true, the framework saves tool_use + tool_result messages to context, * cancels the active stream, and resets the agent to idle. diff --git a/test/mcp-modern-framework.test.ts b/test/mcp-modern-framework.test.ts new file mode 100644 index 000000000..c6b25e5dc --- /dev/null +++ b/test/mcp-modern-framework.test.ts @@ -0,0 +1,258 @@ +/** + * A modern-MCP server inside a real framework (room-284, shelf-487): tool + * listing and status, both dispatch paths, the script view of structured + * results, an MCP-only legacy peer read the same standard way, and the host + * lifetime contract (Tessa #53152, Agnes/Kit #53184, Theo #53178): + * - a modern call already in flight completes while the host quiesces; + * - a list change while quiesced refreshes the inventory, and its wake parks + * until resume; + * - a modern child lost while quiesced reconnects and relists without + * starting a turn. + */ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtempSync, readFileSync, existsSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, dirname } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +import { AgentFramework } from '../src/index.js'; +import type { EventResponse, Module, ModuleContext, ProcessEvent, ProcessState, ToolDefinition, ToolResult } from '../src/index.js'; +import { MockMembrane, createMockResponse } from './helpers/mock-membrane.js'; + +const here = dirname(fileURLToPath(import.meta.url)); +const MODERN = join(here, 'fixtures', 'modern-mcp-server.mjs'); + +/** Turns every external message into a wake, so tool-list diffs wake agents. */ +class Waker implements Module { + readonly name = 'waker'; + async start(_ctx: ModuleContext): Promise {} + async stop(): Promise {} + getTools(): ToolDefinition[] { return []; } + async handleToolCall(): Promise { return { success: false, error: 'no tools', isError: true }; } + async onProcess(event: ProcessEvent, _state: ProcessState): Promise { + if (event.type === 'external-message') { + return { addMessages: [{ participant: 'User', content: [{ type: 'text', text: 'go' }] }], requestInference: true }; + } + return {}; + } +} + +async function waitFor(what: string, cond: () => boolean, timeoutMs = 8000): Promise { + const deadline = Date.now() + timeoutMs; + while (!cond()) { + if (Date.now() > deadline) throw new Error(`timed out waiting for ${what}`); + await new Promise((r) => setTimeout(r, 25)); + } +} + +const lines = (file: string) => (existsSync(file) ? readFileSync(file, 'utf8').split('\n').filter(Boolean) : []); + +interface Harness { framework: AgentFramework; membrane: MockMembrane; dir: string } + +async function withModern(fn: (h: Harness) => Promise, extraServers: Record[] = []): Promise { + const dir = mkdtempSync(join(tmpdir(), 'modern-framework-')); + const membrane = new MockMembrane(); + const framework = await AgentFramework.create({ + storePath: join(dir, 'store'), + membrane: membrane.asMembrane(), + agents: [{ name: 'agent', model: 'test-model', systemPrompt: 'test' }], + modules: [new Waker()], + mcplServers: [ + { + id: 'modern', + command: process.execPath, + args: [MODERN, join(dir, 'starts.log'), join(dir, 'events.log')], + protocol: 'modern', + reconnect: true, + reconnectIntervalMs: 50, + reconnectMaxIntervalMs: 100, + }, + ...(extraServers as never[]), + ], + }); + try { + await fn({ framework, membrane, dir }); + } finally { + await framework.stop(); + rmSync(dir, { recursive: true, force: true }); + } +} + +const call = (name: string, input: Record = {}) => ({ id: `t-${name}-${Math.random()}`, name, input }); + +test('modern tools are listed under the prefix, and status names family, revision and transport', async () => { + await withModern(async ({ framework }) => { + const names = framework.getAllTools().map((t) => t.name); + assert.ok(names.includes('mcpl--modern--echo'), names.join()); + const [status] = framework.listMcplServers(); + assert.equal(status!.family, 'modern'); + assert.equal(status!.protocolVersion, '2026-07-28'); + assert.equal(status!.transport, 'stdio'); + assert.equal(status!.connected, true); + assert.ok(status!.toolCount >= 5); + assert.deepEqual(status!.effectiveGrant, []); + }); +}); + +test('direct path: structured kept by presence, tool errors fail, media is described', async () => { + await withModern(async ({ framework }) => { + const only = await framework.executeToolCall(call('mcpl--modern--structured_only')); + assert.equal(only.success, true); + assert.deepEqual(only.structured, { answer: 42, ok: false, nothing: null }); + assert.equal(only.data, '{"answer":42,"ok":false,"nothing":null}'); + + const zero = await framework.executeToolCall(call('mcpl--modern--structured_zero')); + assert.ok('structured' in zero); + assert.equal(zero.structured, 0); + + const fail = await framework.executeToolCall(call('mcpl--modern--fail')); + assert.equal(fail.success, false); + assert.equal(fail.isError, true); + assert.equal(fail.error, 'the tool failed on purpose'); + + const media = await framework.executeToolCall(call('mcpl--modern--media')); + assert.ok(Array.isArray(media.data), 'an image keeps the array'); + const text = (media.data as Array<{ type: string; text?: string }>).filter((b) => b.type === 'text').map((b) => b.text).join('\n'); + assert.match(text, /\[resource link "report\.pdf": file:\/\/\/srv\/report\.pdf, application\/pdf; not fetched\]/); + assert.match(text, /embedded note text/); + assert.match(text, /\[audio: audio\/wav, \d+ B\. Not shown/); + assert.ok((media.data as Array<{ type: string }>).some((b) => b.type === 'image')); + }); +}); + +test('model path: the result reaches the model, structured-only as JSON', async () => { + await withModern(async ({ framework, membrane }) => { + framework.start(); + membrane.pushResponse(createMockResponse([ + { type: 'tool_use', id: 'toolu_01MODERNSTRUCTURED000000', name: 'mcpl--modern--structured_only', input: {} }, + ] as never, 'tool_use')); + membrane.pushResponse(createMockResponse([{ type: 'text', text: 'Done.' }])); + framework.pushEvent({ type: 'external-message', source: 'test', content: 'go', metadata: {} } as never); + await waitFor('the tool round', () => (membrane.lastStream?.receivedToolResults.length ?? 0) >= 1); + // The framework renders string data JSON-encoded, as it does for every + // string tool result; decoded, it is the structured value's JSON. + const [delivered] = membrane.lastStream!.receivedToolResults[0] as Array<{ content: string }>; + assert.deepEqual(JSON.parse(JSON.parse(delivered!.content)), { answer: 42, ok: false, nothing: null }); + }); +}); + +test('scripts get structured results whole, as one JSON object', async () => { + await withModern(async ({ framework }) => { + framework.start(); + const handle = (framework as unknown as { + handleScriptToolCall: (agent: string, tool: string, args: Record) => Promise; + }).handleScriptToolCall.bind(framework); + const out = JSON.parse(await handle('agent', 'mcpl--modern--structured_and_text', {})); + assert.deepEqual(out, { content: 'Found 2 rows.', structuredContent: { rows: [{ id: 1 }, { id: 2 }] }, isError: false }); + // Without structured content the script contract is unchanged: the + // history rendering, which JSON-encodes a text result. + assert.equal(await handle('agent', 'mcpl--modern--echo', { text: 'x' }), JSON.stringify('echo:x')); + }); +}); + +test('MCPL policy on a modern server is a configuration error at connect', async () => { + await withModern(async ({ framework }) => { + await assert.rejects( + framework.connectMcplServer({ id: 'bad', url: 'https://example.invalid/mcp', enabledFeatureSets: ['x'] }), + /"enabledFeatureSets" is MCPL policy/, + ); + await assert.rejects( + framework.connectMcplServer({ id: 'bad2', url: 'https://example.invalid/mcp', protocol: 'modern' }), + /applies only to stdio/, + ); + assert.ok(!framework.listMcplServers().some((s) => s.id === 'bad' || s.id === 'bad2'), 'nothing registered'); + }); +}); + +test('an MCP-only legacy peer gets the same standard reading', async () => { + const LEGACY_MCP_ONLY = ` +let buf = ''; +process.stdin.on('data', (c) => { + buf += c.toString('utf8'); + let i; + while ((i = buf.indexOf('\\n')) >= 0) { + const line = buf.slice(0, i); buf = buf.slice(i + 1); + if (!line.trim()) continue; + let m; try { m = JSON.parse(line); } catch { continue; } + const reply = (result) => process.stdout.write(JSON.stringify({ jsonrpc: '2.0', id: m.id, result }) + '\\n'); + if (m.method === 'initialize') reply({ protocolVersion: '2024-11-05', capabilities: { tools: {} } }); + else if (m.method === 'tools/list') reply({ tools: [{ name: 'docs', inputSchema: { type: 'object' } }] }); + else if (m.method === 'tools/call') reply({ + content: [ + { type: 'resource_link', uri: 'https://example.invalid/a.md', name: 'a.md' }, + { type: 'resource', resource: { uri: 'memo://n', mimeType: 'text/plain', text: 'note body' } }, + ], + structuredContent: { count: 2 }, + }); + } +}); +`; + await withModern(async ({ framework }) => { + const result = await framework.executeToolCall(call('mcpl--plain--docs')); + assert.equal(result.success, true); + assert.deepEqual(result.structured, { count: 2 }); + assert.equal(result.data, '[resource link "a.md": https://example.invalid/a.md; not fetched]\n[resource memo://n, text/plain]\nnote body'); + const status = framework.listMcplServers().find((s) => s.id === 'plain')!; + assert.equal(status.family, 'legacy'); + assert.equal(status.protocolVersion, '2024-11-05'); + }, [{ id: 'plain', command: process.execPath, args: ['-e', LEGACY_MCP_ONLY] }]); +}); + +test('quiesce: a modern call already in flight completes, and its result reaches the turn', async () => { + await withModern(async ({ framework, membrane, dir }) => { + framework.start(); + membrane.pushResponse(createMockResponse([ + { type: 'tool_use', id: 'toolu_01MODERNSLOW0000000000000', name: 'mcpl--modern--slow', input: { ms: 600 } }, + ] as never, 'tool_use')); + membrane.pushResponse(createMockResponse([{ type: 'text', text: 'Done.' }])); + framework.pushEvent({ type: 'external-message', source: 'test', content: 'go', metadata: {} } as never); + await waitFor('the slow call to start server-side', () => lines(join(dir, 'events.log')).includes('slow-start')); + + const status = await framework.quiesce({ reason: 'maintenance', timeoutMs: 10_000 }); + assert.equal(status.quiesced, true); + assert.equal(status.drained, true, 'the turn drained: its modern call completed'); + assert.ok(lines(join(dir, 'events.log')).includes('slow-finished')); + assert.ok(!lines(join(dir, 'events.log')).includes('slow-aborted'), 'quiesce did not cancel it'); + assert.match(JSON.stringify(membrane.lastStream!.receivedToolResults[0]), /slept 600/); + await framework.resume(); + }); +}); + +test('quiesce: a list change refreshes the inventory, and its wake parks until resume', async () => { + await withModern(async ({ framework, membrane }) => { + framework.start(); + await framework.quiesce({ reason: 'maintenance' }); + const callsBefore = membrane.calls.length; + const gatedBefore = framework.getHostModeStatus().gatedRequests; + + await framework.executeToolCall(call('mcpl--modern--add_tool')); + await waitFor('the refreshed inventory', () => framework.getAllTools().some((t) => t.name === 'mcpl--modern--added_1')); + await waitFor('the wake to park', () => framework.getHostModeStatus().gatedRequests > gatedBefore); + await new Promise((r) => setTimeout(r, 200)); + assert.equal(membrane.calls.length, callsBefore, 'no inference while quiesced'); + + await framework.resume(); + await waitFor('the parked wake to run', () => membrane.calls.length > callsBefore); + }); +}); + +test('quiesce: a lost modern child reconnects and relists without starting a turn', async () => { + await withModern(async ({ framework, membrane, dir }) => { + framework.start(); + await framework.quiesce({ reason: 'maintenance' }); + const callsBefore = membrane.calls.length; + const gatedBefore = framework.getHostModeStatus().gatedRequests; + + await framework.executeToolCall(call('mcpl--modern--die')); + await waitFor('a second launch', () => lines(join(dir, 'starts.log')).length === 2); + await waitFor('reconnected', () => framework.listMcplServers()[0]!.connected); + await waitFor('the relisted inventory', () => framework.getAllTools().some((t) => t.name === 'mcpl--modern--echo')); + await new Promise((r) => setTimeout(r, 200)); + + assert.equal(membrane.calls.length, callsBefore, 'no turn started'); + assert.equal(framework.getHostModeStatus().gatedRequests, gatedBefore, 'no wake parked: the inventory is unchanged'); + assert.deepEqual((await framework.executeToolCall(call('mcpl--modern--echo', { text: 'back' }))).data, 'echo:back'); + await framework.resume(); + }); +}); diff --git a/test/mcp-tool-result-normalize.test.ts b/test/mcp-tool-result-normalize.test.ts new file mode 100644 index 000000000..a5457e9f8 --- /dev/null +++ b/test/mcp-tool-result-normalize.test.ts @@ -0,0 +1,111 @@ +/** + * The standard reading of an MCP tool result (room-284, shelf-487): the + * content array is the model's view, and `structuredContent` the + * programmatic one, kept by presence. A payload the model can't take is + * saved and described by a bounded stub; nothing is fetched. + */ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; + +import { normalizeStandardToolResult, type SaveToolPayload } from '../src/mcpl/tool-result-normalize.js'; + +const b64 = (s: string) => Buffer.from(s).toString('base64'); + +function recordingSave(): { save: SaveToolPayload; saved: Array<{ fileName: string; bytes: string; mimeType: string }> } { + const saved: Array<{ fileName: string; bytes: string; mimeType: string }> = []; + return { + saved, + save: async (fileName, bytes, mimeType) => { + saved.push({ fileName, bytes: bytes.toString('utf8'), mimeType }); + return `home/tool-results/${fileName}`; + }, + }; +} + +test('text only: the joined string, as before, and no structured field', async () => { + const r = await normalizeStandardToolResult({ content: [{ type: 'text', text: 'a' }, { type: 'text', text: 'b' }] }, 'L', null); + assert.deepEqual(r, { success: true, data: 'a\nb', isError: false }); + assert.ok(!('structured' in r)); +}); + +test('structured only: the model sees it as JSON, programs get the value', async () => { + const r = await normalizeStandardToolResult({ content: [], structuredContent: { answer: 42, ok: false, nothing: null } }, 'L', null); + assert.equal(r.success, true); + assert.equal(r.data, '{"answer":42,"ok":false,"nothing":null}'); + assert.deepEqual(r.structured, { answer: 42, ok: false, nothing: null }); +}); + +test('falsy structured values are values: 0, false and null are kept and shown', async () => { + for (const value of [0, false, null, '']) { + const r = await normalizeStandardToolResult({ content: [], structuredContent: value }, 'L', null); + assert.ok('structured' in r, String(value)); + assert.equal(r.structured, value); + assert.equal(r.data, JSON.stringify(value)); + } +}); + +test('text plus structured: the server chose the model\'s text; structured rides beside it', async () => { + const r = await normalizeStandardToolResult({ + content: [{ type: 'text', text: 'Found 2 rows.' }], + structuredContent: { rows: [{ id: 1 }, { id: 2 }] }, + }, 'L', null); + assert.equal(r.data, 'Found 2 rows.'); + assert.deepEqual(r.structured, { rows: [{ id: 1 }, { id: 2 }] }); +}); + +test('an inline image stays native: the array is kept', async () => { + const r = await normalizeStandardToolResult({ + content: [{ type: 'text', text: 'see' }, { type: 'image', data: 'iVBORw0KGgo=', mimeType: 'image/png' }], + }, 'L', null); + assert.deepEqual(r.data, [{ type: 'text', text: 'see' }, { type: 'image', data: 'iVBORw0KGgo=', mimeType: 'image/png' }]); +}); + +test('audio and binary resources are saved to the workspace and described', async () => { + const { save, saved } = recordingSave(); + const r = await normalizeStandardToolResult({ + content: [ + { type: 'audio', data: b64('RIFFdata'), mimeType: 'audio/wav' }, + { type: 'resource', resource: { uri: 'memo://bin', mimeType: 'application/octet-stream', blob: b64('binary!') } }, + ], + }, '2026-10-08-call1', save); + assert.deepEqual(saved, [ + { fileName: '2026-10-08-call1-1.wav', bytes: 'RIFFdata', mimeType: 'audio/wav' }, + { fileName: '2026-10-08-call1-2.bin', bytes: 'binary!', mimeType: 'application/octet-stream' }, + ]); + assert.equal(r.data, + '[audio: audio/wav, 8 B, saved to workspace file home/tool-results/2026-10-08-call1-1.wav]\n' + + '[resource memo://bin: application/octet-stream, 7 B, saved to workspace file home/tool-results/2026-10-08-call1-2.bin]'); +}); + +test('without a workspace the stub says the payload was not shown, and why', async () => { + const r = await normalizeStandardToolResult({ content: [{ type: 'audio', data: b64('x'), mimeType: 'audio/mpeg' }] }, 'L', null); + assert.equal(r.data, "[audio: audio/mpeg, 1 B. Not shown: the model can't take this type, and no workspace is mounted to save it]"); +}); + +test('a resource link is shown as a reference and never fetched; embedded text is shown', async () => { + const r = await normalizeStandardToolResult({ + content: [ + { type: 'resource_link', uri: 'file:///srv/report.pdf', name: 'report.pdf', mimeType: 'application/pdf' }, + { type: 'resource', resource: { uri: 'memo://note', mimeType: 'text/plain', text: 'embedded note text' } }, + ], + }, 'L', null); + assert.equal(r.data, + '[resource link "report.pdf": file:///srv/report.pdf, application/pdf; not fetched]\n' + + '[resource memo://note, text/plain]\nembedded note text'); +}); + +test('a tool error is a failure with its text; structured survives', async () => { + const r = await normalizeStandardToolResult({ content: [{ type: 'text', text: 'nope' }], isError: true, structuredContent: { code: 7 } }, 'L', null); + assert.deepEqual(r, { success: false, error: 'nope', isError: true, structured: { code: 7 } }); +}); + +test('an unknown block is shown as data, and stub fields are bounded', async () => { + const long = `memo://${'x'.repeat(1000)}`; + const r = await normalizeStandardToolResult({ + content: [{ type: 'hologram', depth: 3 }, { type: 'resource_link', uri: long }], + }, 'L', null); + const [unknown, link] = String(r.data).split('\n'); + assert.equal(unknown, '{"type":"hologram","depth":3}'); + assert.ok(link!.length < 260, `bounded: ${link!.length}`); + assert.match(link!, /…; not fetched\]$/); +}); diff --git a/test/mcpl-list-status.test.ts b/test/mcpl-list-status.test.ts index 348275257..e30507121 100644 --- a/test/mcpl-list-status.test.ts +++ b/test/mcpl-list-status.test.ts @@ -42,6 +42,10 @@ test('listMcplServers exposes the live grant layers and host-owned authority', ( id: 'discord', connected: true, retrying: false, + family: 'legacy', + // The stub never handshook, so no revision is established. + protocolVersion: null, + transport: 'stdio', toolPrefix: 'mcpl--discord', toolCount: 1, policyEstablished: true, From b32cd73451ebadf4e3e783ea77b6ddbb8d15825e Mon Sep 17 00:00:00 2001 From: Olive-1804 Date: Thu, 8 Oct 2026 02:51:43 -0400 Subject: [PATCH 0062/1112] fix(mcp): integration review, connect ownership, result views, selectors, reap verdicts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Theo-1805's source review of f9954a5 (room-284 #53865, #53879, #53922). Connect ownership. ModernMcpConnection is now create() (sync, validating) followed by start(). The framework registers and wires the connection before its first connect, so: - stop() and disconnect own and close a connect in flight; - a second connect for the same id is refused synchronously; - a connection closed mid-connect never finishes its setup; - a failed first connect is unregistered, and its cleanup failure is traced beside the connect error, not in its place. start() resolves unconnected when closed. open() launches nothing once closed. Result views stay separate, nothing lowered early: - Direct path (executeToolCall, ModuleContext.callTool). data is the raw content array again, as before. isError is now a failure (success: false, error text, raw content kept). structured is added by presence. - Model path. The structured JSON fallback keys on the server's own text blocks; generated stubs no longer suppress it. A payload that can be neither shown nor saved makes the result a failure to materialize ("result incomplete …; the tool may already have completed"), never a success with a lost payload. - Scripts. A script call returning structuredContent gets the direct reading (detected by the registered script waiter for its id). The script receives {content: , structuredContent, isError, error?} as valid JSON, so the image-plus-scores case keeps its image payload. Over 5 MB the whole object is saved to the workspace, and the script gets {isError, oversized: {chars, savedTo}}. If it can't be saved, an explicit materialization failure. Results without structuredContent keep the existing string contract, pinned by test (JSON-encoded "echo:x"). Selectors. A configuration naming no usable transport is an error for every config, in the framework and in McplServerConnection's handshake. That covers an http(s) url with transport 'websocket', transport 'http' with a ws url or no url (now "transport "http" requires "url""), and unknown schemes. There is no fall-through to another opener. Valid legacy shapes resolve as before. Reap verdicts. StdioTransport.close() is memoized (calls share one attempt) and rejects if the child is still alive 2 s after SIGKILL. ModernMcpConnection.close() closes transport first, then client: during discovery only the transport ends the probe. The transport's verdict is collected, not swallowed, and close() rethrows. stop() uses allSettled: each failure is logged and traced, and the rest of shutdown still runs. disconnect finishes its removal before reporting. Tests: - framework: - raw direct data (echo → [{type:'text'}], media → every block as sent); - an incomplete model result when no workspace is mounted; - an MCP-only legacy peer's model reading, with the structured fallback beside its stubs; - script objects for structured_and_text and scored_image (image payload intact), plus the pinned no-structured contract; - selector rejections; - stop() reaping a connect in flight while a second connect for the id is refused. - normalizer: the materialization failure (no save; failed save), and the structured fallback beside a stub. - outcomes: an unreapable child (signals suppressed) fails close() explicitly, and a second close() shares the verdict. Receipt (Lethe, Node v24.19.0, npx tsc build): every file standalone, 4-way (/tmp/olive-284/run-perfile2.sh, logs in /tmp/olive-284/logs/perfile2/): 134 files, 1445 tests, 1437 pass. The 8 failures were all in checkpoint-attribution, where a prototype-built fake framework has no scriptToolWaiters. Fixed with `?.`, and that file reran 10/10 (checkpoint-attribution.rerun.log). mcpl-coalescing completed 66/66 in this run. --- .../mcp-standard-tool-results.changed.md | 49 +++-- src/framework.ts | 172 ++++++++++++++---- src/mcpl/modern-connection.ts | 83 ++++++--- src/mcpl/protocol-family.ts | 2 + src/mcpl/server-connection.ts | 8 +- src/mcpl/tool-result-normalize.ts | 47 ++++- src/mcpl/transport.ts | 20 +- test/fixtures/modern-raw-server.mjs | 2 + test/fixtures/modern-tools.mjs | 5 + test/mcp-modern-framework.test.ts | 112 ++++++++++-- test/mcp-modern-outcomes.test.ts | 12 ++ test/mcp-protocol-family.test.ts | 1 + test/mcp-tool-result-normalize.test.ts | 28 ++- 13 files changed, 425 insertions(+), 116 deletions(-) diff --git a/changelog.d/mcp-standard-tool-results.changed.md b/changelog.d/mcp-standard-tool-results.changed.md index ae9cad54a..b88e890be 100644 --- a/changelog.d/mcp-standard-tool-results.changed.md +++ b/changelog.d/mcp-standard-tool-results.changed.md @@ -1,24 +1,39 @@ -- MCP tool results from servers that aren't MCPL peers (MCP-only legacy - servers and modern servers) are now read by their content types: +- What the model sees of an MCP tool result from a server that isn't an + MCPL peer (an MCP-only legacy server, or a modern server) now follows its + content types: - text stays text, and inline images stay native; - audio and binary resources are saved to the workspace (`tool-results/`), - with a bounded stub saying where; + with a bounded stub saying where. If a payload can't be saved, the result + is reported as incomplete (`isError`), noting the tool may already have + completed, never as a success with part of it missing; - a `resource_link` is shown as a reference and never fetched; - an embedded text resource shows its text. Before, any non-text block turned the whole result into raw JSON. - `structuredContent` is now kept by presence on the new - `ToolResult.structured`, so `false`, `0` and `null` are values. The model - sees it as JSON only when the content carries no text; a structured-only - result used to reach the model as an empty string. Scripts receive - `{ content, structuredContent, isError }` as one JSON object whenever a - result has structured content. MCPL peers keep their RFC-005 reading and - gain `structured`. + `structuredContent` is kept by presence on the new `ToolResult.structured` + (`false`, `0` and `null` are values). The model sees it as JSON when the + server's content has no text block of its own; a structured-only result + used to reach the model as an empty string. MCPL peers keep their RFC-005 + reading and gain `structured`. +- A script whose tool call returns `structuredContent` receives the + server's result itself, as one JSON object: `{ content, structuredContent, + isError, error? }`, with `content` the blocks as sent, so images and other + payloads survive. If it would exceed the 5 MB script cap, it is saved to the + workspace and the script gets `{ isError, oversized: { chars, savedTo } }`. + If it can't be saved, the script gets a failure that says the tool may have + completed. Results without structured content keep the existing script + contract. - The direct tool path (`executeToolCall`, `ModuleContext.callTool`) now - treats an MCP tool error (`isError: true`) as a failure (`success: false` - with the error text), as the model path does. Before, every answer counted - as success. Its `data` now has the model path's shape too: text joined into - a string, or an array when an image is present. -- Closing a stdio MCP/MCPL server now waits for the child to exit. A child - still alive 2 s after SIGTERM gets SIGKILL. Before, close returned as soon - as the signal was sent. + treats an MCP tool error (`isError: true`) as a failure, with + `success: false` and the error text; before, every answer counted as + success. `data` is still the raw content array, and `structured` is added + when the server sent one. +- Closing a stdio MCP/MCPL server now waits for the child to exit, sending + SIGKILL after 2 s. A child still alive 2 s after SIGKILL fails `close()` + with an explicit error. Before, close returned as soon as a signal was + sent. `stop()` reports such a failure and still finishes shutting + everything else down. +- A server configuration naming no usable transport is now an error before + anything is spawned or dialed. That covers a `transport` that doesn't + match the url's scheme, `transport: 'http'` without a url, and an + unrecognized scheme. Every valid configuration resolves as before. diff --git a/src/framework.ts b/src/framework.ts index 14f5cbe23..871718150 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -2035,7 +2035,15 @@ export class AgentFramework { shutdownPromises.push(this.mcplServerRegistry.closeAll()); } shutdownPromises.push(this.closeModernMcpConnections()); - await Promise.all(shutdownPromises); + // Every part shuts down even if one fails (a child that could not be + // reaped, say); each failure is reported, and the rest of stop() runs. + for (const outcome of await Promise.allSettled(shutdownPromises)) { + if (outcome.status === 'rejected') { + const reason = outcome.reason instanceof Error ? outcome.reason.message : String(outcome.reason); + console.error(`[shutdown] ${reason}`); + this.emitTrace({ type: 'mcpl:server-error', serverId: 'shutdown', error: reason }); + } + } // Streams may queue storage repairs while being cancelled above. Retry // after their teardown, while the store is still open, before final sync. @@ -11215,11 +11223,8 @@ export class AgentFramework { } const args = (call.input && typeof call.input === 'object') ? call.input as Record : {}; try { - // The same reading as the model path, so a tool error is a failure - // here too (this path used to report every answer as success). - if (modern) return await this.mcpToolResult(await modern.callTool(toolName, args), call.id, false); - const result = await server!.sendToolsCall(toolName, args); - return await this.mcpToolResult(result, call.id, server!.capabilities !== null); + const result = modern ? await modern.callTool(toolName, args) : await server!.sendToolsCall(toolName, args); + return AgentFramework.directToolResult(result); } catch (err) { return { success: false, @@ -11866,24 +11871,50 @@ export class AgentFramework { // Scripts get (nearly) full data — the script environment IS the spill: // filtering big results in code is the whole point. 5MB protocol safety // cap only; no context-cap truncation, no file spill. - if (structured) { - // Inside the object, text content is the text itself; arrays get the - // history rendering (images as placeholders). The same 5MB protocol - // safety cap applies, over the whole object. - const content = result.isError - ? (result.error ?? '') - : typeof result.data === 'string' - ? result.data - : result.data === undefined ? '' : toolResultDataToHistoryString(result.data, 5_000_000); - return truncateForHistory(JSON.stringify({ - content, - structuredContent: result.structured, - isError: result.isError === true, - }), 5_000_000); - } + if (structured) return this.scriptStructuredResult(result); return toolResultDataToHistoryString(result.data, 5_000_000); } + /** The 5MB protocol safety cap on what a script receives. */ + private static readonly SCRIPT_RESULT_MAX_CHARS = 5_000_000; + + /** + * A structured MCP result as a script receives it: one JSON object holding + * both of the server's views, `{ content, structuredContent, isError }`, + * plus `error` (the text) on a tool error. `content` is the content array + * as the server sent it (the dispatch gave this call the direct reading), + * so payloads and images survive. Past the size cap the result stays + * valid JSON: the whole object is saved to the workspace, and the script + * gets `{ isError, oversized: { chars, savedTo } }`. If it can't be saved, + * the script gets a failure to materialize the result, saying the tool may + * already have completed, never a success missing its result. + */ + private async scriptStructuredResult(result: ToolResult): Promise { + const object = { + content: Array.isArray(result.data) ? result.data : [], + structuredContent: result.structured, + isError: result.isError === true, + ...(result.isError ? { error: result.error ?? '' } : {}), + }; + const json = JSON.stringify(object); + if (json.length <= AgentFramework.SCRIPT_RESULT_MAX_CHARS) return json; + const workspace = this.getWorkspaceModule(); + const mount = workspace ? this.firstWritableMountName(workspace) : null; + if (workspace && mount) { + const path = `${mount}/tool-results/${new Date().toISOString().slice(0, 10)}-script-${randomUUID().slice(0, 8)}.json`; + const written = await workspace.writeBinary(path, Buffer.from(json, 'utf8'), 'application/json').catch(() => null); + if (written?.success) { + return JSON.stringify({ isError: object.isError, oversized: { chars: json.length, savedTo: path } }); + } + } + return JSON.stringify({ + isError: true, + error: `the result (${json.length} chars) exceeds the ${AgentFramework.SCRIPT_RESULT_MAX_CHARS}-char script result cap ` + + `and could not be saved to the workspace, so it could not be delivered; the tool may already have completed`, + oversized: { chars: json.length, savedTo: null }, + }); + } + /** * Dispatch a script-inner tool call through the NORMAL dispatch path and * resolve with its result. The waiter intercept in handleProcessEvent @@ -13381,7 +13412,10 @@ export class AgentFramework { private async closeModernMcpConnections(): Promise { const modern = [...this.modernServers().values()]; this.modernServers().clear(); - await Promise.all(modern.map((connection) => connection.close())); + const failures = (await Promise.allSettled(modern.map((connection) => connection.close()))) + .filter((o): o is PromiseRejectedResult => o.status === 'rejected') + .map((o) => (o.reason instanceof Error ? o.reason.message : String(o.reason))); + if (failures.length > 0) throw new Error(failures.join('; ')); } /** Reconcile the durable ledger with Chronicle, then deliver every server's work. */ @@ -13741,17 +13775,20 @@ export class AgentFramework { throw new Error('MCPL subsystem is not initialized'); } - // Configuration mistakes the family brought (protocol on a URL, a modern - // deadline of 0, MCPL policy on a modern server) fail here, before any - // routing is registered. A configuration that names no usable transport - // keeps the transport's own error, as before. - let family: 'legacy' | 'modern' | null = null; - try { family = resolveServerBinding(config).family; } catch { /* the transport reports it */ } - if (family !== null) { - const problems = serverConfigProblems(config); - if (problems.length > 0) { - throw new Error(`MCP server "${config.id}" configuration: ${problems.join('; ')}`); - } + // Configuration mistakes fail here, before any routing is registered: a + // combination naming no usable transport (an http(s) url with transport + // 'websocket', transport 'http' with a ws url or with no url at all), + // protocol on a URL, a modern deadline of 0, MCPL policy on a modern + // server. Every valid legacy shape resolves exactly as before. + let family: 'legacy' | 'modern'; + try { + family = resolveServerBinding(config).family; + } catch (error) { + throw new Error(`MCP server "${config.id}" configuration: ${error instanceof Error ? error.message : String(error)}`); + } + const problems = serverConfigProblems(config); + if (problems.length > 0) { + throw new Error(`MCP server "${config.id}" configuration: ${problems.join('; ')}`); } if (this.modernServers().has(config.id) || this.mcplServerRegistry.getServer(config.id)) { throw new Error(`MCPL server "${config.id}" is already registered`); @@ -13764,9 +13801,29 @@ export class AgentFramework { this.mcplServerConfigs.set(config.id, config); if (family === 'modern') { - const modern = await ModernMcpConnection.connect(config); + // Registered and wired before its first connect, so the connect in + // flight is already the framework's: stop() and disconnect close it, + // and a second connect for the same id is refused above. + const modern = ModernMcpConnection.create(config); this.modernServers().set(config.id, modern); this.wireModernEvents(modern); + try { + await modern.start(); + } catch (error) { + if (this.modernServers().get(config.id) === modern) this.modernServers().delete(config.id); + // The connect error is the one to report; a cleanup failure beside + // it is reported too, never in its place. + await modern.close().catch((cleanup: unknown) => { + this.emitTrace({ + type: 'mcpl:server-error', + serverId: config.id, + error: cleanup instanceof Error ? cleanup.message : String(cleanup), + }); + }); + throw error; + } + // Closed (stop, disconnect) while connecting: nothing more to set up. + if (this.modernServers().get(config.id) !== modern) return; this.emitTrace({ type: 'module:added', moduleName: `mcpl:${config.id}` }); return; } @@ -14197,9 +14254,12 @@ export class AgentFramework { const oldToolNames = new Set(this.mcplTools.map(t => t.name)); const modern = this.modernServers().get(id); + // A child that can't be reaped fails the disconnect, but only after the + // rest of the removal is done. + let closeFailure: unknown = null; if (modern) { this.modernServers().delete(id); - await modern.close(); + await modern.close().catch((error: unknown) => { closeFailure = error; }); } await this.mcplServerRegistry.removeServer(id); this.channelRegistry?.removeServer(id); @@ -14217,6 +14277,7 @@ export class AgentFramework { await this.refreshMcplTools(); this.emitMcplToolDiff(oldToolNames, id); + if (closeFailure) throw closeFailure; } /** @@ -15012,10 +15073,20 @@ export class AgentFramework { }); }; + // A script's call that returns structured content gets the server's + // result itself (the direct reading): the script receives both views + // whole, so nothing the model reading lowers (stubs, image placeholders, + // generated JSON) stands in for what the server sent. + const forScript = this.scriptToolWaiters?.has(call.id) ?? false; + const reading = (result: { content?: unknown; isError?: boolean; structuredContent?: unknown }, mcpl: boolean) => + forScript && Object.prototype.hasOwnProperty.call(result, 'structuredContent') + ? Promise.resolve(AgentFramework.directToolResult(result)) + : this.mcpToolResult(result, call.id, mcpl); + if (modern) { // Modern MCP: no state exchange, no channels, no RFC-005 references. modern.callTool(toolName, args) - .then((result) => this.mcpToolResult(result, call.id, false)) + .then((result) => reading(result, false)) .then((result) => { this.emitTrace({ type: 'tool:completed', module: `mcpl:${serverId}`, tool: toolName, callId: call.id, durationMs: Date.now() - startTime }); this.pushEvent({ type: 'tool-result', callId: call.id, agentName, moduleName: `mcpl:${serverId}`, result }); @@ -15084,7 +15155,7 @@ export class AgentFramework { await this.autoFetchReferences(result.content, serverId, true) .catch((e) => console.error('[rfc005] autofetch error:', (e as Error).message)); } - return { result, toolResult: await this.mcpToolResult(result, call.id, mcplPeer) }; + return { result, toolResult: await reading(result, mcplPeer) }; }) .then(({ result, toolResult }) => { const durationMs = Date.now() - startTime; @@ -15167,8 +15238,31 @@ export class AgentFramework { } /** - * The framework's ToolResult for an MCP tool result, on both dispatch - * paths (model and direct). A standard peer, meaning an MCP-only legacy + * The direct path's reading of an MCP tool result (executeToolCall, + * ModuleContext.callTool). `data` stays the raw content array, as module + * callers have always received it. Two things change: a tool error + * (`isError: true`) is now a failure, where every answer used to count as + * success, and `structured` is added by presence. + */ + private static directToolResult(result: { content?: unknown; isError?: boolean; structuredContent?: unknown }): ToolResult { + const structured = Object.prototype.hasOwnProperty.call(result, 'structuredContent') + ? { structured: result.structuredContent } + : {}; + if (result.isError) { + const text = (Array.isArray(result.content) ? result.content : []) + .filter((b): b is { type: 'text'; text: string } => !!b && (b as { type?: unknown }).type === 'text' && typeof (b as { text?: unknown }).text === 'string') + .map((b) => b.text) + .join('\n'); + // The raw content rides along on a failure too: it is the server's + // own account of the error, for a program that wants more than text. + return { success: false, data: result.content, error: text || 'Tool call failed', isError: true, ...structured }; + } + return { success: true, data: result.content, ...structured }; + } + + /** + * The framework's ToolResult for an MCP tool result on the model path + * (model dispatch, and scripts, which ride it). A standard peer, meaning an MCP-only legacy * server or any modern server, gets the standard reading * (tool-result-normalize.ts): typed content blocks, workspace-saved * payloads, and `structured` kept by presence. An MCPL peer keeps the diff --git a/src/mcpl/modern-connection.ts b/src/mcpl/modern-connection.ts index 4df032dc0..98646e724 100644 --- a/src/mcpl/modern-connection.ts +++ b/src/mcpl/modern-connection.ts @@ -349,23 +349,38 @@ export class ModernMcpConnection extends EventEmitter { } /** - * Connect, or with `reconnect` set, return a connection that keeps - * retrying in the background when the first attempt fails (the legacy - * engine's contract). Configuration errors always throw. + * A connection that hasn't started: configuration is validated (and + * throws), nothing is launched or dialed. An owner can register it, and + * wire its events, before {@link start}, so a connect in flight is + * already the owner's to close. */ - static async connect(config: McplServerConfig): Promise { - const connection = new ModernMcpConnection(config); + static create(config: McplServerConfig): ModernMcpConnection { + return new ModernMcpConnection(config); + } + + /** + * The first connect. Without `reconnect` a failure throws. With it, the + * connection keeps retrying in the background, as the legacy engine's + * stub does, and reports `connect-failed`. A connection closed before or + * during its first connect resolves unconnected. + */ + async start(): Promise { try { - await connection.open(); + await this.open(); } catch (error) { - if (!connection.reconnectEnabled) throw error; - console.error(`MCP server "${config.id}" initial connect failed, will retry:`, (error as Error).message); - connection.reconnectAttempts = 1; - // A macrotask, so the caller has wired its listeners (synchronously, - // right after this resolves) before the event fires. - setImmediate(() => connection.emit('connect-failed', { error: (error as Error).message, attempt: 0 })); - connection.scheduleReconnect(); + if (this.closedByHost) return; + if (!this.reconnectEnabled) throw error; + console.error(`MCP server "${this.id}" initial connect failed, will retry:`, (error as Error).message); + this.reconnectAttempts = 1; + this.emit('connect-failed', { error: (error as Error).message, attempt: 0 }); + this.scheduleReconnect(); } + } + + /** {@link create} and {@link start} in one step. */ + static async connect(config: McplServerConfig): Promise { + const connection = ModernMcpConnection.create(config); + await connection.start(); return connection; } @@ -375,6 +390,8 @@ export class ModernMcpConnection extends EventEmitter { /** One connect generation: a fresh client over a fresh transport. */ private async open(): Promise { + // Closed before this connect began: launch nothing. + if (this.closedByHost) return; const generation = ++this.generation; this.cancelRelisten(); const inner: Transport = this.transportKind === 'stdio' @@ -415,16 +432,16 @@ export class ModernMcpConnection extends EventEmitter { try { await done; } catch (error) { - await client.close().catch(() => {}); - await wire.close().catch(() => {}); + const failure = await ModernMcpConnection.closeSession(session); + if (failure) this.emit('error', failure); throw error; } finally { if (this.opening?.session === session) this.opening = null; } if (!this.isCurrent(generation)) { // Superseded or closed while connecting: this generation never serves. - await client.close().catch(() => {}); - await wire.close().catch(() => {}); + const failure = await ModernMcpConnection.closeSession(session); + if (failure) this.emit('error', failure); return; } this.session = session; @@ -593,28 +610,50 @@ export class ModernMcpConnection extends EventEmitter { } } + /** + * Close one generation: the transport first (during discovery the SDK + * hasn't attached it yet, so only closing it ends the probe), then the + * client. The transport's close is the reaping verdict, and its failure is + * returned, never swallowed. + */ + private static async closeSession(session: Session): Promise { + let failure: Error | null = null; + try { + await session.wire.close(); + } catch (error) { + failure = error instanceof Error ? error : new Error(String(error)); + } + await session.client.close().catch(() => { /* SDK teardown; the verdict is the transport's */ }); + return failure; + } + /** Close for good: no reconnect, no reopened subscription, no connect left - * in flight. Awaits the in-flight connect's end. */ + * in flight. Awaits the in-flight connect's end, and rejects if a child + * could not be reaped. */ async close(): Promise { this.closedByHost = true; if (this.reconnectTimer) clearTimeout(this.reconnectTimer); this.reconnectTimer = null; this.cancelRelisten(); + const failures: Error[] = []; const opening = this.opening; if (opening) { // Closing the transport ends the probe; the connect then rejects and // its own cleanup runs. Wait for that, so nothing outlives close(). - await opening.session.wire.close().catch(() => {}); - await opening.session.client.close().catch(() => {}); + const failure = await ModernMcpConnection.closeSession(opening.session); + if (failure) failures.push(failure); await opening.done.catch(() => {}); } const session = this.session; this.session = null; if (session) { - await session.client.close().catch(() => {}); - await session.wire.close().catch(() => {}); + const failure = await ModernMcpConnection.closeSession(session); + if (failure) failures.push(failure); this.emit('close', { reason: 'closed by host' }); } + if (failures.length > 0) { + throw new Error(`MCP server "${this.id}" did not close cleanly: ${failures.map((f) => f.message).join('; ')}`); + } } /** diff --git a/src/mcpl/protocol-family.ts b/src/mcpl/protocol-family.ts index 5037d1205..19cc96a60 100644 --- a/src/mcpl/protocol-family.ts +++ b/src/mcpl/protocol-family.ts @@ -98,6 +98,8 @@ export function resolveServerBinding(config: BindingFields): ServerBinding { `MCP server "${config.id}": transport "${config.transport}" does not match url "${config.url}"`, ); } + } else if (config.transport !== undefined) { + throw new Error(`MCP server "${config.id}": transport "${config.transport}" requires "url"`); } else { throw new Error(`MCP server "${config.id}": needs "command" (stdio) or "url" (WebSocket or HTTP)`); } diff --git a/src/mcpl/server-connection.ts b/src/mcpl/server-connection.ts index 66a2949c0..2e0a03559 100644 --- a/src/mcpl/server-connection.ts +++ b/src/mcpl/server-connection.ts @@ -601,10 +601,10 @@ export class McplServerConnection extends EventEmitter { ): Promise<{ transport: McplTransport; capabilities: McplCapabilities | null; droppedCapabilities: ReadonlySet; mcpToolsAdvertised: boolean; protocolVersion: string }> { // This engine speaks the legacy family only. A configuration that // resolves to the modern family belongs to ModernMcpConnection; refuse it - // here rather than dialing an http(s) url as a WebSocket. A configuration - // that doesn't resolve at all keeps openTransport's existing errors. - let family: string | null = null; - try { family = resolveServerBinding(config).family; } catch { /* openTransport reports it */ } + // here rather than dialing an http(s) url as a WebSocket. One that names + // no usable transport is an error too, before anything is spawned or + // dialed. + const family = resolveServerBinding(config).family; if (family === 'modern') { throw new Error( `MCP server "${config.id}" is configured for modern MCP (${MODERN_MCP_PROTOCOL_VERSION}); ` + diff --git a/src/mcpl/tool-result-normalize.ts b/src/mcpl/tool-result-normalize.ts index 4b82ed85a..2f229c4e5 100644 --- a/src/mcpl/tool-result-normalize.ts +++ b/src/mcpl/tool-result-normalize.ts @@ -9,13 +9,18 @@ * and inline images stay native. Anything the provider format can't carry * (audio, binary blobs) is saved to the workspace, and a bounded stub says * where it went: a stub tells the reader where the payload is; it never - * replaces it silently. A `resource_link` is shown as a reference and never + * replaces it silently. If a payload can't be saved, the result is a + * failure to materialize it, not a success. A `resource_link` is shown as a reference and never * fetched. An embedded resource shows its text, or is saved like a blob. * - **`structuredContent` is the machine-readable view.** It is kept by - * presence on `ToolResult.structured`, for scripts and modules. The model - * sees it rendered as JSON only when the content carries no text, because + * presence on `ToolResult.structured`. The model sees it rendered as JSON + * only when the server's content has no text block of its own, because * otherwise the server already chose what the model reads. * + * This is the model's reading. Programs get the server's result itself: + * the direct path returns the raw content, and a script receives a + * structured result whole (framework.ts). + * * Before this, a structured-only result reached the model as an empty string, * and a mixed result became one JSON string with its image as base64 text * (room-284 probe P1c). @@ -63,7 +68,8 @@ export interface StandardToolResult { /** * Normalize a standard MCP tool result into the framework's `ToolResult`. * `label` names saved payloads (one per call; payloads get an index suffix). - * Without `save`, payloads are described but not saved, and the stub says so. + * A payload that can't be retained (no `save`, or saving failed) makes the + * result a failure to materialize, never a success with part of it lost. */ export async function normalizeStandardToolResult( result: StandardToolResult, @@ -73,16 +79,21 @@ export async function normalizeStandardToolResult( const raw = Array.isArray(result.content) ? result.content : []; const blocks: Block[] = []; let saved = 0; + /** Payloads that could be neither shown nor retained. */ + let lost = 0; const payloadStub = async (kind: string, base64: string, mimeType: string, origin?: string): Promise => { const size = humanSize(decodedSize(base64)); const what = `[${kind}${origin ? ` ${bounded(origin)}` : ''}: ${bounded(mimeType)}, ${size}`; - if (!save) return `${what}. Not shown: the model can't take this type, and no workspace is mounted to save it]`; + if (!save) { + lost++; + return `${what}. Not retained: the model can't take this type, and no workspace is mounted to save it]`; + } const path = await save(`${label}-${++saved}.${extensionFor(mimeType)}`, Buffer.from(base64, 'base64'), mimeType) .catch(() => null); - return path - ? `${what}, saved to workspace file ${path}]` - : `${what}. Not shown: the model can't take this type, and saving it to the workspace failed]`; + if (path) return `${what}, saved to workspace file ${path}]`; + lost++; + return `${what}. Not retained: the model can't take this type, and saving it to the workspace failed]`; }; for (const entry of raw) { @@ -131,8 +142,13 @@ export async function normalizeStandardToolResult( } const hasStructured = Object.prototype.hasOwnProperty.call(result, 'structuredContent'); - const hasText = blocks.some((b) => b.type === 'text' && b.text.trim().length > 0); - if (hasStructured && !hasText) { + // Only the server's own text blocks count as its presentation for the + // model. Stubs this module wrote are notices, not the server's text. + const serverText = raw.some((entry) => { + const block = entry as { type?: unknown; text?: unknown } | null; + return !!block && block.type === 'text' && typeof block.text === 'string' && block.text.trim().length > 0; + }); + if (hasStructured && !serverText) { blocks.push({ type: 'text', text: JSON.stringify(result.structuredContent) ?? 'null' }); } @@ -141,6 +157,17 @@ export async function normalizeStandardToolResult( if (result.isError) { return { success: false, error: text || 'Tool call failed', isError: true, ...structured }; } + if (lost > 0) { + // The server answered, but part of its answer is gone: report that as a + // failure to materialize the result, never as a success. + return { + success: false, + error: `[result incomplete: ${lost} payload(s) could not be retained, so this result is not complete; ` + + `the tool may already have completed]\n${text}`, + isError: true, + ...structured, + }; + } const hasImage = blocks.some((b) => b.type === 'image'); return { success: true, diff --git a/src/mcpl/transport.ts b/src/mcpl/transport.ts index d344e8949..ab36316ba 100644 --- a/src/mcpl/transport.ts +++ b/src/mcpl/transport.ts @@ -222,13 +222,22 @@ export class StdioTransport extends McplTransport { this.child.stdin?.write(json + '\n'); } + private closing: Promise | null = null; + /** * Close and reap: resolves once the child has actually exited, not merely * been signalled (`child.killed` is true as soon as a signal is sent). A - * child that outlives SIGTERM by {@link STDIO_EXIT_GRACE_MS} gets SIGKILL, - * and close waits that long again, so shutdown stays bounded. + * child that outlives SIGTERM by {@link STDIO_EXIT_GRACE_MS} gets SIGKILL. + * If it still hasn't exited after that bound again, close rejects: an + * explicit cleanup failure, rather than waiting forever or claiming an + * exit that wasn't seen. Calls share one attempt. */ - async close(): Promise { + close(): Promise { + this.closing ??= this.reap(); + return this.closing; + } + + private async reap(): Promise { this.markClosed({ reason: 'closed by host' }); this.rl.close(); const child = this.child; @@ -244,7 +253,10 @@ export class StdioTransport extends McplTransport { if (await exitWithin(STDIO_EXIT_GRACE_MS)) return; console.error(`[mcpl] stdio child ${child.pid} ignored SIGTERM for ${STDIO_EXIT_GRACE_MS}ms — sending SIGKILL`); child.kill('SIGKILL'); - await exitWithin(STDIO_EXIT_GRACE_MS); + if (await exitWithin(STDIO_EXIT_GRACE_MS)) return; + throw new Error( + `stdio child ${child.pid} did not exit within ${STDIO_EXIT_GRACE_MS}ms of SIGKILL; it could not be reaped`, + ); } } diff --git a/test/fixtures/modern-raw-server.mjs b/test/fixtures/modern-raw-server.mjs index f63c1a82c..cc4a93210 100644 --- a/test/fixtures/modern-raw-server.mjs +++ b/test/fixtures/modern-raw-server.mjs @@ -7,6 +7,7 @@ // invalid-schema tools/list advertises an outputSchema that doesn't compile // listen-fail-first the first subscriptions/listen of each launch is refused // hang-discover-later launches after the first never answer server/discover +// hang-discover no launch ever answers server/discover // ignore-sigterm the process survives SIGTERM (only SIGKILL ends it) // Tools: op (outputSchema {value: integer}, answers with a string: invalid // structured content), plain (text), err (a JSON-RPC error), touch (announces @@ -40,6 +41,7 @@ createInterface({ input: process.stdin }).on('line', (line) => { switch (m.method) { case 'server/discover': if (flags.has('hang-discover-later') && launch > 1) return; + if (flags.has('hang-discover')) return; return result(m.id, { supportedVersions: ['2026-07-28'], capabilities: { tools: { listChanged: true } }, diff --git a/test/fixtures/modern-tools.mjs b/test/fixtures/modern-tools.mjs index d4e28a724..d156277a6 100644 --- a/test/fixtures/modern-tools.mjs +++ b/test/fixtures/modern-tools.mjs @@ -28,6 +28,11 @@ export function createServer(opts = {}) { async () => ({ content: [{ type: 'text', text: 'Found 2 rows.' }], structuredContent: { rows: [{ id: 1 }, { id: 2 }] } })); server.registerTool('structured_zero', { description: 'Falsy structured value', inputSchema: obj() }, async () => ({ content: [], structuredContent: 0 })); + server.registerTool('scored_image', { description: 'An image with machine-readable scores', inputSchema: obj() }, + async () => ({ + content: [{ type: 'image', data: 'iVBORw0KGgo=', mimeType: 'image/png' }], + structuredContent: { score: 0.9, label: 'cat' }, + })); server.registerTool('media', { description: 'Mixed standard content', inputSchema: obj() }, async () => ({ content: [ diff --git a/test/mcp-modern-framework.test.ts b/test/mcp-modern-framework.test.ts index c6b25e5dc..8636e9eb7 100644 --- a/test/mcp-modern-framework.test.ts +++ b/test/mcp-modern-framework.test.ts @@ -95,29 +95,47 @@ test('modern tools are listed under the prefix, and status names family, revisio }); }); -test('direct path: structured kept by presence, tool errors fail, media is described', async () => { +test('direct path: data stays the raw content; tool errors fail; structured by presence', async () => { await withModern(async ({ framework }) => { const only = await framework.executeToolCall(call('mcpl--modern--structured_only')); assert.equal(only.success, true); + assert.deepEqual(only.data, [], 'the raw content array, as module callers have always received it'); assert.deepEqual(only.structured, { answer: 42, ok: false, nothing: null }); - assert.equal(only.data, '{"answer":42,"ok":false,"nothing":null}'); const zero = await framework.executeToolCall(call('mcpl--modern--structured_zero')); assert.ok('structured' in zero); assert.equal(zero.structured, 0); + const echo = await framework.executeToolCall(call('mcpl--modern--echo', { text: 'x' })); + assert.deepEqual(echo, { success: true, data: [{ type: 'text', text: 'echo:x' }] }); + const fail = await framework.executeToolCall(call('mcpl--modern--fail')); assert.equal(fail.success, false); assert.equal(fail.isError, true); assert.equal(fail.error, 'the tool failed on purpose'); const media = await framework.executeToolCall(call('mcpl--modern--media')); - assert.ok(Array.isArray(media.data), 'an image keeps the array'); - const text = (media.data as Array<{ type: string; text?: string }>).filter((b) => b.type === 'text').map((b) => b.text).join('\n'); - assert.match(text, /\[resource link "report\.pdf": file:\/\/\/srv\/report\.pdf, application\/pdf; not fetched\]/); + assert.deepEqual((media.data as Array<{ type: string }>).map((b) => b.type), + ['text', 'resource_link', 'resource', 'resource', 'audio', 'image'], 'every block as sent'); + }); +}); + +test('model path: a payload the host cannot retain is reported as an incomplete result', async () => { + await withModern(async ({ framework, membrane }) => { + framework.start(); + membrane.pushResponse(createMockResponse([ + { type: 'tool_use', id: 'toolu_01MODERNMEDIA00000000000', name: 'mcpl--modern--media', input: {} }, + ] as never, 'tool_use')); + membrane.pushResponse(createMockResponse([{ type: 'text', text: 'Done.' }])); + framework.pushEvent({ type: 'external-message', source: 'test', content: 'go', metadata: {} } as never); + await waitFor('the tool round', () => (membrane.lastStream?.receivedToolResults.length ?? 0) >= 1); + const [delivered] = membrane.lastStream!.receivedToolResults[0] as Array<{ content: unknown; isError: boolean }>; + // No workspace here: audio and the blob can be neither shown nor kept. + assert.equal(delivered!.isError, true); + const text = JSON.stringify(delivered!.content); + assert.match(text, /result incomplete: 2 payload\(s\) could not be retained/); + assert.match(text, /resource link \\"report\.pdf\\": file:\/\/\/srv\/report\.pdf, application\/pdf; not fetched/); assert.match(text, /embedded note text/); - assert.match(text, /\[audio: audio\/wav, \d+ B\. Not shown/); - assert.ok((media.data as Array<{ type: string }>).some((b) => b.type === 'image')); }); }); @@ -144,9 +162,21 @@ test('scripts get structured results whole, as one JSON object', async () => { handleScriptToolCall: (agent: string, tool: string, args: Record) => Promise; }).handleScriptToolCall.bind(framework); const out = JSON.parse(await handle('agent', 'mcpl--modern--structured_and_text', {})); - assert.deepEqual(out, { content: 'Found 2 rows.', structuredContent: { rows: [{ id: 1 }, { id: 2 }] }, isError: false }); - // Without structured content the script contract is unchanged: the - // history rendering, which JSON-encodes a text result. + assert.deepEqual(out, { + content: [{ type: 'text', text: 'Found 2 rows.' }], + structuredContent: { rows: [{ id: 1 }, { id: 2 }] }, + isError: false, + }); + // Image plus scores: both views reach the script whole, the image's + // payload included (no placeholder stands in for it). + const scored = JSON.parse(await handle('agent', 'mcpl--modern--scored_image', {})); + assert.deepEqual(scored, { + content: [{ type: 'image', data: 'iVBORw0KGgo=', mimeType: 'image/png' }], + structuredContent: { score: 0.9, label: 'cat' }, + isError: false, + }); + // Without structured content the script contract is unchanged (pinned): + // the history rendering, which JSON-encodes a text result. assert.equal(await handle('agent', 'mcpl--modern--echo', { text: 'x' }), JSON.stringify('echo:x')); }); }); @@ -161,7 +191,21 @@ test('MCPL policy on a modern server is a configuration error at connect', async framework.connectMcplServer({ id: 'bad2', url: 'https://example.invalid/mcp', protocol: 'modern' }), /applies only to stdio/, ); - assert.ok(!framework.listMcplServers().some((s) => s.id === 'bad' || s.id === 'bad2'), 'nothing registered'); + // Selector combinations that name no usable transport are errors too, + // not a fall-through to some other opener. + await assert.rejects( + framework.connectMcplServer({ id: 'bad3', url: 'https://example.invalid/mcp', transport: 'websocket' }), + /does not match url/, + ); + await assert.rejects( + framework.connectMcplServer({ id: 'bad4', url: 'ws://127.0.0.1:1/mcpl', transport: 'http' }), + /does not match url/, + ); + await assert.rejects( + framework.connectMcplServer({ id: 'bad5', command: process.execPath, transport: 'http' }), + /transport "http" requires "url"/, + ); + assert.ok(!framework.listMcplServers().some((s) => s.id.startsWith('bad')), 'nothing registered'); }); }); @@ -188,11 +232,19 @@ process.stdin.on('data', (c) => { } }); `; - await withModern(async ({ framework }) => { - const result = await framework.executeToolCall(call('mcpl--plain--docs')); - assert.equal(result.success, true); - assert.deepEqual(result.structured, { count: 2 }); - assert.equal(result.data, '[resource link "a.md": https://example.invalid/a.md; not fetched]\n[resource memo://n, text/plain]\nnote body'); + await withModern(async ({ framework, membrane }) => { + framework.start(); + membrane.pushResponse(createMockResponse([ + { type: 'tool_use', id: 'toolu_01LEGACYDOCS000000000000', name: 'mcpl--plain--docs', input: {} }, + ] as never, 'tool_use')); + membrane.pushResponse(createMockResponse([{ type: 'text', text: 'Done.' }])); + framework.pushEvent({ type: 'external-message', source: 'test', content: 'go', metadata: {} } as never); + await waitFor('the tool round', () => (membrane.lastStream?.receivedToolResults.length ?? 0) >= 1); + const [delivered] = membrane.lastStream!.receivedToolResults[0] as Array<{ content: string; isError: boolean }>; + assert.equal(delivered!.isError, false); + // No server text block: the structured value is shown beside the stubs. + assert.equal(JSON.parse(delivered!.content), + '[resource link "a.md": https://example.invalid/a.md; not fetched]\n[resource memo://n, text/plain]\nnote body\n{"count":2}'); const status = framework.listMcplServers().find((s) => s.id === 'plain')!; assert.equal(status.family, 'legacy'); assert.equal(status.protocolVersion, '2024-11-05'); @@ -252,7 +304,33 @@ test('quiesce: a lost modern child reconnects and relists without starting a tur assert.equal(membrane.calls.length, callsBefore, 'no turn started'); assert.equal(framework.getHostModeStatus().gatedRequests, gatedBefore, 'no wake parked: the inventory is unchanged'); - assert.deepEqual((await framework.executeToolCall(call('mcpl--modern--echo', { text: 'back' }))).data, 'echo:back'); + assert.deepEqual((await framework.executeToolCall(call('mcpl--modern--echo', { text: 'back' }))).data, [{ type: 'text', text: 'echo:back' }]); await framework.resume(); }); }); + +test('the framework owns a modern connect in flight: stop() ends it, and a second connect is refused', async () => { + const RAW = join(here, 'fixtures', 'modern-raw-server.mjs'); + const dir = mkdtempSync(join(tmpdir(), 'modern-framework-')); + const framework = await AgentFramework.create({ + storePath: join(dir, 'store'), + membrane: new MockMembrane().asMembrane(), + agents: [{ name: 'agent', model: 'test-model', systemPrompt: 'test' }], + modules: [], + }); + try { + const log = join(dir, 'hang.jsonl'); + const config = { id: 'hang', command: process.execPath, args: [RAW, 'hang-discover', log], protocol: 'modern' as const }; + const pending = framework.connectMcplServer(config); + await waitFor('the launch waiting on discover', () => lines(log).some((l) => l.includes('server/discover'))); + await assert.rejects(framework.connectMcplServer(config), /already registered/); + const pid = (JSON.parse(lines(log)[0]!) as { pid: number }).pid; + await framework.stop(); + const alive = (p: number) => { try { process.kill(p, 0); return true; } catch { return false; } }; + assert.equal(alive(pid), false, 'the launch in flight was reaped by stop()'); + await pending.catch(() => {}); + assert.ok(!framework.listMcplServers().some((s) => s.id === 'hang' && s.connected), 'nothing installed after stop'); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); diff --git a/test/mcp-modern-outcomes.test.ts b/test/mcp-modern-outcomes.test.ts index 0f5f45ab0..9877fa63c 100644 --- a/test/mcp-modern-outcomes.test.ts +++ b/test/mcp-modern-outcomes.test.ts @@ -18,6 +18,7 @@ import { join, dirname } from 'node:path'; import { fileURLToPath } from 'node:url'; import { ModernMcpConnection } from '../src/mcpl/modern-connection.js'; +import { StdioTransport } from '../src/mcpl/transport.js'; import { McplRequestError } from '../src/mcpl/server-connection.js'; import type { McplServerConfig } from '../src/mcpl/types.js'; @@ -207,3 +208,14 @@ test('close() reaps: a child that ignores SIGTERM is killed before close resolve assert.equal(alive(pid), false, 'dead when close() resolved'); assert.ok(wire(log).some((e) => e.event === 'sigterm-ignored'), 'it really did ignore SIGTERM'); }); + +test('a child that cannot be reaped fails close() explicitly, never a claimed exit', async () => { + const transport = StdioTransport.spawn({ id: 'stuck', command: process.execPath, args: ['-e', 'setInterval(() => {}, 1 << 30)'] }); + const child = (transport as unknown as { child: { pid: number; kill: (signal?: string) => boolean } }).child; + const realKill = child.kill.bind(child); + child.kill = () => true; // signals never arrive: the child outlives both bounds + cleanups.push(() => { realKill('SIGKILL'); }); + await assert.rejects(transport.close(), /did not exit within 2000ms of SIGKILL; it could not be reaped/); + // Calls share the one attempt and its verdict. + await assert.rejects(transport.close(), /could not be reaped/); +}); diff --git a/test/mcp-protocol-family.test.ts b/test/mcp-protocol-family.test.ts index cd53aba66..5b104335c 100644 --- a/test/mcp-protocol-family.test.ts +++ b/test/mcp-protocol-family.test.ts @@ -44,6 +44,7 @@ test('a configuration that names no usable transport throws', () => { assert.throws(() => resolveServerBinding(cfg({ url: 'ws://x', transport: 'http' })), /does not match url/); assert.throws(() => resolveServerBinding(cfg({ url: 'https://x', transport: 'websocket' })), /does not match url/); assert.throws(() => resolveServerBinding(cfg({ url: 'ws://x', transport: 'stdio' })), /requires "command"/); + assert.throws(() => resolveServerBinding(cfg({ command: 'srv', transport: 'http' })), /transport "http" requires "url"/); }); test('every legacy configuration shape passes validation unchanged', () => { diff --git a/test/mcp-tool-result-normalize.test.ts b/test/mcp-tool-result-normalize.test.ts index a5457e9f8..955669120 100644 --- a/test/mcp-tool-result-normalize.test.ts +++ b/test/mcp-tool-result-normalize.test.ts @@ -77,9 +77,31 @@ test('audio and binary resources are saved to the workspace and described', asyn '[resource memo://bin: application/octet-stream, 7 B, saved to workspace file home/tool-results/2026-10-08-call1-2.bin]'); }); -test('without a workspace the stub says the payload was not shown, and why', async () => { - const r = await normalizeStandardToolResult({ content: [{ type: 'audio', data: b64('x'), mimeType: 'audio/mpeg' }] }, 'L', null); - assert.equal(r.data, "[audio: audio/mpeg, 1 B. Not shown: the model can't take this type, and no workspace is mounted to save it]"); +test('a payload that cannot be retained makes the result a failure to materialize, not a success', async () => { + const r = await normalizeStandardToolResult({ + content: [{ type: 'text', text: 'here is the clip' }, { type: 'audio', data: b64('x'), mimeType: 'audio/mpeg' }], + }, 'L', null); + assert.equal(r.success, false); + assert.equal(r.isError, true); + assert.equal(r.error, + '[result incomplete: 1 payload(s) could not be retained, so this result is not complete; the tool may already have completed]\n' + + 'here is the clip\n' + + "[audio: audio/mpeg, 1 B. Not retained: the model can't take this type, and no workspace is mounted to save it]"); + const failedSave = await normalizeStandardToolResult( + { content: [{ type: 'resource', resource: { uri: 'memo://b', blob: b64('zz') } }] }, + 'L', + async () => null, + ); + assert.equal(failedSave.success, false); + assert.match(failedSave.error!, /saving it to the workspace failed/); +}); + +test('generated stubs are not the server\'s text: structured still shows beside them', async () => { + const r = await normalizeStandardToolResult({ + content: [{ type: 'resource_link', uri: 'file:///r.csv', name: 'r.csv' }], + structuredContent: { rows: 3 }, + }, 'L', null); + assert.equal(r.data, '[resource link "r.csv": file:///r.csv; not fetched]\n{"rows":3}'); }); test('a resource link is shown as a reference and never fetched; embedded text is shown', async () => { From 477d993b65388a78b07dbd79f9a011f3110c3a7d Mon Sep 17 00:00:00 2001 From: Olive-1804 Date: Thu, 8 Oct 2026 03:04:59 -0400 Subject: [PATCH 0063/1112] fix(mcp): an unreapable failed launch halts reconnecting instead of orphaning it Theo-1805's lifetime point on b32cd73 (room-284 #53928). A failed connect's cleanup could log a reap failure and let the reconnect loop start another generation, losing ownership of a child that might still be running. That is the same no-orphan contract close() keeps. - ModernMcpConnection keeps every generation whose cleanup couldn't confirm its child exited (`unreaped`), and emits an 'error' that says so. - While any is held, willReconnect is false. No replacement launch starts: the failure is reported as reconnect-failed / connect-failed with `permanent: true`, and the message names the unreaped launch. - close() retries each held generation and reports whatever still fails. - StdioTransport.close() re-checks a failed verdict on a later call: a child that has exited since counts as reaped. No new signals are sent. - The framework's modern failure handler raises the ops alert at once on `permanent`, since no later attempt will come. Test (mcp-modern-outcomes): the first launch serves. Later launches answer server/discover with an error and have their signals suppressed, so the cleanup after the failed reconnect can't reap. The result is a permanent reconnect-failed naming the halt, willReconnect false, and no third launch. Once the stuck child is gone, close() settles cleanly. Receipt (VPS via on-vps, node v24.21.0, npm 11.19.0; sources b32cd73 plus these changes, sha256 942e8b902834ecf2): the seven modern/legacy test files 64/64. --- src/framework.ts | 6 ++-- src/mcpl/modern-connection.ts | 44 +++++++++++++++++++++++++++-- src/mcpl/transport.ts | 15 ++++++++-- test/fixtures/modern-raw-server.mjs | 2 ++ test/mcp-modern-outcomes.test.ts | 37 ++++++++++++++++++++++++ 5 files changed, 96 insertions(+), 8 deletions(-) diff --git a/src/framework.ts b/src/framework.ts index 871718150..d544acefe 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -14746,7 +14746,7 @@ export class AgentFramework { this.emitTrace({ type: 'module:added', moduleName: `mcpl:${connection.id}` }); this.handleToolsListChanged(connection.id); }); - const failed = (params: { error: string; attempt: number }) => { + const failed = (params: { error: string; attempt: number; permanent?: boolean }) => { this.emitTrace({ type: 'mcpl:server-connect-failed', serverId: connection.id, @@ -14754,7 +14754,9 @@ export class AgentFramework { attempt: params.attempt, willRetry: connection.willReconnect, }); - if (params.attempt >= 5) { + // `permanent`: reconnecting halted on a launch that couldn't be reaped. + // No later attempt will come, so alert now. + if (params.permanent || params.attempt >= 5) { this.opsAlert('mcpl-down', connection.id, `MCP server unreachable (attempt ${params.attempt}): ${params.error}`); } }; diff --git a/src/mcpl/modern-connection.ts b/src/mcpl/modern-connection.ts index 98646e724..485a3ee70 100644 --- a/src/mcpl/modern-connection.ts +++ b/src/mcpl/modern-connection.ts @@ -295,6 +295,12 @@ export class ModernMcpConnection extends EventEmitter { /** At most one pending reopen, and only ever for the current generation. */ private relisten: { generation: number; timer: ReturnType } | null = null; private relistenAttempts = 0; + /** + * Generations whose cleanup could not confirm their child exited. While any + * is held, no replacement is started: launching another would orphan the + * first. close() retries them and reports what still fails. + */ + private unreaped: Array<{ session: Session; failure: Error }> = []; /** Child stderr from before anyone listens (the caller attaches only once * connect() returns), delivered to the first listener. Bounded: the * newest lines are kept. */ @@ -345,7 +351,22 @@ export class ModernMcpConnection extends EventEmitter { /** Whether a lost connection will be re-established in the background. */ get willReconnect(): boolean { - return this.reconnectEnabled && !this.closedByHost; + return this.reconnectEnabled && !this.closedByHost && this.unreaped.length === 0; + } + + /** A failed generation's cleanup couldn't reap its child: keep owning it, + * and halt reconnecting until close() settles it. */ + private holdUnreaped(session: Session, failure: Error): void { + this.unreaped.push({ session, failure }); + this.emit('error', new Error( + `MCP server "${this.id}": ${failure.message}. Not starting another launch while it may still be running; close the connection to retry the reap.`, + )); + } + + /** The halt, as a permanent failure for the reconnect events. */ + private unreapedFailure(): string | null { + const held = this.unreaped[0]; + return held ? `the previous launch could not be reaped (${held.failure.message}); reconnecting is halted` : null; } /** @@ -370,6 +391,11 @@ export class ModernMcpConnection extends EventEmitter { } catch (error) { if (this.closedByHost) return; if (!this.reconnectEnabled) throw error; + const halted = this.unreapedFailure(); + if (halted) { + this.emit('connect-failed', { error: `${(error as Error).message}; ${halted}`, attempt: 0, permanent: true }); + return; + } console.error(`MCP server "${this.id}" initial connect failed, will retry:`, (error as Error).message); this.reconnectAttempts = 1; this.emit('connect-failed', { error: (error as Error).message, attempt: 0 }); @@ -433,7 +459,7 @@ export class ModernMcpConnection extends EventEmitter { await done; } catch (error) { const failure = await ModernMcpConnection.closeSession(session); - if (failure) this.emit('error', failure); + if (failure) this.holdUnreaped(session, failure); throw error; } finally { if (this.opening?.session === session) this.opening = null; @@ -441,7 +467,7 @@ export class ModernMcpConnection extends EventEmitter { if (!this.isCurrent(generation)) { // Superseded or closed while connecting: this generation never serves. const failure = await ModernMcpConnection.closeSession(session); - if (failure) this.emit('error', failure); + if (failure) this.holdUnreaped(session, failure); return; } this.session = session; @@ -544,6 +570,11 @@ export class ModernMcpConnection extends EventEmitter { } catch (error) { if (this.closedByHost) return; this.reconnectAttempts = attempt + 1; + const halted = this.unreapedFailure(); + if (halted) { + this.emit('reconnect-failed', { error: `${(error as Error).message}; ${halted}`, attempt, permanent: true }); + return; + } this.emit('reconnect-failed', { error: (error as Error).message, attempt }); this.scheduleReconnect(); } @@ -651,6 +682,13 @@ export class ModernMcpConnection extends EventEmitter { if (failure) failures.push(failure); this.emit('close', { reason: 'closed by host' }); } + // Generations a failed cleanup couldn't reap: try again, report the rest. + const held = this.unreaped; + this.unreaped = []; + for (const { session: stuck } of held) { + const failure = await ModernMcpConnection.closeSession(stuck); + if (failure) failures.push(failure); + } if (failures.length > 0) { throw new Error(`MCP server "${this.id}" did not close cleanly: ${failures.map((f) => f.message).join('; ')}`); } diff --git a/src/mcpl/transport.ts b/src/mcpl/transport.ts index ab36316ba..9d2540c03 100644 --- a/src/mcpl/transport.ts +++ b/src/mcpl/transport.ts @@ -230,11 +230,20 @@ export class StdioTransport extends McplTransport { * child that outlives SIGTERM by {@link STDIO_EXIT_GRACE_MS} gets SIGKILL. * If it still hasn't exited after that bound again, close rejects: an * explicit cleanup failure, rather than waiting forever or claiming an - * exit that wasn't seen. Calls share one attempt. + * exit that wasn't seen. Calls share one attempt; a later call succeeds if + * the child has exited since. */ close(): Promise { - this.closing ??= this.reap(); - return this.closing; + if (!this.closing) { + this.closing = this.reap(); + return this.closing; + } + // A later call re-checks a failed verdict: a child that has exited since + // is reaped now. No new signals are sent. + return this.closing.catch((error: unknown) => { + if (this.child.exitCode !== null || this.child.signalCode !== null) return; + throw error; + }); } private async reap(): Promise { diff --git a/test/fixtures/modern-raw-server.mjs b/test/fixtures/modern-raw-server.mjs index cc4a93210..ac5838b2f 100644 --- a/test/fixtures/modern-raw-server.mjs +++ b/test/fixtures/modern-raw-server.mjs @@ -8,6 +8,7 @@ // listen-fail-first the first subscriptions/listen of each launch is refused // hang-discover-later launches after the first never answer server/discover // hang-discover no launch ever answers server/discover +// reject-discover-later launches after the first answer server/discover with an error // ignore-sigterm the process survives SIGTERM (only SIGKILL ends it) // Tools: op (outputSchema {value: integer}, answers with a string: invalid // structured content), plain (text), err (a JSON-RPC error), touch (announces @@ -42,6 +43,7 @@ createInterface({ input: process.stdin }).on('line', (line) => { case 'server/discover': if (flags.has('hang-discover-later') && launch > 1) return; if (flags.has('hang-discover')) return; + if (flags.has('reject-discover-later') && launch > 1) return error(m.id, -32603, 'not today'); return result(m.id, { supportedVersions: ['2026-07-28'], capabilities: { tools: { listChanged: true } }, diff --git a/test/mcp-modern-outcomes.test.ts b/test/mcp-modern-outcomes.test.ts index 9877fa63c..3f2d7805b 100644 --- a/test/mcp-modern-outcomes.test.ts +++ b/test/mcp-modern-outcomes.test.ts @@ -219,3 +219,40 @@ test('a child that cannot be reaped fails close() explicitly, never a claimed ex // Calls share the one attempt and its verdict. await assert.rejects(transport.close(), /could not be reaped/); }); + +test('a failed launch that cannot be reaped halts reconnecting; close() settles it', async () => { + const log = scratchLog(); + const realSpawn = StdioTransport.spawn; + const stuck: Array<{ pid: number; kill: (signal?: string) => boolean }> = []; + let launches = 0; + (StdioTransport as unknown as { spawn: typeof StdioTransport.spawn }).spawn = (config) => { + const transport = realSpawn.call(StdioTransport, config); + if (++launches > 1) { + // From the second launch on, signals never arrive. + const child = (transport as unknown as { child: { pid: number; kill: (signal?: string) => boolean } }).child; + const realKill = child.kill.bind(child); + child.kill = () => true; + stuck.push({ pid: child.pid, kill: realKill }); + } + return transport; + }; + cleanups.push(() => { + (StdioTransport as unknown as { spawn: typeof StdioTransport.spawn }).spawn = realSpawn; + for (const s of stuck) s.kill('SIGKILL'); + }); + const connection = await connect('reject-discover-later', log, { reconnect: true, reconnectIntervalMs: 30, reconnectMaxIntervalMs: 60 }); + const failures: Array<{ error: string; permanent?: boolean }> = []; + connection.on('reconnect-failed', (f) => failures.push(f)); + connection.on('error', () => {}); + await connection.callTool('die', {}); + await until(() => failures.some((f) => f.permanent), 'a permanent reconnect failure', 9000); + const halted = failures.find((f) => f.permanent)!; + assert.match(halted.error, /the previous launch could not be reaped .*reconnecting is halted/); + assert.equal(connection.willReconnect, false); + await new Promise((r) => setTimeout(r, 300)); + assert.equal(wire(log).filter((e) => e.event === 'start').length, 2, 'no third launch while the second is unreaped'); + // The stuck child goes away; close() re-checks it and settles cleanly. + stuck[0]!.kill('SIGKILL'); + await until(() => { try { process.kill(stuck[0]!.pid, 0); return false; } catch { return true; } }, 'the stuck child gone'); + await connection.close(); +}); From 8b74d2412f8c0e357e21d8f13aeed47992827cbb Mon Sep 17 00:00:00 2001 From: Olive-1804 Date: Thu, 8 Oct 2026 03:16:19 -0400 Subject: [PATCH 0064/1112] fix(mcp): concurrent ModernMcpConnection.close() calls share the teardown Theo-1805 (room-284 #54217): the owning layer wasn't memoized. The first close() cleared `session` before awaiting teardown, so a second concurrent close() could resolve while the teardown was still pending or about to fail. Calls to close() now queue behind the one in flight. Each step closes every generation the connection still owns: a connect in flight, the live one, and any a failed cleanup couldn't reap. A step that can't reap keeps those generations held, so a later close() re-checks them, and StdioTransport's re-check counts a child that has exited since as reaped. The held list is deduplicated: open()'s own cleanup and close() can report the same generation. Test: with a child whose signals are suppressed, two concurrent close() calls both stay unsettled at 1 s and then fail in order with the one verdict. Once the child is gone, a third close() settles cleanly. Receipt (VPS via on-vps, node v24.21.0, without --test-force-exit; sources 477d993 plus this change, sha256 a149e67aa046d80b): the seven modern/legacy files 65/65. --- src/mcpl/modern-connection.ts | 58 ++++++++++++++++++++------------ test/mcp-modern-outcomes.test.ts | 30 +++++++++++++++++ 2 files changed, 66 insertions(+), 22 deletions(-) diff --git a/src/mcpl/modern-connection.ts b/src/mcpl/modern-connection.ts index 485a3ee70..27cde4506 100644 --- a/src/mcpl/modern-connection.ts +++ b/src/mcpl/modern-connection.ts @@ -357,6 +357,7 @@ export class ModernMcpConnection extends EventEmitter { /** A failed generation's cleanup couldn't reap its child: keep owning it, * and halt reconnecting until close() settles it. */ private holdUnreaped(session: Session, failure: Error): void { + if (this.unreaped.some((held) => held.session === session)) return; this.unreaped.push({ session, failure }); this.emit('error', new Error( `MCP server "${this.id}": ${failure.message}. Not starting another launch while it may still be running; close the connection to retry the reap.`, @@ -658,37 +659,50 @@ export class ModernMcpConnection extends EventEmitter { return failure; } - /** Close for good: no reconnect, no reopened subscription, no connect left - * in flight. Awaits the in-flight connect's end, and rejects if a child - * could not be reaped. */ - async close(): Promise { + /** The close in flight, which later calls wait behind. */ + private closing: Promise | null = null; + + /** + * Close for good: no reconnect, no reopened subscription, no connect left + * in flight. Awaits the in-flight connect's end, and rejects if a child + * could not be reaped. Calls queue behind the one in flight, so none + * resolves while teardown is still pending. A call after a failed close + * re-checks what wasn't reaped. + */ + close(): Promise { + const previous = this.closing; + const next = (previous ? previous.catch(() => {}) : Promise.resolve()).then(() => this.closeStep()); + this.closing = next; + return next; + } + + private async closeStep(): Promise { this.closedByHost = true; if (this.reconnectTimer) clearTimeout(this.reconnectTimer); this.reconnectTimer = null; this.cancelRelisten(); - const failures: Error[] = []; const opening = this.opening; - if (opening) { - // Closing the transport ends the probe; the connect then rejects and - // its own cleanup runs. Wait for that, so nothing outlives close(). - const failure = await ModernMcpConnection.closeSession(opening.session); - if (failure) failures.push(failure); - await opening.done.catch(() => {}); - } const session = this.session; this.session = null; - if (session) { - const failure = await ModernMcpConnection.closeSession(session); - if (failure) failures.push(failure); - this.emit('close', { reason: 'closed by host' }); - } - // Generations a failed cleanup couldn't reap: try again, report the rest. - const held = this.unreaped; + // Every generation this connection still owns: a connect in flight, the + // live one, and any a failed cleanup couldn't reap. + const targets: Session[] = []; + if (opening) targets.push(opening.session); + if (session) targets.push(session); + for (const held of this.unreaped) if (!targets.includes(held.session)) targets.push(held.session); this.unreaped = []; - for (const { session: stuck } of held) { - const failure = await ModernMcpConnection.closeSession(stuck); - if (failure) failures.push(failure); + const failures: Error[] = []; + for (const target of targets) { + // Closing the transport ends a probe in flight too: its connect then + // rejects and its own cleanup runs (sharing this verdict). + const failure = await ModernMcpConnection.closeSession(target); + if (failure) { + failures.push(failure); + if (!this.unreaped.some((held) => held.session === target)) this.unreaped.push({ session: target, failure }); + } } + if (opening) await opening.done.catch(() => {}); + if (session) this.emit('close', { reason: 'closed by host' }); if (failures.length > 0) { throw new Error(`MCP server "${this.id}" did not close cleanly: ${failures.map((f) => f.message).join('; ')}`); } diff --git a/test/mcp-modern-outcomes.test.ts b/test/mcp-modern-outcomes.test.ts index 3f2d7805b..b91f31680 100644 --- a/test/mcp-modern-outcomes.test.ts +++ b/test/mcp-modern-outcomes.test.ts @@ -256,3 +256,33 @@ test('a failed launch that cannot be reaped halts reconnecting; close() settles await until(() => { try { process.kill(stuck[0]!.pid, 0); return false; } catch { return true; } }, 'the stuck child gone'); await connection.close(); }); + +test('concurrent close() calls share one teardown and its verdict; a later close re-checks', async () => { + const log = scratchLog(); + const realSpawn = StdioTransport.spawn; + let stuck: { pid: number; kill: (signal?: string) => boolean } | null = null; + (StdioTransport as unknown as { spawn: typeof StdioTransport.spawn }).spawn = (config) => { + const transport = realSpawn.call(StdioTransport, config); + const child = (transport as unknown as { child: { pid: number; kill: (signal?: string) => boolean } }).child; + const realKill = child.kill.bind(child); + child.kill = () => true; // teardown can't reap it + stuck = { pid: child.pid, kill: realKill }; + return transport; + }; + cleanups.push(() => { + (StdioTransport as unknown as { spawn: typeof StdioTransport.spawn }).spawn = realSpawn; + stuck?.kill('SIGKILL'); + }); + const connection = await connect('', log); + const settled: string[] = []; + const first = connection.close().then(() => settled.push('first:ok'), () => settled.push('first:failed')); + const second = connection.close().then(() => settled.push('second:ok'), () => settled.push('second:failed')); + await new Promise((r) => setTimeout(r, 1000)); + assert.deepEqual(settled, [], 'neither resolves while teardown is pending'); + await Promise.all([first, second]); + assert.deepEqual(settled, ['first:failed', 'second:failed'], 'one verdict, in order'); + // Once the child is gone, a later close re-checks and settles. + stuck!.kill('SIGKILL'); + await until(() => { try { process.kill(stuck!.pid, 0); return false; } catch { return true; } }, 'the child gone'); + await connection.close(); +}); From 3aeea4c9b47a3e276cf5a8a6ba5b2fedf895bb66 Mon Sep 17 00:00:00 2001 From: Olive-1804 Date: Thu, 8 Oct 2026 03:25:57 -0400 Subject: [PATCH 0065/1112] fix(mcp): a stopping or stopped framework admits no new server connection Theo-1805 (room-284 #54276): after `await framework.stop()`, a call to connectMcplServer() still reached StdioTransport.spawn. Registering before start() makes a connect the framework has already admitted its own, but it did nothing to close admission to new connects arriving during or after stop(). stop() now sets mcpServerAdmissionClosed as its first act, and never resets it. connectMcplServer() and connectMcplServerInternal() refuse while it is set, before anything initializes, spawns or dials: "was not connected: the framework is stopping or stopped". That covers both engines, restartMcplServer's reconnect, and a call that arrives after teardown has collected the modern map. A legacy connect whose handshake was already running when stop() began is removed and refused once the handshake returns: the registry teardown had already collected it, so it would otherwise outlive stop(). This was the same lifetime gap on the legacy side, predating this work. Test (mcp-modern-framework, "the framework owns a modern connect in flight"): after stop(), a new connect is refused, with no launch in the fixture's log. Receipt (VPS via on-vps, node v24.21.0, without --test-force-exit; sources 8b74d24 plus this change, sha256 867b325e5920ead5): mcp-modern-framework, mcp-modern-outcomes and mcpl-runtime-lifecycle, 31/31. --- src/framework.ts | 18 ++++++++++++++++++ test/mcp-modern-framework.test.ts | 8 ++++++++ 2 files changed, 26 insertions(+) diff --git a/src/framework.ts b/src/framework.ts index d544acefe..3a899ff24 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -1012,6 +1012,10 @@ export class AgentFramework { private providerAccelerationJitterMs = PROVIDER_ACCELERATION_JITTER_MS; private providerHoldHook: ProviderHoldHook | undefined; private providerAdmissionClosed = false; + /** Set when stop() begins, and never reset: a stopping or stopped + * framework admits no new MCP/MCPL server connection, so nothing is + * spawned or dialed after teardown has collected what it closes. */ + private mcpServerAdmissionClosed = false; /** Last time we reported stale (busy-requeued) inference requests, per agent. */ private staleWarnAt = new Map(); /** Per-agent last inference activity (epoch ms), for /healthz + doctor tooling. */ @@ -1956,6 +1960,7 @@ export class AgentFramework { * Stop the event loop. */ async stop(): Promise { + this.mcpServerAdmissionClosed = true; this.pushCoalescer?.suspend(); this.flushCoalescingSnapshot(); this.running = false; @@ -13771,6 +13776,9 @@ export class AgentFramework { config: import('./mcpl/types.js').McplServerConfig, deferAwareness = false, ): Promise { + if (this.mcpServerAdmissionClosed) { + throw new Error(`MCP server "${config.id}" was not connected: the framework is stopping or stopped`); + } if (!this.mcplServerRegistry || !this.mcplHostCapabilities) { throw new Error('MCPL subsystem is not initialized'); } @@ -13838,6 +13846,12 @@ export class AgentFramework { } const connection = await this.mcplServerRegistry.addServer(config, this.mcplHostCapabilities); + // stop() began while the handshake ran: its teardown has already + // collected the registry, so this connection would outlive it. Close it. + if (this.mcpServerAdmissionClosed) { + await this.mcplServerRegistry.removeServer(config.id).catch(() => {}); + throw new Error(`MCP server "${config.id}" was not connected: the framework stopped while it was connecting`); + } // Wire listeners before either startup staging or the runtime global gate // releases control traffic needed for registration and marker service. @@ -14212,6 +14226,10 @@ export class AgentFramework { async connectMcplServer( config: import('./mcpl/types.js').McplServerConfig, ): Promise { + // Refused before anything initializes, spawns or dials. + if (this.mcpServerAdmissionClosed) { + throw new Error(`MCP server "${config.id}" was not connected: the framework is stopping or stopped`); + } // Lazily bring up the MCPL subsystem — a framework that started with zero // configured servers can still deploy its first one at runtime. if (!this.mcplServerRegistry || !this.mcplHostCapabilities) { diff --git a/test/mcp-modern-framework.test.ts b/test/mcp-modern-framework.test.ts index 8636e9eb7..2a0ecb9aa 100644 --- a/test/mcp-modern-framework.test.ts +++ b/test/mcp-modern-framework.test.ts @@ -330,6 +330,14 @@ test('the framework owns a modern connect in flight: stop() ends it, and a secon assert.equal(alive(pid), false, 'the launch in flight was reaped by stop()'); await pending.catch(() => {}); assert.ok(!framework.listMcplServers().some((s) => s.id === 'hang' && s.connected), 'nothing installed after stop'); + // A connect that arrives after stop() is refused before anything spawns. + const before = lines(log).length; + await assert.rejects( + framework.connectMcplServer({ ...config, id: 'late' }), + /was not connected: the framework is stopping or stopped/, + ); + await new Promise((r) => setTimeout(r, 200)); + assert.equal(lines(log).length, before, 'no launch after stop'); } finally { rmSync(dir, { recursive: true, force: true }); } From ee4d424803b5bf67560bf6f3f3ca333890a6f0b6 Mon Sep 17 00:00:00 2001 From: Olive-1804 Date: Thu, 8 Oct 2026 03:32:12 -0400 Subject: [PATCH 0066/1112] fix(mcp): keep a cleanup failure in the legacy stopped-while-connecting refusal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Theo-1805 (room-284 #54366): the new legacy branch in 3aeea4c, for a connect whose handshake returned after stop() began, used `removeServer(...).catch(() => {})`. That dropped a reap failure and reported only that the framework stopped. The refusal now carries it: "…stopped while it was connecting; its cleanup also failed: ". This follows the explicit cleanup-failure rule. Legacy connection cancellation is not redesigned. Receipt (VPS via on-vps, node v24.21.0, no force-exit; sources 3aeea4c plus this change, sha256 ab943f271bfe9118): build clean; mcp-modern-framework + mcpl-runtime-lifecycle 18/18. --- src/framework.ts | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/framework.ts b/src/framework.ts index 3a899ff24..99da96447 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -13849,8 +13849,15 @@ export class AgentFramework { // stop() began while the handshake ran: its teardown has already // collected the registry, so this connection would outlive it. Close it. if (this.mcpServerAdmissionClosed) { - await this.mcplServerRegistry.removeServer(config.id).catch(() => {}); - throw new Error(`MCP server "${config.id}" was not connected: the framework stopped while it was connecting`); + // A cleanup that can't reap the child is reported with the refusal, + // not in place of it and not dropped. + let cleanup = ''; + try { + await this.mcplServerRegistry.removeServer(config.id); + } catch (error) { + cleanup = `; its cleanup also failed: ${error instanceof Error ? error.message : String(error)}`; + } + throw new Error(`MCP server "${config.id}" was not connected: the framework stopped while it was connecting${cleanup}`); } // Wire listeners before either startup staging or the runtime global gate From 91d2411da440cb6a34ca2cc9c570a87d48cfa066 Mon Sep 17 00:00:00 2001 From: Oscar-1826 Date: Thu, 8 Oct 2026 05:06:39 -0400 Subject: [PATCH 0067/1112] fix(mcp): one owner for every launch and every server id Greptile's review of #262 at ee4d424 (findings 1, 2, 3, 4, 6, 7), and two related edges from Iris-1827's independent assessment (room-293 #54849). Design agreed in room-293 #55051, #55064, #55135, #55137. The rule is the one the modern engine already kept: nothing launches while an earlier launch may still be running, and a teardown that can't confirm a child exited isn't finished. Framework (findings 1 and 2; Iris's edge on failed modern connects): - holdMcpServerId: a connect or disconnect in flight holds its server id. Together with the live connections (MCPL registry, modern map) that is what holds an id, from a connect's admission until its teardown is done. - A connect is refused while anything holds the id, in the same synchronous step as the check: a legacy handshake still running (finding 2), a connect of either family, or a disconnect still tearing down ("still being disconnected", finding 1). Fail-fast, as the existing test pins. - A disconnect takes the hold over. It starts closing a modern connection (which ends a connect in flight), then waits for the operation it displaced, so a legacy connect in flight lands whole before it is removed. Each release frees only its own claim. - A rejected close is not a finished teardown (Iris, #55064). Disconnect now removes nothing until the close verdict is clean; this reverses the earlier "fail only after the rest of the removal" order, which released the id. On failure the server stays registered, closed and listed, the error says so, a connect for the id is refused ("not connected and not reconnecting: disconnect it first"), and a retried disconnect completes once the exit is confirmed. A failed modern connect whose cleanup can't reap keeps its entry the same way, and its error carries both failures. - A modern map entry leaves only after its close settles, so stop() still sees a teardown in flight; the modern 'close' handler removes only a connection lost on its own (ModernMcpConnection.isClosed), never one the host is closing. A modern connect closed by stop() or a disconnect while connecting now rejects with "was not connected", as the legacy path does. - The legacy 'connect-failed' handler raises the ops alert on `permanent`. Legacy McplServerConnection (findings 3, 4, 6; Iris's reconnect edge): - connect() and connectWithReconnect() run checkServerConfig plus the modern-family refusal before anything launches, outside the retry path, so `protocol` on a WebSocket url is refused and a configuration error is never turned into a retry stub (finding 6). - handshake() takes an AbortSignal (checked once the transport is open, raced against the initialize answer), and no longer swallows a cleanup failure: it throws HandshakeCleanupError carrying the transport. - Launches whose close couldn't confirm the child exited are held (`unreaped`). While any is held, reconnecting is halted (`permanent`) and close() retries the reap (finding 4). connect() throws the new exported McplUnreapedLaunchError, whose `connection` owns the launch and can retry the reap; connectWithReconnect() returns that connection, halted. - Reconnect attempts are tracked and abortable. An attempt first waits for the previous transport's close (after reconnectAfterFailure the child may still be reaping), and an attempt aborted by close() closes its new transport instead of adopting it. - close() keeps its synchronous effects and also aborts a reconnect in flight. Its teardown is shared: calls queue behind the one in flight, each waits for the aborted attempt, closes the transport (memoized: the reaping verdict) and re-checks every held launch, and rejects naming what still can't be reaped (finding 3). 'close' is still emitted once, when a live transport is torn down. - McplServerRegistry keeps a failed launch's owner registered, and re-registers a connection whose removal failed, so the id stays held and a retry finds it. Modern (finding 7): start() runs once; later calls get the first promise. Tests (all on the VPS via on-vps, node v24.21.0): - mcpl-legacy-lifetime (new, 8): config refused before any dial (a real WebSocket server counts 0 dials), never retried; concurrent close() waits for the reap; failed reaps reject for every concurrent call and a later close re-checks; connect() hands over its unreaped owner; a halted stub; a halted reconnect loop; close() during a reconnect handshake ends that launch promptly; no reconnect launch while the previous child lives. - mcp-server-ownership (new, 9): connect during disconnect refused, then listed and routed; two simultaneous disconnects; a hold releases only its own claim; modern connect refused during a legacy handshake; disconnect during a legacy connect; Iris's regression for both families (cleanup fails, connect refused, child exits, retried disconnect completes, id free); failed connects whose cleanup can't reap, both families. - mcp-modern-outcomes: start() runs once. - Fixtures: legacy server modes error/hang/slow and ok-then-, IGNORE_SIGTERM; raw modern server flag reject-discover. - Mutation check: ten mutations, each reverting one of these fixes, each failed the tests aimed at it (hold ignored; close not shared; handshake swallowing; close not aborting; no wait for the previous child; registry or framework dropping a failed teardown; failed modern connect dropping its entry; start not memoized; no config check). - This tree alone: mcpl-legacy-lifetime 8/8, mcp-server-ownership 9/9, mcp-modern-outcomes 14/14, mcp-modern-framework 11/11, mcpl-legacy-version 11/11. --- .../mcp-standard-tool-results.changed.md | 26 +- changelog.d/modern-mcp-client.added.md | 4 +- src/framework.ts | 228 ++++++++--- src/index.ts | 2 +- src/mcpl/index.ts | 2 +- src/mcpl/modern-connection.ts | 19 +- src/mcpl/server-connection.ts | 362 ++++++++++++++---- src/mcpl/server-registry.ts | 41 +- test/fixtures/legacy-version-server.mjs | 13 +- test/fixtures/modern-raw-server.mjs | 2 + test/mcp-modern-outcomes.test.ts | 17 + test/mcp-server-ownership.test.ts | 272 +++++++++++++ test/mcpl-legacy-lifetime.test.ts | 229 +++++++++++ 13 files changed, 1062 insertions(+), 155 deletions(-) create mode 100644 test/mcp-server-ownership.test.ts create mode 100644 test/mcpl-legacy-lifetime.test.ts diff --git a/changelog.d/mcp-standard-tool-results.changed.md b/changelog.d/mcp-standard-tool-results.changed.md index b88e890be..724ed01f2 100644 --- a/changelog.d/mcp-standard-tool-results.changed.md +++ b/changelog.d/mcp-standard-tool-results.changed.md @@ -32,8 +32,32 @@ SIGKILL after 2 s. A child still alive 2 s after SIGKILL fails `close()` with an explicit error. Before, close returned as soon as a signal was sent. `stop()` reports such a failure and still finishes shutting - everything else down. + everything else down. Concurrent `close()` calls share one teardown, and a + later call re-checks a child that couldn't be reaped. A launch whose exit + can't be confirmed is never orphaned, and nothing launches beside it: + - `McplServerConnection.connect()` throws the new `McplUnreapedLaunchError`, + whose `connection` owns the launch; its `close()` retries the reap. + `connectWithReconnect()` returns that connection, halted + (`connect-failed` with `permanent: true`, which raises the ops alert), + and a reconnect attempt halts the loop the same way + (`reconnect-failed`, `permanent: true`). A reconnect attempt never + starts before the previous child has exited. + - `close()` during a reconnect handshake ends that launch before it + resolves. + - In the framework, a disconnect or failed connect whose cleanup can't + reap the child keeps the server registered, closed and listed as + disconnected, and its error says so. A connect for that id is refused + until a disconnect, retried once the child is gone, confirms the exit + and completes the removal. +- One owner per server id, across both families: a connect is refused while + another connect for that id is in flight (a legacy handshake included) or + a disconnect is still tearing it down, and a disconnect waits for a + connect in flight to settle before removing it, so a teardown never + removes what a later connection registered. - A server configuration naming no usable transport is now an error before anything is spawned or dialed. That covers a `transport` that doesn't match the url's scheme, `transport: 'http'` without a url, and an unrecognized scheme. Every valid configuration resolves as before. + `McplServerConnection.connect()` and `connectWithReconnect()` apply the + framework's whole check (so `protocol` on a WebSocket url is refused too), + and a configuration error is never turned into a retry stub. diff --git a/changelog.d/modern-mcp-client.added.md b/changelog.d/modern-mcp-client.added.md index 285d8bc13..ec5225fd4 100644 --- a/changelog.d/modern-mcp-client.added.md +++ b/changelog.d/modern-mcp-client.added.md @@ -23,6 +23,8 @@ - **Lifetime:** reconnects use the legacy backoff settings. A tool-list change subscription is kept open, reopened if refused or lost, and its changes go through ordinary admission, so they park under quiesce like - any wake. `close()` and `stop()` end any connect in flight. + any wake. `close()` and `stop()` end any connect in flight. A + `ModernMcpConnection` starts once: a second `start()` returns the first + call's promise. - **Status:** `listMcplServers()` entries gain `family`, `protocolVersion` and `transport`. diff --git a/src/framework.ts b/src/framework.ts index 99da96447..86de9c349 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -391,7 +391,7 @@ import { EventGate, formatShadowWarning } from './gate/event-gate.js'; import { UsageTracker, type PersistedUsageState } from './usage/usage-tracker.js'; import type { SessionUsageSnapshot, UsageUpdatedEvent } from './usage/types.js'; import type { McplServerConnection } from './mcpl/server-connection.js'; -import { McplProtocolVersionError } from './mcpl/server-connection.js'; +import { McplProtocolVersionError, McplUnreapedLaunchError } from './mcpl/server-connection.js'; import { ModernMcpConnection } from './mcpl/modern-connection.js'; import { resolveServerBinding, serverConfigProblems } from './mcpl/protocol-family.js'; import { normalizeStandardToolResult } from './mcpl/tool-result-normalize.js'; @@ -1386,6 +1386,9 @@ export class AgentFramework { * only through the tool paths, and their list changes reach agents * through ordinary admission. */ private modernMcpConnections: Map | undefined; + /** Connects and disconnects in flight, by server id (holdMcpServerId); + * created on first use, like modernMcpConnections. */ + private mcpServerOps: Map }> | undefined; /** Host capabilities advertised during the MCP handshake — stored so servers * can be connected at runtime (connectMcplServer) after initialization. */ private mcplHostCapabilities: McplHostCapabilities | null = null; @@ -13412,6 +13415,44 @@ export class AgentFramework { return (this.modernMcpConnections ??= new Map()); } + /** + * Hold a server id for one connect or disconnect, until `release()`. + * Together with the live connections (the MCPL registry and the modern + * map) this is what holds an id: from a connect's admission until its + * teardown is done, one owner at a time. So a connect is refused while + * anything holds its id, and a teardown never removes what a later + * connection registered. A disconnect takes the hold over, and waits for + * the operation it displaced (`previous`). Each release frees only its own + * hold. + */ + private holdMcpServerId(id: string, kind: 'connect' | 'disconnect'): { previous: Promise | undefined; release: () => void } { + const ops = (this.mcpServerOps ??= new Map()); + const previous = ops.get(id)?.settled; + let settle!: () => void; + const op = { kind, settled: new Promise((resolve) => { settle = resolve; }) }; + ops.set(id, op); + return { + previous, + release: () => { + if (ops.get(id) === op) ops.delete(id); + settle(); + }, + }; + } + + /** + * A connect whose failed launch couldn't be confirmed exited. Its + * connection stays registered, closed, and holds the id, so nothing + * launches beside a child that may still be running; a disconnect retries + * the reap, and completes once the exit is confirmed. + */ + private static unreapedConnectFailure(id: string, failure: Error): Error { + return new Error( + `${failure.message}. MCP server "${id}" stays registered, closed, until its exit is confirmed: disconnect it to retry the reap`, + { cause: failure }, + ); + } + /** Close every modern-MCP connection, including any connect still in * flight, so none outlives a failed startup or stop(). */ private async closeModernMcpConnections(): Promise { @@ -13798,8 +13839,34 @@ export class AgentFramework { if (problems.length > 0) { throw new Error(`MCP server "${config.id}" configuration: ${problems.join('; ')}`); } - if (this.modernServers().has(config.id) || this.mcplServerRegistry.getServer(config.id)) { - throw new Error(`MCPL server "${config.id}" is already registered`); + // One owner per id, across both engines (holdMcpServerId): a connect or + // disconnect in flight holds it, and so does a registered connection, + // including one kept because its launch couldn't be confirmed exited. + const op = this.mcpServerOps?.get(config.id); + if (op?.kind === 'disconnect') { + throw new Error(`MCP server "${config.id}" is still being disconnected; connect it once that has finished`); + } + const registered = this.modernServers().get(config.id) ?? this.mcplServerRegistry.getServer(config.id); + if (op || registered) { + const idle = !op && registered && !registered.isConnected && !registered.willReconnect; + throw new Error(`MCPL server "${config.id}" is already registered${idle ? ' (not connected and not reconnecting: disconnect it first)' : ''}`); + } + const hold = this.holdMcpServerId(config.id, 'connect'); + try { + await this.connectHeldMcplServer(config, family, deferAwareness); + } finally { + hold.release(); + } + } + + /** The part of a connect that runs holding its id. */ + private async connectHeldMcplServer( + config: import('./mcpl/types.js').McplServerConfig, + family: 'legacy' | 'modern', + deferAwareness: boolean, + ): Promise { + if (!this.mcplServerRegistry || !this.mcplHostCapabilities) { + throw new Error('MCPL subsystem is not initialized'); } // Register prefix + config for tool dispatch routing (idempotent with @@ -13815,25 +13882,38 @@ export class AgentFramework { const modern = ModernMcpConnection.create(config); this.modernServers().set(config.id, modern); this.wireModernEvents(modern); + let failure: Error | null = null; try { await modern.start(); } catch (error) { - if (this.modernServers().get(config.id) === modern) this.modernServers().delete(config.id); - // The connect error is the one to report; a cleanup failure beside - // it is reported too, never in its place. - await modern.close().catch((cleanup: unknown) => { - this.emitTrace({ - type: 'mcpl:server-error', - serverId: config.id, - error: cleanup instanceof Error ? cleanup.message : String(cleanup), - }); - }); - throw error; + failure = error instanceof Error ? error : new Error(String(error)); } - // Closed (stop, disconnect) while connecting: nothing more to set up. - if (this.modernServers().get(config.id) !== modern) return; - this.emitTrace({ type: 'module:added', moduleName: `mcpl:${config.id}` }); - return; + if (!failure && !modern.isClosed) { + this.emitTrace({ type: 'module:added', moduleName: `mcpl:${config.id}` }); + return; + } + // It failed, or stop() or a disconnect closed it while connecting: it + // never serves. The connect error is the one to report. + const reason = failure ?? new Error( + `MCP server "${config.id}" was not connected: ` + + (this.mcpServerAdmissionClosed ? 'the framework stopped while it was connecting' : 'it was disconnected while connecting'), + ); + try { + await modern.close(); + } catch (cleanup) { + // A cleanup failure is reported beside it, never in its place. The + // launch may still be running, so the connection stays registered + // and holds the id: nothing launches beside it, and a disconnect + // retries the reap. + const message = cleanup instanceof Error ? cleanup.message : String(cleanup); + this.emitTrace({ type: 'mcpl:server-error', serverId: config.id, error: message }); + if (this.modernServers().get(config.id) === modern) { + throw AgentFramework.unreapedConnectFailure(config.id, new Error(`${reason.message}; its cleanup also failed: ${message}`, { cause: reason })); + } + throw reason; + } + if (this.modernServers().get(config.id) === modern) this.modernServers().delete(config.id); + throw reason; } // Record per-server channel subscription policy before the server @@ -13845,7 +13925,15 @@ export class AgentFramework { ); } - const connection = await this.mcplServerRegistry.addServer(config, this.mcplHostCapabilities); + let connection: McplServerConnection; + try { + connection = await this.mcplServerRegistry.addServer(config, this.mcplHostCapabilities); + } catch (error) { + // The registry keeps a failed launch that may still be running, as the + // closed connection that owns it. + if (error instanceof McplUnreapedLaunchError) throw AgentFramework.unreapedConnectFailure(config.id, error); + throw error; + } // stop() began while the handshake ran: its teardown has already // collected the registry, so this connection would outlive it. Close it. if (this.mcpServerAdmissionClosed) { @@ -14270,39 +14358,61 @@ export class AgentFramework { * transport close, which preserves checkpoints for the reconnect), remove * its channels from the registry, drop routing entries, and refresh tools. * No-op-ish if the server is not connected (still clears routing state). + * + * It holds the id throughout (holdMcpServerId): a connect for it is + * refused meanwhile, and a connect already in flight settles first (a + * modern one is closed, which ends it). A close that can't confirm a child + * exited fails the disconnect before anything is removed: the server stays + * registered, closed, so nothing launches beside a child that may still be + * running, and disconnecting again retries the reap. */ async disconnectMcplServer(id: string): Promise { if (!this.mcplServerRegistry) { throw new Error('MCPL subsystem is not initialized'); } - const config = this.mcplServerConfigs.get(id); - const oldToolNames = new Set(this.mcplTools.map(t => t.name)); + const registry = this.mcplServerRegistry; + const hold = this.holdMcpServerId(id, 'disconnect'); + try { + // Closing a modern connection now ends a connect in flight, rather than + // waiting out its handshake. + const modern = this.modernServers().get(id); + const modernClosed = modern?.close(); + modernClosed?.catch(() => { /* its verdict is awaited below */ }); + // What held the id before (a connect still running, an earlier + // disconnect) settles first, so what it registered is removed whole. + await hold.previous; + const config = this.mcplServerConfigs.get(id); + const oldToolNames = new Set(this.mcplTools.map(t => t.name)); - const modern = this.modernServers().get(id); - // A child that can't be reaped fails the disconnect, but only after the - // rest of the removal is done. - let closeFailure: unknown = null; - if (modern) { - this.modernServers().delete(id); - await modern.close().catch((error: unknown) => { closeFailure = error; }); + try { + await modernClosed; + await registry.removeServer(id); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + throw new Error( + `${message}. MCP server "${id}" stays registered, closed, until its exit is confirmed: disconnect it again to retry the reap`, + { cause: error }, + ); + } + if (modern && this.modernServers().get(id) === modern) this.modernServers().delete(id); + this.channelRegistry?.removeServer(id); + + // Permanent removal: also destroy feature-set and checkpoint state + // explicitly. The 'close' handler usually does this, but a connection that + // already transiently closed (reconnect pending) emits no second 'close' + // from close(), and the transient path deliberately preserves checkpoints. + this.featureSetManager?.removeServer(id); + this.checkpointManager?.removeServer(id); + + const prefix = config?.toolPrefix ?? `mcpl--${id}`; + this.mcplPrefixMap.delete(prefix); + this.mcplServerConfigs.delete(id); + + await this.refreshMcplTools(); + this.emitMcplToolDiff(oldToolNames, id); + } finally { + hold.release(); } - await this.mcplServerRegistry.removeServer(id); - this.channelRegistry?.removeServer(id); - - // Permanent removal: also destroy feature-set and checkpoint state - // explicitly. The 'close' handler usually does this, but a connection that - // already transiently closed (reconnect pending) emits no second 'close' - // from close(), and the transient path deliberately preserves checkpoints. - this.featureSetManager?.removeServer(id); - this.checkpointManager?.removeServer(id); - - const prefix = config?.toolPrefix ?? `mcpl--${id}`; - this.mcplPrefixMap.delete(prefix); - this.mcplServerConfigs.delete(id); - - await this.refreshMcplTools(); - this.emitMcplToolDiff(oldToolNames, id); - if (closeFailure) throw closeFailure; } /** @@ -14661,7 +14771,7 @@ export class AgentFramework { // Surface connect/reconnect failures. Before these traces existed the // only receipt was a console.error on the host's own stderr — invisible // unless someone ssh'd in and read the process log. - connection.on('connect-failed', (params: { error: string; attempt: number }) => { + connection.on('connect-failed', (params: { error: string; attempt: number; permanent?: boolean }) => { this.emitTrace({ type: 'mcpl:server-connect-failed', serverId: connection.id, @@ -14669,6 +14779,11 @@ export class AgentFramework { attempt: params.attempt, willRetry: connection.willReconnect, }); + // `permanent`: the failed launch couldn't be reaped, so no retry will + // start beside it. Alert now, as for a permanent reconnect failure. + if (params.permanent) { + this.opsAlert('mcpl-down', connection.id, `MCPL server unreachable (attempt ${params.attempt}): ${params.error}`); + } }); connection.on('reconnect-failed', (params: { error: string; attempt: number; permanent?: boolean }) => { this.emitTrace({ @@ -14678,12 +14793,13 @@ export class AgentFramework { attempt: params.attempt, willRetry: connection.willReconnect, }); - // The reconnect loop gives up only on a protocol-version verdict - // (`permanent`), which no later attempt can change — alert at once, - // since no fifth attempt will come. Otherwise backoff caps at ~300s, - // so "the server is effectively down" is an attempt-count judgment: - // 5 failed attempts ≈ a few minutes of outage. Throttled per - // (serverId, kind), so a long outage re-posts every ~15 min. + // The reconnect loop gives up only on a protocol-version verdict or a + // launch that couldn't be reaped (`permanent`), which no later attempt + // changes — alert at once, since no fifth attempt will come. Otherwise + // backoff caps at ~300s, so "the server is effectively down" is an + // attempt-count judgment: 5 failed attempts ≈ a few minutes of outage. + // Throttled per (serverId, kind), so a long outage re-posts every + // ~15 min. if (params.permanent || params.attempt >= 5) { this.opsAlert( 'mcpl-down', @@ -14801,9 +14917,11 @@ export class AgentFramework { willReconnect: connection.willReconnect, }); this.emitTrace({ type: 'module:removed', moduleName: `mcpl:${connection.id}` }); - // Without reconnect a closed server is gone, as the MCPL registry - // drops one; its configuration stays listed as disconnected. - if (!connection.willReconnect && this.modernServers().get(connection.id) === connection) { + // Without reconnect a server lost on its own is gone, as the MCPL + // registry drops one; its configuration stays listed as disconnected. + // One the host is closing stays until that teardown is done: whoever + // closes it removes it, so the id stays held meanwhile. + if (!connection.isClosed && !connection.willReconnect && this.modernServers().get(connection.id) === connection) { this.modernServers().delete(connection.id); } }); diff --git a/src/index.ts b/src/index.ts index 34b61109a..39d1a6938 100644 --- a/src/index.ts +++ b/src/index.ts @@ -90,7 +90,7 @@ export type { McplServerConfig } from './mcpl/index.js'; // MCPL client connection (exposed so external tooling — server playtests, // health probes, protocol harnesses — can dial an MCPL server exactly the // way the framework host does, over stdio or WebSocket) -export { McplServerConnection, McplRequestError, McplProtocolVersionError } from './mcpl/index.js'; +export { McplServerConnection, McplRequestError, McplProtocolVersionError, McplUnreapedLaunchError } from './mcpl/index.js'; export { ModernMcpConnection } from './mcpl/index.js'; export type { ModernToolCallResult } from './mcpl/index.js'; export { diff --git a/src/mcpl/index.ts b/src/mcpl/index.ts index 8e09c869e..aeca314a1 100644 --- a/src/mcpl/index.ts +++ b/src/mcpl/index.ts @@ -106,7 +106,7 @@ export { } from './errors.js'; // Server connection and registry -export { McplServerConnection, McplRequestError, McplProtocolVersionError } from './server-connection.js'; +export { McplServerConnection, McplRequestError, McplProtocolVersionError, McplUnreapedLaunchError } from './server-connection.js'; export { ModernMcpConnection } from './modern-connection.js'; export type { ModernToolCallResult } from './modern-connection.js'; export { diff --git a/src/mcpl/modern-connection.ts b/src/mcpl/modern-connection.ts index 27cde4506..d4bd74e25 100644 --- a/src/mcpl/modern-connection.ts +++ b/src/mcpl/modern-connection.ts @@ -354,6 +354,12 @@ export class ModernMcpConnection extends EventEmitter { return this.reconnectEnabled && !this.closedByHost && this.unreaped.length === 0; } + /** Whether close() has been called: closed for good, or closing. A + * connection lost on its own is not closed in this sense. */ + get isClosed(): boolean { + return this.closedByHost; + } + /** A failed generation's cleanup couldn't reap its child: keep owning it, * and halt reconnecting until close() settles it. */ private holdUnreaped(session: Session, failure: Error): void { @@ -380,13 +386,22 @@ export class ModernMcpConnection extends EventEmitter { return new ModernMcpConnection(config); } + /** The first connect, shared by every call to start(). */ + private starting: Promise | null = null; + /** * The first connect. Without `reconnect` a failure throws. With it, the * connection keeps retrying in the background, as the legacy engine's * stub does, and reports `connect-failed`. A connection closed before or - * during its first connect resolves unconnected. + * during its first connect resolves unconnected. A connection starts once: + * later calls return the first call's promise, so a second session never + * opens beside the first. */ - async start(): Promise { + start(): Promise { + return (this.starting ??= this.firstConnect()); + } + + private async firstConnect(): Promise { try { await this.open(); } catch (error) { diff --git a/src/mcpl/server-connection.ts b/src/mcpl/server-connection.ts index 2e0a03559..778b91fbe 100644 --- a/src/mcpl/server-connection.ts +++ b/src/mcpl/server-connection.ts @@ -18,7 +18,7 @@ import { CAPABILITY_DISABLED } from './errors.js'; import { LEGACY_MCP_PROTOCOL_VERSION, MODERN_MCP_PROTOCOL_VERSION, - resolveServerBinding, + checkServerConfig, } from './protocol-family.js'; import type { @@ -154,6 +154,43 @@ export class McplRequestError extends Error { } } +const errorOf = (value: unknown): Error => (value instanceof Error ? value : new Error(String(value))); + +/** + * A handshake that failed, and whose cleanup couldn't confirm that the child + * exited. The transport goes to whoever owns the launch. + */ +class HandshakeCleanupError extends Error { + constructor( + readonly failure: Error, + readonly cleanup: Error, + readonly transport: McplTransport, + ) { + super(`${failure.message}; its cleanup also failed: ${cleanup.message}`, { cause: failure }); + this.name = 'HandshakeCleanupError'; + } +} + +/** + * A connect that failed, and whose cleanup couldn't confirm that the child + * exited: it may still be running. `connection` owns that launch, closed and + * not reconnecting. Its close() retries the reap and resolves once the exit + * is confirmed, so the caller can still clean up, and should start nothing + * else for this server until then. connect() throws this; + * connectWithReconnect() returns the connection instead, halted. `cause` is + * the connect's own failure. + */ +export class McplUnreapedLaunchError extends Error { + constructor( + readonly connection: McplServerConnection, + failure: Error, + cleanup: Error, + ) { + super(`${failure.message}; its cleanup also failed: ${cleanup.message}`, { cause: failure }); + this.name = 'McplUnreapedLaunchError'; + } +} + /** * Represents a pending JSON-RPC request awaiting a response. */ @@ -336,9 +373,21 @@ export class McplServerConnection extends EventEmitter { /** Consecutive failed connect attempts since the last successful handshake. * Drives the exponential backoff and is reported on reconnect events. */ private reconnectAttempts = 0; + /** The reconnect attempt in flight, so close() can abort it and wait for + * its cleanup. */ + private reconnecting: { abort: AbortController; done: Promise } | null = null; + /** + * Launches whose cleanup couldn't confirm that the child exited. While any + * is held, reconnecting is halted, since another launch could run beside + * one that's still alive. close() retries their reap. + */ + private unreaped: Array<{ transport: McplTransport; failure: Error }> = []; + /** The teardown in flight, which later close() calls wait behind. */ + private closing: Promise | null = null; /** Whether a background reconnect loop will revive this connection after a - * drop. False once close() has been called (explicit close stops retrying). */ + * drop. False once close() has been called (explicit close stops retrying), + * after a protocol-version verdict, and while a launch is unreaped. */ get willReconnect(): boolean { return this.reconnectEnabled; } @@ -560,14 +609,29 @@ export class McplServerConnection extends EventEmitter { * capability negotiation, and return a ready-to-use connection. * * Works over stdio (spawned child) or WebSocket, selected by `config` (see - * {@link openTransport}). Throws if the transport can't be opened, the server - * closes/errors before the handshake, or the handshake times out. + * {@link openTransport}). Throws if the configuration can't be used (before + * anything is spawned or dialed), the transport can't be opened, the server + * closes/errors before the handshake, or the handshake times out. If the + * failed launch's cleanup can't confirm its child exited, it throws + * {@link McplUnreapedLaunchError}, whose connection owns that launch. */ static async connect( config: McplServerConfig, hostCapabilities: McplHostCapabilities, ): Promise { - const { transport, capabilities, droppedCapabilities, mcpToolsAdvertised, protocolVersion } = await McplServerConnection.handshake(config, hostCapabilities); + McplServerConnection.checkConfig(config); + let established: Awaited>; + try { + established = await McplServerConnection.handshake(config, hostCapabilities); + } catch (error) { + if (!(error instanceof HandshakeCleanupError)) throw error; + // The launch may still be running. A connection owns it, closed and + // halted, so the caller can retry the reap by closing it. + const owner = McplServerConnection.createDisconnectedStub(config, hostCapabilities); + owner.holdUnreaped(error.transport, error.cleanup); + throw new McplUnreapedLaunchError(owner, error.failure, error.cleanup); + } + const { transport, capabilities, droppedCapabilities, mcpToolsAdvertised, protocolVersion } = established; const connection = new McplServerConnection(config.id, capabilities, transport); connection.droppedCapabilities = droppedCapabilities; @@ -588,32 +652,45 @@ export class McplServerConnection extends EventEmitter { return connection; } + /** + * This engine's configuration check, made before anything is spawned or + * dialed: every problem {@link checkServerConfig} finds, the same check the + * framework applies, and the modern family, which belongs to + * ModernMcpConnection (an http(s) url is never dialed as a WebSocket). + * Every configuration that worked before still passes. + */ + private static checkConfig(config: McplServerConfig): void { + const { family } = checkServerConfig(config); + if (family === 'modern') { + throw new Error( + `MCP server "${config.id}" is configured for modern MCP (${MODERN_MCP_PROTOCOL_VERSION}); ` + + `McplServerConnection speaks the legacy family (${MCP_PROTOCOL_VERSION} + MCPL) only`, + ); + } + } + /** * Open a fresh transport and run the MCP/MCPL initialize handshake over it, * returning the connected transport plus the negotiated capabilities. Shared * by {@link connect} and {@link attemptReconnect} so both transports handshake * identically. On any failure the transport is closed so a spawned child / - * open socket never leaks. + * open socket never leaks. If that close can't confirm the child exited, + * the failure is a {@link HandshakeCleanupError} carrying the transport, so + * the caller keeps owning it. `signal` ends a handshake whose connection is + * closed meanwhile: it is checked once the transport is open (a WebSocket + * dial isn't cut short) and races the initialize answer. */ private static async handshake( config: McplServerConfig, hostCapabilities: McplHostCapabilities, + signal?: AbortSignal, ): Promise<{ transport: McplTransport; capabilities: McplCapabilities | null; droppedCapabilities: ReadonlySet; mcpToolsAdvertised: boolean; protocolVersion: string }> { - // This engine speaks the legacy family only. A configuration that - // resolves to the modern family belongs to ModernMcpConnection; refuse it - // here rather than dialing an http(s) url as a WebSocket. One that names - // no usable transport is an error too, before anything is spawned or - // dialed. - const family = resolveServerBinding(config).family; - if (family === 'modern') { - throw new Error( - `MCP server "${config.id}" is configured for modern MCP (${MODERN_MCP_PROTOCOL_VERSION}); ` + - `McplServerConnection speaks the legacy family (${MCP_PROTOCOL_VERSION} + MCPL) only`, - ); - } + const closedWhileConnecting = () => new Error(`MCPL server "${config.id}" was closed while connecting`); + if (signal?.aborted) throw closedWhileConnecting(); const transport = await openTransport(config); try { + if (signal?.aborted) throw closedWhileConnecting(); const initId = 0; // use id=0 for the handshake const initRequest: JsonRpcRequest = { jsonrpc: '2.0', @@ -626,15 +703,20 @@ export class McplServerConnection extends EventEmitter { }, }; - // Await the initialize response, racing an early transport close/error and - // a timeout. All three paths clean up their listeners. + // Await the initialize response, racing an early transport close/error, + // a timeout and the owner's close. Every path cleans up its listeners. const { mcpl: rawCapabilities, mcpToolsAdvertised, protocolVersion } = await new Promise<{ mcpl: McplCapabilities | null; mcpToolsAdvertised: boolean; protocolVersion: string }>((resolve, reject) => { const cleanup = () => { transport.off('line', onLine); transport.off('close', onClose); transport.off('error', onError); + signal?.removeEventListener('abort', onAbort); clearTimeout(timer); }; + const onAbort = () => { + cleanup(); + reject(closedWhileConnecting()); + }; const onLine = (line: string) => { let msg: JsonRpcResponse; try { @@ -689,6 +771,7 @@ export class McplServerConnection extends EventEmitter { reject(new Error(`MCPL server "${config.id}" initialize handshake timed out`)); }, INITIALIZE_TIMEOUT_MS); + signal?.addEventListener('abort', onAbort, { once: true }); transport.on('line', onLine); transport.on('close', onClose); transport.on('error', onError); @@ -713,8 +796,14 @@ export class McplServerConnection extends EventEmitter { return { transport, capabilities: scoped, droppedCapabilities: new Set(dropped), mcpToolsAdvertised, protocolVersion }; } catch (err) { - // Never leak the child / socket if the handshake fails. - await transport.close().catch(() => { /* best effort */ }); + // Never leak the child / socket if the handshake fails. A cleanup that + // can't confirm the child exited is part of the failure, never + // swallowed: the launch may still be running, so the caller keeps it. + try { + await transport.close(); + } catch (cleanup) { + throw new HandshakeCleanupError(errorOf(err), errorOf(cleanup), transport); + } throw err; } } @@ -723,12 +812,18 @@ export class McplServerConnection extends EventEmitter { * Connect with reconnect support. * When `config.reconnect` is true and the initial connection fails, * resolves immediately with null capabilities and retries in the background. + * A configuration this engine can't use, or a protocol-version verdict, + * still throws: retrying can't change either. A failed launch that can't be + * reaped resolves with a halted stub that owns it: no retry starts beside + * a child that may still be running, and close() retries the reap. * Adapted from Anarchid/agent-framework@mcpl-module-proto. */ static async connectWithReconnect( config: McplServerConfig, hostCapabilities: McplHostCapabilities, ): Promise { + // Outside the retry path below: a configuration error is never retried. + McplServerConnection.checkConfig(config); try { return await McplServerConnection.connect(config, hostCapabilities); } catch (error) { @@ -738,20 +833,28 @@ export class McplServerConnection extends EventEmitter { throw error; } - // Non-blocking start: create a disconnected stub that will reconnect in background - console.error(`MCPL server "${config.id}" initial connect failed, will retry:`, (error as Error).message); - const stub = McplServerConnection.createDisconnectedStub(config, hostCapabilities); // Surface the initial failure. The event is buffered by the emit() // override until the host wires listeners and calls ready(), so it is // not lost in the window before wireMcplEvents runs. + if (error instanceof McplUnreapedLaunchError) { + const halted = error.connection; + console.error(`MCPL server "${config.id}" initial connect failed, not retrying:`, error.message); + halted.emit('connect-failed', { error: `${(error.cause as Error).message}; ${halted.unreapedFailure()}`, attempt: 0, permanent: true }); + return halted; + } + // Non-blocking start: create a disconnected stub that will reconnect in background + console.error(`MCPL server "${config.id}" initial connect failed, will retry:`, (error as Error).message); + const stub = McplServerConnection.createDisconnectedStub(config, hostCapabilities); stub.emit('connect-failed', { error: (error as Error).message, attempt: 0 }); + stub.scheduleReconnect(); return stub; } } /** - * Create a disconnected stub connection that will reconnect in the background. - * Used when initial connect fails and reconnect is enabled. + * Create a disconnected stub connection, for a failed initial connect: one + * that will reconnect in the background (its caller schedules that), or one + * that owns a launch that couldn't be reaped. * @internal */ private static createDisconnectedStub( @@ -764,15 +867,31 @@ export class McplServerConnection extends EventEmitter { stub.requestTimeoutMs = config.requestTimeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS; stub.config = config; stub.hostCapabilities = hostCapabilities; - stub.reconnectEnabled = true; + stub.reconnectEnabled = config.reconnect === true; stub.reconnectIntervalMs = config.reconnectIntervalMs ?? 5000; stub.reconnectMaxIntervalMs = config.reconnectMaxIntervalMs ?? 300_000; stub.reconnectAttempts = 1; // the initial connect already failed + return stub; + } - // Schedule background reconnect - stub.scheduleReconnect(); + /** + * Keep a launch whose cleanup couldn't confirm its child exited, and halt + * reconnecting: another launch could run beside one that's still alive. + * close() retries the reap. + */ + private holdUnreaped(transport: McplTransport, failure: Error): void { + this.reconnectEnabled = false; + if (this.reconnectTimer) { + clearTimeout(this.reconnectTimer); + this.reconnectTimer = null; + } + if (!this.unreaped.some((held) => held.transport === transport)) this.unreaped.push({ transport, failure }); + } - return stub; + /** The halt, as a reconnect event's failure text. */ + private unreapedFailure(): string { + const held = this.unreaped[0]; + return `the previous launch could not be reaped (${held?.failure.message ?? 'unknown'}); reconnecting is halted`; } // ========================================================================== @@ -992,37 +1111,72 @@ export class McplServerConnection extends EventEmitter { // Lifecycle // ========================================================================== - /** Close the connection: disable reconnect, kill the child process, and clean up resources. */ - async close(): Promise { + /** + * Close the connection for good: disable reconnect, kill the child + * process, and clean up resources. At once, as before, reconnecting stops + * and pending requests fail; a reconnect handshake in flight is aborted. + * + * The teardown is shared. Calls queue behind the one in flight, so none + * resolves while it is pending. Each waits for the aborted attempt's own + * cleanup, closes the transport (its close is the reaping verdict, and a + * transport the host is already closing shares it), and re-checks every + * launch an earlier cleanup couldn't reap. It rejects, naming them, while + * any child still can't be confirmed exited; a later call re-checks. + */ + close(): Promise { // Disable reconnect before closing — explicit close means stop retrying this.reconnectEnabled = false; if (this.reconnectTimer) { clearTimeout(this.reconnectTimer); this.reconnectTimer = null; } + this.reconnecting?.abort.abort(); - if (this.closed) { - return; - } - this.closed = true; - this.resetPolicyForTransportBoundary(); + const live = !this.closed; + if (live) { + this.closed = true; + this.resetPolicyForTransportBoundary(); - // Reject all pending requests - for (const [id, pending] of this.pendingRequests) { - if (pending.timer) clearTimeout(pending.timer); - pending.reject(new McplRequestError(`Connection to MCPL server "${this.id}" closed while awaiting response for ${pending.method} (id=${id})`, 'no-response')); + // Reject all pending requests + for (const [id, pending] of this.pendingRequests) { + if (pending.timer) clearTimeout(pending.timer); + pending.reject(new McplRequestError(`Connection to MCPL server "${this.id}" closed while awaiting response for ${pending.method} (id=${id})`, 'no-response')); + } + this.pendingRequests.clear(); + this.orphanedRequests.clear(); } - this.pendingRequests.clear(); - this.orphanedRequests.clear(); - // Tear down the transport (kills the child / closes the socket). The - // transport's own 'close' event is short-circuited by `this.closed` in - // setupLifecycle, so this is the single source of the connection 'close'. - if (this.transport) { - await this.transport.close(); - } + const transport = this.transport; + const previous = this.closing; + const step = (previous ? previous.catch(() => {}) : Promise.resolve()).then(() => this.closeStep(transport, live)); + this.closing = step; + return step; + } - this.emit('close'); + private async closeStep(transport: McplTransport | null, live: boolean): Promise { + // An aborted reconnect cleans up its own launch, holding it if it can't. + await this.reconnecting?.done; + const targets = transport ? [transport] : []; + for (const held of this.unreaped) if (!targets.includes(held.transport)) targets.push(held.transport); + const failures: Error[] = []; + for (const target of targets) { + try { + // Kills the child / closes the socket. + await target.close(); + this.unreaped = this.unreaped.filter((held) => held.transport !== target); + } catch (error) { + const failure = errorOf(error); + failures.push(failure); + if (!this.unreaped.some((held) => held.transport === target)) this.unreaped.push({ transport: target, failure }); + } + } + // The transport's own 'close' event is short-circuited by `this.closed` + // in setupLifecycle, so this is the single source of the connection + // 'close' when the host tears down a live transport. + if (live) this.emit('close'); + if (failures.length > 0) { + throw new Error(`MCPL server "${this.id}" did not close cleanly: ${failures.map((f) => f.message).join('; ')}`); + } } /** @@ -1074,52 +1228,96 @@ export class McplServerConnection extends EventEmitter { * Attempt to reconnect by opening a fresh transport and performing the * handshake, then adopting that transport in place. Unlike the previous * approach this builds no throwaway connection instance, so there are no - * leaked listeners on a dead object. + * leaked listeners on a dead object. The attempt is close()'s to abort, + * and it settles only once its own launch is adopted, failed or cleaned up. */ private async attemptReconnect(): Promise { if (!this.reconnectEnabled || !this.config || !this.hostCapabilities) return; + const abort = new AbortController(); + const done = this.reconnectOnce(this.config, this.hostCapabilities, abort.signal); + this.reconnecting = { abort, done }; + await done; + if (this.reconnecting?.done === done) this.reconnecting = null; + } + /** One reconnect attempt. It never rejects: each outcome is an event, a + * scheduled retry, or a launch held as unreaped. */ + private async reconnectOnce( + config: McplServerConfig, + hostCapabilities: McplHostCapabilities, + signal: AbortSignal, + ): Promise { // Ordinal of this attempt: 0 was the initial connect, so the Nth retry // reports attempt N (reconnectAttempts counts failures so far). const attempt = Math.max(1, this.reconnectAttempts); - try { - const { transport, capabilities, droppedCapabilities, mcpToolsAdvertised, protocolVersion } = await McplServerConnection.handshake( - this.config, - this.hostCapabilities, - ); - - // Close the data plane before the new transport can emit. The reconnect - // lifecycle event bypasses buffering; its framework listener installs the - // awareness barrier synchronously, then releases control traffic needed - // to establish the server while data remains held. - this.pauseDataPlane(); - this.resetPolicyForTransportBoundary(); - this.transport = transport; - this.capabilities = capabilities; - this.droppedCapabilities = droppedCapabilities; - this.mcpToolsAdvertised = mcpToolsAdvertised; - this.protocolVersion = protocolVersion; - this.allowHostCommands = this.config?.allowHostCommands === true; - this.closed = false; - this.nextRequestId = 1; - this.pendingRequests.clear(); - this.wireTransport(transport); + // The previous launch must be gone before another starts. A transport + // the host closed itself (reconnectAfterFailure) may still be reaping its + // child; one whose child already exited resolves at once. + const previous = this.transport; + if (previous) { + try { + await previous.close(); + } catch (error) { + this.holdUnreaped(previous, errorOf(error)); + if (!signal.aborted) this.emit('reconnect-failed', { error: this.unreapedFailure(), attempt, permanent: true }); + return; + } + } + if (signal.aborted) return; - console.error(`MCPL server "${this.id}" reconnected successfully`); - this.reconnectAttempts = 0; - this.emit('reconnect', { attempts: attempt }); + let established: Awaited>; + try { + established = await McplServerConnection.handshake(config, hostCapabilities, signal); } catch (error) { - console.error(`MCPL server "${this.id}" reconnect failed:`, (error as Error).message); + const unreaped = error instanceof HandshakeCleanupError; + if (unreaped) this.holdUnreaped(error.transport, error.cleanup); + // Closed by the host meanwhile: not a failed attempt. + if (signal.aborted) return; + const failure = unreaped ? error.failure : errorOf(error); + console.error(`MCPL server "${this.id}" reconnect failed:`, failure.message); this.reconnectAttempts = attempt + 1; // No common protocol revision: every further attempt would restart the // server to hear the same answer. Stop retrying; the configuration is - // what has to change. Every other failure keeps the backoff loop. - const permanent = error instanceof McplProtocolVersionError; - if (permanent) this.reconnectEnabled = false; - this.emit('reconnect-failed', { error: (error as Error).message, attempt, ...(permanent ? { permanent: true } : {}) }); + // what has to change. A launch that couldn't be reaped halts retrying + // too, until close() confirms its exit. Every other failure keeps the + // backoff loop. + if (error instanceof McplProtocolVersionError) this.reconnectEnabled = false; + const permanent = unreaped || error instanceof McplProtocolVersionError; + const message = unreaped ? `${failure.message}; ${this.unreapedFailure()}` : failure.message; + this.emit('reconnect-failed', { error: message, attempt, ...(permanent ? { permanent: true } : {}) }); if (!permanent) this.scheduleReconnect(); + return; } + const { transport, capabilities, droppedCapabilities, mcpToolsAdvertised, protocolVersion } = established; + + if (signal.aborted) { + // Closed by the host while the handshake finished: this launch never + // serves. close() is waiting on this attempt, so it hears the verdict. + await transport.close().catch((error: unknown) => this.holdUnreaped(transport, errorOf(error))); + return; + } + + // Close the data plane before the new transport can emit. The reconnect + // lifecycle event bypasses buffering; its framework listener installs the + // awareness barrier synchronously, then releases control traffic needed + // to establish the server while data remains held. + this.pauseDataPlane(); + this.resetPolicyForTransportBoundary(); + this.transport = transport; + this.capabilities = capabilities; + this.droppedCapabilities = droppedCapabilities; + this.mcpToolsAdvertised = mcpToolsAdvertised; + this.protocolVersion = protocolVersion; + this.allowHostCommands = this.config?.allowHostCommands === true; + this.closed = false; + this.nextRequestId = 1; + this.pendingRequests.clear(); + this.wireTransport(transport); + + console.error(`MCPL server "${this.id}" reconnected successfully`); + this.reconnectAttempts = 0; + this.emit('reconnect', { attempts: attempt }); } // ========================================================================== diff --git a/src/mcpl/server-registry.ts b/src/mcpl/server-registry.ts index 62b3854d5..fc773df9f 100644 --- a/src/mcpl/server-registry.ts +++ b/src/mcpl/server-registry.ts @@ -5,7 +5,7 @@ */ import type { McplServerConfig, McplHostCapabilities } from './types.js'; -import { McplServerConnection } from './server-connection.js'; +import { McplServerConnection, McplUnreapedLaunchError } from './server-connection.js'; /** * Capability keys that can be queried via `getServersWithCapability`. @@ -30,7 +30,10 @@ export class McplServerRegistry { * Connect to an MCPL server and register it. * * Throws if a server with the same id is already registered, or if - * the connection/handshake fails. + * the connection/handshake fails. A failed launch whose child couldn't be + * confirmed exited stays registered all the same, as the closed connection + * that owns it ({@link McplUnreapedLaunchError}), so the id stays held + * and removeServer can retry the reap. */ async addServer( config: McplServerConfig, @@ -40,14 +43,27 @@ export class McplServerRegistry { throw new Error(`MCPL server "${config.id}" is already registered`); } - const connection = config.reconnect - ? await McplServerConnection.connectWithReconnect(config, hostCapabilities) - : await McplServerConnection.connect(config, hostCapabilities); + let connection: McplServerConnection; + try { + connection = config.reconnect + ? await McplServerConnection.connectWithReconnect(config, hostCapabilities) + : await McplServerConnection.connect(config, hostCapabilities); + } catch (error) { + if (error instanceof McplUnreapedLaunchError && !this.servers.has(config.id)) { + this.register(config, error.connection); + } + throw error; + } + this.register(config, connection); + return connection; + } + + private register(config: McplServerConfig, connection: McplServerConnection): void { this.servers.set(config.id, connection); // Auto-remove on unexpected close (unless reconnect will re-add) connection.on('close', () => { - if (!config.reconnect) { + if (!config.reconnect && this.servers.get(config.id) === connection) { this.servers.delete(config.id); } }); @@ -56,14 +72,14 @@ export class McplServerRegistry { connection.on('reconnect', () => { // Connection already in the map — capabilities may have changed }); - - return connection; } /** * Disconnect and remove a server by id. * - * No-op if the server is not registered. + * No-op if the server is not registered. If its close can't confirm the + * child exited, it stays registered and the failure is thrown: removing it + * again retries the reap. */ async removeServer(id: string): Promise { const connection = this.servers.get(id); @@ -71,7 +87,12 @@ export class McplServerRegistry { return; } this.servers.delete(id); - await connection.close(); + try { + await connection.close(); + } catch (error) { + if (!this.servers.has(id)) this.servers.set(id, connection); + throw error; + } } /** diff --git a/test/fixtures/legacy-version-server.mjs b/test/fixtures/legacy-version-server.mjs index 9a84796f1..e2968ff90 100644 --- a/test/fixtures/legacy-version-server.mjs +++ b/test/fixtures/legacy-version-server.mjs @@ -7,7 +7,12 @@ // reject-nodata error -32022 without data // mismatch protocolVersion 2025-06-18 // missing no protocolVersion at all -// ok-then-reject `ok` on the first start recorded in startLog, `reject` after +// error a JSON-RPC error (-32603), not a version verdict +// hang never answers +// slow `ok`, after SLOW_INIT_MS (default 600) +// ok-then- `ok` on the first start recorded in startLog, after +// +// IGNORE_SIGTERM=1 makes the process survive SIGTERM (only SIGKILL ends it). // // tools/list pages: by default three pages (cursors p2, p3). MODE_LIST picks // an edge case: loop (always nextCursor "same"), empty (nextCursor "" then a @@ -20,8 +25,9 @@ const startLog = process.argv[3]; if (startLog) { const priorStarts = existsSync(startLog) ? readFileSync(startLog, 'utf8').split('\n').filter(Boolean).length : 0; appendFileSync(startLog, `start ${process.pid}\n`); - if (mode === 'ok-then-reject') mode = priorStarts === 0 ? 'ok' : 'reject'; + if (mode.startsWith('ok-then-')) mode = priorStarts === 0 ? 'ok' : mode.slice('ok-then-'.length); } +if (process.env.IGNORE_SIGTERM === '1') process.on('SIGTERM', () => {}); const listMode = process.env.MODE_LIST ?? 'pages'; const send = (m) => process.stdout.write(JSON.stringify(m) + '\n'); @@ -46,6 +52,9 @@ process.stdin.on('data', (chunk) => { else if (mode === 'reject-nodata') error(m.id, -32022, `Unsupported protocol version: ${requested}`); else if (mode === 'mismatch') reply(m.id, { protocolVersion: '2025-06-18', capabilities: { tools: {} } }); else if (mode === 'missing') reply(m.id, { capabilities: { tools: {} } }); + else if (mode === 'error') error(m.id, -32603, 'initialize failed on purpose'); + else if (mode === 'hang') { /* never answers */ } + else if (mode === 'slow') setTimeout(() => reply(m.id, { protocolVersion: '2024-11-05', capabilities: { tools: {} } }), Number(process.env.SLOW_INIT_MS ?? 600)); else reply(m.id, { protocolVersion: '2024-11-05', capabilities: { tools: {} } }); } else if (m.method === 'tools/list') { const cursor = m.params?.cursor; diff --git a/test/fixtures/modern-raw-server.mjs b/test/fixtures/modern-raw-server.mjs index ac5838b2f..00866dc40 100644 --- a/test/fixtures/modern-raw-server.mjs +++ b/test/fixtures/modern-raw-server.mjs @@ -9,6 +9,7 @@ // hang-discover-later launches after the first never answer server/discover // hang-discover no launch ever answers server/discover // reject-discover-later launches after the first answer server/discover with an error +// reject-discover every launch answers server/discover with an error // ignore-sigterm the process survives SIGTERM (only SIGKILL ends it) // Tools: op (outputSchema {value: integer}, answers with a string: invalid // structured content), plain (text), err (a JSON-RPC error), touch (announces @@ -44,6 +45,7 @@ createInterface({ input: process.stdin }).on('line', (line) => { if (flags.has('hang-discover-later') && launch > 1) return; if (flags.has('hang-discover')) return; if (flags.has('reject-discover-later') && launch > 1) return error(m.id, -32603, 'not today'); + if (flags.has('reject-discover')) return error(m.id, -32603, 'not today'); return result(m.id, { supportedVersions: ['2026-07-28'], capabilities: { tools: { listChanged: true } }, diff --git a/test/mcp-modern-outcomes.test.ts b/test/mcp-modern-outcomes.test.ts index b91f31680..651f6fc7e 100644 --- a/test/mcp-modern-outcomes.test.ts +++ b/test/mcp-modern-outcomes.test.ts @@ -286,3 +286,20 @@ test('concurrent close() calls share one teardown and its verdict; a later close await until(() => { try { process.kill(stuck!.pid, 0); return false; } catch { return true; } }, 'the child gone'); await connection.close(); }); + +test('start() runs once: a second call shares the first connect, and no second session opens beside it', async () => { + const log = scratchLog(); + const connection = ModernMcpConnection.create({ id: 'raw', command: process.execPath, args: [SERVER, '', log], protocol: 'modern' }); + cleanups.push(() => connection.close()); + const first = connection.start(); + assert.equal(connection.start(), first, 'the same promise while connecting'); + await first; + assert.equal(connection.start(), first, 'and once connected'); + await connection.start(); + assert.equal(connection.isConnected, true); + const launches = wire(log).filter((e) => e.event === 'start'); + assert.equal(launches.length, 1, 'one launch'); + await connection.close(); + const alive = (p: number) => { try { process.kill(p, 0); return true; } catch { return false; } }; + assert.equal(alive(launches[0]!.pid!), false, 'close() reaped the only child'); +}); diff --git a/test/mcp-server-ownership.test.ts b/test/mcp-server-ownership.test.ts new file mode 100644 index 000000000..f5f7e4607 --- /dev/null +++ b/test/mcp-server-ownership.test.ts @@ -0,0 +1,272 @@ +/** + * One owner per MCP server id across both engines (room-293, greptile's + * review of PR #262 and Iris-1827's invariant): from a connect's admission + * until its teardown is done, exactly one owner holds an id. + * - a connect is refused while a connect or disconnect for the id is in + * flight, a legacy handshake included; + * - a disconnect waits for what held the id before it, so nothing it removes + * belongs to a later connection, and each release frees only its own claim; + * - a teardown that can't confirm a child exited isn't finished: the server + * stays registered, closed, a connect is refused, and a retried disconnect + * completes once the exit is confirmed. + */ +import { test, afterEach } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtempSync, readFileSync, existsSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, dirname } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +import { AgentFramework } from '../src/index.js'; +import type { McplServerConfig } from '../src/mcpl/types.js'; +import { StdioTransport } from '../src/mcpl/transport.js'; +import { MockMembrane } from './helpers/mock-membrane.js'; + +const here = dirname(fileURLToPath(import.meta.url)); +const RAW = join(here, 'fixtures', 'modern-raw-server.mjs'); +const LEGACY = join(here, 'fixtures', 'legacy-version-server.mjs'); + +const cleanups: Array<() => Promise | void> = []; +afterEach(async () => { + while (cleanups.length) await cleanups.pop()!(); +}); + +const lines = (file: string) => (existsSync(file) ? readFileSync(file, 'utf8').split('\n').filter(Boolean) : []); +/** Launch pids from a raw modern server's JSONL log, or a legacy start log. */ +const launches = (file: string): number[] => lines(file).flatMap((l) => { + if (l.startsWith('start ')) return [Number(l.slice('start '.length))]; + const entry = JSON.parse(l) as { event?: string; pid?: number }; + return entry.event === 'start' ? [entry.pid!] : []; +}); +const alive = (pid: number) => { try { process.kill(pid, 0); return true; } catch { return false; } }; + +async function until(cond: () => boolean, what: string, ms = 10_000): Promise { + const deadline = Date.now() + ms; + while (!cond() && Date.now() < deadline) await new Promise((r) => setTimeout(r, 20)); + assert.ok(cond(), `in time: ${what}`); +} + +async function withFramework(fn: (h: { framework: AgentFramework; dir: string }) => Promise): Promise { + const dir = mkdtempSync(join(tmpdir(), 'mcp-ownership-')); + const framework = await AgentFramework.create({ + storePath: join(dir, 'store'), + membrane: new MockMembrane().asMembrane(), + agents: [{ name: 'agent', model: 'test-model', systemPrompt: 'test' }], + modules: [], + }); + try { + await fn({ framework, dir }); + } finally { + await framework.stop(); + rmSync(dir, { recursive: true, force: true }); + } +} + +const modern = (id: string, flags: string, log: string): McplServerConfig => + ({ id, command: process.execPath, args: [RAW, flags, log], protocol: 'modern' }); +const legacy = (id: string, mode: string, log: string, env: Record = {}): McplServerConfig => + ({ id, command: process.execPath, args: [LEGACY, mode, log], env }); +const status = (framework: AgentFramework, id: string) => framework.listMcplServers().find((s) => s.id === id); +const toolsOf = (framework: AgentFramework, id: string) => framework.getAllTools().map((t) => t.name).filter((n) => n.startsWith(`mcpl--${id}--`)); +const call = (name: string) => ({ id: `t-${Math.random()}`, name, input: {} }); + +/** + * Stdio launches can't be signalled while `stuck.active` is set, so their + * cleanup can't reap them; `kill` really ends one (Olive-1804's pattern). + */ +function unreapable(): { active: boolean; stuck: Array<{ pid: number; kill: () => void }> } { + const realSpawn = StdioTransport.spawn; + const state = { active: true, stuck: [] as Array<{ pid: number; kill: () => void }> }; + (StdioTransport as unknown as { spawn: typeof StdioTransport.spawn }).spawn = (c) => { + const transport = realSpawn.call(StdioTransport, c); + if (state.active) { + const child = (transport as unknown as { child: { pid: number; kill: (signal?: string) => boolean } }).child; + const realKill = child.kill.bind(child); + child.kill = () => true; + state.stuck.push({ pid: child.pid, kill: () => { realKill('SIGKILL'); } }); + } + return transport; + }; + cleanups.push(() => { + (StdioTransport as unknown as { spawn: typeof StdioTransport.spawn }).spawn = realSpawn; + for (const s of state.stuck) s.kill(); + }); + return state; +} + +async function killAndWait(stuck: { pid: number; kill: () => void }): Promise { + stuck.kill(); + await until(() => !alive(stuck.pid), `child ${stuck.pid} gone`); +} + +test('a connect during a disconnect is refused; once the teardown is done, the new server is listed and routed', async () => { + await withFramework(async ({ framework, dir }) => { + const log = join(dir, 'raw.jsonl'); + // A child that ignores SIGTERM keeps the teardown in flight for ~2 s. + const config = modern('m', 'ignore-sigterm', log); + await framework.connectMcplServer(config); + const disconnecting = framework.disconnectMcplServer('m'); + await assert.rejects(framework.connectMcplServer(config), /MCP server "m" is still being disconnected/); + assert.equal(launches(log).length, 1, 'nothing launched beside the closing child'); + await disconnecting; + assert.equal(status(framework, 'm'), undefined); + + await framework.connectMcplServer(config); + assert.equal(status(framework, 'm')?.connected, true, 'listed: its routing was not erased'); + assert.ok(toolsOf(framework, 'm').includes('mcpl--m--plain')); + assert.deepEqual((await framework.executeToolCall(call('mcpl--m--plain'))).data, [{ type: 'text', text: 'plain' }]); + }); +}); + +test('two simultaneous disconnects both settle, in order, and leave the id free', async () => { + await withFramework(async ({ framework, dir }) => { + const log = join(dir, 'raw.jsonl'); + const config = modern('m', 'ignore-sigterm', log); + await framework.connectMcplServer(config); + const order: string[] = []; + const first = framework.disconnectMcplServer('m').then(() => order.push('first')); + const second = framework.disconnectMcplServer('m').then(() => order.push('second')); + await Promise.all([first, second]); + assert.deepEqual(order, ['first', 'second']); + assert.equal(status(framework, 'm'), undefined); + assert.equal(alive(launches(log)[0]!), false); + await framework.connectMcplServer(config); + assert.equal(status(framework, 'm')?.connected, true); + }); +}); + +test('a hold releases only its own claim: a disconnect queued behind another still holds the id', async () => { + await withFramework(async ({ framework, dir }) => { + const hold = (framework as unknown as { + holdMcpServerId: (id: string, kind: 'connect' | 'disconnect') => { previous: Promise | undefined; release: () => void }; + }).holdMcpServerId.bind(framework); + const first = hold('x', 'disconnect'); + const second = hold('x', 'disconnect'); + assert.equal(first.previous, undefined); + let secondMayRun = false; + void second.previous!.then(() => { secondMayRun = true; }); + first.release(); + await new Promise((r) => setImmediate(r)); + assert.equal(secondMayRun, true, 'the second runs once the first has settled'); + const config = modern('x', '', join(dir, 'raw.jsonl')); + await assert.rejects(framework.connectMcplServer(config), /"x" is still being disconnected/, 'the first release left the second claim'); + second.release(); + await framework.connectMcplServer(config); + assert.equal(status(framework, 'x')?.connected, true); + }); +}); + +test('a modern connect is refused while a legacy connect for the same id is still handshaking', async () => { + await withFramework(async ({ framework, dir }) => { + const legacyLog = join(dir, 'legacy.log'); + const modernLog = join(dir, 'raw.jsonl'); + const pending = framework.connectMcplServer(legacy('same', 'slow', legacyLog, { SLOW_INIT_MS: '800' })); + await until(() => launches(legacyLog).length === 1, 'the legacy launch, handshaking'); + await assert.rejects(framework.connectMcplServer(modern('same', '', modernLog)), /MCPL server "same" is already registered/); + assert.equal(launches(modernLog).length, 0, 'the modern server was never launched'); + await pending; + const served = status(framework, 'same')!; + assert.equal(served.family, 'legacy'); + assert.equal(served.connected, true); + assert.deepEqual((await framework.executeToolCall(call('mcpl--same--a1'))).data, [{ type: 'text', text: 'ok' }], 'its tools reach the legacy server'); + }); +}); + +test('a disconnect during a legacy connect waits for it to land, then removes it whole', async () => { + await withFramework(async ({ framework, dir }) => { + const log = join(dir, 'legacy.log'); + const pending = framework.connectMcplServer(legacy('same', 'slow', log, { SLOW_INIT_MS: '600' })); + await until(() => launches(log).length === 1, 'the legacy launch, handshaking'); + const disconnecting = framework.disconnectMcplServer('same'); + await assert.rejects(framework.connectMcplServer(legacy('same', 'ok', log)), /"same" is still being disconnected/); + await Promise.all([pending, disconnecting]); + assert.equal(status(framework, 'same'), undefined); + assert.deepEqual(toolsOf(framework, 'same'), []); + assert.equal(alive(launches(log)[0]!), false, 'its child was reaped'); + }); +}); + +test('modern: a disconnect that cannot reap keeps the server registered; a connect is refused until a retried disconnect confirms the exit', async () => { + await withFramework(async ({ framework, dir }) => { + const log = join(dir, 'raw.jsonl'); + const config = modern('m', '', log); + const reaping = unreapable(); + await framework.connectMcplServer(config); + await assert.rejects( + framework.disconnectMcplServer('m'), + /could not be reaped\. MCP server "m" stays registered, closed, until its exit is confirmed: disconnect it again to retry the reap/, + ); + assert.equal(status(framework, 'm')?.connected, false, 'still listed, closed'); + await assert.rejects(framework.connectMcplServer(config), /"m" is already registered \(not connected and not reconnecting: disconnect it first\)/); + assert.equal(launches(log).length, 1, 'no launch beside the unreaped child'); + + await killAndWait(reaping.stuck[0]!); + reaping.active = false; + await framework.disconnectMcplServer('m'); + assert.equal(status(framework, 'm'), undefined, 'the retried disconnect completed'); + await framework.connectMcplServer(config); + assert.equal(status(framework, 'm')?.connected, true, 'the id is free'); + }); +}); + +test('legacy: a disconnect that cannot reap keeps the server registered; a connect is refused until a retried disconnect confirms the exit', async () => { + await withFramework(async ({ framework, dir }) => { + const log = join(dir, 'legacy.log'); + const config = legacy('l', 'ok', log); + const reaping = unreapable(); + await framework.connectMcplServer(config); + await assert.rejects( + framework.disconnectMcplServer('l'), + /could not be reaped\. MCP server "l" stays registered, closed, until its exit is confirmed: disconnect it again to retry the reap/, + ); + assert.equal(status(framework, 'l')?.connected, false, 'still listed, closed'); + await assert.rejects(framework.connectMcplServer(config), /"l" is already registered \(not connected and not reconnecting: disconnect it first\)/); + assert.equal(launches(log).length, 1, 'no launch beside the unreaped child'); + + await killAndWait(reaping.stuck[0]!); + reaping.active = false; + await framework.disconnectMcplServer('l'); + assert.equal(status(framework, 'l'), undefined, 'the retried disconnect completed'); + await framework.connectMcplServer(config); + assert.equal(status(framework, 'l')?.connected, true, 'the id is free'); + }); +}); + +test('modern: a failed connect whose cleanup cannot reap stays registered until a disconnect confirms the exit', async () => { + await withFramework(async ({ framework, dir }) => { + const log = join(dir, 'raw.jsonl'); + const reaping = unreapable(); + // The SDK reports a refused server/discover as a failed version negotiation. + await assert.rejects( + framework.connectMcplServer(modern('m', 'reject-discover', log)), + /server\/discover .*; its cleanup also failed: .*could not be reaped\. MCP server "m" stays registered, closed, until its exit is confirmed: disconnect it to retry the reap/, + ); + await assert.rejects(framework.connectMcplServer(modern('m', '', log)), /"m" is already registered \(not connected and not reconnecting/); + assert.equal(launches(log).length, 1); + + await killAndWait(reaping.stuck[0]!); + reaping.active = false; + await framework.disconnectMcplServer('m'); + await framework.connectMcplServer(modern('m', '', log)); + assert.equal(status(framework, 'm')?.connected, true); + }); +}); + +test('legacy: a failed connect whose cleanup cannot reap stays registered until a disconnect confirms the exit', async () => { + await withFramework(async ({ framework, dir }) => { + const log = join(dir, 'legacy.log'); + const reaping = unreapable(); + await assert.rejects( + framework.connectMcplServer(legacy('l', 'error', log)), + /initialize failed on purpose; its cleanup also failed: .*could not be reaped\. MCP server "l" stays registered, closed, until its exit is confirmed: disconnect it to retry the reap/, + ); + await assert.rejects(framework.connectMcplServer(legacy('l', 'ok', log)), /"l" is already registered \(not connected and not reconnecting/); + assert.equal(launches(log).length, 1); + + await killAndWait(reaping.stuck[0]!); + reaping.active = false; + await framework.disconnectMcplServer('l'); + await framework.connectMcplServer(legacy('l', 'ok', log)); + assert.equal(status(framework, 'l')?.connected, true); + }); +}); diff --git a/test/mcpl-legacy-lifetime.test.ts b/test/mcpl-legacy-lifetime.test.ts new file mode 100644 index 000000000..8e836bb61 --- /dev/null +++ b/test/mcpl-legacy-lifetime.test.ts @@ -0,0 +1,229 @@ +/** + * The legacy engine's lifetime contract (room-293, greptile's review of + * PR #262): one owner for every launch, the rule the modern engine keeps. + * - a configuration it can't use is refused before anything is dialed, and + * never turned into a retry stub; + * - concurrent close() calls share one teardown and its verdict; + * - a failed launch that can't be reaped is held, never orphaned: connect() + * hands its owner to the caller, connectWithReconnect() and the reconnect + * loop halt, and close() retries the reap; + * - close() during a reconnect handshake ends that launch before resolving; + * - no reconnect launches while the previous child may still be running. + */ +import { test, afterEach } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtempSync, readFileSync, existsSync, rmSync } from 'node:fs'; +import type { AddressInfo } from 'node:net'; +import { tmpdir } from 'node:os'; +import { join, dirname } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { WebSocketServer } from 'ws'; + +import { McplServerConnection, McplUnreapedLaunchError } from '../src/mcpl/server-connection.js'; +import { StdioTransport } from '../src/mcpl/transport.js'; +import type { McplHostCapabilities, McplServerConfig } from '../src/mcpl/types.js'; + +const here = dirname(fileURLToPath(import.meta.url)); +const SERVER = join(here, 'fixtures', 'legacy-version-server.mjs'); +const HOST_CAPS: McplHostCapabilities = { version: '0.5', pushEvents: true }; + +const cleanups: Array<() => Promise | void> = []; +afterEach(async () => { + while (cleanups.length) await cleanups.pop()!(); +}); + +function scratchLog(): string { + const dir = mkdtempSync(join(tmpdir(), 'legacy-lifetime-')); + cleanups.push(() => rmSync(dir, { recursive: true, force: true })); + return join(dir, 'starts.log'); +} + +/** The pid of each launch, in order. */ +const starts = (log: string): number[] => + existsSync(log) ? readFileSync(log, 'utf8').split('\n').filter(Boolean).map((l) => Number(l.split(' ')[1])) : []; +const alive = (pid: number) => { try { process.kill(pid, 0); return true; } catch { return false; } }; + +async function until(cond: () => boolean, what: string, ms = 10_000): Promise { + const deadline = Date.now() + ms; + while (!cond() && Date.now() < deadline) await new Promise((r) => setTimeout(r, 20)); + assert.ok(cond(), `in time: ${what}`); +} + +function config(mode: string, log: string, extra: Partial = {}): McplServerConfig { + return { id: 'legacy', command: process.execPath, args: [SERVER, mode, log], ...extra }; +} +const RECONNECT = { reconnect: true, reconnectIntervalMs: 20, reconnectMaxIntervalMs: 40 } as const; + +/** + * Launches from the `from`-th on can't be signalled, so their cleanup can't + * reap them (Olive-1804's pattern in mcp-modern-outcomes). `kill` really ends + * one; cleanup restores spawning and ends any left. + */ +function unreapableFrom(from: number): Array<{ pid: number; kill: () => void }> { + const realSpawn = StdioTransport.spawn; + const stuck: Array<{ pid: number; kill: () => void }> = []; + let launches = 0; + (StdioTransport as unknown as { spawn: typeof StdioTransport.spawn }).spawn = (c) => { + const transport = realSpawn.call(StdioTransport, c); + if (++launches >= from) { + const child = (transport as unknown as { child: { pid: number; kill: (signal?: string) => boolean } }).child; + const realKill = child.kill.bind(child); + child.kill = () => true; + stuck.push({ pid: child.pid, kill: () => { realKill('SIGKILL'); } }); + } + return transport; + }; + cleanups.push(() => { + (StdioTransport as unknown as { spawn: typeof StdioTransport.spawn }).spawn = realSpawn; + for (const s of stuck) s.kill(); + }); + return stuck; +} + +async function killAndWait(stuck: { pid: number; kill: () => void }): Promise { + stuck.kill(); + await until(() => !alive(stuck.pid), `child ${stuck.pid} gone`); +} + +test('a configuration the engine cannot use is refused before anything is dialed, and never retried', async () => { + const wss = new WebSocketServer({ host: '127.0.0.1', port: 0 }); + await new Promise((r) => wss.once('listening', r)); + cleanups.push(() => new Promise((r) => wss.close(() => r()))); + let dials = 0; + wss.on('connection', (socket) => { dials++; socket.close(); }); + const url = `ws://127.0.0.1:${(wss.address() as AddressInfo).port}/mcpl`; + + const modernOnWs = { id: 'ws', url, protocol: 'modern' as const }; + await assert.rejects(McplServerConnection.connect(modernOnWs, HOST_CAPS), /"protocol" applies only to stdio/); + // With reconnect too: a configuration error is thrown, not turned into a + // stub that retries it forever. + await assert.rejects(McplServerConnection.connectWithReconnect({ ...modernOnWs, ...RECONNECT }, HOST_CAPS), /"protocol" applies only to stdio/); + await assert.rejects( + McplServerConnection.connectWithReconnect({ id: 'ftp', url: 'ftp://127.0.0.1/x', ...RECONNECT }, HOST_CAPS), + /url must be ws:\/\/ or wss:\/\//, + ); + await new Promise((r) => setTimeout(r, 200)); + assert.equal(dials, 0, 'nothing was dialed'); +}); + +test('concurrent close() calls share the teardown: none resolves while the child still runs', async () => { + const log = scratchLog(); + const connection = await McplServerConnection.connect(config('ok', log, { env: { IGNORE_SIGTERM: '1' } }), HOST_CAPS); + connection.ready(); + const [pid] = starts(log); + const order: string[] = []; + const first = connection.close().then(() => order.push(`first:${alive(pid!) ? 'alive' : 'gone'}`)); + const second = connection.close().then(() => order.push(`second:${alive(pid!) ? 'alive' : 'gone'}`)); + await Promise.all([first, second]); + assert.deepEqual(order, ['first:gone', 'second:gone']); +}); + +test('a close that cannot reap fails, and so do concurrent ones; a later close re-checks and settles', async () => { + const log = scratchLog(); + const stuck = unreapableFrom(1); + const connection = await McplServerConnection.connect(config('ok', log), HOST_CAPS); + connection.ready(); + const settled: string[] = []; + const outcome = (name: string) => [() => settled.push(`${name}:ok`), (e: Error) => settled.push(`${name}:${e.message}`)] as const; + const first = connection.close().then(...outcome('first')); + const second = connection.close().then(...outcome('second')); + await new Promise((r) => setTimeout(r, 1000)); + assert.deepEqual(settled, [], 'neither resolves while the teardown is pending'); + await Promise.all([first, second]); + assert.equal(settled.length, 2); + assert.match(settled[0]!, /^first:MCPL server "legacy" did not close cleanly: .*could not be reaped/); + assert.match(settled[1]!, /^second:.*could not be reaped/, 'the second re-checked, rather than claiming success'); + await killAndWait(stuck[0]!); + await connection.close(); +}); + +test('a failed connect that cannot be reaped hands its owner to the caller, who can retry the reap', async () => { + const log = scratchLog(); + const stuck = unreapableFrom(1); + const error = await McplServerConnection.connect(config('error', log), HOST_CAPS).then(() => null, (e: unknown) => e); + assert.ok(error instanceof McplUnreapedLaunchError, String(error)); + assert.match(error.message, /initialize failed on purpose; its cleanup also failed: .*could not be reaped/); + assert.match((error.cause as Error).message, /initialize failed on purpose/); + const owner = error.connection; + assert.equal(owner.isConnected, false); + assert.equal(owner.willReconnect, false); + await assert.rejects(owner.close(), /could not be reaped/, 'still running: the reap fails again'); + await killAndWait(stuck[0]!); + await owner.close(); +}); + +test('with reconnect, a failed first launch that cannot be reaped halts instead of retrying', async () => { + const log = scratchLog(); + const stuck = unreapableFrom(1); + const connection = await McplServerConnection.connectWithReconnect(config('error', log, RECONNECT), HOST_CAPS); + cleanups.push(() => connection.close().catch(() => {})); + const failures: Array<{ error: string; attempt: number; permanent?: boolean }> = []; + connection.on('connect-failed', (f) => failures.push(f)); + connection.ready(); + await until(() => failures.length > 0, 'the connect-failed event'); + assert.equal(failures[0]!.permanent, true); + assert.match(failures[0]!.error, /initialize failed on purpose; the previous launch could not be reaped .*reconnecting is halted/); + assert.equal(connection.willReconnect, false); + await new Promise((r) => setTimeout(r, 300)); + assert.equal(starts(log).length, 1, 'no launch beside the unreaped one'); + await killAndWait(stuck[0]!); + await connection.close(); +}); + +test('a reconnect launch that cannot be reaped halts reconnecting; close() settles it once it has exited', async () => { + const log = scratchLog(); + const stuck = unreapableFrom(2); + const connection = await McplServerConnection.connectWithReconnect(config('ok-then-error', log, RECONNECT), HOST_CAPS); + cleanups.push(() => connection.close().catch(() => {})); + connection.ready(); + const failures: Array<{ error: string; attempt: number; permanent?: boolean }> = []; + connection.on('reconnect-failed', (f) => failures.push(f)); + await assert.rejects(connection.sendToolsCall('die', {})); + await until(() => failures.some((f) => f.permanent), 'a permanent reconnect failure'); + assert.match(failures.find((f) => f.permanent)!.error, /initialize failed on purpose; the previous launch could not be reaped .*reconnecting is halted/); + assert.equal(connection.willReconnect, false); + await new Promise((r) => setTimeout(r, 300)); + assert.equal(starts(log).length, 2, 'no third launch while the second is unreaped'); + await assert.rejects(connection.close(), /could not be reaped/); + await killAndWait(stuck[0]!); + await connection.close(); +}); + +test('close() during a reconnect handshake ends that launch before resolving, and nothing is adopted', async () => { + const log = scratchLog(); + const connection = await McplServerConnection.connectWithReconnect(config('ok-then-hang', log, RECONNECT), HOST_CAPS); + connection.ready(); + let reconnected = false; + connection.on('reconnect', () => { reconnected = true; }); + await assert.rejects(connection.sendToolsCall('die', {})); + await until(() => starts(log).length === 2, 'the reconnect launch, waiting on initialize'); + const second = starts(log)[1]!; + const began = Date.now(); + await connection.close(); + assert.ok(Date.now() - began < 5_000, 'promptly: the handshake was ended, not waited out (30 s)'); + assert.equal(alive(second), false, 'the launch in flight was reaped before close() resolved'); + assert.equal(reconnected, false); + assert.equal(connection.isConnected, false); + await new Promise((r) => setTimeout(r, 200)); + assert.equal(starts(log).length, 2, 'and no later launch'); +}); + +test('a reconnect never launches while the previous child may still be running', async () => { + const log = scratchLog(); + const connection = await McplServerConnection.connectWithReconnect( + config('ok', log, { ...RECONNECT, env: { IGNORE_SIGTERM: '1' } }), + HOST_CAPS, + ); + cleanups.push(() => connection.close().catch(() => {})); + connection.ready(); + const [first] = starts(log); + let reconnected = false; + connection.on('reconnect', () => { reconnected = true; }); + // Recycle it as the framework does after an awareness failure: the host + // closes the transport itself, and the child ignores SIGTERM for 2 s. + const recycled = connection.reconnectAfterFailure(); + await until(() => starts(log).length === 2, 'the next launch'); + assert.equal(alive(first!), false, 'it started only once the first child was gone'); + await recycled; + await until(() => reconnected, 'reconnected'); +}); From 7e0a97c2934c5218c3f7f6d5438cde71f88a69ed Mon Sep 17 00:00:00 2001 From: Oscar-1826 Date: Thu, 8 Oct 2026 05:06:59 -0400 Subject: [PATCH 0068/1112] fix(mcp): saved tool results never overwrite each other; pin the script cap Greptile's review of #262 at ee4d424, findings 5 and 8. Finding 5: a saved payload's name was the date plus the call id, sanitized and cut to 80 characters, so distinct ids could share it (`call:a` and `call/a`, or ids that differ only past the kept prefix), and a later writeBinary replaced the earlier result's file. The name now ends in a full randomUUID() after the readable part. The oversized-script save had the same property with 8 hex characters; it now uses a full UUID too. Finding 8: the script view of structured results had no coverage at its size cap or for either workspace outcome. A fixture tool `sized` pads structuredContent so the serialized object's length is exact (the cap measures that length, Iris-1827, room-293 #54849). Through a real modern server and a real WorkspaceModule (store wired as the host wires it): - exactly 5,000,000 characters arrive inline, whole; - one more is saved, and the script gets {isError, oversized} only; the saved object, read back through the workspace, is the whole {content, structuredContent, isError}, and `error` too on a tool error, whose isError stays visible without reading the file; - no workspace, or a write that fails, gives the explicit failure to deliver (savedTo null, "the tool may already have completed"), never a success. A payload test saves four results whose ids collide after sanitizing or truncation, and reads each file back with its own bytes. Mutation check: dropping the UUID from the label fails the payload test. Receipt, both commits (VPS via on-vps, node v24.21.0, npm 11.19.0; clean tree, sources sha256 b9e5424066c10bd5; `npm run clean && npm run build`, then every test file standalone, 4 at a time): 136 files, 1469 tests, 1468 pass. The one failure is conversation-routing.test.js at file level ("Queue is closed" after the fork channel_publish test ends), which fails the same way on the base ee4d424, 3 of 3, and which Petra-1779 recorded as pre-existing in room-272. --- .../mcp-standard-tool-results.changed.md | 7 +- src/framework.ts | 8 +- test/fixtures/modern-tools.mjs | 8 ++ test/mcp-modern-framework.test.ts | 118 +++++++++++++++++- 4 files changed, 133 insertions(+), 8 deletions(-) diff --git a/changelog.d/mcp-standard-tool-results.changed.md b/changelog.d/mcp-standard-tool-results.changed.md index 724ed01f2..5294e3bcd 100644 --- a/changelog.d/mcp-standard-tool-results.changed.md +++ b/changelog.d/mcp-standard-tool-results.changed.md @@ -3,9 +3,10 @@ content types: - text stays text, and inline images stay native; - audio and binary resources are saved to the workspace (`tool-results/`), - with a bounded stub saying where. If a payload can't be saved, the result - is reported as incomplete (`isError`), noting the tool may already have - completed, never as a success with part of it missing; + each under a unique name, with a bounded stub saying where. If a payload + can't be saved, the result is reported as incomplete (`isError`), noting + the tool may already have completed, never as a success with part of it + missing; - a `resource_link` is shown as a reference and never fetched; - an embedded text resource shows its text. diff --git a/src/framework.ts b/src/framework.ts index 86de9c349..3ffec37a3 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -11909,7 +11909,7 @@ export class AgentFramework { const workspace = this.getWorkspaceModule(); const mount = workspace ? this.firstWritableMountName(workspace) : null; if (workspace && mount) { - const path = `${mount}/tool-results/${new Date().toISOString().slice(0, 10)}-script-${randomUUID().slice(0, 8)}.json`; + const path = `${mount}/tool-results/${new Date().toISOString().slice(0, 10)}-script-${randomUUID()}.json`; const written = await workspace.writeBinary(path, Buffer.from(json, 'utf8'), 'application/json').catch(() => null); if (written?.success) { return JSON.stringify({ isError: object.isError, oversized: { chars: json.length, savedTo: path } }); @@ -15422,7 +15422,11 @@ export class AgentFramework { ): Promise { const hasStructured = Object.prototype.hasOwnProperty.call(result, 'structuredContent'); if (!mcplPeer) { - const label = `${new Date().toISOString().slice(0, 10)}-${callId}`.replace(/[^A-Za-z0-9._-]/g, '_').slice(0, 80); + // The date and call id make a saved payload's name readable; the UUID + // makes it unique. Sanitizing and truncating map distinct ids to one + // label (`call:a` and `call/a`), and a later write would replace the + // earlier result's file. + const label = `${new Date().toISOString().slice(0, 10)}-${callId}`.replace(/[^A-Za-z0-9._-]/g, '_').slice(0, 80) + `-${randomUUID()}`; const workspace = this.getWorkspaceModule(); const mount = workspace ? this.firstWritableMountName(workspace) : null; const save = workspace && mount diff --git a/test/fixtures/modern-tools.mjs b/test/fixtures/modern-tools.mjs index d156277a6..5fb435165 100644 --- a/test/fixtures/modern-tools.mjs +++ b/test/fixtures/modern-tools.mjs @@ -28,6 +28,14 @@ export function createServer(opts = {}) { async () => ({ content: [{ type: 'text', text: 'Found 2 rows.' }], structuredContent: { rows: [{ id: 1 }, { id: 2 }] } })); server.registerTool('structured_zero', { description: 'Falsy structured value', inputSchema: obj() }, async () => ({ content: [], structuredContent: 0 })); + server.registerTool('sized', { + description: 'Structured content padded to `pad` characters, optionally a tool error', + inputSchema: obj({ pad: { type: 'number' }, isError: { type: 'boolean' } }), + }, async (args) => ({ + content: [], + structuredContent: { pad: 'x'.repeat(Number(args.pad ?? 0)) }, + ...(args.isError ? { isError: true } : {}), + })); server.registerTool('scored_image', { description: 'An image with machine-readable scores', inputSchema: obj() }, async () => ({ content: [{ type: 'image', data: 'iVBORw0KGgo=', mimeType: 'image/png' }], diff --git a/test/mcp-modern-framework.test.ts b/test/mcp-modern-framework.test.ts index 2a0ecb9aa..570993c81 100644 --- a/test/mcp-modern-framework.test.ts +++ b/test/mcp-modern-framework.test.ts @@ -16,7 +16,7 @@ import { tmpdir } from 'node:os'; import { join, dirname } from 'node:path'; import { fileURLToPath } from 'node:url'; -import { AgentFramework } from '../src/index.js'; +import { AgentFramework, WorkspaceModule } from '../src/index.js'; import type { EventResponse, Module, ModuleContext, ProcessEvent, ProcessState, ToolDefinition, ToolResult } from '../src/index.js'; import { MockMembrane, createMockResponse } from './helpers/mock-membrane.js'; @@ -50,14 +50,18 @@ const lines = (file: string) => (existsSync(file) ? readFileSync(file, 'utf8').s interface Harness { framework: AgentFramework; membrane: MockMembrane; dir: string } -async function withModern(fn: (h: Harness) => Promise, extraServers: Record[] = []): Promise { +async function withModern( + fn: (h: Harness) => Promise, + extraServers: Record[] = [], + extraModules: Module[] = [], +): Promise { const dir = mkdtempSync(join(tmpdir(), 'modern-framework-')); const membrane = new MockMembrane(); const framework = await AgentFramework.create({ storePath: join(dir, 'store'), membrane: membrane.asMembrane(), agents: [{ name: 'agent', model: 'test-model', systemPrompt: 'test' }], - modules: [new Waker()], + modules: [new Waker(), ...extraModules], mcplServers: [ { id: 'modern', @@ -181,6 +185,114 @@ test('scripts get structured results whole, as one JSON object', async () => { }); }); +// The script cap measures the serialized object's length (Iris-1827, +// room-293): `sized` pads structuredContent so that length is exact. +const SCRIPT_CAP = 5_000_000; +const sizedObject = (pad: number, isError = false) => ({ + content: [], + structuredContent: { pad: 'x'.repeat(pad) }, + isError, + // A tool error with no text carries the framework's own error text. + ...(isError ? { error: 'Tool call failed' } : {}), +}); +/** The pad that makes the script's JSON exactly `chars` long. */ +const padFor = (chars: number, isError = false) => chars - JSON.stringify(sizedObject(0, isError)).length; +const scriptHandle = (framework: AgentFramework) => (framework as unknown as { + handleScriptToolCall: (agent: string, tool: string, args: Record) => Promise; +}).handleScriptToolCall.bind(framework); + +async function withWorkspace(fn: (h: Harness & { workspace: WorkspaceModule }) => Promise): Promise { + const board = mkdtempSync(join(tmpdir(), 'modern-board-')); + const workspace = new WorkspaceModule({ mounts: [{ name: 'board', path: board, mode: 'read-write', watch: 'never' }] }); + try { + await withModern((h) => { + // The host wires the store after create(), as conhost does. + workspace.initStore(h.framework.getStore()); + return fn({ ...h, workspace }); + }, [], [workspace]); + } finally { + rmSync(board, { recursive: true, force: true }); + } +} + +/** A saved file's bytes, read back through the workspace as a resident would. */ +async function readSaved(workspace: WorkspaceModule, path: string): Promise { + const read = await workspace.readBinary(path); + assert.ok('data' in read, `readable: ${path} (${'error' in read ? read.error : ''})`); + return read.data.toString('utf8'); +} + +async function savedObject(workspace: WorkspaceModule, savedTo: string): Promise { + assert.match(savedTo, /^board\/tool-results\/\d{4}-\d{2}-\d{2}-script-[0-9a-f-]{36}\.json$/); + return JSON.parse(await readSaved(workspace, savedTo)); +} + +test('script results: exactly at the cap arrive inline; one past it are saved whole, a tool error included', async () => { + await withWorkspace(async ({ framework, workspace }) => { + framework.start(); + const handle = scriptHandle(framework); + + const at = await handle('agent', 'mcpl--modern--sized', { pad: padFor(SCRIPT_CAP) }); + assert.equal(at.length, SCRIPT_CAP); + assert.deepEqual(JSON.parse(at), sizedObject(padFor(SCRIPT_CAP))); + + const over = JSON.parse(await handle('agent', 'mcpl--modern--sized', { pad: padFor(SCRIPT_CAP + 1) })); + assert.deepEqual(Object.keys(over).sort(), ['isError', 'oversized']); + assert.equal(over.isError, false); + assert.equal(over.oversized.chars, SCRIPT_CAP + 1); + assert.deepEqual(await savedObject(workspace, over.oversized.savedTo), sizedObject(padFor(SCRIPT_CAP + 1)), 'the whole object, recoverable'); + + const failed = JSON.parse(await handle('agent', 'mcpl--modern--sized', { pad: padFor(SCRIPT_CAP + 1, true), isError: true })); + assert.equal(failed.isError, true, 'a tool error stays visible without reading the file'); + assert.equal(failed.oversized.chars, SCRIPT_CAP + 1); + assert.notEqual(failed.oversized.savedTo, over.oversized.savedTo); + assert.deepEqual(await savedObject(workspace, failed.oversized.savedTo), sizedObject(padFor(SCRIPT_CAP + 1, true), true)); + }); +}); + +test('script results past the cap: no workspace, or a failed write, is a failure to deliver, never a success', async () => { + const assertUndelivered = (out: { isError: boolean; error: string; oversized: unknown }) => { + assert.equal(out.isError, true); + assert.match(out.error, /exceeds the 5000000-char script result cap and could not be saved to the workspace, so it could not be delivered; the tool may already have completed/); + assert.deepEqual(out.oversized, { chars: SCRIPT_CAP + 1, savedTo: null }); + }; + await withModern(async ({ framework }) => { + framework.start(); + assertUndelivered(JSON.parse(await scriptHandle(framework)('agent', 'mcpl--modern--sized', { pad: padFor(SCRIPT_CAP + 1) }))); + }); + await withWorkspace(async ({ framework, workspace }) => { + framework.start(); + workspace.writeBinary = async () => ({ success: false, error: 'disk full', isError: true }); + assertUndelivered(JSON.parse(await scriptHandle(framework)('agent', 'mcpl--modern--sized', { pad: padFor(SCRIPT_CAP + 1) }))); + }); +}); + +test('saved payloads: call ids that sanitize to one label still get distinct files', async () => { + await withWorkspace(async ({ framework, workspace }) => { + const read = (framework as unknown as { + mcpToolResult: (result: unknown, callId: string, mcplPeer: boolean) => Promise; + }).mcpToolResult.bind(framework); + const audio = (bytes: string) => ({ content: [{ type: 'audio', data: Buffer.from(bytes).toString('base64'), mimeType: 'audio/wav' }] }); + const savedTo = (result: ToolResult) => /saved to workspace file (\S+)\]/.exec(String(result.data))?.[1]; + // `call:a` and `call/a` sanitize alike; so do ids that differ only past the kept prefix. + const long = 'x'.repeat(90); + const results = [ + await read(audio('first'), 'call:a', false), + await read(audio('second'), 'call/a', false), + await read(audio('third'), `${long}-1`, false), + await read(audio('fourth'), `${long}-2`, false), + ]; + const paths = results.map(savedTo); + assert.ok(paths.every((p) => typeof p === 'string'), JSON.stringify(results)); + assert.equal(new Set(paths).size, 4, `distinct paths: ${paths.join(', ')}`); + assert.deepEqual( + await Promise.all(paths.map((p) => readSaved(workspace, p!))), + ['first', 'second', 'third', 'fourth'], + 'each file keeps its own call\'s bytes', + ); + }); +}); + test('MCPL policy on a modern server is a configuration error at connect', async () => { await withModern(async ({ framework }) => { await assert.rejects( From c35d28692cf6c561fb7cd897754c75ab5a5ed3f2 Mon Sep 17 00:00:00 2001 From: Oscar-1826 Date: Thu, 8 Oct 2026 05:28:04 -0400 Subject: [PATCH 0069/1112] fix(mcp): one reconnect attempt at a time; a disconnect ends a connect in flight Iris-1827's review of 7e0a97c (room-293 #55410) found a blocking hole in the legacy retry ownership, reproduced through public methods on the VPS: connectWithReconnect with an `ok-then-hang` server, `die`, wait for launch 2 (initialize pending), `reconnectAfterFailure()`, wait for launch 3, then `close()`. The recycle scheduled a second attempt while the first was in flight; attemptReconnect overwrote `reconnecting`, so close() aborted and reaped only the newer handshake and resolved with launch 2 still alive. - attemptReconnect is single-owner: while an attempt is in flight, a new request starts nothing. That attempt either adopts its launch or schedules the next retry itself, so nothing needed is dropped; a timer can't fire inside an attempt's tail, since the tail is microtasks. Found in my own re-read while the review ran: a disconnect waited out a legacy connect's post-handshake setup on a live connection, as long as that setup's requests took, where the old code's disconnect ended it by closing. (Iris checked one such request: featureSets/update has its own 15 s bound, so it isn't left unbounded by requestTimeoutMs 0.) A disconnect now ends a connect in flight for both families: - it closes what the id has at once: a modern connection (which ends its connect), and a registered legacy one through registry.removeServer, whose failure path keeps the connection registered; - a legacy connect still handshaking checks, when the handshake finishes, whether a disconnect has taken its hold (`displaced`, beside the existing stop() check), closes its connection and fails with "was not connected: it was disconnected while connecting"; - the disconnect then waits for it to settle, and removes what it registered, re-checking anything kept after a failed close. A connect already past its handshake when the disconnect closes it may resolve (its setup tolerates the closed connection) before the removal; one still handshaking rejects. Both end in the same teardown (Iris exercised the first with a server hanging on featureSets/update and requestTimeoutMs 0: the disconnect completed in 8 ms and reaped the child). Tests: - mcpl-legacy-lifetime: Iris's interleaving. No second attempt starts beside the one in flight; close() resolves promptly with every launch reaped. - mcp-server-ownership: a disconnect during a legacy handshake ends that connect when the handshake finishes (it rejects "disconnected while connecting"), removes it whole, and nothing else launches; a disconnect during a hanging modern connect ends it at once (under 5 s, not the 30 s connect timeout). - Mutations: without the single-owner guard, Iris's test fails (and the orphaned child outlives the file); without the displaced check, the legacy-handshake test fails; the earlier ownership mutations still fail their tests with the reworked disconnect. Receipt (VPS via on-vps, node v24.21.0): mcpl-legacy-lifetime 9/9, mcp-server-ownership 10/10, mcp-modern-framework 14/14, mcp-modern-outcomes 14/14, mcpl-legacy-version 11/11. --- .../mcp-standard-tool-results.changed.md | 12 ++-- src/framework.ts | 57 ++++++++++++------- src/mcpl/server-connection.ts | 6 +- test/mcp-server-ownership.test.ts | 20 ++++++- test/mcpl-legacy-lifetime.test.ts | 17 ++++++ 5 files changed, 85 insertions(+), 27 deletions(-) diff --git a/changelog.d/mcp-standard-tool-results.changed.md b/changelog.d/mcp-standard-tool-results.changed.md index 5294e3bcd..27782b0a5 100644 --- a/changelog.d/mcp-standard-tool-results.changed.md +++ b/changelog.d/mcp-standard-tool-results.changed.md @@ -41,8 +41,8 @@ `connectWithReconnect()` returns that connection, halted (`connect-failed` with `permanent: true`, which raises the ops alert), and a reconnect attempt halts the loop the same way - (`reconnect-failed`, `permanent: true`). A reconnect attempt never - starts before the previous child has exited. + (`reconnect-failed`, `permanent: true`). Reconnect attempts run one at + a time, and never start before the previous child has exited. - `close()` during a reconnect handshake ends that launch before it resolves. - In the framework, a disconnect or failed connect whose cleanup can't @@ -52,9 +52,11 @@ and completes the removal. - One owner per server id, across both families: a connect is refused while another connect for that id is in flight (a legacy handshake included) or - a disconnect is still tearing it down, and a disconnect waits for a - connect in flight to settle before removing it, so a teardown never - removes what a later connection registered. + a disconnect is still tearing it down. A disconnect closes a registered + connection immediately, and a legacy connect still handshaking is cleaned + up when its handshake settles (a modern connect in flight, or such a + legacy one, fails with "was not connected"); then it removes what was + registered. A teardown never removes what a later connection registered. - A server configuration naming no usable transport is now an error before anything is spawned or dialed. That covers a `transport` that doesn't match the url's scheme, `transport: 'http'` without a url, and an diff --git a/src/framework.ts b/src/framework.ts index 3ffec37a3..394fc991c 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -13422,10 +13422,15 @@ export class AgentFramework { * teardown is done, one owner at a time. So a connect is refused while * anything holds its id, and a teardown never removes what a later * connection registered. A disconnect takes the hold over, and waits for - * the operation it displaced (`previous`). Each release frees only its own - * hold. + * the operation it displaced (`previous`); a connect that finds itself + * `displaced` ends rather than setting up a server being removed. Each + * release frees only its own hold. */ - private holdMcpServerId(id: string, kind: 'connect' | 'disconnect'): { previous: Promise | undefined; release: () => void } { + private holdMcpServerId(id: string, kind: 'connect' | 'disconnect'): { + previous: Promise | undefined; + displaced: () => boolean; + release: () => void; + } { const ops = (this.mcpServerOps ??= new Map()); const previous = ops.get(id)?.settled; let settle!: () => void; @@ -13433,6 +13438,7 @@ export class AgentFramework { ops.set(id, op); return { previous, + displaced: () => ops.get(id) !== op, release: () => { if (ops.get(id) === op) ops.delete(id); settle(); @@ -13853,17 +13859,19 @@ export class AgentFramework { } const hold = this.holdMcpServerId(config.id, 'connect'); try { - await this.connectHeldMcplServer(config, family, deferAwareness); + await this.connectHeldMcplServer(config, family, deferAwareness, hold.displaced); } finally { hold.release(); } } - /** The part of a connect that runs holding its id. */ + /** The part of a connect that runs holding its id. `displaced`: a + * disconnect has taken the id over and is waiting for this connect. */ private async connectHeldMcplServer( config: import('./mcpl/types.js').McplServerConfig, family: 'legacy' | 'modern', deferAwareness: boolean, + displaced: () => boolean, ): Promise { if (!this.mcplServerRegistry || !this.mcplHostCapabilities) { throw new Error('MCPL subsystem is not initialized'); @@ -13935,17 +13943,21 @@ export class AgentFramework { throw error; } // stop() began while the handshake ran: its teardown has already - // collected the registry, so this connection would outlive it. Close it. - if (this.mcpServerAdmissionClosed) { + // collected the registry, so this connection would outlive it. Or a + // disconnect took the id over and is waiting for this connect. Either + // way, close it rather than set up a server that is being removed. + if (this.mcpServerAdmissionClosed || displaced()) { + const why = this.mcpServerAdmissionClosed ? 'the framework stopped while it was connecting' : 'it was disconnected while connecting'; // A cleanup that can't reap the child is reported with the refusal, - // not in place of it and not dropped. + // not in place of it and not dropped. The registry keeps such a + // connection, so the id stays held. let cleanup = ''; try { await this.mcplServerRegistry.removeServer(config.id); } catch (error) { cleanup = `; its cleanup also failed: ${error instanceof Error ? error.message : String(error)}`; } - throw new Error(`MCP server "${config.id}" was not connected: the framework stopped while it was connecting${cleanup}`); + throw new Error(`MCP server "${config.id}" was not connected: ${why}${cleanup}`); } // Wire listeners before either startup staging or the runtime global gate @@ -14360,11 +14372,12 @@ export class AgentFramework { * No-op-ish if the server is not connected (still clears routing state). * * It holds the id throughout (holdMcpServerId): a connect for it is - * refused meanwhile, and a connect already in flight settles first (a - * modern one is closed, which ends it). A close that can't confirm a child - * exited fails the disconnect before anything is removed: the server stays - * registered, closed, so nothing launches beside a child that may still be - * running, and disconnecting again retries the reap. + * refused meanwhile, and a connect already in flight is ended (closed, or + * displaced when its legacy handshake finishes) and settles before + * anything is removed. A close that can't confirm a child exited fails the + * disconnect before anything is removed: the server stays registered, + * closed, so nothing launches beside a child that may still be running, + * and disconnecting again retries the reap. */ async disconnectMcplServer(id: string): Promise { if (!this.mcplServerRegistry) { @@ -14373,11 +14386,15 @@ export class AgentFramework { const registry = this.mcplServerRegistry; const hold = this.holdMcpServerId(id, 'disconnect'); try { - // Closing a modern connection now ends a connect in flight, rather than - // waiting out its handshake. + // Close what the id has now, so a connect in flight ends rather than + // being waited out: a modern connect is ended by its close, and a + // legacy one already registered (setting up after its handshake) + // fails its requests. A legacy connect still handshaking ends when its + // handshake does, finding itself displaced. The registry's removal is + // the legacy close: a failed one stays registered. const modern = this.modernServers().get(id); - const modernClosed = modern?.close(); - modernClosed?.catch(() => { /* its verdict is awaited below */ }); + const closing = [modern?.close(), registry.removeServer(id)]; + for (const pending of closing) pending?.catch(() => { /* its verdict is awaited below */ }); // What held the id before (a connect still running, an earlier // disconnect) settles first, so what it registered is removed whole. await hold.previous; @@ -14385,7 +14402,9 @@ export class AgentFramework { const oldToolNames = new Set(this.mcplTools.map(t => t.name)); try { - await modernClosed; + for (const pending of closing) await pending; + // Whatever was registered since: a connect that was still + // handshaking, whose own removal couldn't reap, is re-checked here. await registry.removeServer(id); } catch (error) { const message = error instanceof Error ? error.message : String(error); diff --git a/src/mcpl/server-connection.ts b/src/mcpl/server-connection.ts index 778b91fbe..1a87641f7 100644 --- a/src/mcpl/server-connection.ts +++ b/src/mcpl/server-connection.ts @@ -1230,9 +1230,13 @@ export class McplServerConnection extends EventEmitter { * approach this builds no throwaway connection instance, so there are no * leaked listeners on a dead object. The attempt is close()'s to abort, * and it settles only once its own launch is adopted, failed or cleaned up. + * One attempt at a time: a request that arrives while one is in flight (a + * recycle, say) starts nothing, since that attempt either adopts its + * launch or schedules the next retry itself. A second attempt would take + * close()'s ownership of the first (Iris-1827, room-293 #55410). */ private async attemptReconnect(): Promise { - if (!this.reconnectEnabled || !this.config || !this.hostCapabilities) return; + if (this.reconnecting || !this.reconnectEnabled || !this.config || !this.hostCapabilities) return; const abort = new AbortController(); const done = this.reconnectOnce(this.config, this.hostCapabilities, abort.signal); this.reconnecting = { abort, done }; diff --git a/test/mcp-server-ownership.test.ts b/test/mcp-server-ownership.test.ts index f5f7e4607..65def7234 100644 --- a/test/mcp-server-ownership.test.ts +++ b/test/mcp-server-ownership.test.ts @@ -172,17 +172,33 @@ test('a modern connect is refused while a legacy connect for the same id is stil }); }); -test('a disconnect during a legacy connect waits for it to land, then removes it whole', async () => { +test('a disconnect during a legacy handshake ends that connect when the handshake finishes, and removes it whole', async () => { await withFramework(async ({ framework, dir }) => { const log = join(dir, 'legacy.log'); const pending = framework.connectMcplServer(legacy('same', 'slow', log, { SLOW_INIT_MS: '600' })); await until(() => launches(log).length === 1, 'the legacy launch, handshaking'); const disconnecting = framework.disconnectMcplServer('same'); await assert.rejects(framework.connectMcplServer(legacy('same', 'ok', log)), /"same" is still being disconnected/); - await Promise.all([pending, disconnecting]); + await assert.rejects(pending, /MCP server "same" was not connected: it was disconnected while connecting/); + await disconnecting; assert.equal(status(framework, 'same'), undefined); assert.deepEqual(toolsOf(framework, 'same'), []); assert.equal(alive(launches(log)[0]!), false, 'its child was reaped'); + assert.equal(launches(log).length, 1, 'and nothing else launched'); + }); +}); + +test('a disconnect during a modern connect ends it at once', async () => { + await withFramework(async ({ framework, dir }) => { + const log = join(dir, 'raw.jsonl'); + const pending = framework.connectMcplServer(modern('m', 'hang-discover', log)); + await until(() => lines(log).some((l) => l.includes('server/discover')), 'the launch waiting on discover'); + const began = Date.now(); + await framework.disconnectMcplServer('m'); + await assert.rejects(pending, /MCP server "m" was not connected: it was disconnected while connecting/); + assert.ok(Date.now() - began < 5_000, 'not after the 30 s connect timeout'); + assert.equal(status(framework, 'm'), undefined); + assert.equal(alive(launches(log)[0]!), false, 'its child was reaped'); }); }); diff --git a/test/mcpl-legacy-lifetime.test.ts b/test/mcpl-legacy-lifetime.test.ts index 8e836bb61..0ee93a278 100644 --- a/test/mcpl-legacy-lifetime.test.ts +++ b/test/mcpl-legacy-lifetime.test.ts @@ -208,6 +208,23 @@ test('close() during a reconnect handshake ends that launch before resolving, an assert.equal(starts(log).length, 2, 'and no later launch'); }); +test('a recycle during a reconnect in flight starts no second attempt, and close() still ends the one in flight', async () => { + // Iris-1827's interleaving (room-293 #55410): before, the recycle's attempt + // took close()'s ownership of the first, which outlived a successful close. + const log = scratchLog(); + const connection = await McplServerConnection.connectWithReconnect(config('ok-then-hang', log, RECONNECT), HOST_CAPS); + connection.ready(); + await assert.rejects(connection.sendToolsCall('die', {})); + await until(() => starts(log).length === 2, 'the reconnect launch, waiting on initialize'); + await connection.reconnectAfterFailure(); + await new Promise((r) => setTimeout(r, 400)); + assert.equal(starts(log).length, 2, 'no second attempt beside the one in flight'); + const began = Date.now(); + await connection.close(); + assert.ok(Date.now() - began < 5_000, 'promptly'); + assert.deepEqual(starts(log).filter(alive), [], 'every launch reaped when close() resolved'); +}); + test('a reconnect never launches while the previous child may still be running', async () => { const log = scratchLog(); const connection = await McplServerConnection.connectWithReconnect( From 315d12c2ed8ebf31cc6fac9b1ba4a17ba26ec57a Mon Sep 17 00:00:00 2001 From: Rupert-1859 Date: Thu, 8 Oct 2026 14:28:00 -0400 Subject: [PATCH 0070/1112] fix(drafts): refuse a dismissal while its draft's attempt is out; the receipt keeps a dismissed draft's own uncertain attempt MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit slimepriestess's approving review of #247 at 1d6d2bb noted that a `dismiss` issued while that draft's resend is in flight is accepted (the dismiss path doesn't consult draftsInFlight, while resend and {{unsent}} do), and that if the send then comes back unknown, draftState reports `dismissed`, so the receipt's one-word state hides a possibly-posted draft. Two routes record a dismissal: the drafts tool's `dismiss`, and dismissLatestBounce (`>>skip_reply {{unsent}}`, its hybrid form, and the held hybrid path beside a send). A round's calls are dispatched without awaiting each other (`drafts` is fire-and-forget), and a round's prose is routed alongside its tools, so either route can land while a resend's publish, or an {{unsent}} delivery, is out. Neither remedy as worded fits: - consulting draftsInFlight would also refuse a draft only queued in a resend batch, whose dismissal the batch deliberately honours when it reaches it (test: "a draft dismissed while the batch waits is not published"); - ranking an uncertain attempt above `dismissed` inside draftState would make a draft with an unknown attempt impossible to dismiss: it would stay `unconfirmed` and listed, its only way out a confirmDuplicate resend. So: - draftsSending: the drafts in draftsInFlight whose attempt is out right now. It is set around the awaited publish in the resend loop and in deliverWithUnsent, cleared as each publish resolves, and cleared again in the batch's finally as a backstop. The tool's dismiss is refused up front, dismissing nothing, when any draft it names is sending, and the resident is told to wait for that result; dismissLatestBounce leaves the draft and tells the resident privately (takeUnsent's busy-notice pattern). No dismissal is then recorded while that draft's attempt is out, so the resend's own reply ("The draft stays held.", "that needs confirmDuplicate: true") can't contradict a mid-send dismissal, and no "dismissed" reply is given over words that go out. A queued draft can still be dismissed. - The turn-end receipt names a dismissed draft whose own attempt may have been posted (uncertainAttempt; not inherited risk: a dismissed copy was never sent itself, and its source's risk is reported where that attempt is) with its riskText, and counts it toward "nothing confirmed". This also covers the ordinary, knowing case: an unknown resend followed by a deliberate dismiss used to leave "[delivered] nothing — draft d dismissed by you". - The changelog fragment says both. Tests (prose-drafts.test.ts): - the race through real round dispatch, `[resend d, dismiss d]` in one round, delivered and no-receipt; - a dismissal naming a sending and a queued draft dismisses nothing; - the prose route's busy notice; - a tool dismissal while an {{unsent}} delivery of that draft is out (the ownership harness's registry stub now holds deliverSpeech like publish); - the knowing case's receipt, as a third turn of the receipt test. All five fail on 1d6d2bb's source, and the fourth also fails when only deliverWithUnsent's sending mark is removed. The new assertions don't pin destinationText's rendering, which #256 changes. Design worked out with Humphrey-1858, who reviewed it. His probes showed the resend replies contradicting a mid-send dismissal (failed: "The draft stays held."; unknown: "that needs confirmDuplicate: true" for a draft a resend then refuses; delivered: a "dismissed" reply over words that went out), which is why the dismissal is refused rather than only the receipt changing, and his probe U is the {{unsent}} test. --- changelog.d/prose-drafts.added.md | 14 +++- src/framework.ts | 60 ++++++++++++-- test/prose-drafts.test.ts | 130 ++++++++++++++++++++++++++++-- 3 files changed, 188 insertions(+), 16 deletions(-) diff --git a/changelog.d/prose-drafts.added.md b/changelog.d/prose-drafts.added.md index 7e0a0a73c..85c6607ad 100644 --- a/changelog.d/prose-drafts.added.md +++ b/changelog.d/prose-drafts.added.md @@ -29,8 +29,12 @@ `confirmDuplicate` its resend needs), and the turn-end `[delivered]` receipt names every draft held in the turn by its state at turn end: still held ("held as drafts d-… (not sent …)"), delivered by the resident's resend - (listed where it landed), unconfirmed, or dismissed. A draft no notice - reached (a crash, an aborted turn) is named at the next turn's start. + (listed where it landed), unconfirmed, or dismissed. A dismissed draft + whose own attempt may already have been posted is named with that risk, + and counts like an unconfirmed one: a receipt that names one, with nothing + confirmed delivered, reads "nothing confirmed", not "nothing". A draft no + notice reached (a crash, an aborted turn) is named at the next turn's + start. - A draft is never published except by its resident's explicit resend (or `{{unsent}}`), and only that resident can list or act on it. Its notices go into the resident's own history, which residents sharing one message slot @@ -43,7 +47,11 @@ instead of sending again; a draft with any attempt whose outcome is unknown (and none confirmed) needs `confirmDuplicate: true`, and a resend owns its drafts until it finishes, re-checking each before it is sent. - Unused fields may be passed as null. + Dismissing a draft (with the tool or `>>skip_reply {{unsent}}`) is + refused while that draft's attempt is out, since a dismissal can't call + back words already on their way; the resident is told to wait for that + result. A draft only queued in a resend can still be dismissed, and is + then not sent. Unused fields may be passed as null. - Prose segments also break at XML tool mode's refused attempts and their notices (`tool_attempt`, `tool_notice`): the words before an all-refused call and after its notice are two messages, routed or held separately. diff --git a/src/framework.ts b/src/framework.ts index bd582fa12..8534ce4c6 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -9205,6 +9205,12 @@ export class AgentFramework { /** Drafts with a delivery attempt in flight, `${agent}\u0000${id}`: a * parallel resend (or `{{unsent}}`) of the same draft is refused. */ private draftsInFlight = new Set(); + /** The drafts in draftsInFlight whose attempt is out right now (journaled, + * its publish awaited). A dismissal of one is refused, and the resident + * told to wait for that result, since a dismissal can't call back words + * already on their way. A draft only queued in a resend batch can still + * be dismissed: the batch honours that when it reaches the draft. */ + private draftsSending = new Set(); /** * The resident's `drafts` tool: list, read, resend and dismiss its own held @@ -9301,6 +9307,14 @@ export class AgentFramework { } if (action === 'dismiss') { + // Refuse up front, before anything is dismissed, rather than half-way. + const sending = drafts.find((d) => this.draftsSending.has(`${agentName}\u0000${d.id}`)); + if (sending) { + return refuse( + `${sending.id} is being sent right now, and dismissing it can't call those words back; ` + + 'wait for that result before dismissing it. Nothing was dismissed.', + ); + } const lines: string[] = []; for (const d of drafts) { const state = draftState(d); @@ -9409,7 +9423,10 @@ export class AgentFramework { stopped = true; continue; } + const sendingKey = `${agentName}\u0000${d.id}`; + this.draftsSending.add(sendingKey); const outcome = await registry.publish(agentName, d.text, { serverId: destination.serverId, channelId: destination.channelId }); + this.draftsSending.delete(sendingKey); let recorded = true; try { this.proseDrafts.recordOutcome(agentName, d.id, attemptId, outcome); @@ -9450,7 +9467,10 @@ export class AgentFramework { : ' The draft stays held.')); } } finally { - for (const claim of claims) this.draftsInFlight.delete(claim); + for (const claim of claims) { + this.draftsInFlight.delete(claim); + this.draftsSending.delete(claim); + } } const allDelivered = !stopped; return allDelivered ? ok(lines.join('\n')) : refuse(lines.join('\n')); @@ -9534,8 +9554,15 @@ export class AgentFramework { for (const d of unconfirmed) { notes.push(`draft ${d.id} is unconfirmed: ${this.riskText(d)} — check that channel before sending it again (resend needs confirmDuplicate: true)`); } + // A dismissal sets a draft aside; it doesn't unsay that the draft's own + // attempt may already have been posted. const dismissed = inState('dismissed'); - if (dismissed.length > 0) notes.push(`draft${dismissed.length === 1 ? '' : 's'} ${idList(dismissed)} dismissed by you`); + const dismissedAtRisk = dismissed.filter((d) => uncertainAttempt(d)); + const dismissedQuietly = dismissed.filter((d) => !uncertainAttempt(d)); + for (const d of dismissedAtRisk) notes.push(`draft ${d.id} dismissed by you, but ${this.riskText(d)}`); + if (dismissedQuietly.length > 0) { + notes.push(`draft${dismissedQuietly.length === 1 ? '' : 's'} ${idList(dismissedQuietly)} dismissed by you`); + } for (const d of inState('delivered')) { const attempt = d.attempts.find((a) => a.outcome?.status === 'delivered')!; const how = attempt.via === 'resend' ? 'by your resend' : 'by your {{unsent}}'; @@ -9560,11 +9587,12 @@ export class AgentFramework { : `${suppressed} plain-speech segment(s) suppressed`); } const suppressedNote = notes.join(' · '); - // Nothing is confirmed delivered, but an unconfirmed draft may have been. + // Nothing is confirmed delivered, but an unconfirmed draft (or a dismissed + // one whose own attempt is uncertain) may have been. const text = shown.length > 0 ? `[delivered] plain speech → ${shown.join(' · ')}${suppressedNote ? ` · ${suppressedNote}` : ''}` - : `[delivered] nothing${unconfirmed.length > 0 ? ' confirmed' : ''} — ${suppressedNote}`; + : `[delivered] nothing${unconfirmed.length > 0 || dismissedAtRisk.length > 0 ? ' confirmed' : ''} — ${suppressedNote}`; try { const mid = agent.getContextManager().addMessage( 'user', @@ -9766,12 +9794,18 @@ export class AgentFramework { } } const flight = unsent ? `${agent.name}\u0000${unsent.id}` : undefined; - if (flight) this.draftsInFlight.add(flight); + if (flight) { + this.draftsInFlight.add(flight); + this.draftsSending.add(flight); + } let outcome: PublishOutcome; try { outcome = await registry.deliverSpeech(agent.name, body, channelId); } finally { - if (flight) this.draftsInFlight.delete(flight); + if (flight) { + this.draftsInFlight.delete(flight); + this.draftsSending.delete(flight); + } } if (unsent && attemptId) { try { @@ -9879,10 +9913,22 @@ export class AgentFramework { } } - /** `>>skip_reply {{unsent}}` and its hybrid form: set the latest bounce draft aside. */ + /** `>>skip_reply {{unsent}}` and its hybrid form: set the latest bounce + * draft aside — unless it is being sent right now, which a dismissal + * can't call back: the resident is told instead. */ private dismissLatestBounce(agent: Agent): void { const draft = this.proseDrafts.latestBounce(agent.name); if (!draft) return; + if (this.draftsSending.has(`${agent.name}\u0000${draft.id}`)) { + const busy = `[prose-routing] {{unsent}} is draft ${draft.id}, which is being sent right now. It was not set aside, ` + + 'since that can\'t call back words already on their way; wait for that result.'; + try { + this.addMessage('user', [{ type: 'text', text: busy }], { system: true, kind: 'prose-bounce' }, { forAgent: agent.name }); + } catch (err) { + console.error('[drafts] in-flight-dismiss notice failed:', err); + } + return; + } try { this.proseDrafts.dismiss(agent.name, draft.id); this.emitTrace({ type: 'prose:drafts-dismissed', agentName: agent.name, draftIds: [draft.id] }); diff --git a/test/prose-drafts.test.ts b/test/prose-drafts.test.ts index bfea62ed2..09ec8f432 100644 --- a/test/prose-drafts.test.ts +++ b/test/prose-drafts.test.ts @@ -668,6 +668,46 @@ describe('held prose drafts, end to end', () => { } }); + it('a dismissal in the same round as its draft\'s resend is refused until that result is in: it can\'t call back words on their way', async () => { + const h = await harness(); + try { + await h.turn([ + createMockResponse([text('words that land'), explicitSend('s1'), text('words in doubt'), call('l1', 'channel_list', {})], 'tool_use'), + createMockResponse([]), + ]); + const sure = h.drafts().find((d) => d.text === 'words that land')!; + const unsure = h.drafts().find((d) => d.text === 'words in doubt')!; + // The resend is dispatched first, so its attempt is out when the dismissal runs. + const raced = (id: string) => [ + drafts(`r-${id}`, { action: 'resend', draftIds: [id], destination: ROOM }), + drafts(`x-${id}`, { action: 'dismiss', draftIds: [id] }), + ]; + const waits = (id: string) => new RegExp( + `${id} is being sent right now, and dismissing it can't call those words back; wait for that result before dismissing it\\. Nothing was dismissed\\.`); + + await h.turn([createMockResponse(raced(sure.id), 'tool_use'), createMockResponse([])]); + let [resent, dismissal] = h.toolResults(); + assert.match(resent!, /delivered to/); + assert.match(dismissal!, waits(sure.id), 'no "dismissed" reply over words that went out'); + assert.equal(draftState(h.store().get('scout', sure.id)!), 'delivered'); + + h.command({ op: 'publish-mode', mode: 'no-receipt' }); + await h.turn([createMockResponse(raced(unsure.id), 'tool_use'), createMockResponse([])]); + [resent, dismissal] = h.toolResults(); + assert.match(dismissal!, waits(unsure.id)); + assert.match(resent!, /NOT confirmed .* Check the channel before sending it again \(that needs confirmDuplicate: true\)/, + 'the resend\'s advice holds: the draft is still open'); + assert.equal(draftState(h.store().get('scout', unsure.id)!), 'unconfirmed'); + + // With the result in, the resident can set it aside knowingly. + await h.turn([createMockResponse([drafts('x2', { action: 'dismiss', draftIds: [unsure.id] })], 'tool_use'), createMockResponse([])]); + assert.match(h.toolResults()[0]!, new RegExp(`${unsure.id}: dismissed`)); + assert.equal(draftState(h.store().get('scout', unsure.id)!), 'dismissed'); + } finally { + await h.close(); + } + }); + it('refuses unusable calls before sending anything', async () => { const h = await harness(); try { @@ -805,14 +845,17 @@ describe('drafts resend ownership', () => { modules: [], }); const pending: Array<{ text: string; resolve: (o: unknown) => void }> = []; + const held = (text: string, channelId: string) => + new Promise((resolve) => pending.push({ + text, + resolve: (o) => resolve({ destination: { serverId: 'discord', channelId }, at: Date.now(), ...(o as object) }), + })); const stub: Record = { resolveProseTarget: (spec: string) => ({ channelId: spec }), resolveDestination: ({ channelId }: { channelId: string }) => ({ destination: { serverId: 'discord', channelId } }), - publish: (_agent: string, text: string, target: { channelId: string }) => - new Promise((resolve) => pending.push({ - text, - resolve: (o) => resolve({ destination: { serverId: 'discord', channelId: target.channelId }, at: Date.now(), ...(o as object) }), - })), + publish: (_agent: string, text: string, target: { channelId: string }) => held(text, target.channelId), + // An envelope's delivery, `{{unsent}}` included. + deliverSpeech: (_agent: string, text: string, channelId: string) => held(text, channelId), getChannelTools: () => [], }; (framework as unknown as { channelRegistry: unknown }).channelRegistry = new Proxy(stub, { @@ -821,6 +864,7 @@ describe('drafts resend ownership', () => { const internals = framework as unknown as { proseDrafts: ProseDraftStore; handleDraftsTool(agent: string, input: unknown): Promise<{ success: boolean; data?: Array<{ text: string }>; error?: string }>; + deliverProseEnvelope(agent: unknown, text: string, hold: { notice: 'later' }): Promise; }; const say = (r: { success: boolean; data?: Array<{ text: string }>; error?: string }) => r.success ? r.data!.map((b) => b.text).join('') : r.error!; @@ -891,6 +935,68 @@ describe('drafts resend ownership', () => { await t.close(); } }); + + it('a dismissal naming a draft whose attempt is out dismisses nothing, not even the queued one beside it', async () => { + const t = await setup(); + try { + const batch = t.internals.handleDraftsTool('scout', { action: 'resend', draftIds: [t.a.id, t.b.id], destination: 'chan' }); + await t.tick(); + assert.deepEqual(t.pending.map((p) => p.text), ['A words'], 'A is out; B is queued'); + const refused = t.say(await t.internals.handleDraftsTool('scout', { action: 'dismiss', draftIds: [t.b.id, t.a.id] })); + assert.match(refused, new RegExp(`${t.a.id} is being sent right now, and dismissing it can't call those words back; .*Nothing was dismissed\\.`)); + assert.equal(t.internals.proseDrafts.get('scout', t.b.id)!.dismissedAt, undefined); + t.pending.shift()!.resolve({ status: 'unknown', reason: 'no receipt' }); + const result = t.say(await batch); + assert.match(result, new RegExp(`${t.a.id}: delivery to .+ NOT confirmed`)); + assert.equal(draftState(t.internals.proseDrafts.get('scout', t.a.id)!), 'unconfirmed'); + assert.match(t.say(await t.internals.handleDraftsTool('scout', { action: 'dismiss', draftIds: [t.a.id] })), new RegExp(`${t.a.id}: dismissed`), + 'with the result in, it can be dismissed'); + } finally { + await t.close(); + } + }); + + it('>>skip_reply {{unsent}} leaves the latest bounce alone while it is being sent, and tells the resident', async () => { + const t = await setup(); + try { + const [bounce] = t.internals.proseDrafts.hold('scout', [{ text: 'bounced words', source }], 'bounced').held; + const sending = t.internals.handleDraftsTool('scout', { action: 'resend', draftIds: [bounce!.id], destination: 'chan' }); + await t.tick(); + const agent = t.framework.getAgent('scout')!; + const envelope = (text: string) => t.internals.deliverProseEnvelope(agent, text, { notice: 'later' }); + await envelope('>>skip_reply {{unsent}}'); + assert.equal(t.internals.proseDrafts.get('scout', bounce!.id)!.dismissedAt, undefined); + const words = agent.getContextManager().getAllMessages().flatMap((m) => m.content).map((b) => (b as { text?: string }).text ?? ''); + assert.ok(words.includes(`[prose-routing] {{unsent}} is draft ${bounce!.id}, which is being sent right now. It was not set aside, ` + + 'since that can\'t call back words already on their way; wait for that result.'), words.join('\n')); + t.pending.shift()!.resolve({ status: 'unknown', reason: 'no receipt' }); + await sending; + await envelope('>>skip_reply {{unsent}}'); + assert.equal(draftState(t.internals.proseDrafts.get('scout', bounce!.id)!), 'dismissed', 'with the result in, it is set aside'); + } finally { + await t.close(); + } + }); + + it('a dismissal is refused while an {{unsent}} delivery of that draft is out', async () => { + const t = await setup(); + try { + const [bounce] = t.internals.proseDrafts.hold('scout', [{ text: 'bounced words', source }], 'bounced').held; + const agent = t.framework.getAgent('scout')!; + const delivering = t.internals.deliverProseEnvelope(agent, '>>chan {{unsent}}', { notice: 'later' }); + await t.tick(); + assert.deepEqual(t.pending.map((p) => p.text), ['bounced words'], 'the bounce is out through {{unsent}}'); + const refused = t.say(await t.internals.handleDraftsTool('scout', { action: 'dismiss', draftIds: [bounce!.id] })); + assert.match(refused, new RegExp(`${bounce!.id} is being sent right now, and dismissing it can't call those words back; .*Nothing was dismissed\\.`)); + t.pending.shift()!.resolve({ status: 'unknown', reason: 'no receipt' }); + await delivering; + assert.equal(draftState(t.internals.proseDrafts.get('scout', bounce!.id)!), 'unconfirmed'); + assert.match(t.say(await t.internals.handleDraftsTool('scout', { action: 'dismiss', draftIds: [bounce!.id] })), new RegExp(`${bounce!.id}: dismissed`), + 'with the result in, it can be dismissed'); + } finally { + await t.close(); + } + }); }); describe('held drafts: exact runs, hybrid envelopes beside a send, and turn-end state', () => { @@ -1000,7 +1106,7 @@ describe('held drafts: exact runs, hybrid envelopes beside a send, and turn-end it('the turn-end receipt reports each draft as it stands: resent, dismissed, unconfirmed or still held', async () => { const h = await harness(); try { - const ids = ['d-rcpta', 'd-rcptb', 'd-rcptc', 'd-rcptd']; + const ids = ['d-rcpta', 'd-rcptb', 'd-rcptc', 'd-rcptd', 'd-rcpte']; fixIds(h, ids); await h.turn([ createMockResponse([text('resend me'), explicitSend('s1'), text('dismiss me'), call('l1', 'channel_list', {}), text('keep me'), call('l2', 'channel_list', {})], 'tool_use'), @@ -1023,6 +1129,18 @@ describe('held drafts: exact runs, hybrid envelopes beside a send, and turn-end assert.match(h.texts().filter((t) => t.startsWith('[delivered]')).at(-1)!, new RegExp( `^\\[delivered\\] nothing confirmed — draft ${ids[3]} is unconfirmed: ${ids[3]}'s attempt to #room \\(Guild One\\) \\(${ROOM}\\) at .+ ` + 'may already have been posted: .+ — check that channel before sending it again \\(resend needs confirmDuplicate: true\\)$')); + + // Dismissed knowingly after an unknown resend: the dismissal sets it + // aside, and the receipt still says its attempt may have been posted. + await h.turn([ + createMockResponse([text('dismissed after doubt'), explicitSend('s3')], 'tool_use'), + createMockResponse([drafts('r3', { action: 'resend', draftIds: [ids[4]], destination: ROOM })], 'tool_use'), + createMockResponse([drafts('x2', { action: 'dismiss', draftIds: [ids[4]] })], 'tool_use'), + createMockResponse([]), + ]); + assert.match(h.texts().filter((t) => t.startsWith('[delivered]')).at(-1)!, new RegExp( + `^\\[delivered\\] nothing confirmed — draft ${ids[4]} dismissed by you, but ${ids[4]}'s attempt to #room \\(Guild One\\) \\(.*${ROOM}\\) at .+ ` + + 'may already have been posted: .+$')); } finally { await h.close(); } From a2accc1d8de1686de26ef629eb518938edff5e0d Mon Sep 17 00:00:00 2001 From: Ogden-1862 Date: Thu, 8 Oct 2026 14:28:43 -0400 Subject: [PATCH 0071/1112] fix(mcpl): digest a body as the store keeps it, so an untouched stored copy matches its storedBodyDigest slimepriestess's review of #252 (pullrequestreview-5460863368) found an untouched image copy that permanently failed its storedBodyDigest: storage takes an image's media type from its bytes' signature and re-encodes its base64, so the digest of the blocks handed to storage didn't match the copy read back. Writing bodies through the store directly (context-manager 0.13.0 over chronicle 0.4.0, read back live and after reopening) showed the class is wider than images: - every inline media block (an image whose source isn't a URL; any document, audio or video) is kept as a blob and comes back as { type, source: { type: 'base64', data, mediaType } } alone: its base64 re-encoded from the decoded bytes (padding, whitespace, URL-safe alphabet, unused bits), an image's media type taken from its signature, and its other fields (sourceUrl, tokenEstimate, filename, duration) dropped; - the store writes strings as UTF-8: a lone surrogate comes back as U+FFFD, and an entry whose key has one is dropped. sourceBodyDigest now hashes each block as the store keeps it. storedBlock maps inline media to its stored form; its base64 is re-encoded only when it isn't already canonical, so a stored copy's data is never decoded in full, and the store's signature test is copied (context-manager doesn't export it). canonicalJson writes strings well-formed and drops entries whose keys aren't. The normalization is inside the shared digest, not only in how storedBodyDigest is taken, so every copy of a body hashes alike: as delivered, as handed to storage, as injected into a live turn (#253's injectedEvidence hashes those very blocks), and as read back. Had only the stored digest been taken over the stored form, the injected copy would have failed it instead of the read-back one. An undecorated body's two digests stay equal whatever its media, and a legacy copy's recovered digest matches the delivered one. Docs: canonicalJson says it expects JSON-shaped values (review item 3): any other object is written by its own enumerable entries, which the store's own write matches (a Date as {}, a typed array by its indices, as probed) but a deferred write recovered from its JSON file doesn't. sourceBodyDigest says to check copies read back with blobs resolved (item 2). tsconfig gains ES2024.String for isWellFormed/toWellFormed, which Node has from 20, the engines floor. Tests: the reviewer's reproduction through the real framework's push lane (a PNG labeled image/jpeg, base64 broken by a newline), and a store contract test that writes nineteen bodies through a real ContextManager and checks that each hashes as it was handed over, live and after reopening, and which ones the store hands back changed. Its rows of image bytes the store has no signature for (SVG, BMP, AVIF) are labeled image/png and expected back unchanged, so a context-manager that learns to sniff one fails the test rather than letting the copied sniff drift. All three new tests fail at 88a24e6 and pass here. Reviewed independently by Lottie-1863 (room-310 #57630), who found that the store blobs an image of any non-URL source, not only a base64 one, and asked for the unsniffed rows. --- changelog.d/ingestion-body-digests.added.md | 20 ++-- src/framework.ts | 4 +- src/mcpl/inbound-source.ts | 96 ++++++++++++++++-- test/inbound-source.test.ts | 102 ++++++++++++++++++++ tsconfig.json | 2 +- 5 files changed, 205 insertions(+), 19 deletions(-) diff --git a/changelog.d/ingestion-body-digests.added.md b/changelog.d/ingestion-body-digests.added.md index f97bc691a..6e181ed8c 100644 --- a/changelog.d/ingestion-body-digests.added.md +++ b/changelog.d/ingestion-body-digests.added.md @@ -3,15 +3,21 @@ - `metadata.sourceBodyDigest` is the version identity of the body as delivered, taken at ingestion, after MCPL conversion and before any host decoration or storage sharding. It is the SHA-256 (hex) of `[blocks]` - as canonical JSON (keys sorted at every level, `undefined` dropped), the - framing an undecorated, unsharded stored copy already hashes to. + as canonical JSON (keys sorted at every level, `undefined` dropped), with + the blocks as the store keeps them: inline media as the store hands it + back (base64 re-encoded from its bytes, an image's media type taken from + its bytes' signature, only `type` and `source` kept) and strings + well-formed (a lone surrogate as U+FFFD). So the body hashes alike as + delivered, as handed to storage and as read back, and an undecorated, + unsharded stored copy hashes to it. - `metadata.storedBodyDigest` uses the same function over exactly the blocks handed to storage, decorations included, taken where each path stores. Both are always the host's own: values an adapter supplies are replaced. - A copy that no longer hashes to its `storedBodyDigest` was changed after it - arrived (`editMessage` keeps metadata). `sourceBodyDigest(blocks)` is - exported from the package root as the one implementation consumers share; - it carries the framing, so check a copy with it rather than hashing your - own serialization. + A stored copy, read back with its blobs resolved, that no longer hashes to + its `storedBodyDigest` was changed after it arrived (`editMessage` keeps + metadata). `sourceBodyDigest(blocks)` is exported from the package root as + the one implementation consumers share; it carries the framing and the + store's normalization, so check a copy with it rather than hashing your own + serialization. diff --git a/src/framework.ts b/src/framework.ts index 81957b3bf..5e44cc63c 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -7837,7 +7837,9 @@ export class AgentFramework { * An inbound item's metadata with its stored copy's own digest * (`storedBodyDigest`): the delivered-body digest's function over exactly * the blocks handed to storage here — every decoration included, before - * storage shards them. Taken at each storage site, never earlier, because + * storage shards them. That function hashes blocks as the store keeps them + * (inline media re-encoded and relabeled from its bytes), so the copy read + * back hashes to this too. Taken at each storage site, never earlier, because * a path can still decorate the body after ingestion stamped it (the * closed-channel invitation; room-220 #48282). An edit through * editMessage keeps metadata but not this hash. diff --git a/src/mcpl/inbound-source.ts b/src/mcpl/inbound-source.ts index 4f51a1860..b7c39c107 100644 --- a/src/mcpl/inbound-source.ts +++ b/src/mcpl/inbound-source.ts @@ -156,9 +156,18 @@ export function conversationKey(source: InboundSource): string | undefined { } /** - * JSON with object keys sorted at every level and `undefined` values dropped, - * so a value hashes alike however its keys were ordered (content read back - * from the store has its keys in a different order than it was written). + * JSON with object keys sorted at every level, written as the store keeps a + * value: `undefined` values are dropped, a string is well-formed (the store + * writes UTF-8, so each lone surrogate comes back as U+FFFD), and an entry + * whose key isn't well-formed is dropped (the store doesn't keep it). So a + * value hashes alike however its keys were ordered (content read back from + * the store has them in a different order than it was written), and alike + * before and after the store. + * + * It expects JSON-shaped values, as the MCPL lanes deliver them (parsed from + * JSON). Any other object is written by its own enumerable entries, unlike + * JSON.stringify: a Date becomes `{}`, as the store's own write keeps it, but + * a deferred write recovered from its JSON file brings it back as a string. * * sourceBodyDigest's serializer, kept out of the package API: serialization * and framing both stay inside sourceBodyDigest, so consumers don't @@ -168,33 +177,100 @@ export function canonicalJson(value: unknown): string { if (Array.isArray(value)) return `[${value.map(canonicalJson).join(',')}]`; if (value && typeof value === 'object') { const entries = Object.entries(value as Record) - .filter(([, v]) => v !== undefined) + .filter(([k, v]) => v !== undefined && k.isWellFormed()) .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)); return `{${entries.map(([k, v]) => `${JSON.stringify(k)}:${canonicalJson(v)}`).join(',')}}`; } + if (typeof value === 'string') return JSON.stringify(value.toWellFormed()); return JSON.stringify(value) ?? 'null'; } +/** + * A block as the store hands it back. Context-manager keeps inline media as + * blobs (blob-manager.ts): an image whose source isn't a URL, and every + * document, audio or video block, comes back as `{ type, source: { type: + * 'base64', data, mediaType } }` and nothing else, its data re-encoded from + * the decoded bytes, and an image's media type taken from its bytes' + * signature where they have one. Any other block comes back as it was given. + */ +function storedBlock(block: unknown): unknown { + const media = block as { type?: unknown; source?: { type?: unknown; data?: unknown; mediaType?: unknown } } | null; + const blob = media?.type === 'image' + ? media.source?.type !== 'url' + : media?.type === 'document' || media?.type === 'audio' || media?.type === 'video'; + const source = media?.source; + if (!blob || typeof source?.data !== 'string') return block; + const data = canonicalBase64(source.data); + const mediaType = media!.type === 'image' + ? sniffRasterImageMediaType(Buffer.from(data.slice(0, 32), 'base64')) ?? source.mediaType + : source.mediaType; + return { type: media!.type, source: { type: 'base64', data, mediaType } }; +} + +/** + * Base64 as the store writes it back from the decoded bytes: padded, with no + * whitespace, the standard alphabet and zero unused bits. Data already in + * that form, as every stored copy's is, is returned without decoding it all. + */ +function canonicalBase64(data: string): string { + if (data.length % 4 === 0 && /^[A-Za-z0-9+/]*={0,2}$/.test(data)) { + const last = data.slice(-4); + if (Buffer.from(last, 'base64').toString('base64') === last) return data; + } + return Buffer.from(data, 'base64').toString('base64'); +} + +/** + * The store's own test for raster image types with an unambiguous byte + * signature, copied from context-manager's blob-manager.ts (it isn't + * exported): the store relabels an image whose bytes match one of these. + */ +function sniffRasterImageMediaType(bytes: Uint8Array): string | null { + if (bytes.length >= 8 && + bytes[0] === 0x89 && bytes[1] === 0x50 && bytes[2] === 0x4e && bytes[3] === 0x47 && + bytes[4] === 0x0d && bytes[5] === 0x0a && bytes[6] === 0x1a && bytes[7] === 0x0a) { + return 'image/png'; + } + if (bytes.length >= 3 && bytes[0] === 0xff && bytes[1] === 0xd8 && bytes[2] === 0xff) { + return 'image/jpeg'; + } + if (bytes.length >= 6) { + const signature = Buffer.from(bytes.subarray(0, 6)).toString('ascii'); + if (signature === 'GIF87a' || signature === 'GIF89a') return 'image/gif'; + } + if (bytes.length >= 12 && + Buffer.from(bytes.subarray(0, 4)).toString('ascii') === 'RIFF' && + Buffer.from(bytes.subarray(8, 12)).toString('ascii') === 'WEBP') { + return 'image/webp'; + } + return null; +} + /** * The version identity of a delivered body (agreed with the receipts lane, * room-220 #46752–#47210): SHA-256 hex of `canonicalJson([blocks])` over the * ContentBlock[] the framework stores for the body — after MCPL conversion, - * before any host decoration is added and before storage shards it. The - * one-element array keeps the framing an undecorated, unsharded stored copy - * has always hashed to, so a stamped copy and an older copy of the same body - * share a version. + * before any host decoration is added and before storage shards it — with + * each block as the store keeps it (storedBlock, canonicalJson). So a body + * hashes alike as delivered, as handed to storage, as injected into a live + * turn, and as read back from the store. The one-element array keeps the + * framing an undecorated, unsharded stored copy has always hashed to, so a + * stamped copy and an older copy of the same body share a version. * * Ingestion stamps it as `metadata.sourceBodyDigest`, and the same function * over exactly the blocks handed to storage (decorations included) as * `metadata.storedBodyDigest`, on both MCPL lanes. Both live outside the * frozen admission envelope: they describe the body actually delivered (a * materialization, a correction), not the admission. Neither proves later - * presence or completeness; a copy that no longer hashes to its + * presence or completeness; a stored copy that no longer hashes to its * storedBodyDigest was changed after delivery (editMessage keeps metadata). + * Check a copy read back with its blobs resolved (getAllMessages, getMessage, + * or getMessageWindow without `resolveBlobs: false`): a blob reference never + * matches. * * Exported from the package root, so a consumer checks a copy against either * field with this same function rather than a copy of it. */ export function sourceBodyDigest(blocks: readonly unknown[]): string { - return createHash('sha256').update(canonicalJson([blocks])).digest('hex'); + return createHash('sha256').update(canonicalJson([blocks.map(storedBlock)])).digest('hex'); } diff --git a/test/inbound-source.test.ts b/test/inbound-source.test.ts index 8a8b797a6..65232a7ca 100644 --- a/test/inbound-source.test.ts +++ b/test/inbound-source.test.ts @@ -18,6 +18,7 @@ import { conversationKey, // The package's public digest: consumers import it from the root. sourceBodyDigest, + PassthroughStrategy, type InboundSource, type Module, type ModuleContext, @@ -28,6 +29,7 @@ import { type ToolCall, type ToolResult, } from '../src/index.js'; +import { ContextManager } from '@animalabs/context-manager'; import { MockMembrane } from './helpers/mock-membrane.js'; import { canonicalJson } from '../src/mcpl/inbound-source.js'; import { createHash } from 'node:crypto'; @@ -36,6 +38,9 @@ import { fixture, eventually, TS } from './helpers/coalescing-fixture.js'; const FIXTURE = join(import.meta.dirname, 'fixtures/speech-route-mcpl-server.mjs'); const ROOM = 'discord:g1:room'; const RAW_DM = '1548000000000000001'; +/** Bytes that open with PNG's signature, which is all the store reads of an + * image. 14 bytes, so their base64 is padded and has a '/' in it. */ +const PNG = Buffer.from('89504e470d0a1a0afbefbeffffff', 'hex'); async function waitFor(cond: () => boolean, what: string, timeoutMs = 15_000): Promise { const deadline = Date.now() + timeoutMs; @@ -256,6 +261,29 @@ describe('inbound source envelope', () => { assert.notEqual(meta.storedBodyDigest, meta.sourceBodyDigest, 'the invitation decorates the stored copy'); }); + it('an image the store relabels or re-encodes still matches both its digests once read back', async () => { + // slimepriestess's review of #252: the store takes an image's media type + // from its bytes and re-encodes its base64, so a digest of the blocks as + // handed to storage failed against the untouched copy read back. + const png = PNG.toString('base64'); + const pushes: Record = { + 'img-labeled': { type: 'image', data: png, mimeType: 'image/png' }, + 'img-mislabeled': { type: 'image', data: png, mimeType: 'image/jpeg' }, + 'img-wrapped': { type: 'image', data: `${png.slice(0, 11)}\n${png.slice(11)}`, mimeType: 'image/png' }, + }; + for (const [eventId, block] of Object.entries(pushes)) command({ op: 'push', eventId, content: [block] }); + await waitFor(() => Object.keys(pushes).every((id) => !!storedWith((m) => m.eventId === id)), 'images stored'); + for (const eventId of Object.keys(pushes)) { + const message = storedWith((m) => m.eventId === eventId)!; + const meta = message.metadata as Record; + const image = message.content.find((b) => b.type === 'image'); + assert.deepEqual(image, { type: 'image', source: { type: 'base64', data: png, mediaType: 'image/png' } }, + `${eventId}: stored as the bytes' own type, in canonical base64`); + assert.equal(sourceBodyDigest(message.content), meta.storedBodyDigest, `${eventId}: the untouched copy matches its stored digest`); + assert.equal(meta.sourceBodyDigest, sourceBodyDigest([image]), `${eventId}: the delivered body hashes as the image stored for it`); + } + }); + it('keeps a stored envelope when the channel is renamed later', async () => { command({ op: 'incoming', channelId: ROOM, messageId: 'm-2', mode: 'ambient', text: 'before rename' }); await waitFor(() => !!storedWith((m) => m.messageId === 'm-2'), 'first message stored'); @@ -443,3 +471,77 @@ describe('inbound source envelope under RFC-006 coalescing', () => { assert.deepEqual(observed, [stored]); }); }); + +describe('sourceBodyDigest hashes a body as the store keeps it', () => { + // Every agent stores through context-manager over chronicle. Whatever the + // store changes on the way in (an image's label, base64's spelling, fields + // it doesn't keep, a lone surrogate), the body read back must hash as it + // did when it was handed over, live and after the store is reopened. + const png = PNG.toString('base64'); + const image = (data: string, mediaType = 'image/png', extra: Record = {}) => + ({ type: 'image', source: { type: 'base64', data, mediaType }, ...extra }); + const media = (type: string, data: string, mediaType: string, extra: Record = {}) => + ({ type, source: { type: 'base64', data, mediaType }, ...extra }); + /** Each body, and whether the store hands it back changed. */ + const bodies: Record = { + 'a labeled image': { blocks: [image(png)], changed: false }, + 'PNG bytes labeled image/jpeg': { blocks: [image(png, 'image/jpeg')], changed: true }, + 'JPEG bytes labeled image/png': { blocks: [image(Buffer.from('ffd8ffe000104a464946', 'hex').toString('base64'))], changed: true }, + 'base64 with a newline in it': { blocks: [image(`${png.slice(0, 11)}\n${png.slice(11)}`)], changed: true }, + 'base64 with whitespace around it': { blocks: [image(` ${png}\n`)], changed: true }, + 'base64 without its padding': { blocks: [image(png.replace(/=+$/, ''))], changed: true }, + 'base64 with non-zero unused bits': { blocks: [image(png.replace(/8=$/, '9='))], changed: true }, + 'base64 in the URL-safe alphabet': { blocks: [image(png.replace(/\//g, '_'))], changed: true }, + // Bytes the store has no signature for keep their label. Labeled + // image/png, so a context-manager that learns to sniff one fails here. + 'SVG bytes labeled image/png': { blocks: [image(Buffer.from('').toString('base64'))], changed: false }, + 'BMP bytes labeled image/png': { blocks: [image(Buffer.from('424d3a0000000000000036000000', 'hex').toString('base64'))], changed: false }, + 'AVIF bytes labeled image/png': { blocks: [image(Buffer.from('0000001c6674797061766966000000006d696631', 'hex').toString('base64'))], changed: false }, + 'an image whose source is neither base64 nor a URL': { blocks: [{ type: 'image', source: { type: 'file', data: png, mediaType: 'image/png' } }], changed: true }, + 'an image with fields the store drops': { blocks: [image(png, 'image/png', { sourceUrl: 'https://cdn.example/a.png', tokenEstimate: 85 })], changed: true }, + 'an image by URL, its fields kept': { blocks: [{ type: 'image', source: { type: 'url', url: 'https://cdn.example/a.png' }, sourceUrl: 'https://cdn.example/a.png' }], changed: false }, + 'audio, wrapped, with a duration': { blocks: [media('audio', 'SUQz\nAwAAAAAA', 'audio/mpeg', { duration: 3 })], changed: true }, + 'a document with a filename': { blocks: [media('document', 'JVBERi0xLjQ=', 'application/pdf', { filename: 'a.pdf' })], changed: true }, + 'text with lone surrogates': { blocks: [{ type: 'text', text: 'cut \ud83d here, \ude00 there, \ud83d\ud83d twice' }], changed: true }, + 'a key that is not well-formed': { blocks: [{ type: 'text', text: 'x', rawItem: { 'k\ud800': 1, kept: 2 } }], changed: true }, + 'a relabeled image with a text decoration': { blocks: [image(png, 'image/jpeg'), { type: 'text', text: '[invitation]' }], changed: true }, + }; + + it('every body hashes alike as handed to the store and as read back, live and after reopening', async () => { + const dir = mkdtempSync(join(tmpdir(), 'stored-digest-')); + const path = join(dir, 'store.chronicle'); + try { + const strategy = () => new PassthroughStrategy(); + const ids = new Map(); + let cm = await ContextManager.open({ path, strategy: strategy() }); + for (const [name, { blocks }] of Object.entries(bodies)) { + ids.set(name, cm.addMessage('user', blocks as never, { storedBodyDigest: sourceBodyDigest(blocks) })); + } + const check = (when: string) => { + for (const [name, { blocks, changed }] of Object.entries(bodies)) { + const back = cm.getMessage(ids.get(name)!)!; + if (changed) assert.notDeepEqual(back.content, blocks, `${when}, ${name}: the store hands it back changed`); + else assert.deepEqual(back.content, blocks, `${when}, ${name}: the store hands it back as it was`); + assert.equal(sourceBodyDigest(back.content), (back.metadata as Record).storedBodyDigest, `${when}, ${name}: it still hashes as handed over`); + } + }; + check('live'); + cm.close(); + cm = await ContextManager.open({ path, strategy: strategy() }); + check('reopened'); + cm.close(); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it('what the store keeps still changes the digest', () => { + const digest = (blocks: unknown[]) => sourceBodyDigest(blocks); + assert.notEqual(digest([image(png)]), digest([image(Buffer.from([...PNG, 0]).toString('base64'))]), 'other bytes'); + const svg = Buffer.from('').toString('base64'); + assert.notEqual(digest([image(svg, 'image/svg+xml')]), digest([image(svg, 'image/png')]), 'a label the store keeps'); + assert.notEqual(digest([media('audio', 'SUQzAwAAAAAA', 'audio/mpeg')]), digest([media('audio', 'SUQzAwAAAAAA', 'audio/wav')]), 'an audio label'); + assert.notEqual(digest([{ type: 'text', text: 'a' }]), digest([{ type: 'text', text: 'b' }]), 'other text'); + assert.equal(digest([image(png, 'image/jpeg')]), digest([image(png)]), 'a label the store replaces is not part of the body'); + }); +}); diff --git a/tsconfig.json b/tsconfig.json index c10092cff..beb230fc8 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -3,7 +3,7 @@ "target": "ES2022", "module": "NodeNext", "moduleResolution": "NodeNext", - "lib": ["ES2022"], + "lib": ["ES2022", "ES2024.String"], "outDir": "./dist", "rootDir": ".", "strict": true, From d3eb0a92316439c4f742aeb1300e08a923ac6c8c Mon Sep 17 00:00:00 2001 From: Nigel-1885 Date: Thu, 8 Oct 2026 16:33:22 -0400 Subject: [PATCH 0072/1112] fix(awareness): a drain pass stops being joinable in the same step as its last claim MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit drainDiscordAwarenessOutbox kept one pass per server in discordAwarenessDrains and returned it to any caller while it was registered (`if (existing) return existing`), but removed it only in a `.finally` on its promise. That ran at least one microtask after the pass's last claimDispatch had come back empty and it had returned. A request in that gap (a retract, then deliverDiscordAwarenessInBackground -> syncDiscordAwarenessMarkers) was handed a pass that would never claim again, and its work stayed due until an unrelated trigger: another surgery, an operator act, a reconnect, tools/list_changed or a restart. For a retract, a 💤 stays on someone's message that long. Everywhere else in the loop the join is safe, since each claim re-reads the journal. The pattern predates this PR: main 02adf27, #246 8b530ee and the 0.21.0 release reuse a pass the same way. This PR rewrote the drain and added the operator retract, which is what the gap strands. Found by slimepriestess's review of #250 at 485a3c7 (https://github.com/anima-research/agent-framework/pull/250#pullrequestreview-5462105514): their second reader's stubbed-connection probe left `remove:m1` due, with nothing draining, for a retract queued one microtask after the last reply. They suggested a "rerun requested" flag read when the pass ends. The fix instead makes "this pass will claim nothing more" and "the next request starts a new pass" one synchronous step. The pass body moves into runDiscordAwarenessDrain, whose try/finally (around the whole body, so it covers the empty claim, the route-gone break and every throw) marks the pass ended and removes it from the registry, identity-checked so it never removes a successor. A pass that ends before its first await (nothing due, no route, or a failed first claim) is never registered. So a registered pass always has a claim ahead of it, and a caller who joins one is served by it, and syncDiscordAwarenessMarkers' "wait for this pass" holds for a caller who joins. A rerun flag works too, as the reviewer's chained re-drain showed; this was preferred because it needs no second piece of state and no extra pass after a join. The documented synchronous start (the first write-ahead before deliverDiscordAwarenessInBackground returns) is kept. The body is unchanged apart from moving; its indentation is the same. stop() still awaits whatever passes are registered. The same gap had a second face, which Cordelia-1884's review probes found: a pass with nothing due completes before its first await, but stayed registered until its `.finally`, so a second call in the same synchronous run joined a pass that never claims. Such a pass is now never registered. Tests: - A retract through retractDiscordAwareness, injected when the drain's claim comes back empty, at depths 0-3, depth 0 being the next microtask after the empty claim. At 485a3c7 depth 0 fails (only add_reaction:m1 sent, the remove still due) and 1-3 pass: the `.finally` reaction is queued when the pass returns, just behind the injected callback, so the gap is that one microtask. Counted from the reply's resolution, as the reviewer's probe counted, the failing depth is 1. With this commit all four depths send the remove. - Single flight, which nothing else pinned (dropping the registration passed every other test, Cordelia-1884 found): with m1's reply held open and work for m2 queued, a second call returns the same promise and sends nothing; once the reply comes, the same pass sends m2. The first request leaves before the first call returns, and a pass with nothing due is never registered. At 485a3c7 only that last assertion fails. Mutants on the VPS copy, each caught: deferring the registry delete by a microtask fails the retract test; dropping the registration fails the single-flight test; registering a pass that already ended fails the single-flight test and "framework drain classifies outcomes" (its offline pass would stay registered and swallow the next drain). --- .../surgery-marks-delivery-handoff.fixed.md | 7 ++ src/framework.ts | 45 +++++++-- test/discord-awareness-outbox.test.ts | 99 +++++++++++++++++++ 3 files changed, 144 insertions(+), 7 deletions(-) create mode 100644 changelog.d/surgery-marks-delivery-handoff.fixed.md diff --git a/changelog.d/surgery-marks-delivery-handoff.fixed.md b/changelog.d/surgery-marks-delivery-handoff.fixed.md new file mode 100644 index 000000000..975d09120 --- /dev/null +++ b/changelog.d/surgery-marks-delivery-handoff.fixed.md @@ -0,0 +1,7 @@ +- Awareness marks queued just as a delivery pass finishes are now sent on + their own trigger. A pass that had found nothing more to send stayed + joinable for one more microtask, so a request made then (an operator's + retract, say) joined a pass that would never look again, and its work + waited for the next surgery, operator act, reconnect, `tools/list_changed` + or restart. A pass now stops being joinable in the same step as its last + claim. diff --git a/src/framework.ts b/src/framework.ts index ab4c36cc6..d1c3327d2 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -519,6 +519,14 @@ type DiscordAwarenessDrainOutcome = | { status: 'delivered'; delivered: number; failed: number } | { status: 'unavailable'; accounted: number }; +/** One drain pass's place in the registry (see runDiscordAwarenessDrain). */ +interface DiscordAwarenessDrainPass { + /** Set as the pass stops claiming, in the same synchronous step. */ + ended: boolean; + /** Its promise, once registered. */ + drain?: Promise; +} + function normalizeDiscordAwarenessDeadline(value: number | undefined): number { if (value === undefined || !Number.isFinite(value) || value <= 0) { return DEFAULT_DISCORD_AWARENESS_DEADLINE_MS; @@ -13752,7 +13760,30 @@ export class AgentFramework { const existing = this.discordAwarenessDrains.get(serverId); if (existing) return existing; - const drain = (async () => { + const pass: DiscordAwarenessDrainPass = { ended: false }; + const drain = this.runDiscordAwarenessDrain(serverId, pass); + // A pass that ended before its first await (nothing was due, the route + // was down, or its first claim failed) is never registered. + if (!pass.ended) { + pass.drain = drain; + this.discordAwarenessDrains.set(serverId, drain); + } + return drain; + } + + /** + * One pass of drainDiscordAwarenessOutbox. It leaves the registry in the + * same synchronous step as its last claim, so a registered pass always has + * a claim ahead of it: a request that joins it is served by it, and one + * made after its last claim starts a new pass. Leaving any later, when its + * promise settles, would let a request join a pass that will never claim + * again, and that work would stay due until some unrelated trigger. + */ + private async runDiscordAwarenessDrain( + serverId: string, + pass: DiscordAwarenessDrainPass, + ): Promise { + try { const outbox = this.discordAwarenessOutbox!; const connection = this.mcplServerRegistry?.getServer(serverId); if (!connection?.isConnected) { @@ -13854,12 +13885,12 @@ export class AgentFramework { ); } return { status: 'delivered' as const, delivered, failed }; - })().finally(() => { - this.discordAwarenessDrains.delete(serverId); - }); - - this.discordAwarenessDrains.set(serverId, drain); - return drain; + } finally { + pass.ended = true; + if (pass.drain !== undefined && this.discordAwarenessDrains.get(serverId) === pass.drain) { + this.discordAwarenessDrains.delete(serverId); + } + } } private readDiscordAwarenessDispatches(serverId?: string) { diff --git a/test/discord-awareness-outbox.test.ts b/test/discord-awareness-outbox.test.ts index 7e47bb3f0..1c19282be 100644 --- a/test/discord-awareness-outbox.test.ts +++ b/test/discord-awareness-outbox.test.ts @@ -782,6 +782,105 @@ test('a drain whose route goes away stops claiming: the rest stays queued, with } }); +test('a retract made just after a drain finds nothing due is delivered on its own trigger', async () => { + // A drain that had made its final claim but not yet left the registry was + // handed the retract's trigger, and never looked again: the removal stayed + // due until some unrelated trigger. Each depth puts the retract one + // microtask further from that final claim. + const sent: Record = {}; + const due: Record = {}; + for (const depth of [0, 1, 2, 3]) { + const h = storeHarness(); + try { + const batch = activeBatch(h.outbox, [ref('m1')]); + const calls: string[] = []; + const connection = { + isConnected: true, + sendToolsCallWithDeadline: async (name: string, args: Record) => { + calls.push(`${name}:${args.messageId}`); + return { content: [{ type: 'text', text: 'ok' }] }; + }, + }; + const framework = Object.create(AgentFramework.prototype) as any; + framework.discordAwarenessOutbox = h.outbox; + framework.discordAwarenessDrains = new Map(); + framework.discordAwarenessDeadlineMs = 1000; + framework.mcplServerRegistry = { getServer: () => connection }; + framework.operatorLog = new OperatorLog(undefined); + framework.opsAlert = () => {}; + const claim = h.outbox.claimDispatch.bind(h.outbox); + let retracted: Promise | undefined; + h.outbox.claimDispatch = (serverId, skip) => { + const claimed = claim(serverId, skip); + if (!claimed && !retracted) { + retracted = Promise.resolve().then(async () => { + for (let i = 0; i < depth; i++) await null; + framework.retractDiscordAwareness(batch.id); + }); + } + return claimed; + }; + await muted(async () => { + await framework.drainDiscordAwarenessOutbox('discord'); + await retracted; + while (framework.discordAwarenessDrains.size > 0) { + await Promise.allSettled([...framework.discordAwarenessDrains.values()]); + } + }); + sent[depth] = calls; + due[depth] = h.outbox.pendingDispatches('discord').length; + } finally { + h.cleanup(); + } + } + const delivered = ['add_reaction:m1', 'remove_reaction:m1']; + assert.deepEqual(sent, { 0: delivered, 1: delivered, 2: delivered, 3: delivered }); + assert.deepEqual(due, { 0: 0, 1: 0, 2: 0, 3: 0 }); +}); + +test('one drain pass per server: a request while its reply is out joins the pass, which sends the new work next', async () => { + const h = storeHarness(); + try { + activeBatch(h.outbox, [ref('m1')]); + const calls: string[] = []; + let answerFirst!: () => void; + const firstAnswered = new Promise((resolve) => { answerFirst = resolve; }); + const connection = { + isConnected: true, + sendToolsCallWithDeadline: async (name: string, args: Record) => { + calls.push(`${name}:${args.messageId}`); + if (calls.length === 1) await firstAnswered; + return { content: [{ type: 'text', text: 'ok' }] }; + }, + }; + const framework = Object.create(AgentFramework.prototype) as any; + framework.discordAwarenessOutbox = h.outbox; + framework.discordAwarenessDrains = new Map(); + framework.discordAwarenessDeadlineMs = 1000; + framework.mcplServerRegistry = { getServer: () => connection }; + await muted(async () => { + const first = framework.drainDiscordAwarenessOutbox('discord'); + assert.deepEqual(calls, ['add_reaction:m1'], 'the first request leaves before the call returns'); + // Work for another message while m1's reply is out. Only the registry + // holds it back: the per-key guard would let a second pass send it. + activeBatch(h.outbox, [ref('m2')], { targetBranch: 'rollback/cairn/2' }); + const second = framework.drainDiscordAwarenessOutbox('discord'); + assert.equal(second, first, 'the running pass is joined'); + await new Promise((resolve) => setImmediate(resolve)); + assert.deepEqual(calls, ['add_reaction:m1'], 'nothing more leaves while the reply is out'); + answerFirst(); + assert.deepEqual(await first, { status: 'delivered', delivered: 2, failed: 0 }); + assert.deepEqual(calls, ['add_reaction:m1', 'add_reaction:m2'], 'the joined pass sent the new work'); + assert.equal(framework.discordAwarenessDrains.size, 0); + // A pass with nothing due ends before its first await: never registered. + void framework.drainDiscordAwarenessOutbox('discord'); + assert.equal(framework.discordAwarenessDrains.size, 0); + }); + } finally { + h.cleanup(); + } +}); + test('framework resumes an interrupted suppression before activating its marks', async () => { const h = storeHarness(); try { From 763c932b0e71c6053ba0593254049b502422eeeb Mon Sep 17 00:00:00 2001 From: Nigel-1885 Date: Thu, 8 Oct 2026 16:33:33 -0400 Subject: [PATCH 0073/1112] fix(awareness): import a restored copy of a legacy ledger once per batch, not once per file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pre-journal ledger import was guarded only by the file's sha256 (`importedSources`). After import the file is renamed `.migrated-v2`, but a byte-different copy of the same ledger put back at the legacy path (a pretty-printed or older backup) hashed differently and imported again. Its `batch` records were no-ops at fold (applyRecords skips a known id), but `legacy-activated`, `held` and `legacy-evidence` all applied to the existing batch. A released batch read `status: held` again (with `released` still set) and offered its release actions, and its evidence was counted twice. Releasing it a second time would authorize the adds after the operator's retract, so they would win over it and re-add the 💤 the operator had removed. It took a manual restore and a release, so it was never a send without operator action, but that release would have acted on false history. Found by slimepriestess's review of #250 at 485a3c7 (https://github.com/anima-research/agent-framework/pull/250#pullrequestreview-5462105514), whose second reader's probe (import, release, a confirmed add, retract, the same ledger pretty-printed back) gave `status=held released=true` and doubled evidence. They suggested skipping `held`, `legacy-activated` and evidence for batch ids the journal already holds. importLegacy now takes the journal's batches and skips every record for a batch it already holds, and for a second listing of one id within the file. It still ends with `import-complete` and the new hash, and the file is still renamed, so the copy is never read again. Batches new to the journal import as before (held when they carry undelivered work). That covers a pre-journal build writing a fresh ledger at the old path after the rename (a downgrade, then an upgrade): its batches stay visible to the operator. Replay of an existing journal is unchanged. README and the breaking changelog fragment say a copy put back later imports only batches the journal doesn't already hold. This import is new in this PR, so no separate fixed fragment. Test: import, release, a confirmed add, retract, then the same ledger pretty-printed at the path. The reopened journal's view() and operations() equal the ones before, a release is refused (the batch is active, not held), and the file is moved aside. Then a file holding that batch plus a new one listed twice imports the new one alone, once: held, with one release action and one evidence entry. At 485a3c7 the first assertion fails (held, entries 2, outcomesUnrecorded 2). Mutants on the VPS copy: dropping the known-batch skip, or only the in-file one, each fail this test. --- README.md | 5 ++- changelog.d/surgery-marks-opt-in.breaking.md | 7 +-- src/recovery/discord-awareness-outbox.ts | 25 +++++++++-- test/discord-awareness-outbox.test.ts | 45 ++++++++++++++++++++ 4 files changed, 74 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 311852747..9ba8b06f8 100644 --- a/README.md +++ b/README.md @@ -409,8 +409,9 @@ const result = await framework.rollbackToMessage('cairn', { A pre-journal awareness ledger (`discord-awareness-outbox.json` under the store, or `discordAwarenessOutboxPath`) is imported once on first use and the -file renamed `.migrated-v2`. What it recorded about each message is kept as -evidence (attempt count; the last action and the outcome its error field +file renamed `.migrated-v2`; a copy put back at that path later imports only +batches the journal doesn't already hold. What it recorded about each message +is kept as evidence (attempt count; the last action and the outcome its error field establishes; the old writer's delivery status, labelled as the reconciliation state it was; and how many attempt outcomes it leaves unrecorded), never as a claim about what is on Discord, diff --git a/changelog.d/surgery-marks-opt-in.breaking.md b/changelog.d/surgery-marks-opt-in.breaking.md index a0648b270..34d954a6a 100644 --- a/changelog.d/surgery-marks-opt-in.breaking.md +++ b/changelog.d/surgery-marks-opt-in.breaking.md @@ -36,7 +36,8 @@ `view()`, `cancel()`, `retract()`, `release()` and `settleActivation()`. `discordAwarenessOutboxPath` now only names a previous JSON ledger to import. Such a file (by default `/recovery/discord-awareness-outbox.json`) - is imported once on first use and renamed `.migrated-v2`. What it recorded - is kept as evidence, never as synthesized attempts or as proof that a - suppression's body completed, and its undelivered work is held for an + is imported once on first use and renamed `.migrated-v2`; a copy put back + there later imports only batches the journal doesn't already hold. What it + recorded is kept as evidence, never as synthesized attempts or as proof that + a suppression's body completed, and its undelivered work is held for an explicit release. diff --git a/src/recovery/discord-awareness-outbox.ts b/src/recovery/discord-awareness-outbox.ts index 8850c3df4..e94ba37f9 100644 --- a/src/recovery/discord-awareness-outbox.ts +++ b/src/recovery/discord-awareness-outbox.ts @@ -1294,7 +1294,12 @@ export class DiscordAwarenessOutbox { return state; } - /** Import a pre-journal ledger beside the store once (by hash), then rename it. */ + /** + * Import a pre-journal ledger beside the store, then rename it. A file + * whose hash was imported before is only renamed, and a batch the journal + * already holds is never imported again, so a restored or re-saved copy of + * an imported ledger changes nothing (see importLegacy). + */ private importLegacyLedger(state: JournalState): void { if (!this.legacyPath) return; let raw: string; @@ -1314,7 +1319,7 @@ export class DiscordAwarenessOutbox { } catch { throw new Error(`Invalid Discord awareness ledger ${this.legacyPath}`); } - this.append(importLegacy(parsed, this.legacyPath, sha256), { durable: true }); + this.append(importLegacy(parsed, this.legacyPath, sha256, state.batches), { durable: true }); } // Imported (now or by an earlier process that stopped before renaming): // the file is inert; move it out of the way. @@ -1557,18 +1562,32 @@ interface LegacyBatch { * attempts and never a judgment about what is on Discord. Pending work, and * batches still prepared, are held for an operator: they were queued before * marks were an explicit choice. Nothing imported is ever dispatched. + * + * Each batch is imported once. A batch the journal already holds, or a + * second listing of one in this file, is skipped whole: its evidence would + * be counted twice, and holding it again would offer the operator a release + * of adds they may since have retracted. Batches new to the journal import + * as above. */ -function importLegacy(parsed: unknown, path: string, sha256: string): JournalRecord[] { +function importLegacy( + parsed: unknown, + path: string, + sha256: string, + journalBatches: ReadonlyMap, +): JournalRecord[] { const document = parsed as { version?: unknown; batches?: unknown }; if (!document || (document.version !== 1 && document.version !== 2) || !Array.isArray(document.batches)) { throw new Error(`Invalid Discord awareness ledger ${path}`); } const at = Date.now(); const records: JournalRecord[] = []; + const imported = new Set(); for (const raw of document.batches as LegacyBatch[]) { if (!raw || typeof raw.id !== 'string' || !Array.isArray(raw.refs)) { throw new Error(`Invalid Discord awareness ledger ${path}`); } + if (journalBatches.has(raw.id) || imported.has(raw.id)) continue; + imported.add(raw.id); const legacyActive = raw.status === 'active' || raw.status === 'pending'; const refs = raw.refs.map((entry) => ({ serverId: entry.serverId, diff --git a/test/discord-awareness-outbox.test.ts b/test/discord-awareness-outbox.test.ts index 1c19282be..b891f48ba 100644 --- a/test/discord-awareness-outbox.test.ts +++ b/test/discord-awareness-outbox.test.ts @@ -628,6 +628,51 @@ test('v2 import keeps what the old ledger recorded as evidence, holds pending wo assert.equal(existsSync(h.legacyPath), false); })); +test('a restored copy of an imported ledger changes nothing: import is once per batch, not once per file', withJournal((_outbox, h) => { + const pending = (id: string, messageId: string) => ({ + id, status: 'active', agentName: 'resident', sourceBranch: 'main', targetBranch: `rollback/resident/${id}`, + emoji: '💤', createdAt: 1, + refs: [{ ...ref(messageId), desired: true, markerPresent: false, deliveryStatus: 'pending', attempts: 1, lastAction: 'add', + lastError: 'MCPL server "discord" did not respond within 10000ms' }], + }); + const legacy = { version: 2, batches: [pending('restored', 'm1')] }; + mkdirSync(join(h.dir, 'recovery'), { recursive: true }); + writeFileSync(h.legacyPath, JSON.stringify(legacy)); + const outbox = h.reopen(); + // The operator releases the held add, it lands, and then they retract it. + outbox.release('restored', 'operator'); + answer(outbox, 'm1', 'confirmed'); + outbox.retract('restored', 'operator'); + const view = outbox.view(); + const operations = outbox.operations(); + + // A byte-different copy of the same ledger (pretty-printed, as a backup + // may be) is put back at the old path: it is moved aside and changes + // nothing. Before, the batch read as held again, with its evidence + // doubled, and a second release would have re-added the retracted mark. + writeFileSync(h.legacyPath, JSON.stringify(legacy, null, 2)); + const restored = h.reopen(); + assert.deepEqual(restored.view(), view); + assert.deepEqual(restored.operations(), operations); + assert.throws(() => restored.release('restored'), /is active, not held/); + assert.equal(existsSync(h.legacyPath), false); + + // A file that also holds a batch the journal has never seen imports that + // batch alone, once even if the file lists it twice, held like any import. + writeFileSync(h.legacyPath, JSON.stringify({ + version: 2, + batches: [pending('restored', 'm1'), pending('later', 'm2'), pending('later', 'm2')], + })); + const widened = h.reopen(); + const later = batches(widened).find((v) => v.id === 'later')!; + assert.equal(later.status, 'held'); + assert.equal(later.held!.releaseActions, 1); + assert.deepEqual(later.legacy, { entries: 1, lastAddConfirmed: 0, lastRemoveConfirmed: 0, outcomesUnrecorded: 1 }); + assert.deepEqual(widened.view().filter((v) => v.id !== 'later'), view); + assert.deepEqual(widened.operations(), operations); + assert.equal(existsSync(h.legacyPath), false); +})); + test('legacy evidence reads outcomes from the old writer\'s lastError, never from its reconciliation state', withJournal((_outbox, h) => { // Entries exactly as the 64c480b writer leaves them (recordSuccess clears // lastError, recordFailure sets it; setDesired rewrites deliveryStatus From df209f249a228113df80c0c6aa3b108af61dee3c Mon Sep 17 00:00:00 2001 From: Otis-1924 Date: Thu, 8 Oct 2026 18:22:49 -0400 Subject: [PATCH 0074/1112] fix(mcp): a call that ends before the server's final answer is no-response slimepriestess's review of #262 at c35d286 (CHANGES_REQUESTED, review 5462380482) found a deadline misreported as an answer. With requestTimeoutMs 600 and a server that answers leg 1 with a requestState-only input_required late in the budget, SDK 2.3.1 sleeps REQUEST_STATE_ONLY_LEG_PACING_MS (250) before leg 2, and the deadline fires inside that sleep. The sleep rejects with SdkError(RequestTimeout, "TimeoutError: The operation was aborted due to timeout"), but requestFailure() checked `record.answer === 'result'` first, so the model and modules got a plain Error: "answered tools/call ..., but the result can't be used", with no outcome and no "verify before retrying". The cause is wider than the deadline. The wire record treated any result as the call's answer, and an input_required result is not one: it asks for another round. Nigel-1925's probe (room-341 #58500) showed the same branch misreading a connection lost between rounds. The SDK's _onclose clears _transport, so the retry leg's _requestWithSchemaViaCodec rejects with a plain Error("Not connected") before anything reaches ObservedTransport.send, and leg 1's answer still stood. A host close() during the pause goes the same way. What changed: - WireRecord.answer is now 'none' | 'error' | 'complete' | 'input-required' | 'unrecognized', read from the answer's resultType on the wire. Only an error or resultType 'complete' is a final answer. Keying "final" on 'complete', so that anything this revision doesn't define (a task-style result, a missing or non-string resultType) is 'unrecognized' and reads as unknown, was Nigel-1925's suggestion (#58520). - requestFailure(): a complete result that can't be used stays a plain error. A latest answer that doesn't finish the call is now no-response, outcome unknown, whatever ended the call: the deadline in the pause between rounds, the connection gone, or this client unable to go on (an input request it can't supply, the 10-round cap). That last part reverses what the comment documented ("input this client can't supply, a rounds cap" were plain errors). Each round re-invokes the server's handler with the echoed requestState, so a multi-step flow can have acted by the cap, and the outcome is unknown. Keeping the category on the wire record alone also keeps the module's own rule: classified by what crossed the boundary, never by the error's class. The error class now only picks the wording, as RequestTimeout already did. - Wording. Between rounds nothing is in flight, so the deadline message no longer claims cancellation was requested: "did not finish X within Nms: it asked for another round (input_required), and the deadline passed before that round was sent, so nothing was in flight to cancel. The outcome is unknown: the server may already have acted on the call. Verify state before retrying; ...". Other endings name the SDK's reason ("... the call ended before that round was sent: Not connected."). The in-flight deadline message is unchanged. - The module header, README and changelog.d/modern-mcp-client.added.md said "at the deadline cancellation is requested" unconditionally. They now say that only a leg in flight is cancelled, and that a call ending after an input_required answer is no-response. changelog.d/mcpl-request-outcome.added.md is Petra-1012's 10380dd, shared with #247-#257, and is left alone. Tests (mcp-modern-outcomes, raw fixture): `cont` gains arguments.leg1 'exit' (exits just after answering leg 1) and 'elicit' (asks for elicitation input), and leg2 'again' (sheds load on every round). New `task` and `bare` tools answer with resultType 'task' and with no resultType. Four new tests: - deadline between rounds: requestTimeoutMs 240, below the 250 ms pause, so the deadline always falls inside it and leg 1 has the whole budget to answer. Asserts no-response, no cancellation claim, one leg, and no notifications/cancelled on the wire; - connection lost between rounds: no-response "Not connected", one leg; - an elicitation request this client can't supply and the rounds cap (1 + 11 legs): both no-response; - a result type this revision doesn't define, and a result with none: both no-response. The existing "second leg decides" test (an error on leg 2 is error-response, a hung leg 2 is no-response with cancellation) passes unchanged. Evidence, VPS, node 24.21.0: mcp-modern-outcomes + mcp-modern-connection 27/27 (23 before plus 4). Six mutations, each failing exactly the tests aimed at it: c35d286's file (the 4 new tests), input_required read as complete (3), an unrecognized type read as complete (1), a missing resultType read as complete (1; Nigel-1925 found this one unpinned in review, hence `bare`), the between-rounds deadline wording (1), and no unfinished-answer branch (4). Nigel-1925's probe of the review's table now reads no-response in every row: 600 ms with leg 1 at 380 or 450 ms, and 1000 ms with leg 1 at 900 ms, each with 1 leg and 0 cancels. The 100 ms row keeps the in-flight cancellation (2 legs, 1 cancel). --- README.md | 12 ++-- changelog.d/modern-mcp-client.added.md | 12 ++-- src/mcpl/modern-connection.ts | 59 ++++++++++++++---- test/fixtures/modern-raw-server.mjs | 26 +++++++- test/mcp-modern-outcomes.test.ts | 85 ++++++++++++++++++++++++++ 5 files changed, 171 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index 545279646..6ca53d160 100644 --- a/README.md +++ b/README.md @@ -297,10 +297,14 @@ two ways: - `requestTimeoutMs` is one deadline per tool call and must be 1 to 2^31−1; 0 is refused, where MCPL reads it as no watchdog. -At the deadline cancellation is requested, the outcome is reported as -unknown, and the call is never retried. An MCPL server that refuses -`2024-11-05` with `-32022` fails with `McplProtocolVersionError`, which names -the fix, and is not retried. +At the deadline, cancellation is requested for a leg still in flight; +between the rounds of an `input_required` continuation nothing is in flight +to cancel. Either way the outcome is reported as unknown (`no-response`), +and the call is never retried. A call that ends for any other reason after +the server asked for another round, before its final answer, is reported +the same way, since the server may already have acted. An MCPL server that +refuses `2024-11-05` with `-32022` fails with `McplProtocolVersionError`, +which names the fix, and is not retried. #### Feature sets diff --git a/changelog.d/modern-mcp-client.added.md b/changelog.d/modern-mcp-client.added.md index ec5225fd4..8d4a38ac6 100644 --- a/changelog.d/modern-mcp-client.added.md +++ b/changelog.d/modern-mcp-client.added.md @@ -14,12 +14,16 @@ - **HTTP credentials:** come from `token`/`accessProvider`, as a cached bearer refreshed once on 401. - **Deadline:** each call gets one `requestTimeoutMs` deadline across every - leg. At the deadline cancellation is requested, the outcome is reported - as unknown, and nothing is replayed. + leg and the pauses between them. At the deadline cancellation is + requested for a leg in flight (between continuation rounds none is), the + outcome is reported as unknown, and nothing is replayed. - **Failures:** reported as `McplRequestError`, with the outcome taken from what actually crossed the transport: `not-sent`, `error-response` (HTTP - 401/403 with `data.httpStatus`) or `no-response`. A server answer that - can't be used is a plain error. + 401/403 with `data.httpStatus`) or `no-response`. Only an error or a + `complete` result is a final answer: a call that ends after an + `input_required` answer, or after a result type this revision doesn't + define, is `no-response`. A complete result that can't be used is a + plain error. - **Lifetime:** reconnects use the legacy backoff settings. A tool-list change subscription is kept open, reopened if refused or lost, and its changes go through ordinary admission, so they park under quiesce like diff --git a/src/mcpl/modern-connection.ts b/src/mcpl/modern-connection.ts index d4bd74e25..2ad44f6d8 100644 --- a/src/mcpl/modern-connection.ts +++ b/src/mcpl/modern-connection.ts @@ -14,13 +14,17 @@ * - **HTTP credentials** come from `token`/`accessProvider`: a cached bearer * token, re-resolved once when the server answers 401. * - **One deadline per tool call** (`requestTimeoutMs`, always positive) - * bounds every leg of the call. At the deadline the SDK requests - * cancellation and the outcome is unknown. Calls are never replayed. + * bounds every leg of the call and the pauses between them. At the + * deadline the SDK requests cancellation of a leg in flight; between + * continuation rounds nothing is in flight. Either way the outcome is + * unknown. Calls are never replayed. * - **Request outcomes from the wire.** A failed call is classified by what * crossed Connectome's transport boundary, never by the error's class. The * SDK raises the same error classes locally (an invalid outputSchema before * dispatch, output validation after a successful result) as it does for - * server answers. + * server answers. A call's answer is the server's final one: an + * `input_required` answer asks for another round, so a call that ends + * after one has no answer. * - **Lifetime.** On a lost transport the connection restarts with the same * backoff settings as legacy, when `reconnect` is set. List-change * awareness is kept while the connection lives: a subscription the server @@ -79,12 +83,15 @@ export interface ModernToolCallResult { * any of its requests handed to the transport, and how did the server answer * the most recent one? A logical call can take several legs: a * header-mismatch retry, or the rounds of an `input_required` continuation. + * Only an error or a `complete` result is the server's final answer. + * `input-required` asks for another round, and a result of any other type, + * or of none (`unrecognized`), doesn't say the call finished. */ interface WireRecord { readonly method: string; handedOff: boolean; currentId?: string | number; - answer: 'none' | 'result' | 'error'; + answer: 'none' | 'error' | 'complete' | 'input-required' | 'unrecognized'; error?: { code: number; message: string; data?: unknown }; } @@ -173,7 +180,9 @@ class ObservedTransport implements Transport { record.answer = 'error'; record.error = message.error as WireRecord['error']; } else { - record.answer = 'result'; + const result = (message as { result?: unknown }).result; + const type = typeof result === 'object' && result !== null ? (result as { resultType?: unknown }).resultType : undefined; + record.answer = type === 'complete' ? 'complete' : type === 'input_required' ? 'input-required' : 'unrecognized'; } return record; } @@ -725,7 +734,9 @@ export class ModernMcpConnection extends EventEmitter { /** * Map a failure onto the request-outcome contract by what crossed the - * transport boundary for this request, claiming no more than that shows: + * transport boundary for this request, claiming no more than that shows. + * The call's answer is the server's final one: an error or a `complete` + * result. * - nothing handed to the transport: `not-sent`, whatever the SDK raised * (a pre-dispatch validation, a signal already aborted, no transport); * - an HTTP 401/403 refusal: `error-response`. The server answered, at the @@ -733,9 +744,17 @@ export class ModernMcpConnection extends EventEmitter { * JSON-RPC code space's (absent); * - the most recent leg answered with a JSON-RPC error: `error-response` * with that error's code and data; - * - the most recent leg answered with a result: the server acted, but what - * it returned can't be used (output validation, input this client can't - * supply, a rounds cap). That's a plain error, not a request outcome; + * - the most recent leg answered with a complete result: the server + * finished, but what it returned can't be used (output validation). + * That's a plain error, not a request outcome; + * - the most recent leg answered without finishing the call: `no-response`, + * since the server may have acted on what it received. An + * `input_required` answer asks for another round, and the call can end + * before that round is sent: at the deadline, when the connection is + * gone, or when this client can't go on (input it can't supply, the + * rounds cap). Nothing is in flight then, so nothing is cancelled. A + * result of a type this client doesn't recognize ends the call where it + * is; * - handed off and unanswered: `no-response`. At the deadline, cancellation * was requested and the outcome is unknown. */ @@ -753,12 +772,28 @@ export class ModernMcpConnection extends EventEmitter { const { code, message: wireMessage, data } = record.error; return new McplRequestError(`${prefix} returned error for ${what}: [${code}] ${wireMessage}`, 'error-response', code, data); } - if (record.answer === 'result') { + if (record.answer === 'complete') { return new Error(`${prefix} answered ${what}, but the result can't be used: ${message}`); } - if (error instanceof SdkError && error.code === SdkErrorCode.RequestTimeout) { + const deadline = error instanceof SdkError && error.code === SdkErrorCode.RequestTimeout; + const within = deadline && timeoutMs ? ` within ${timeoutMs}ms` : ''; + if (record.answer === 'input-required' || record.answer === 'unrecognized') { + const reason = /[.!?]$/.test(message) ? message : `${message}.`; + const ending = record.answer === 'unrecognized' + ? `its answer was not a complete result: ${reason}` + : deadline + ? 'it asked for another round (input_required), and the deadline passed before that round was sent, so nothing was in flight to cancel.' + : `it asked for another round (input_required), and the call ended before that round was sent: ${reason}`; return new McplRequestError( - `${prefix} did not answer ${what}${timeoutMs ? ` within ${timeoutMs}ms` : ''}. Cancellation was requested; ` + + `${prefix} did not finish ${what}${within}: ${ending} ` + + `The outcome is unknown: the server may already have acted on the call. Verify state before retrying; ` + + `a blind retry of a side-effecting tool may duplicate it.`, + 'no-response', + ); + } + if (deadline) { + return new McplRequestError( + `${prefix} did not answer ${what}${within}. Cancellation was requested; ` + `the outcome is unknown: the tool may still complete server-side. Verify state before retrying; ` + `a blind retry of a side-effecting tool may duplicate it.`, 'no-response', diff --git a/test/fixtures/modern-raw-server.mjs b/test/fixtures/modern-raw-server.mjs index 00866dc40..da7abbfb5 100644 --- a/test/fixtures/modern-raw-server.mjs +++ b/test/fixtures/modern-raw-server.mjs @@ -16,7 +16,10 @@ // a tool list change on the open subscription), nullframe (writes JSON lines // that aren't JSON-RPC messages, then answers), cont (a state-only // input_required first leg; the second leg does what arguments.leg2 says: -// 'error', 'hang' or 'ok'), die (exits). +// 'error', 'hang', 'ok', or 'again' for another state-only round every time. +// arguments.leg1 'exit' exits just after the first leg's answer, and +// 'elicit' asks for elicitation input instead), task (a result type this +// revision doesn't define), bare (a result with no resultType), die (exits). import { createInterface } from 'node:readline'; import { appendFileSync, existsSync, readFileSync } from 'node:fs'; @@ -82,6 +85,8 @@ createInterface({ input: process.stdin }).on('line', (line) => { { name: 'touch', inputSchema: { type: 'object' } }, { name: 'nullframe', inputSchema: { type: 'object' } }, { name: 'cont', inputSchema: { type: 'object' } }, + { name: 'task', inputSchema: { type: 'object' } }, + { name: 'bare', inputSchema: { type: 'object' } }, { name: 'die', inputSchema: { type: 'object' } }, ], }); @@ -96,17 +101,32 @@ createInterface({ input: process.stdin }).on('line', (line) => { return result(m.id, { content: [{ type: 'text', text: subscription === null ? 'no subscription' : 'touched' }] }); } if (name === 'die') { setTimeout(() => process.exit(5), 10); return result(m.id, { content: [{ type: 'text', text: 'bye' }] }); } + if (name === 'task') return send({ jsonrpc: '2.0', id: m.id, result: { resultType: 'task', taskId: 'task-1' } }); + if (name === 'bare') return send({ jsonrpc: '2.0', id: m.id, result: { content: [{ type: 'text', text: 'no resultType' }] } }); if (name === 'nullframe') { process.stdout.write('null\n[1,2]\n"just a string"\n{"no":"jsonrpc"}\n'); return result(m.id, { content: [{ type: 'text', text: 'after the noise' }] }); } if (name === 'cont') { - const leg2 = m.params?.arguments?.leg2; + const { leg1, leg2 } = m.params?.arguments ?? {}; if (m.params?.requestState === undefined) { - return send({ jsonrpc: '2.0', id: m.id, result: { resultType: 'input_required', requestState: `state-for-${leg2}` } }); + if (leg1 === 'elicit') { + return send({ + jsonrpc: '2.0', + id: m.id, + result: { + resultType: 'input_required', + inputRequests: { confirm: { method: 'elicitation/create', params: { message: 'Proceed?', requestedSchema: { type: 'object', properties: {} } } } }, + }, + }); + } + send({ jsonrpc: '2.0', id: m.id, result: { resultType: 'input_required', requestState: `state-for-${leg2}` } }); + if (leg1 === 'exit') setTimeout(() => process.exit(5), 10); + return; } if (leg2 === 'error') return error(m.id, -32603, 'second leg failed', { leg: 2, state: m.params.requestState }); if (leg2 === 'hang') return; + if (leg2 === 'again') return send({ jsonrpc: '2.0', id: m.id, result: { resultType: 'input_required', requestState: `${m.params.requestState}+` } }); return result(m.id, { content: [{ type: 'text', text: `leg 2 done (${m.params.requestState})` }] }); } return result(m.id, { content: [{ type: 'text', text: 'plain' }] }); diff --git a/test/mcp-modern-outcomes.test.ts b/test/mcp-modern-outcomes.test.ts index 651f6fc7e..bd555d669 100644 --- a/test/mcp-modern-outcomes.test.ts +++ b/test/mcp-modern-outcomes.test.ts @@ -7,6 +7,11 @@ * - a result that fails validation is an answer that can't be used (1 call, * answered), never a claimed server error; * - a JSON-RPC error on the wire is `error-response` with its code and data; + * - a call that ends after an `input_required` answer, before a final one, is + * `no-response`: at the deadline (with nothing in flight to cancel), with + * the connection gone, or when this client can't go on (PR #262 review); + * so is one whose answer has a result type this revision doesn't define, + * or none; * - a list-change subscription refused at connect is reopened; * - `close()` during a reconnect handshake ends the launch it was waiting on. */ @@ -166,6 +171,86 @@ test('an input_required continuation keeps its evidence: the second leg decides assert.equal(calls(log, 'cont'), 4, 'two legs each'); }); +test('a deadline between continuation rounds is no-response, and claims no cancellation', async () => { + // The SDK pauses 250 ms before a state-only round. With a shorter budget, + // the deadline passes inside that pause whenever leg 1 answers in time. + const log = scratchLog(); + const connection = await connect('', log, { requestTimeoutMs: 240 }); + await connection.listTools(); + await assert.rejects(connection.callTool('cont', { leg2: 'ok' }), (err: unknown) => { + assert.ok(err instanceof McplRequestError, String(err)); + assert.equal(err.outcome, 'no-response'); + assert.match( + err.message, + /did not finish tools\/call "cont" within 240ms: it asked for another round \(input_required\), and the deadline passed before that round was sent, so nothing was in flight to cancel\. The outcome is unknown/, + ); + assert.doesNotMatch(err.message, /Cancellation was requested|answered/); + return true; + }); + await new Promise((r) => setTimeout(r, 100)); + assert.equal(calls(log, 'cont'), 1, 'only leg 1 reached the wire'); + assert.equal(wire(log).filter((e) => e.method === 'notifications/cancelled').length, 0, 'nothing was cancelled'); +}); + +test('a connection lost between continuation rounds is no-response, not an answer', async () => { + const log = scratchLog(); + const connection = await connect('', log); + await connection.listTools(); + await assert.rejects(connection.callTool('cont', { leg1: 'exit', leg2: 'ok' }), (err: unknown) => { + assert.ok(err instanceof McplRequestError, String(err)); + assert.equal(err.outcome, 'no-response'); + assert.match( + err.message, + /did not finish tools\/call "cont": it asked for another round \(input_required\), and the call ended before that round was sent: Not connected\. The outcome is unknown/, + ); + return true; + }); + assert.equal(calls(log, 'cont'), 1); +}); + +test('a continuation this client cannot go on with is no-response: the server may have acted', async () => { + const log = scratchLog(); + const connection = await connect('', log); + await connection.listTools(); + // Input this client can't supply: it advertises no elicitation. + await assert.rejects(connection.callTool('cont', { leg1: 'elicit' }), (err: unknown) => { + assert.ok(err instanceof McplRequestError, String(err)); + assert.equal(err.outcome, 'no-response'); + assert.match(err.message, /it asked for another round \(input_required\), and the call ended before that round was sent: Cannot fulfil input request 'confirm'/); + return true; + }); + assert.equal(calls(log, 'cont'), 1); + // A server that sheds load on every round meets the rounds cap: the first + // leg and ten retries. + await assert.rejects(connection.callTool('cont', { leg2: 'again' }), (err: unknown) => { + assert.ok(err instanceof McplRequestError, String(err)); + assert.equal(err.outcome, 'no-response'); + assert.match(err.message, /the call ended before that round was sent: Multi-round-trip request 'tools\/call' still required input after 10 rounds/); + return true; + }); + assert.equal(calls(log, 'cont'), 1 + 11); +}); + +test('a result type this revision does not define, or none, is not a final answer: no-response', async () => { + const log = scratchLog(); + const connection = await connect('', log); + await connection.listTools(); + await assert.rejects(connection.callTool('task', {}), (err: unknown) => { + assert.ok(err instanceof McplRequestError, String(err)); + assert.equal(err.outcome, 'no-response'); + assert.match(err.message, /did not finish tools\/call "task": its answer was not a complete result: Unsupported result type 'task'.*The outcome is unknown/); + return true; + }); + await assert.rejects(connection.callTool('bare', {}), (err: unknown) => { + assert.ok(err instanceof McplRequestError, String(err)); + assert.equal(err.outcome, 'no-response'); + assert.match(err.message, /did not finish tools\/call "bare": its answer was not a complete result: .*missing required resultType.*The outcome is unknown/); + return true; + }); + assert.equal(calls(log, 'task'), 1); + assert.equal(calls(log, 'bare'), 1); +}); + test('concurrent continuations never cross their evidence', async () => { const log = scratchLog(); const connection = await connect('', log, { requestTimeoutMs: 2000 }); From 0adf92e74d3746b4fc7b4dae456d929f9ffb64b4 Mon Sep 17 00:00:00 2001 From: Otis-1924 Date: Thu, 8 Oct 2026 18:22:50 -0400 Subject: [PATCH 0075/1112] feat(mcp): warn when a server's credential would cross the network unencrypted From slimepriestess's review of #262 (non-blocking note 3). A plain listener on a LAN address received "Authorization: Bearer SECRET" from a modern http:// server config with nothing logged. The legacy ws:// path does the same with ?token=, so this is parity, not a new hole. A refusal would break deliberate setups (container networks, tailnets, a TLS-terminating proxy), so this warns. It covers ws:// too, as Nigel-1925 suggested (room-341 #58500), so the two families behave alike. - protocol-family.ts: serverConfigWarnings(config) returns a message when a token or accessProvider is configured for an http:// or ws:// url whose host isn't loopback (localhost, *.localhost, 127.0.0.0/8, [::1], and the unspecified 0.0.0.0 and [::], which connect to this host). Stdio configurations, encrypted schemes and unusable configurations give no warning; an unusable one is serverConfigProblems' to report. checkServerConfig() now logs each warning with console.warn ("[mcp] ..."), after the problems check, so both engines warn once per connection at admission. That is ModernMcpConnection's constructor and McplServerConnection.connect()/connectWithReconnect(). It's exported beside serverConfigProblems for hosts' own validation. - server-connection.ts: connectWithReconnect() checked the configuration itself and then called connect(), which checked it again, so the warning would have been logged twice. The body of connect() is now connectChecked(), which both public entries call after their single check. There is no behaviour change otherwise. No test pins the single warning on this path: that needs a dial to a non-loopback host that fails at once. The cheap candidate, ws://0.0.0.0:1 (Nigel-1925's suggestion), is in fact local, so it is quiet here rather than a trigger. - The access provider's error message is still logged verbatim, as the WebSocket dial already does (transport.ts). Redacting it would drop the diagnosis, and only the provider knows what is secret in it. Tests (mcp-protocol-family): the warning cases for both schemes and the quiet ones (https, wss, loopback forms including 127.0.0.2 and 0.0.0.0, no credential, stdio with a url, an unusable scheme); checkServerConfig logs exactly one line, and ModernMcpConnection.create() logs one before anything is dialed. Mutations, each caught: no warnings at all (2 tests), checkServerConfig not logging (1), nothing counted as loopback (1), only 127.0.0.1 counted (1; from Nigel-1925's review), and the unspecified addresses not counted (1). --- .../cleartext-mcp-credential-warning.added.md | 6 +++ src/index.ts | 1 + src/mcpl/index.ts | 1 + src/mcpl/protocol-family.ts | 38 +++++++++++++++- src/mcpl/server-connection.ts | 11 ++++- test/mcp-protocol-family.test.ts | 45 +++++++++++++++++++ 6 files changed, 100 insertions(+), 2 deletions(-) create mode 100644 changelog.d/cleartext-mcp-credential-warning.added.md diff --git a/changelog.d/cleartext-mcp-credential-warning.added.md b/changelog.d/cleartext-mcp-credential-warning.added.md new file mode 100644 index 000000000..4851f6685 --- /dev/null +++ b/changelog.d/cleartext-mcp-credential-warning.added.md @@ -0,0 +1,6 @@ +- Connecting to an MCP server warns once when its credential (`token` or + `accessProvider`) would cross the network unencrypted: an `http://` or + `ws://` url whose host isn't loopback. It warns rather than refuses, since a + private network or a TLS-terminating proxy can make that deliberate. + `serverConfigWarnings` is exported beside `serverConfigProblems`, so a host + can apply the same check in its own configuration validation. diff --git a/src/index.ts b/src/index.ts index 39d1a6938..3b5de57a3 100644 --- a/src/index.ts +++ b/src/index.ts @@ -98,6 +98,7 @@ export { MODERN_MCP_PROTOCOL_VERSION, resolveServerBinding, serverConfigProblems, + serverConfigWarnings, checkServerConfig, } from './mcpl/index.js'; export type { McpProtocolFamily, McpTransportKind, ServerBinding } from './mcpl/index.js'; diff --git a/src/mcpl/index.ts b/src/mcpl/index.ts index aeca314a1..02230b523 100644 --- a/src/mcpl/index.ts +++ b/src/mcpl/index.ts @@ -116,6 +116,7 @@ export { MCPL_ONLY_POLICY_FIELDS, resolveServerBinding, serverConfigProblems, + serverConfigWarnings, checkServerConfig, } from './protocol-family.js'; export type { McpProtocolFamily, McpTransportKind, ServerBinding } from './protocol-family.js'; diff --git a/src/mcpl/protocol-family.ts b/src/mcpl/protocol-family.ts index 19cc96a60..cea22cedb 100644 --- a/src/mcpl/protocol-family.ts +++ b/src/mcpl/protocol-family.ts @@ -154,9 +154,45 @@ export function serverConfigProblems(config: McplServerConfig): string[] { return problems; } -/** Throw one error listing every problem, or return the resolved binding. */ +/** A host a connection to which never leaves this machine. The unspecified + * addresses count: a connection to 0.0.0.0 or [::] reaches this host. */ +function isLoopbackHost(hostname: string): boolean { + const host = hostname.toLowerCase(); + return host === 'localhost' || host.endsWith('.localhost') || host === '[::1]' || /^127(\.\d{1,3}){3}$/.test(host) || + host === '0.0.0.0' || host === '[::]'; +} + +/** + * What is usable in a server configuration but worth an operator's attention, + * as messages; empty when there is nothing. A credential (`token` or + * `accessProvider`) for an `http://` or `ws://` url whose host isn't loopback + * crosses the network in cleartext, in the `Authorization` header or the + * WebSocket `?token=` query. That can be deliberate, on a private network or + * behind a TLS-terminating proxy, so it is a warning and never a refusal. + */ +export function serverConfigWarnings(config: McplServerConfig): string[] { + if (!config.token && !config.accessProvider) return []; + let transport: McpTransportKind; + try { + transport = resolveServerBinding(config).transport; + } catch { + return []; // an unusable configuration is serverConfigProblems' to report + } + if (transport === 'stdio') return []; + const url = new URL(config.url!); + if ((url.protocol !== 'http:' && url.protocol !== 'ws:') || isLoopbackHost(url.hostname)) return []; + const secure = url.protocol === 'http:' ? 'https' : 'wss'; + return [ + `MCP server "${config.id}": its credential goes to ${url.host} unencrypted (${url.protocol}//), where anything on the ` + + `network path can read it; use ${secure}:// unless that network is trusted`, + ]; +} + +/** Throw one error listing every problem. Otherwise log each warning and + * return the resolved binding. Both engines admit a configuration here. */ export function checkServerConfig(config: McplServerConfig): ServerBinding { const problems = serverConfigProblems(config); if (problems.length > 0) throw new Error(problems.join('; ')); + for (const warning of serverConfigWarnings(config)) console.warn(`[mcp] ${warning}`); return resolveServerBinding(config); } diff --git a/src/mcpl/server-connection.ts b/src/mcpl/server-connection.ts index 1a87641f7..e1c5b1463 100644 --- a/src/mcpl/server-connection.ts +++ b/src/mcpl/server-connection.ts @@ -620,6 +620,15 @@ export class McplServerConnection extends EventEmitter { hostCapabilities: McplHostCapabilities, ): Promise { McplServerConnection.checkConfig(config); + return McplServerConnection.connectChecked(config, hostCapabilities); + } + + /** {@link connect}, for a configuration its caller has already checked + * (and warned about) once. */ + private static async connectChecked( + config: McplServerConfig, + hostCapabilities: McplHostCapabilities, + ): Promise { let established: Awaited>; try { established = await McplServerConnection.handshake(config, hostCapabilities); @@ -825,7 +834,7 @@ export class McplServerConnection extends EventEmitter { // Outside the retry path below: a configuration error is never retried. McplServerConnection.checkConfig(config); try { - return await McplServerConnection.connect(config, hostCapabilities); + return await McplServerConnection.connectChecked(config, hostCapabilities); } catch (error) { // No common protocol revision is a configuration fact, not an outage: // a retry stub would restart the server forever for nothing. diff --git a/test/mcp-protocol-family.test.ts b/test/mcp-protocol-family.test.ts index 5b104335c..5a0d31d20 100644 --- a/test/mcp-protocol-family.test.ts +++ b/test/mcp-protocol-family.test.ts @@ -10,9 +10,12 @@ import assert from 'node:assert/strict'; import { resolveServerBinding, serverConfigProblems, + serverConfigWarnings, + checkServerConfig, MCPL_ONLY_POLICY_FIELDS, MAX_TIMER_MS, } from '../src/mcpl/protocol-family.js'; +import { ModernMcpConnection } from '../src/mcpl/modern-connection.js'; import type { McplServerConfig } from '../src/mcpl/types.js'; const cfg = (c: Partial): McplServerConfig => ({ id: 's', ...c }); @@ -102,3 +105,45 @@ test('MCPL policy on a modern server is rejected, empty values are not', () => { }); assert.deepEqual(serverConfigProblems(empty), []); }); + +test('a credential sent in cleartext to a host that is not loopback is warned about, never refused', () => { + const provider = async () => 'fresh'; + const warned = (c: Partial) => serverConfigWarnings(cfg(c)); + // The Authorization header over http, and the ?token= query over ws. + assert.match(warned({ url: 'http://10.0.0.5:8080/mcp', token: 't' })[0]!, /credential goes to 10\.0\.0\.5:8080 unencrypted \(http:\/\/\).*use https:\/\//); + assert.match(warned({ url: 'ws://mcp.lan/mcpl', accessProvider: provider })[0]!, /credential goes to mcp\.lan unencrypted \(ws:\/\/\).*use wss:\/\//); + assert.equal(serverConfigProblems(cfg({ url: 'http://10.0.0.5:8080/mcp', token: 't' })).length, 0, 'still usable'); + // Nothing to warn about: encrypted, loopback, no credential, or not dialed. + for (const quiet of [ + { url: 'https://10.0.0.5/mcp', token: 't' }, + { url: 'wss://mcp.lan/mcpl', token: 't' }, + { url: 'http://localhost:8080/mcp', token: 't' }, + { url: 'http://127.0.0.1:8080/mcp', token: 't' }, + { url: 'http://127.0.0.2:8080/mcp', token: 't' }, + { url: 'ws://0.0.0.0:8080/mcpl', token: 't' }, + { url: 'http://[::1]:8080/mcp', token: 't' }, + { url: 'ws://app.localhost/mcpl', token: 't' }, + { url: 'http://10.0.0.5:8080/mcp' }, + { command: 'srv', url: 'http://10.0.0.5/mcp', token: 't' }, + { url: 'ftp://10.0.0.5/mcp', token: 't' }, + ]) assert.deepEqual(warned(quiet), [], JSON.stringify(quiet)); +}); + +test('both engines admit a configuration through checkServerConfig, which logs its warnings', () => { + const lines: string[] = []; + const realWarn = console.warn; + console.warn = (...args: unknown[]) => { lines.push(args.join(' ')); }; + try { + checkServerConfig(cfg({ url: 'ws://mcp.lan/mcpl', token: 't' })); + assert.equal(lines.length, 1); + assert.match(lines[0]!, /^\[mcp\] MCP server "s": its credential goes to mcp\.lan unencrypted/); + // The modern engine checks at create(), before anything is dialed. + ModernMcpConnection.create(cfg({ id: 'm', url: 'http://10.0.0.5:8080/mcp', token: 't' })); + assert.equal(lines.length, 2); + assert.match(lines[1]!, /MCP server "m": its credential goes to 10\.0\.0\.5:8080 unencrypted/); + checkServerConfig(cfg({ url: 'https://10.0.0.5/mcp', token: 't' })); + assert.equal(lines.length, 2, 'nothing for https'); + } finally { + console.warn = realWarn; + } +}); From 660ed6fc101abf8a3921f6d08793eb0154113451 Mon Sep 17 00:00:00 2001 From: Otis-1924 Date: Thu, 8 Oct 2026 18:22:50 -0400 Subject: [PATCH 0076/1112] fix(mcp): a protocol-version verdict at startup raises the ops alert From slimepriestess's review of #262 (non-blocking note 3). main ignored an MCPL server's protocolVersion, and this PR's legacy engine now requires 2024-11-05 back. So a hand-written resident server that omits it "vanishes at startup with one console.error line (connected:false, retrying:false, 0 tools)". The strict check stays: room-284 chose it after surveying real servers, and the review found every house MCPL server answering exactly 2024-11-05. Part of the gap was ours, as Nigel-1925 found (room-341 #58500). The same verdict on reconnect raises the mcpl-down ops alert at once, because no later attempt comes. At startup it only logged and traced mcpl:server-connect-failed (willRetry false), since no stub exists to emit the permanent event. initializeMcpl's fail-open catch now raises the same alert for a McplProtocolVersionError: failures.log, an ops:alert trace, and the ops webhook when configured. A runtime connectMcplServer() needs no alert: its caller gets the error. Not taken: a one-release warning mode (the version check is the point, and a transitional mode is a deferral), and a failure reason in listMcplServers(). Every kind of connect failure lacks that today, so it belongs to its own change. Test (mcpl-legacy-version): a framework started with a 'missing'-mode server and reconnect: true. A module subscribes to traces in start(), which runs before the MCP servers connect, and sees mcpl:server-connect-failed with willRetry false, then ops:alert kind mcpl-down for that server, naming "protocolVersion (none)". Mutation (the alert removed): that test fails, nothing else. --- .../legacy-mcp-protocol-boundary.breaking.md | 4 ++- src/framework.ts | 5 +++ test/mcpl-legacy-version.test.ts | 35 ++++++++++++++++++- 3 files changed, 42 insertions(+), 2 deletions(-) diff --git a/changelog.d/legacy-mcp-protocol-boundary.breaking.md b/changelog.d/legacy-mcp-protocol-boundary.breaking.md index 92d319362..84cf28ddd 100644 --- a/changelog.d/legacy-mcp-protocol-boundary.breaking.md +++ b/changelog.d/legacy-mcp-protocol-boundary.breaking.md @@ -10,7 +10,9 @@ with `protocol: 'modern'`. A protocol-version verdict is never retried: no reconnect stub, and a reconnecting connection stops its backoff loop (`reconnect-failed` carries `permanent: true` and raises the ops alert at - once). The established revision is on `connection.protocolVersion`. + once). At startup the verdict raises the same `mcpl-down` ops alert, so a + server refused for its revision doesn't drop out with only a log line. The + established revision is on `connection.protocolVersion`. - `tools/list` now follows `nextCursor` across pages, so a paginating server's whole inventory reaches the agent; before, every page after the first was silently dropped. A repeated cursor, or more than 100 pages, is an error diff --git a/src/framework.ts b/src/framework.ts index 394fc991c..194b0fee1 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -13388,6 +13388,11 @@ export class AgentFramework { // only configuration can fix it). willRetry: config.reconnect === true && !(error instanceof McplProtocolVersionError), }); + // So the server doesn't vanish quietly: alert at once, as the + // reconnect path does for the same verdict, since no attempt follows. + if (error instanceof McplProtocolVersionError) { + this.opsAlert('mcpl-down', config.id, `MCPL server unreachable at startup: ${err.message}`); + } } } diff --git a/test/mcpl-legacy-version.test.ts b/test/mcpl-legacy-version.test.ts index d672fea31..050cd5b65 100644 --- a/test/mcpl-legacy-version.test.ts +++ b/test/mcpl-legacy-version.test.ts @@ -2,7 +2,8 @@ * The legacy engine's protocol boundary and inventory (room-284, shelf-487): * - it offers and accepts exactly MCP 2024-11-05; anything else is a typed * McplProtocolVersionError carrying what each side said; - * - such a verdict is never retried (no reconnect stub, no backoff loop); + * - such a verdict is never retried (no reconnect stub, no backoff loop), and + * at startup it raises the ops alert at once, as it does on reconnect; * - tools/list follows nextCursor to the complete inventory, and refuses a * server whose cursors don't terminate. */ @@ -15,6 +16,8 @@ import { fileURLToPath } from 'node:url'; import { McplServerConnection, McplProtocolVersionError, McplRequestError } from '../src/mcpl/server-connection.js'; import type { McplHostCapabilities, McplServerConfig } from '../src/mcpl/types.js'; +import { AgentFramework } from '../src/framework.js'; +import type { Module, ModuleContext, ToolDefinition, ToolResult, TraceEvent } from '../src/index.js'; const here = dirname(fileURLToPath(import.meta.url)); const SERVER = join(here, 'fixtures', 'legacy-version-server.mjs'); @@ -126,6 +129,36 @@ test('a version verdict on reconnect stops the backoff loop', async () => { assert.equal(failures.length, 1); }); +test('a version verdict at startup raises the ops alert at once, as on reconnect', async () => { + const dir = mkdtempSync(join(tmpdir(), 'legacy-version-')); + cleanups.push(() => rmSync(dir, { recursive: true, force: true })); + // Modules start before the MCP servers connect, so this one sees startup. + const traces: TraceEvent[] = []; + const watcher: Module = { + name: 'trace-watcher', + async start(ctx: ModuleContext) { ctx.onTrace((event) => traces.push(event)); }, + async stop() {}, + getTools: (): ToolDefinition[] => [], + handleToolCall: async (): Promise => ({ success: false, error: 'no tools', isError: true }), + onProcess: async () => ({}), + }; + const framework = await AgentFramework.create({ + storePath: join(dir, 'store'), + membrane: { complete: async () => { throw new Error('not used'); } } as unknown as import('@animalabs/membrane').Membrane, + agents: [{ name: 'agent', model: 'test-model', systemPrompt: 'test' }], + modules: [watcher], + mcplServers: [config('missing', { reconnect: true })], + }); + cleanups.push(() => framework.stop()); + const failed = traces.find((t) => t.type === 'mcpl:server-connect-failed') as { serverId?: string; willRetry?: boolean } | undefined; + assert.equal(failed?.serverId, 'legacy'); + assert.equal(failed?.willRetry, false); + const alert = traces.find((t) => t.type === 'ops:alert') as { kind?: string; agentName?: string; message?: string } | undefined; + assert.equal(alert?.kind, 'mcpl-down'); + assert.equal(alert?.agentName, 'legacy'); + assert.match(alert?.message ?? '', /unreachable at startup: .*protocolVersion \(none\)/); +}); + test('tools/list follows nextCursor to the complete inventory', async () => { const connection = await connect(config('ok')); const { tools } = await connection.sendToolsList(); From d981e494806b69ef1b2a4ed17b281d14c24edab2 Mon Sep 17 00:00:00 2001 From: Petra-1012 Date: Fri, 9 Oct 2026 13:07:10 -0400 Subject: [PATCH 0077/1112] test(mcpl): pin every branch of the copied raster sniffer with a mislabeled fixture slimepriestess approved #252 at a2accc1 with one non-blocking ask: the sniffer that sourceBodyDigest uses (sniffRasterImageMediaType, copied from context-manager's unexported blob-manager code) was pinned by real-store round-trip fixtures only for png and jpeg (and svg as unsniffed), so a context-manager change to its gif or webp branch would silently bring back the stored-digest mismatch for those formats. Three rows join the real-store table, each labeled as something else so the store relabels it on the way in: GIF87a bytes labeled image/png, GIF89a bytes labeled image/jpeg, and WebP (RIFF....WEBP) bytes labeled image/png. Each must come back changed and still hash as handed over, live and after the store is reopened, against the installed @animalabs/context-manager 0.13.0. Red checks, each a one-line edit to the copied sniffer, rebuilt and run: - the gif branch removed fails at "GIF87a bytes labeled image/png"; - only GIF87a kept fails at "GIF89a bytes labeled image/jpeg"; - the webp branch removed fails at "WebP bytes labeled image/png". Restored, inbound-source passes 22/22. The existing SVG, BMP and AVIF rows already pin the other direction (a store that starts sniffing a new type). Test-only; no changelog fragment. Co-Authored-By: Claude Opus 5.5 --- test/inbound-source.test.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/inbound-source.test.ts b/test/inbound-source.test.ts index 65232a7ca..b50b14a70 100644 --- a/test/inbound-source.test.ts +++ b/test/inbound-source.test.ts @@ -487,6 +487,11 @@ describe('sourceBodyDigest hashes a body as the store keeps it', () => { 'a labeled image': { blocks: [image(png)], changed: false }, 'PNG bytes labeled image/jpeg': { blocks: [image(png, 'image/jpeg')], changed: true }, 'JPEG bytes labeled image/png': { blocks: [image(Buffer.from('ffd8ffe000104a464946', 'hex').toString('base64'))], changed: true }, + // One fixture per branch of the copied sniffer, each labeled as something + // else, so a context-manager that stops sniffing one fails here. + 'GIF87a bytes labeled image/png': { blocks: [image(Buffer.from('GIF87a\x01\x00\x01\x00\x00\x00\x00', 'latin1').toString('base64'))], changed: true }, + 'GIF89a bytes labeled image/jpeg': { blocks: [image(Buffer.from('GIF89a\x01\x00\x01\x00\x00\x00\x00', 'latin1').toString('base64'), 'image/jpeg')], changed: true }, + 'WebP bytes labeled image/png': { blocks: [image(Buffer.from('RIFF\x1a\x00\x00\x00WEBPVP8L\x0d\x00\x00\x00', 'latin1').toString('base64'))], changed: true }, 'base64 with a newline in it': { blocks: [image(`${png.slice(0, 11)}\n${png.slice(11)}`)], changed: true }, 'base64 with whitespace around it': { blocks: [image(` ${png}\n`)], changed: true }, 'base64 without its padding': { blocks: [image(png.replace(/=+$/, ''))], changed: true }, From 8fb4c7848c7136de7e0ebfc71feb0de17cb5b12a Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Fri, 9 Oct 2026 17:49:30 -0400 Subject: [PATCH 0078/1112] fix(mcpl): a channels/open answered for another channel fails the open MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit slimepriestess suggested this host-side check in her review of discord-mcpl #62 (room-410 #64335, relayed by Agnes-984 in room-203 #64343): #210 checks host state around an open but never compared the answer's channel with the one requested. Her probe on discord-mcpl's base showed the substitution is real: a channels/open of a stale `discord:g1:c9` fell back to the first registered channel of the type, opened `c1`, and returned c1's history. slack-mcpl's handleChannelOpen has the same first-of-type fallback today. discord-mcpl #62 closes it on that server; this closes it for every server, on the host. In applyDesiredChannelState, right after the open RPC returns (before the stale-receipt checks, since a substituted answer is wrong whatever the target did meanwhile): when the result names a channel id (`result.channel.id` present) that isn't the requested channelId, - the current entry's transport confirmation is invalidated (open=false, unconfirmed): an answer for another channel confirms nothing about this one, and a server that no longer maps our id probably hasn't got it open. Incoming traffic from the channel still re-confirms it, and a later open retries it; - a ChannelOpenSubstitutionError is thrown, so nothing in the answer is used: channel_open returns a failure with no history, delivery-forced opens fail the route visibly, reply opens report open-failed; - the host logs `[channel-open-substituted] server=… requested=… answered=…` (console.error, so headless hosts without a trace client see it), with the server-chosen id printed only when it's a short visible-ASCII token and otherwise described (plainIdForLog), so it can't add a line or a field; - an `mcpl:channel-open-substituted` trace carries serverId, channelId, answeredFor and the message. reconcileChannels skips its generic reconcile-failed trace for this error, as it does for exhausted convergence: one diagnostic per incident. waitForOpenChannel is unchanged on purpose: joiners fail through the invalidated open flag, and a later operation that does establish the channel still lets them through. A result that names no channel (several test doubles return `{}` or nothing) can't be checked and is accepted as before. Checked servers: discord-mcpl and slack-mcpl return their registered descriptor on the normal path, so a matching id is what production sends. Tests (test/channel-open-substitution.test.ts, 5): the tool open fails with no history and neither channel marked open, one trace, one log line; a policy-open channel stays closed, reports once (no reconcile-failed), and speech into it fails with the reason and sends nothing; a known-open channel loses confirmation and reopens once the server answers for it; an odd id (newline plus a forged log line) is described, not printed; controls: a same-id answer and a no-channel answer open as before. With src reverted, the first four fail and the control passes. Related files (substitution, lifecycle-order, registry-routing, machine-close-provenance, mcpl-gate-roundtrip, channels-changed-respond-first): 79/79. #210's author, Jasper-314, is archived; Ada-1017 adds this as agent-framework's owner (room-203 #64353). --- .../210-channel-lifecycle-order.fixed.md | 2 + src/mcpl/channel-registry.ts | 44 +++++- test/channel-open-substitution.test.ts | 146 ++++++++++++++++++ 3 files changed, 190 insertions(+), 2 deletions(-) create mode 100644 test/channel-open-substitution.test.ts diff --git a/changelog.d/210-channel-lifecycle-order.fixed.md b/changelog.d/210-channel-lifecycle-order.fixed.md index 50b39afd1..f7060528f 100644 --- a/changelog.d/210-channel-lifecycle-order.fixed.md +++ b/changelog.d/210-channel-lifecycle-order.fixed.md @@ -5,3 +5,5 @@ - Open receipts confirm only the current registration, connection, type, and address. Retargeting invalidates an older open flag; failed corrective opens leave the new target unconfirmed and fail joined delivery, while same-target backscroll failures retain known-open state. Explicit close retries an unconfirmed target instead of reporting it already closed. - Lifecycle convergence makes at most five transport attempts per operation. Continued descriptor/intent supersession fails with a diagnostic, releases the queue, and gives joined speech or reply preparation a visible failure instead of hanging. Later stable operations can retry. + +- A `channels/open` answered for a different channel than the one requested fails the open. The requested channel isn't marked open (its open state becomes unconfirmed), nothing in the answer is used, its history included, and the host logs `[channel-open-substituted]` and emits an `mcpl:channel-open-substituted` trace. An answer that names no channel is accepted as before. diff --git a/src/mcpl/channel-registry.ts b/src/mcpl/channel-registry.ts index 0873fd438..c47a82015 100644 --- a/src/mcpl/channel-registry.ts +++ b/src/mcpl/channel-registry.ts @@ -50,8 +50,22 @@ const MAX_CHANNEL_LIFECYCLE_ATTEMPTS = 5; /** A successful RPC can still fail to establish the latest channel target. */ class ChannelLifecycleConvergenceError extends Error {} +/** A channels/open answered for a channel other than the one it named. */ +class ChannelOpenSubstitutionError extends Error {} const CHANNEL_LIFECYCLE_LOG_ID = 'mcpl/channel-lifecycle'; +/** + * A server-chosen channel id for a host log line: as it is when it's a short + * token of visible ASCII, as composite ids such as `discord:g1:c1` are, and + * otherwise only described, so an odd id can't add a line or a field. + */ +function plainIdForLog(id: unknown): string { + if (typeof id === 'string' && /^[\x21-\x7e]{1,200}$/.test(id)) return id; + return typeof id === 'string' + ? `(not a plain id: a ${id.length}-character string)` + : `(not a plain id: ${id === null ? 'null' : typeof id})`; +} + /** * Durable "which label did this channelId have, each time we saw it" log. * `resolveProseTarget()` resolves a label to an id only for channels @@ -2161,6 +2175,30 @@ export class ChannelRegistry { if (this.channels.get(channelKey) === entry && !sameTarget()) this.invalidateTransportConfirmation(entry); throw error; } + // A server can answer for a channel other than the one named: one + // that falls back to its first channel of the type for an id it no + // longer knows opens that channel and returns its history. That + // receipt confirms nothing about this channel, and the history is + // another channel's. So the open fails, nothing is marked open, the + // history goes nowhere, and this channel's open state is unconfirmed. + // A result that names no channel can't be checked and is taken as + // before. + const answeredFor = result?.channel?.id; + if (answeredFor !== undefined && answeredFor !== channelId) { + if (this.channels.get(channelKey) === entry) this.invalidateTransportConfirmation(entry); + const error = new ChannelOpenSubstitutionError( + `Server "${serverId}" answered channels/open for ${channelId} with a different channel; ` + + `${channelId} is not marked open, and nothing in the answer is used`, + ); + console.error( + `[channel-open-substituted] server=${serverId} requested=${channelId} answered=${plainIdForLog(answeredFor)}`, + ); + this.emitTraceFn({ + type: 'mcpl:channel-open-substituted', + serverId, channelId, answeredFor, error: error.message, + }); + throw error; + } // Never apply an old receipt to the current target, even transiently: // a corrective open might fail before that stale bit is replaced. if (this.channels.get(channelKey) !== entry) continue; @@ -2279,9 +2317,11 @@ export class ChannelRegistry { await this.applyDesiredChannelState(serverId, channel.id); } catch (err) { // Removal while queued is cancellation, not a failed subscription. - // Exhausted convergence already emitted its diagnostic at the bound. + // Exhausted convergence and a substituted answer already emitted + // their own diagnostics. if (!this.channels.has(`${serverId}:${channel.id}`) || - err instanceof ChannelLifecycleConvergenceError) continue; + err instanceof ChannelLifecycleConvergenceError || + err instanceof ChannelOpenSubstitutionError) continue; this.emitTraceFn({ type: 'mcpl:channel-reconcile-failed', serverId, diff --git a/test/channel-open-substitution.test.ts b/test/channel-open-substitution.test.ts new file mode 100644 index 000000000..d4a4b8260 --- /dev/null +++ b/test/channel-open-substitution.test.ts @@ -0,0 +1,146 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { ChannelRegistry } from '../src/mcpl/channel-registry.js'; +import { CapabilityGrant, ALL_CAPABILITY_PATHS } from '../src/mcpl/capability-grant.js'; +import type { McplServerRegistry } from '../src/mcpl/server-registry.js'; +import type { FeatureSetManager } from '../src/mcpl/feature-set-manager.js'; +import type { ChannelDescriptor, ChannelIncomingMessage, ChannelsOpenResult } from '../src/mcpl/types.js'; + +// A server that answers channels/open for another channel than the one named, +// as discord-mcpl's base did for a stale id: it fell back to its first channel +// of the type, opened that, and returned that channel's history. + +function descriptor(id: string, initiallyOpen = false): ChannelDescriptor { + return { id, type: 'test', label: id, direction: 'bidirectional', initiallyOpen, address: { value: id } }; +} + +function historyOf(channelId: string): ChannelIncomingMessage[] { + return [{ + channelId, messageId: `${channelId}-1`, author: { id: 'u1', name: 'someone' }, + timestamp: '2026-10-09T00:00:00Z', content: [{ type: 'text', text: `from ${channelId}` }], + } as ChannelIncomingMessage]; +} + +function fixture(answer: (requested: string) => ChannelsOpenResult) { + const calls: Array<{ kind: 'open' | 'close'; id: string }> = []; + const published: string[] = []; + const traces: Array<{ type: string; [key: string]: unknown }> = []; + const routeFailures: Array<{ reason: string }> = []; + const server = { + grant: new CapabilityGrant(new Set(ALL_CAPABILITY_PATHS), []), + sendChannelsOpen: async (params: { channelId?: string }) => { + calls.push({ kind: 'open', id: params.channelId! }); + return answer(params.channelId!); + }, + sendChannelsClose: async (params: { channelId: string }) => { + calls.push({ kind: 'close', id: params.channelId }); + return { closed: true }; + }, + sendChannelsPublish: async (params: { channelId: string }) => { + published.push(params.channelId); + return { delivered: true }; + }, + }; + const registry = new ChannelRegistry( + { getServer: () => server } as unknown as McplServerRegistry, + {} as FeatureSetManager, () => {}, (event) => traces.push(event), + { onRouteFailure: (info) => routeFailures.push(info) }, + ); + const isOpen = (id: string) => registry.getOpenChannels().some((e) => e.descriptor.id === id); + const substituted = () => traces.filter((t) => t.type === 'mcpl:channel-open-substituted'); + return { registry, calls, published, traces, routeFailures, isOpen, substituted }; +} + +/** Run `work` with console.error captured, returning what it printed. */ +async function capturingErrors(work: () => Promise): Promise<{ value: T; lines: string[] }> { + const lines: string[] = []; + const original = console.error; + console.error = (...args: unknown[]) => { lines.push(args.map(String).join(' ')); }; + try { + return { value: await work(), lines }; + } finally { + console.error = original; + } +} + +test('an open answered for another channel fails, and that channel\'s history goes nowhere', async () => { + const f = fixture(() => ({ channel: descriptor('y'), history: historyOf('y') })); + await f.registry.handleChanged('test', { added: [descriptor('x'), descriptor('y')] }); + const { value: result, lines } = await capturingErrors(() => + f.registry.handleChannelToolCall('channel_open', { serverId: 'test', channelId: 'x', backscroll: 5 })); + + assert.equal(result.success, false); + assert.match(String(result.error), /answered channels\/open for x with a different channel/); + assert.equal(result.data, undefined, 'no history, or anything else from the answer, reaches the resident'); + assert.equal(f.isOpen('x'), false, 'the requested channel is not marked open'); + assert.equal(f.isOpen('y'), false, 'nor is the channel the server answered for'); + + const [trace, ...more] = f.substituted(); + assert.equal(more.length, 0); + assert.equal(trace?.serverId, 'test'); + assert.equal(trace?.channelId, 'x'); + assert.equal(trace?.answeredFor, 'y'); + assert.deepEqual(lines.filter((l) => l.startsWith('[channel-open-substituted]')), + ['[channel-open-substituted] server=test requested=x answered=y']); +}); + +test('a channel admitted open stays closed on a substituted answer, reports it once, and speech into it fails loudly', async () => { + const f = fixture(() => ({ channel: descriptor('y') })); + await capturingErrors(() => f.registry.handleChanged('test', { added: [descriptor('x', true), descriptor('y')] })); + assert.equal(f.isOpen('x'), false); + assert.equal(f.substituted().length, 1); + assert.equal(f.traces.filter((t) => t.type === 'mcpl:channel-reconcile-failed').length, 0, + 'the substitution is its own diagnostic, not also a generic reconcile failure'); + + const { value: delivery } = await capturingErrors(() => f.registry.routeSpeech('resident', 'hello', 'x')); + assert.equal(delivery, null); + assert.deepEqual(f.published, [], 'nothing is sent into a channel the server would not confirm'); + assert.equal(f.routeFailures.length, 1); + assert.match(f.routeFailures[0]!.reason, /answered channels\/open for x with a different channel/); +}); + +test('a known-open channel loses its confirmation on a substituted answer, and reopens once the server answers for it', async () => { + let answerFor: string | undefined; + const f = fixture((requested) => ({ channel: descriptor(answerFor ?? requested) })); + await f.registry.handleChanged('test', { added: [descriptor('x', true), descriptor('y')] }); + assert.equal(f.isOpen('x'), true); + + answerFor = 'y'; + const { value: backscroll } = await capturingErrors(() => + f.registry.handleChannelToolCall('channel_open', { serverId: 'test', channelId: 'x', backscroll: 5 })); + assert.equal(backscroll.success, false); + assert.equal(f.isOpen('x'), false, 'an answer for another channel confirms nothing about this one'); + + answerFor = undefined; + f.calls.length = 0; + const delivery = await f.registry.routeSpeech('resident', 'hello', 'x'); + assert.equal(delivery?.delivered, true); + assert.deepEqual(f.calls, [{ kind: 'open', id: 'x' }], 'delivery reopens the channel before sending'); + assert.deepEqual(f.published, ['x']); +}); + +test('an odd answered id is described in the log line, never printed raw', async () => { + const forged = 'y\n[channel-open-substituted] server=other requested=x answered=x'; + const f = fixture(() => ({ channel: { ...descriptor('y'), id: forged } })); + await f.registry.handleChanged('test', { added: [descriptor('x')] }); + const { lines } = await capturingErrors(() => + f.registry.handleChannelToolCall('channel_open', { serverId: 'test', channelId: 'x' })); + const logged = lines.filter((l) => l.includes('channel-open-substituted')); + assert.deepEqual(logged, [ + `[channel-open-substituted] server=test requested=x answered=(not a plain id: a ${forged.length}-character string)`, + ]); +}); + +test('an answer for the requested channel, or one naming no channel, opens it as before', async () => { + for (const answer of [ + (requested: string) => ({ channel: descriptor(requested), history: historyOf(requested) }), + () => ({}) as ChannelsOpenResult, + ]) { + const f = fixture(answer); + await f.registry.handleChanged('test', { added: [descriptor('x')] }); + const result = await f.registry.handleChannelToolCall('channel_open', { serverId: 'test', channelId: 'x', backscroll: 5 }); + assert.equal(result.success, true); + assert.equal(f.isOpen('x'), true); + assert.equal(f.substituted().length, 0); + } +}); From 1997b120339bfc9dacbd90e39cf8f14f12c9e83e Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Fri, 9 Oct 2026 18:00:18 -0400 Subject: [PATCH 0079/1112] fix(mcpl): undo the open a substituting server made; check history ids; retry stale receipts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Basil-1007's review of 8fb4c78 (room-376 #64491), all five points taken, one adjusted: 1. The server really opened the channel it answered for, and nothing closed it: Basil's probe showed y (policy-closed) then forwarding a message, being marked open by handleIncoming, reaching the resident and becoming the default publish channel. Now closeSubstitutedChannel closes it through y's own lifecycle queue when y is registered on this server and its desired state is 'closed'. Adjusted from Basil's `onlyIfClosed: true` suggestion: a channel the host wants open is left alone, so the correction only ever sends a close, and a close can't be answered for another channel. That rules out ping-pong between two channels a pathological server keeps swapping. An id the host doesn't know needs nothing: §14.5 refuses its incoming (channel-registry ~950). Its failure is traced as mcpl:channel-reconcile-failed (convergence exhaustion already traced itself). 2. Invalidation kept, as Basil agreed. 3. substitutedChannel() also reads history items: a history item whose channelId names another channel is substitution, with or without `channel` in the answer. discord-mcpl's history items carry desc.id, the same id the answer's channel carries. Absent ids still can't be checked and pass, the same rule for both fields. The ChannelsOpenParams.channelId doc now says the id is exact and a server that doesn't know it should refuse, instead of "preferred over type/address matching", which read as permitting the fallback. 4. Placement: the log line, trace and corrective close happen as soon as the answer names another channel; the failure itself comes after the two stale-receipt checks. A stale receipt is retried whatever it says, so a substitution on it now retries instead of failing; order only matters there, and it turns a failure into a retry, never a success into a failure. The current-entry guard on the invalidation is gone (always true after the checks). 5. plainIdForLog now guards both ids on the log line: requested ids come from server registration too, which only checks for a non-empty string. The trace no longer carries `error` (it's emitted before the error exists); serverId, channelId and answeredFor remain. Tests: 9 in test/channel-open-substitution.test.ts (4 new: history-only substitution, the corrective close, no correction for a wanted-open channel, a stale substituted receipt retried and still corrected). Red check: with src at 24a84f5, 7 fail (the wanted-open guard and the controls pass); with src at 8fb4c78, the three new behaviours fail. Related channel files 83/83. Full suite at 8fb4c78: 1158 tests, 19 fail and 5 cancelled, all in mcpl-awareness-barrier, mcpl-tool-lifecycle-e2e and tune-out-e2e, whose child MCPL servers don't start in this worktree; those files fail the same at #210's own head 24a84f5 (19 fail, 4 cancelled; tune-out-e2e cancelled). --- src/mcpl/channel-registry.ts | 77 +++++++++++++++------ src/mcpl/types.ts | 7 +- test/channel-open-substitution.test.ts | 93 +++++++++++++++++++++----- 3 files changed, 142 insertions(+), 35 deletions(-) diff --git a/src/mcpl/channel-registry.ts b/src/mcpl/channel-registry.ts index c47a82015..b9edc829b 100644 --- a/src/mcpl/channel-registry.ts +++ b/src/mcpl/channel-registry.ts @@ -66,6 +66,23 @@ function plainIdForLog(id: unknown): string { : `(not a plain id: ${id === null ? 'null' : typeof id})`; } +/** + * The other channel a channels/open answer names, if it names one: its + * `channel.id`, or else the `channelId` of a history item. Each is evidence of + * which channel the server opened. An answer that names no channel at all + * can't be checked, and gives undefined. + */ +function substitutedChannel(result: ChannelsOpenResult | undefined, requested: string): unknown { + const answered = result?.channel?.id; + if (answered !== undefined && answered !== requested) return answered; + const history: unknown[] = Array.isArray(result?.history) ? result.history : []; + for (const item of history) { + const from = (item as { channelId?: unknown } | null)?.channelId; + if (from !== undefined && from !== requested) return from; + } + return undefined; +} + /** * Durable "which label did this channelId have, each time we saw it" log. * `resolveProseTarget()` resolves a label to an id only for channels @@ -2177,27 +2194,16 @@ export class ChannelRegistry { } // A server can answer for a channel other than the one named: one // that falls back to its first channel of the type for an id it no - // longer knows opens that channel and returns its history. That - // receipt confirms nothing about this channel, and the history is - // another channel's. So the open fails, nothing is marked open, the - // history goes nowhere, and this channel's open state is unconfirmed. - // A result that names no channel can't be checked and is taken as - // before. - const answeredFor = result?.channel?.id; - if (answeredFor !== undefined && answeredFor !== channelId) { - if (this.channels.get(channelKey) === entry) this.invalidateTransportConfirmation(entry); - const error = new ChannelOpenSubstitutionError( - `Server "${serverId}" answered channels/open for ${channelId} with a different channel; ` + - `${channelId} is not marked open, and nothing in the answer is used`, - ); + // longer knows opens that channel and returns its history. Whatever + // becomes of this receipt, the server did that: say so, and undo the + // open it made if the host wants that channel closed. + const substitute = open ? substitutedChannel(result, channelId) : undefined; + if (substitute !== undefined) { console.error( - `[channel-open-substituted] server=${serverId} requested=${channelId} answered=${plainIdForLog(answeredFor)}`, + `[channel-open-substituted] server=${serverId} requested=${plainIdForLog(channelId)} answered=${plainIdForLog(substitute)}`, ); - this.emitTraceFn({ - type: 'mcpl:channel-open-substituted', - serverId, channelId, answeredFor, error: error.message, - }); - throw error; + this.emitTraceFn({ type: 'mcpl:channel-open-substituted', serverId, channelId, answeredFor: substitute }); + if (typeof substitute === 'string') this.closeSubstitutedChannel(serverId, substitute); } // Never apply an old receipt to the current target, even transiently: // a corrective open might fail before that stale bit is replaced. @@ -2206,6 +2212,18 @@ export class ChannelRegistry { this.invalidateTransportConfirmation(entry); continue; } + // A current receipt for another channel confirms nothing about this + // one, and its history is another channel's. So the open fails, + // nothing in the answer is used, and this channel's open state is + // unconfirmed: a server that doesn't resolve this id may not resolve + // its sends either. + if (substitute !== undefined) { + this.invalidateTransportConfirmation(entry); + throw new ChannelOpenSubstitutionError( + `Server "${serverId}" answered channels/open for ${channelId} with a different channel; ` + + `${channelId} is not marked open, and nothing in the answer is used`, + ); + } if (open && !entry.open) opened = true; if (!open) opened = false; entry.open = open; @@ -2228,6 +2246,27 @@ export class ChannelRegistry { return operation; } + /** + * A server that answered an open with another channel really opened that + * one. When it's registered here and the host wants it closed, close it + * through its own lifecycle queue: otherwise its traffic would reach the + * resident, marked open by its first incoming message. A channel the host + * wants open is left alone, and a close can't be answered for another + * channel, so this never starts another substitution. An id the host + * doesn't know needs nothing: §14.5 refuses its traffic. + */ + private closeSubstitutedChannel(serverId: string, channelId: string): void { + if (!this.channels.has(`${serverId}:${channelId}`)) return; + if (this.getDesiredState(serverId, channelId) !== 'closed') return; + this.applyDesiredChannelState(serverId, channelId).catch((err) => { + if (err instanceof ChannelLifecycleConvergenceError) return; + this.emitTraceFn({ + type: 'mcpl:channel-reconcile-failed', + serverId, channelId, desired: 'closed', error: (err as Error).message, + }); + }); + } + /** * Join outstanding lifecycle work without recording new intent. Retargeting * may have invalidated open confirmation while the decision remains pending. diff --git a/src/mcpl/types.ts b/src/mcpl/types.ts index c080af76d..d6c9e21cd 100644 --- a/src/mcpl/types.ts +++ b/src/mcpl/types.ts @@ -1048,7 +1048,12 @@ export interface ChannelsListResult { * Spec Section 14.3. */ export interface ChannelsOpenParams { - /** Exact registered id. Preferred over type/address matching. */ + /** + * The exact registered id to open. A server that doesn't know it should + * refuse rather than open another channel: the host fails an open whose + * answer names any other channel, by its `channel.id` or a history item's + * `channelId`. + */ channelId?: string; type: string; address?: Record; diff --git a/test/channel-open-substitution.test.ts b/test/channel-open-substitution.test.ts index d4a4b8260..f6d2a8b2c 100644 --- a/test/channel-open-substitution.test.ts +++ b/test/channel-open-substitution.test.ts @@ -10,18 +10,20 @@ import type { ChannelDescriptor, ChannelIncomingMessage, ChannelsOpenResult } fr // as discord-mcpl's base did for a stale id: it fell back to its first channel // of the type, opened that, and returned that channel's history. -function descriptor(id: string, initiallyOpen = false): ChannelDescriptor { - return { id, type: 'test', label: id, direction: 'bidirectional', initiallyOpen, address: { value: id } }; +const tick = () => new Promise((resolve) => setImmediate(resolve)); + +function descriptor(id: string, initiallyOpen = false, address = id): ChannelDescriptor { + return { id, type: 'test', label: id, direction: 'bidirectional', initiallyOpen, address: { value: address } }; } function historyOf(channelId: string): ChannelIncomingMessage[] { return [{ channelId, messageId: `${channelId}-1`, author: { id: 'u1', name: 'someone' }, timestamp: '2026-10-09T00:00:00Z', content: [{ type: 'text', text: `from ${channelId}` }], - } as ChannelIncomingMessage]; + }]; } -function fixture(answer: (requested: string) => ChannelsOpenResult) { +function fixture(answer: (requested: string) => ChannelsOpenResult | Promise) { const calls: Array<{ kind: 'open' | 'close'; id: string }> = []; const published: string[] = []; const traces: Array<{ type: string; [key: string]: unknown }> = []; @@ -48,7 +50,9 @@ function fixture(answer: (requested: string) => ChannelsOpenResult) { ); const isOpen = (id: string) => registry.getOpenChannels().some((e) => e.descriptor.id === id); const substituted = () => traces.filter((t) => t.type === 'mcpl:channel-open-substituted'); - return { registry, calls, published, traces, routeFailures, isOpen, substituted }; + const openX = (backscroll?: number) => + registry.handleChannelToolCall('channel_open', { serverId: 'test', channelId: 'x', ...(backscroll ? { backscroll } : {}) }); + return { registry, calls, published, traces, routeFailures, isOpen, substituted, openX }; } /** Run `work` with console.error captured, returning what it printed. */ @@ -66,8 +70,7 @@ async function capturingErrors(work: () => Promise): Promise<{ value: T; l test('an open answered for another channel fails, and that channel\'s history goes nowhere', async () => { const f = fixture(() => ({ channel: descriptor('y'), history: historyOf('y') })); await f.registry.handleChanged('test', { added: [descriptor('x'), descriptor('y')] }); - const { value: result, lines } = await capturingErrors(() => - f.registry.handleChannelToolCall('channel_open', { serverId: 'test', channelId: 'x', backscroll: 5 })); + const { value: result, lines } = await capturingErrors(() => f.openX(5)); assert.equal(result.success, false); assert.match(String(result.error), /answered channels\/open for x with a different channel/); @@ -84,6 +87,39 @@ test('an open answered for another channel fails, and that channel\'s history go ['[channel-open-substituted] server=test requested=x answered=y']); }); +test('an answer naming no channel but carrying another channel\'s history fails the same way', async () => { + const f = fixture(() => ({ history: historyOf('y') }) as unknown as ChannelsOpenResult); + await f.registry.handleChanged('test', { added: [descriptor('x'), descriptor('y')] }); + const { value: result } = await capturingErrors(() => f.openX(5)); + assert.equal(result.success, false); + assert.equal(result.data, undefined); + assert.equal(f.isOpen('x'), false); + assert.equal(f.substituted()[0]?.answeredFor, 'y'); +}); + +test('the channel opened instead is closed again when the host wants it closed', async () => { + const f = fixture(() => ({ channel: descriptor('y') })); + await f.registry.handleChanged('test', { added: [descriptor('x'), descriptor('y')] }); + f.calls.length = 0; + await capturingErrors(() => f.openX()); + await tick(); + assert.deepEqual(f.calls, [{ kind: 'open', id: 'x' }, { kind: 'close', id: 'y' }], + 'the server opened y, so the host closes it through y\'s own lifecycle'); + assert.equal(f.registry.getDesiredState('test', 'y'), 'closed'); + assert.equal(f.isOpen('y'), false); +}); + +test('the channel opened instead is left alone when the host wants it open', async () => { + const f = fixture((requested) => ({ channel: descriptor(requested === 'x' ? 'y' : requested) })); + await f.registry.handleChanged('test', { added: [descriptor('x'), descriptor('y', true)] }); + assert.equal(f.isOpen('y'), true); + f.calls.length = 0; + await capturingErrors(() => f.openX()); + await tick(); + assert.deepEqual(f.calls, [{ kind: 'open', id: 'x' }], 'no corrective close, and no reopening of y'); + assert.equal(f.isOpen('y'), true); +}); + test('a channel admitted open stays closed on a substituted answer, reports it once, and speech into it fails loudly', async () => { const f = fixture(() => ({ channel: descriptor('y') })); await capturingErrors(() => f.registry.handleChanged('test', { added: [descriptor('x', true), descriptor('y')] })); @@ -106,12 +142,12 @@ test('a known-open channel loses its confirmation on a substituted answer, and r assert.equal(f.isOpen('x'), true); answerFor = 'y'; - const { value: backscroll } = await capturingErrors(() => - f.registry.handleChannelToolCall('channel_open', { serverId: 'test', channelId: 'x', backscroll: 5 })); + const { value: backscroll } = await capturingErrors(() => f.openX(5)); assert.equal(backscroll.success, false); assert.equal(f.isOpen('x'), false, 'an answer for another channel confirms nothing about this one'); answerFor = undefined; + await tick(); f.calls.length = 0; const delivery = await f.registry.routeSpeech('resident', 'hello', 'x'); assert.equal(delivery?.delivered, true); @@ -119,28 +155,55 @@ test('a known-open channel loses its confirmation on a substituted answer, and r assert.deepEqual(f.published, ['x']); }); -test('an odd answered id is described in the log line, never printed raw', async () => { +test('a substituted answer on a receipt made stale by retargeting is retried, not failed', async () => { + let release!: (answer: ChannelsOpenResult) => void; + let first = true; + const f = fixture((requested) => { + if (!first) return { channel: descriptor(requested) }; + first = false; + return new Promise((resolve) => { release = resolve; }); + }); + await f.registry.handleChanged('test', { added: [descriptor('x'), descriptor('y')] }); + f.calls.length = 0; + const { value: opened } = await capturingErrors(async () => { + const opening = f.openX(); + await tick(); + await f.registry.handleChanged('test', { updated: [descriptor('x', false, 'new-target')] }); + release({ channel: descriptor('y') }); + return opening; + }); + assert.equal(opened.success, true, 'the stale receipt is not trusted either way; the retry answers for x'); + assert.equal(f.isOpen('x'), true); + assert.equal(f.substituted().length, 1, 'the substitution the server made is still reported'); + await tick(); + assert.deepEqual(f.calls.filter((c) => c.id === 'y'), [{ kind: 'close', id: 'y' }], + 'and the channel it opened instead is still closed again'); +}); + +test('odd ids are described in the log line, never printed raw', async () => { const forged = 'y\n[channel-open-substituted] server=other requested=x answered=x'; const f = fixture(() => ({ channel: { ...descriptor('y'), id: forged } })); await f.registry.handleChanged('test', { added: [descriptor('x')] }); - const { lines } = await capturingErrors(() => - f.registry.handleChannelToolCall('channel_open', { serverId: 'test', channelId: 'x' })); + const { lines } = await capturingErrors(() => f.openX()); const logged = lines.filter((l) => l.includes('channel-open-substituted')); assert.deepEqual(logged, [ `[channel-open-substituted] server=test requested=x answered=(not a plain id: a ${forged.length}-character string)`, ]); }); -test('an answer for the requested channel, or one naming no channel, opens it as before', async () => { +test('an answer for the requested channel, with or without its history, or one naming no channel, opens it as before', async () => { for (const answer of [ (requested: string) => ({ channel: descriptor(requested), history: historyOf(requested) }), + (requested: string) => ({ channel: descriptor(requested) }), () => ({}) as ChannelsOpenResult, ]) { const f = fixture(answer); - await f.registry.handleChanged('test', { added: [descriptor('x')] }); - const result = await f.registry.handleChannelToolCall('channel_open', { serverId: 'test', channelId: 'x', backscroll: 5 }); + await f.registry.handleChanged('test', { added: [descriptor('x'), descriptor('y')] }); + f.calls.length = 0; + const result = await f.openX(5); assert.equal(result.success, true); assert.equal(f.isOpen('x'), true); assert.equal(f.substituted().length, 0); + assert.deepEqual(f.calls, [{ kind: 'open', id: 'x' }]); } }); From c46e853691472e53cda369d201c7f1925073bd3d Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Fri, 9 Oct 2026 18:00:32 -0400 Subject: [PATCH 0080/1112] docs(changelog): the substitution check reads history ids, closes the channel opened instead, retries stale receipts Brings #210's fragment in line with 1997b12 (Basil-1007's review round). --- changelog.d/210-channel-lifecycle-order.fixed.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.d/210-channel-lifecycle-order.fixed.md b/changelog.d/210-channel-lifecycle-order.fixed.md index f7060528f..7b5e4a269 100644 --- a/changelog.d/210-channel-lifecycle-order.fixed.md +++ b/changelog.d/210-channel-lifecycle-order.fixed.md @@ -6,4 +6,4 @@ - Lifecycle convergence makes at most five transport attempts per operation. Continued descriptor/intent supersession fails with a diagnostic, releases the queue, and gives joined speech or reply preparation a visible failure instead of hanging. Later stable operations can retry. -- A `channels/open` answered for a different channel than the one requested fails the open. The requested channel isn't marked open (its open state becomes unconfirmed), nothing in the answer is used, its history included, and the host logs `[channel-open-substituted]` and emits an `mcpl:channel-open-substituted` trace. An answer that names no channel is accepted as before. +- A `channels/open` answered for a different channel than the one requested, by the answer's `channel.id` or a history item's `channelId`, fails the open. The requested channel isn't marked open (its open state becomes unconfirmed), nothing in the answer is used, its history included, and the host logs `[channel-open-substituted]` and emits an `mcpl:channel-open-substituted` trace. When the channel the server opened instead is registered and its desired state is closed, the host closes it again, so its traffic doesn't reach the resident. A substituted answer on a receipt already made stale by retargeting is retried like any stale receipt. An answer that names no channel is accepted as before. `ChannelsOpenParams.channelId` is documented as exact: a server that doesn't know the id should refuse. From 1ad55766211d8563595cfae4ae29a0f323cd2d61 Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Fri, 9 Oct 2026 19:11:38 -0400 Subject: [PATCH 0081/1112] test(mcpl): pin the containment pieces no test reached Removal checks on the head merged with main 02adf27 (VPS, Node 24.21.0; dist/test/mcpl-request-handler-errors.test.js) left five pieces of this PR that could be removed with every test still passing. Each now has a test that fails without it: - the connected reader's non-object guard (server-connection.ts, before dispatchMessage): without it, `null`/42/"hi"/true reach `'id' in msg`, throw, and are reported as inbound dispatch failures (console.error and an 'error' emit) instead of being ignored like a non-JSON line. The dispatch catch masked this for host survival, so the old tests passed. New: four "non-object inbound X is ignored like a non-JSON line, with no failure report" tests (no console.error, no 'error' event). `[]` is left out: it doesn't throw in `in`, and the existing control already covers it. - the try/catch around dispatchMessage in live routing: listener faults inside emit are already caught by dispatchInboundEvent, so only throws from dispatchMessage's own work reach this catch, such as handleResponse's 'orphaned-response' emit. New: a throwing orphaned-response listener is contained (no throw out of the transport's 'line' event), reported as one mcpl:server-error, and the next notification still arrives. - the method-name lookup guard (typeof string + Object.hasOwn): without it an array-valued method that stringifies to a known method name (["notifications/tools/list_changed"]) is dispatched. New test asserts it isn't. The hasOwn half alone has no observable effect here: a prototype-named method dispatches to no listener either way. - the onRequest wrapping of tools-observe and model-info: no test made either handler fail. New: a throwing toolObserveFilter setter is answered with one -32603 and traced; a throwing model-info error write is traced with responseAttempted: true, no second write, nothing escapes emit. With the new tests, each of those five removals fails at least one test (M2: 4, M3: 1, M5: 1, tools-observe: 1, model-info: 1). The file is 42/42 on the merged tree. --- test/mcpl-request-handler-errors.test.ts | 59 ++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/test/mcpl-request-handler-errors.test.ts b/test/mcpl-request-handler-errors.test.ts index d6f76a81c..fb57f3247 100644 --- a/test/mcpl-request-handler-errors.test.ts +++ b/test/mcpl-request-handler-errors.test.ts @@ -399,3 +399,62 @@ for (const release of ['live', 'ready', 'readyControlPlane'] as const) { assert.equal(errors.length, 1); }); } + +for (const value of [null, 42, 'hi', true]) { + test('non-object inbound ' + JSON.stringify(value) + ' is ignored like a non-JSON line, with no failure report', (t) => { + const { connection, transport } = wireHarness(); + const errors: unknown[][] = []; + t.mock.method(console, 'error', (...args: unknown[]) => { errors.push(args); }); + const reported: unknown[] = []; + connection.on('error', (error: unknown) => { reported.push(error); }); + transport.emit('line', JSON.stringify(value)); + assert.deepEqual(errors, []); + assert.deepEqual(reported, []); + }); +} + +test('a throwing orphaned-response listener is contained, and later messages still arrive', (t) => { + const { connection, transport, traces } = wireHarness(); + t.mock.method(console, 'error', () => {}); + connection.prependListener('orphaned-response', () => { throw new Error('orphan listener failed'); }); + let received = false; + connection.removeAllListeners('tools-list-changed'); + connection.on('tools-list-changed', () => { received = true; }); + // A response to no pending request, carrying state, is surfaced as orphaned. + assert.doesNotThrow(() => transport.emit('line', JSON.stringify({ jsonrpc: '2.0', id: 999, result: { state: {} } }))); + transport.emit('line', JSON.stringify({ jsonrpc: '2.0', method: 'notifications/tools/list_changed' })); + assert.equal(received, true); + assert.equal(traces.filter((e) => e.type === 'mcpl:server-error').length, 1); +}); + +test('a method that is not a string is not dispatched, even if it stringifies to a known method', () => { + const { connection, transport } = wireHarness(); + let received = false; + connection.removeAllListeners('tools-list-changed'); + connection.on('tools-list-changed', () => { received = true; }); + transport.emit('line', JSON.stringify({ jsonrpc: '2.0', method: ['notifications/tools/list_changed'] })); + assert.equal(received, false); +}); + +test('tools-observe: a synchronous failure is answered with one internal error', async () => { + const { connection, traces } = harness(); + Object.defineProperty(connection, 'toolObserveFilter', { set() { throw new Error('filter store failed'); } }); + const reply = responder(); + assert.doesNotThrow(() => connection.emit('tools-observe', { rules: [] }, reply)); + await settle(); + assert.deepEqual(reply.errors, [{ code: -32603, message: 'filter store failed', data: undefined }]); + assert.equal(failureTraces(traces).length, 1); +}); + +test('model-info: a throwing error write is traced without escaping or a second write', async () => { + const { connection, traces } = harness(); + const reply = responder(); + let writes = 0; + reply.respondError = () => { writes++; throw new Error('response transport failed'); }; + assert.doesNotThrow(() => connection.emit('model-info', {}, reply)); + await settle(); + assert.equal(writes, 1); + const failures = failureTraces(traces); + assert.equal(failures.length, 1); + assert.equal(failures[0].responseAttempted, true); +}); From 03f043c7c4e24e6379e45fa0b38618cd9808ee45 Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Fri, 9 Oct 2026 19:33:01 -0400 Subject: [PATCH 0082/1112] test(mcpl): pin the Object.hasOwn half of the method lookup Basil-1007's review of 1ad5576 (room-203 #65148) found that the hasOwn half does have an effect on its own: while a connection's planes are paused, a method named after an Object.prototype member (constructor, __proto__, toString) resolves to a prototype value without it, and the event is buffered, taking a slot under the 1,000-event cap before being flushed to no listener. His probe: bufferedEvents.length 0 at the head, 3 with only `&& Object.hasOwn(...)` removed. New test: before ready(), send constructor, __proto__, toString and an unknown method; assert nothing was buffered (white-box read of bufferedEvents, as the file already reads allowHostCommands). Removal checks (VPS, Node 24.21.0, the file alone): with only the hasOwn clause removed, this test fails (42/43); with the whole string-key lookup removed, it and the array-method test fail (41/43). Against main's src the file is 2 pass / 41 fail of 43; the two passes are the existing controls. --- test/mcpl-request-handler-errors.test.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/test/mcpl-request-handler-errors.test.ts b/test/mcpl-request-handler-errors.test.ts index fb57f3247..18b8858c6 100644 --- a/test/mcpl-request-handler-errors.test.ts +++ b/test/mcpl-request-handler-errors.test.ts @@ -458,3 +458,14 @@ test('model-info: a throwing error write is traced without escaping or a second assert.equal(failures.length, 1); assert.equal(failures[0].responseAttempted, true); }); + +test('a method named after an Object.prototype member is not buffered while the planes are paused', () => { + const transport = new ResponseTransport(); + const connection = new (McplServerConnection as any)('srv', null, transport) as McplServerConnection; + connection.establishGrant(new CapabilityGrant(new Set(ALL_CAPABILITY_PATHS), [])); + // Before ready(), inbound events wait in the buffer; an unknown method must not take a slot. + for (const method of ['constructor', '__proto__', 'toString', 'no/such-method']) { + transport.emit('line', JSON.stringify({ jsonrpc: '2.0', method })); + } + assert.equal((connection as any).bufferedEvents.length, 0); +}); From e5cb38ad279db4a071717ad3b7e798fbbc858d1f Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Fri, 9 Oct 2026 20:21:58 -0400 Subject: [PATCH 0083/1112] fix(mcpl): announce a delivery's open only when it turned a closed channel open; drop pieces later fixes made redundant MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The upstream-PR skill pass removed each piece of #210 on its own (42 removals over the 23 channel test files). Several survivors weren't missing tests: they were added for problems that later commits on this branch solved more generally, and each now changes nothing, or changes something for the worse. - `onlyIfClosed` and its shortcut in applyDesiredChannelState, with the receipt-time `opened` tracking (`open && !entry.open`), came with d64c803 so concurrent delivery-forced opens skipped redundant RPCs and reported one transition. cd726dd then made both delivery paths join any pending lifecycle work, so a second delivery never reaches openChannelNow while the first is in flight. What was left: the shortcut fired only when the transport was known open but the decision was closed (a failed channel_close), where a reply flipped the decision back to open and reported 'already-open', with no trace and no notice. main said 'opened' there and told the resident. And routeSpeech's `opened` was false only when incoming traffic marked the entry open during the open RPC. - `opened` now means what the notices tell the resident: this call turned a channel decided closed open. openChannelNow reads the desired state before recording its own; a channel already decided 'open' or 'tuned-out' isn't one. Compared with c46e853: - a reply after a failed close: 'opened', the trace, and one open RPC that reconfirms the transport (was 'already-open', silent, no RPC); - a delivery or reply into a channel decided open whose transport lost confirmation (retarget, substitution, failed open): no "Now open … your own message into the closed channel engaged it" notice and no trace, 'already-open' (c46e853 and main both announced it as closed); - the same into a tuned-out channel: the tune-out is kept, as before, and nothing claims "ongoing traffic will reach you" (c46e853 did); - a fresh delivery into a settled closed channel: unchanged, one notice. - The convergence cap's invalidateTransportConfirmation came with 9c94d0a for descriptor churn. de1a9f3 then made retargeting invalidate in handleChanged and after a stale receipt. At the cap, every retry has followed a replaced registration (a fresh entry), a retarget (already invalidated), or a receipt for this very target (a valid confirmation the line threw away). Its comment, "reuse an earlier target's open flag", can't happen there. - reconcileChannels' `currentAdmissions` re-filtered the admissions it had just recorded, with no await in between, so it always equalled `policyOpened`. The notice now uses `policyOpened`. Tests (channel-lifecycle-order.test.ts), five new: - a reply into a channel whose close failed opens it again, and says so; - speech/reply into a channel decided open whose transport lost confirmation is not announced as opening it; - speech/reply into a tuned-out channel whose transport lost confirmation keeps the tune-out, unannounced. All five fail against c46e853's src, each on the claim it names ('already-open' vs 'opened'; 1 notice vs 0; 'opened' vs 'already-open'), and the file's other 32 tests pass there. Locally (node 24.19) the two #210 files pass 46/46 with typecheck clean. Changelog: the fixed fragment's delivery sentence and the breaking fragment's `opened-by-delivery` sentence now say which deliveries are announced. --- .../210-channel-lifecycle-order.fixed.md | 2 +- .../210-policy-admission-notice.breaking.md | 2 +- src/mcpl/channel-registry.ts | 46 +++++++------- test/channel-lifecycle-order.test.ts | 61 +++++++++++++++++++ 4 files changed, 84 insertions(+), 27 deletions(-) diff --git a/changelog.d/210-channel-lifecycle-order.fixed.md b/changelog.d/210-channel-lifecycle-order.fixed.md index 7b5e4a269..fb6f6546f 100644 --- a/changelog.d/210-channel-lifecycle-order.fixed.md +++ b/changelog.d/210-channel-lifecycle-order.fixed.md @@ -1,4 +1,4 @@ -- Failed backscroll requests preserve the registry's known-open state rather than claiming a close. Automatic delivery joins pending lifecycle work without reopening a channel closed during the wait, and reports only actual delivery-forced opens using the current registration label. +- Failed backscroll requests preserve the registry's known-open state rather than claiming a close. Automatic delivery joins pending lifecycle work without reopening a channel closed during the wait. A delivery or reply is announced as opening a channel only when it turned a channel decided closed open, using the current registration label; reconfirming the transport of a channel already decided open or tuned out isn't announced. - Channel lifecycle operations now run in order per channel and reconcile the latest desired state and registration after an in-flight operation completes. Successful reconciliation confirms the current target and desired state at its receipt boundary; a superseded tool request reports that its requested state no longer applies. Automatic delivery preserves active tune-out attention state while opening or awaiting transport. diff --git a/changelog.d/210-policy-admission-notice.breaking.md b/changelog.d/210-policy-admission-notice.breaking.md index f47b54371..a324c147a 100644 --- a/changelog.d/210-policy-admission-notice.breaking.md +++ b/changelog.d/210-policy-admission-notice.breaking.md @@ -1 +1 @@ -- `onChannelAutoOpened` with `source: 'subscription-policy'` now reports a durable admission decision before transport reconciliation. Treat it as intent; await the registration call and inspect `getOpenChannels()` when confirmed live state is needed. `opened-by-delivery` still reports a confirmed transition. The built-in host notice now distinguishes selected-for-opening from currently open. +- `onChannelAutoOpened` with `source: 'subscription-policy'` now reports a durable admission decision before transport reconciliation. Treat it as intent; await the registration call and inspect `getOpenChannels()` when confirmed live state is needed. `opened-by-delivery` reports a delivery that turned a closed channel open, once its transport is confirmed; a delivery that only reconfirms the transport of a channel already decided open or tuned out isn't reported. The built-in host notice now distinguishes selected-for-opening from currently open. diff --git a/src/mcpl/channel-registry.ts b/src/mcpl/channel-registry.ts index b9edc829b..e0911c70c 100644 --- a/src/mcpl/channel-registry.ts +++ b/src/mcpl/channel-registry.ts @@ -526,8 +526,10 @@ interface ChannelRegistryOptions { }) => void; /** * Announces either a durable subscription-policy admission (before its - * transport RPC; not an open confirmation), or a confirmed delivery-forced - * closed-to-open transition. The host records a durable, non-waking notice. + * transport RPC; not an open confirmation), or a delivery that turned a + * closed channel open, once its transport is confirmed. Reconfirming the + * transport of a channel already decided open or tuned out isn't one. + * The host records a durable, non-waking notice. * Policy admissions fire once per channel: the persisted desired-state * decision prevents re-announcement on retry or reboot. channel_close * supersedes policy; a fresh delivery into a closed locus engages it again. @@ -2135,14 +2137,13 @@ export class ChannelRegistry { private applyDesiredChannelState( serverId: string, channelId: string, - options?: { history?: ChannelHistoryRequest; onlyIfClosed?: boolean }, - ): Promise<{ open: boolean; opened: boolean; result?: ChannelsOpenResult }> { + options?: { history?: ChannelHistoryRequest }, + ): Promise<{ open: boolean; result?: ChannelsOpenResult }> { const lifecycleKey = this.lifecycleKey(serverId, channelId); const channelKey = `${serverId}:${channelId}`; const previous = this.channelLifecycleTails.get(lifecycleKey) ?? Promise.resolve(); const operation = previous.catch(() => {}).then(async () => { let attempts = 0; - let opened = false; for (;;) { const entry = this.channels.get(channelKey); if (!entry) throw new Error(`Channel is no longer registered: ${channelId}`); @@ -2154,16 +2155,13 @@ export class ChannelRegistry { const descriptor = entry.descriptor; const desired = this.getDesiredState(serverId, channelId); const open = desired === 'open' || desired === 'tuned-out'; - if (attempts === 0 && options?.onlyIfClosed && open && entry.open) { - return { open: true, opened: false }; - } // Per-RPC timeouts do not bound a stream of promptly successful but // continually superseded receipts. Bound this whole operation, then // release its queue slot and surface the failure to delivery joiners. + // Its open flag needs no change here: every retry follows a replaced + // registration, a retarget (already invalidated), or a receipt for + // this very target. if (attempts >= MAX_CHANNEL_LIFECYCLE_ATTEMPTS) { - // No receipt confirmed the current target/intent. A later delivery - // must reconcile it rather than reuse an earlier target's open flag. - this.invalidateTransportConfirmation(entry); const error = new ChannelLifecycleConvergenceError( `Channel ${channelId} did not converge after ${attempts} lifecycle attempts; its target or desired state kept changing`, ); @@ -2224,13 +2222,11 @@ export class ChannelRegistry { `${channelId} is not marked open, and nothing in the answer is used`, ); } - if (open && !entry.open) opened = true; - if (!open) opened = false; entry.open = open; this.unconfirmedChannelTargets.delete(entry); const now = this.getDesiredState(serverId, channelId); if (open !== (now === 'open' || now === 'tuned-out')) continue; - return { open, opened, result }; + return { open, result }; } }); // Keep the rejection visible to existing joiners, while observing it @@ -2332,18 +2328,12 @@ export class ChannelRegistry { // Announce the durable admission decision before awaiting transport. // This is intent, not confirmation that an open RPC succeeded. It stays // truthful if the first attempt fails, and a retry needs no new admission. - // channel_close supersedes policy; use current entries at this boundary. - const currentAdmissions = policyOpened.flatMap(({ channelId }) => { - const entry = this.channels.get(`${serverId}:${channelId}`); - if (!entry || this.getDesiredState(serverId, channelId) !== 'open') return []; - return [{ channelId, label: entry.descriptor.label }]; - }); - if (currentAdmissions.length > 0) { + if (policyOpened.length > 0) { try { this.onChannelAutoOpened?.({ serverId, source: 'subscription-policy', - channels: currentAdmissions, + channels: policyOpened, }); } catch (err) { console.error('onChannelAutoOpened (policy) failed:', err); @@ -2500,6 +2490,10 @@ export class ChannelRegistry { * even if the subscribe fails, and reconciliation retries later. * Automatic delivery needs open transport, not a change of attention: * preserve tune-out epochs both before and during the round-trip. + * + * `opened` says whether this call turned a closed channel open, which is + * what the delivery and reply notices tell the resident. Reconfirming the + * transport of a channel already decided open or tuned out isn't one. */ private async openChannelNow( entry: ChannelEntry, @@ -2507,15 +2501,17 @@ export class ChannelRegistry { history?: ChannelHistoryRequest, ): Promise<{ result: ChannelsOpenResult; opened: boolean }> { const transportOnly = source !== 'agent-tool'; - if (!transportOnly || this.getDesiredState(entry.serverId, entry.descriptor.id) !== 'tuned-out') { + const before = this.getDesiredState(entry.serverId, entry.descriptor.id); + const opened = before !== 'open' && before !== 'tuned-out'; + if (!transportOnly || before !== 'tuned-out') { this.setDesiredState(entry.serverId, entry.descriptor.id, 'open', source); } - const applied = await this.applyDesiredChannelState(entry.serverId, entry.descriptor.id, { history, onlyIfClosed: transportOnly }); + const applied = await this.applyDesiredChannelState(entry.serverId, entry.descriptor.id, { history }); const desired = this.getDesiredState(entry.serverId, entry.descriptor.id); if (!applied.open || (desired !== 'open' && !(transportOnly && desired === 'tuned-out'))) { throw new Error('Channel open was superseded by a newer lifecycle decision'); } - return { result: applied.result!, opened: applied.opened }; + return { result: applied.result!, opened }; } /** diff --git a/test/channel-lifecycle-order.test.ts b/test/channel-lifecycle-order.test.ts index cb7c25b9b..e4a581b76 100644 --- a/test/channel-lifecycle-order.test.ts +++ b/test/channel-lifecycle-order.test.ts @@ -609,3 +609,64 @@ test('a newer tune-out decision keeps transport open when an older close complet assert.equal(f.isOpen(), true); assert.equal((await close).success, false, 'superseded close must not report current state as closed'); }); + +test('a reply into a channel whose close failed opens it again, and says so', async () => { + const f = fixture(); + await f.registry.handleChanged('test', { added: [descriptor('x', true)] }); + f.calls.length = 0; + f.held.add('x'); + const closing = f.tool('close'); + await tick(); + f.pending.shift()!.fail(); + assert.equal((await closing).success, false); + await tick(); + assert.equal(f.registry.getDesiredState('test', 'x'), 'closed'); + assert.equal(f.isOpen(), true, 'a failed same-target close leaves the transport known open'); + f.held.delete('x'); + const reply = await f.registry.openIfClosedForSend('x', 'test'); + assert.equal(reply.status, 'opened', 'the reply turned a channel decided closed open again'); + assert.equal(f.registry.getDesiredState('test', 'x'), 'open'); + assert.equal(f.traces.filter((e) => e.type === 'mcpl:channel-opened-by-send').length, 1); + assert.deepEqual(f.calls.map((c) => c.kind), ['close', 'open'], 'the transport is confirmed again after the failed close'); +}); + +for (const delivery of ['speech', 'reply'] as const) { + test(delivery + ' into a channel decided open whose transport lost confirmation is not announced as opening it', async () => { + const f = fixture(); + await f.registry.handleChanged('test', { added: [descriptor('x', true, 'A')] }); + await f.registry.handleChanged('test', { updated: [descriptor('x', true, 'B')] }); + assert.equal(f.isOpen(), false, 'retargeting leaves the new target unconfirmed'); + f.calls.length = 0; + if (delivery === 'speech') { + assert.equal((await f.registry.routeSpeech('resident', 'hello', 'x'))?.delivered, true); + assert.deepEqual(f.publishedWhileOpen, [true]); + } else { + assert.equal((await f.registry.openIfClosedForSend('x', 'test')).status, 'already-open'); + } + assert.deepEqual(f.calls.map((c) => [c.kind, c.address]), [['open', { value: 'B' }]], 'the new target is confirmed first'); + assert.equal(f.notices.length, 0, 'the channel was never closed'); + assert.equal(f.traces.filter((e) => e.type === 'mcpl:channel-opened-by-send').length, 0); + assert.equal(f.registry.getDesiredState('test', 'x'), 'open'); + }); + + test(delivery + ' into a tuned-out channel whose transport lost confirmation keeps the tune-out, unannounced', async () => { + const f = fixture(); + await f.registry.handleChanged('test', { added: [descriptor('x', true, 'A')] }); + f.registry.enterTuneOut('test', 'x', { + epochId: 'kept-epoch', cadenceSeconds: 60, backlogCap: 20, maxWakes: 3, startedAtSequence: 1, + }, 'agent-tool'); + await f.registry.handleChanged('test', { updated: [descriptor('x', true, 'B')] }); + assert.equal(f.isOpen(), false); + if (delivery === 'speech') { + assert.equal((await f.registry.routeSpeech('resident', 'hello', 'x'))?.delivered, true); + assert.deepEqual(f.publishedWhileOpen, [true]); + } else { + assert.equal((await f.registry.openIfClosedForSend('x', 'test')).status, 'already-open'); + } + assert.equal(f.registry.getDesiredState('test', 'x'), 'tuned-out'); + assert.equal(f.registry.getTuneOutState('test', 'x')?.params.epochId, 'kept-epoch'); + assert.equal(f.isOpen(), true); + assert.equal(f.notices.length, 0, 'its traffic is still diverted, so nothing new reaches the resident'); + assert.equal(f.traces.filter((e) => e.type === 'mcpl:channel-opened-by-send').length, 0); + }); +} From a6130d03c94c79465fe163f1f67ef1c88b650b74 Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Fri, 9 Oct 2026 20:22:40 -0400 Subject: [PATCH 0084/1112] test(mcpl): pin the lifecycle pieces no test reached Removing each piece of #210 on its own left these with every test passing, across all 23 channel test files. Each now has a test that fails without it: - The stale-entry retry (`channels.get(channelKey) !== entry`): "an explicit open answers only once the current registration is open". The server registers the channel again while channel_open is in flight; without the check, the old registration's receipt marks the replaced entry open and the tool answers while the current one is still closed. - channel_open's pending-work condition on its already-open shortcut: "an explicit open waits for a close in flight rather than answering already open", the open-side twin of the existing already-closed test. - The removal clause in reconcile's catch: the removal test now also asserts no mcpl:channel-reconcile-failed trace. - The superseded-open throw in openChannelNow: the overlapping-intent tests now assert that the first tool result says it was superseded. Without the throw, the open variant answers with a TypeError about `history`. - closeSubstitutedChannel's registration check: "the channel opened instead needs nothing once it is no longer registered" (substitution file). Without it, a removed channel whose durable desired state is closed gets a lifecycle operation that fails at once and emits a reconcile-failed trace. - closeSubstitutedChannel's quiet catch for exhausted convergence: "a corrective close that cannot converge reports its exhaustion once". The fixture gains `answeredFor`, a server answering an open for another channel; y's corrective close is retargeted before each receipt, and the cap's own trace is the only one. The join's removed-registration check is caught by typecheck. The tail cleanup's removal hangs (an awaited settled tail spins in microtasks, where no test timeout can fire), so it can't have a failing test. Locally (node 24.19), the two #210 files pass 50/50: 41 at c46e853, 5 in the commit before this, and 4 here, plus new assertions in two existing tests (the removal test and the overlapping-intent pair). --- test/channel-lifecycle-order.test.ts | 74 ++++++++++++++++++++++++-- test/channel-open-substitution.test.ts | 13 +++++ 2 files changed, 84 insertions(+), 3 deletions(-) diff --git a/test/channel-lifecycle-order.test.ts b/test/channel-lifecycle-order.test.ts index e4a581b76..fbc9b180e 100644 --- a/test/channel-lifecycle-order.test.ts +++ b/test/channel-lifecycle-order.test.ts @@ -27,13 +27,16 @@ function fixture(onNotice?: NoticeHandler, store?: JsStore) { const publishedWhileOpen: boolean[] = []; const notices: Array<{ source: string; channels: Array<{ channelId: string; label?: string }> }> = []; const routeFailures: Array<{ reason: string }> = []; + // A server that answers an open for another channel, by requested id. + const answeredFor = new Map(); const serve = (kind: 'open' | 'close', params: { channelId?: string; address?: unknown }) => { const id = params.channelId!; calls.push({ kind, id, address: params.address }); return new Promise((resolve, reject) => { const finish = () => { - actual.set(id, kind === 'open'); - resolve(kind === 'open' ? { channel: descriptor(id) } : { closed: true }); + const target = kind === 'open' ? answeredFor.get(id) ?? id : id; + actual.set(target, kind === 'open'); + resolve(kind === 'open' ? { channel: descriptor(target) } : { closed: true }); }; if (held.has(id)) pending.push({ finish, fail: () => reject(new Error('fixture lifecycle failure')) }); else finish(); @@ -71,7 +74,7 @@ function fixture(onNotice?: NoticeHandler, store?: JsStore) { assert.equal(settled, true, 'lifecycle operations must settle'); await result; }; - return { registry, actual, calls, pending, held, traces, isOpen, tool, drain, publishedWhileOpen, notices, routeFailures, replaceServer }; + return { registry, actual, calls, pending, held, traces, isOpen, tool, drain, publishedWhileOpen, notices, routeFailures, replaceServer, answeredFor }; } test('remove/re-add during reconcile converges on the replacement desired state, with ACK first (#185)', async () => { @@ -115,6 +118,8 @@ for (const firstKind of ['open', 'close'] as const) { const lastKind = firstKind === 'open' ? 'close' : 'open'; const last = f.tool(lastKind); await f.drain(first, last); + assert.equal((await first).success, false); + assert.match(String((await first).error), /superseded by a newer lifecycle decision/); assert.equal((await last).success, true); const desired = lastKind === 'open'; assert.equal(f.registry.getDesiredState('test', 'x'), lastKind === 'open' ? 'open' : 'closed'); @@ -149,6 +154,8 @@ test('removal cancels queued work rather than recreating or mutating an unregist assert.equal((await open).success, false); assert.equal(f.registry.listChannelsRaw().length, 0); assert.equal(f.calls.filter((c) => c.kind === 'open').length, 0); + assert.equal(f.traces.filter((e) => e.type === 'mcpl:channel-reconcile-failed').length, 0, + 'removal while queued is cancellation, not a failed subscription'); }); test('a failed operation releases the queue for a later decision', async () => { @@ -610,6 +617,39 @@ test('a newer tune-out decision keeps transport open when an older close complet assert.equal((await close).success, false, 'superseded close must not report current state as closed'); }); +test('an explicit open answers only once the current registration is open', async () => { + const f = fixture(); + await f.registry.handleChanged('test', { added: [descriptor()] }); + f.held.add('x'); + let openWhenAnswered: boolean | undefined; + const opening = f.tool('open').then((result) => { openWhenAnswered = f.isOpen(); return result; }); + await tick(); + assert.equal(f.pending.length, 1); + // The server registers the channel again while the open is in flight. + const reRegistered = f.registry.handleChanged('test', { added: [descriptor()] }); + await f.drain(opening, reRegistered); + assert.equal((await opening).success, true); + assert.equal(openWhenAnswered, true, 'a receipt for the replaced registration confirms nothing about the new one'); + assert.equal(f.isOpen(), true); +}); + +test('an explicit open waits for a close in flight rather than answering already open', async () => { + const f = fixture(); + await f.registry.handleChanged('test', { added: [descriptor('x', true)] }); + f.held.add('x'); + const closing = f.tool('close'); + await tick(); + const reopening = f.tool('open'); + let settled = false; + const again = f.tool('open').then((result) => { settled = true; return result; }); + await tick(); + assert.equal(settled, false, 'entry.open is provisional while the close is in flight'); + await f.drain(closing, reopening, again); + assert.equal((await again).success, true); + assert.equal(f.actual.get('x'), true); + assert.equal(f.isOpen(), true); +}); + test('a reply into a channel whose close failed opens it again, and says so', async () => { const f = fixture(); await f.registry.handleChanged('test', { added: [descriptor('x', true)] }); @@ -670,3 +710,31 @@ for (const delivery of ['speech', 'reply'] as const) { assert.equal(f.traces.filter((e) => e.type === 'mcpl:channel-opened-by-send').length, 0); }); } + +test('a corrective close that cannot converge reports its exhaustion once', async () => { + const f = fixture(); + await f.registry.handleChanged('test', { added: [descriptor('x'), descriptor('y')] }); + f.calls.length = 0; + // Asked to open x, the server opens y instead, and y's corrective close is + // retargeted before each receipt. + f.answeredFor.set('x', 'y'); + f.held.add('y'); + assert.equal((await f.tool('open')).success, false); + await tick(); + assert.equal(f.pending.length, 1, 'the corrective close for y is in flight'); + try { + for (let i = 0; i < 12 && f.pending.length > 0; i++) { + await f.registry.handleChanged('test', { updated: [descriptor('y', false, 'churn-' + i)] }); + f.pending.shift()!.finish(); + await tick(); + } + assert.equal(f.pending.length, 0, 'the corrective close is bounded'); + assert.equal(f.calls.filter((c) => c.id === 'y' && c.kind === 'close').length, 5); + const failures = f.traces.filter((e) => e.type === 'mcpl:channel-reconcile-failed' && e.channelId === 'y'); + assert.equal(failures.length, 1); + assert.match(String(failures[0].error), /did not converge/); + } finally { + f.held.delete('y'); + while (f.pending.length > 0) f.pending.shift()!.finish(); + } +}); diff --git a/test/channel-open-substitution.test.ts b/test/channel-open-substitution.test.ts index f6d2a8b2c..bfecb4578 100644 --- a/test/channel-open-substitution.test.ts +++ b/test/channel-open-substitution.test.ts @@ -120,6 +120,19 @@ test('the channel opened instead is left alone when the host wants it open', asy assert.equal(f.isOpen('y'), true); }); +test('the channel opened instead needs nothing once it is no longer registered', async () => { + const f = fixture(() => ({ channel: descriptor('y') })); + await f.registry.handleChanged('test', { added: [descriptor('x'), descriptor('y')] }); + await f.registry.handleChanged('test', { removed: ['y'] }); + f.calls.length = 0; + await capturingErrors(() => f.openX()); + await tick(); + assert.equal(f.substituted().length, 1, 'the substitution is still reported'); + assert.deepEqual(f.calls, [{ kind: 'open', id: 'x' }], 'nothing is sent for y'); + assert.equal(f.traces.filter((t) => t.type === 'mcpl:channel-reconcile-failed').length, 0, + 'an unregistered channel has no lifecycle to fail'); +}); + test('a channel admitted open stays closed on a substituted answer, reports it once, and speech into it fails loudly', async () => { const f = fixture(() => ({ channel: descriptor('y') })); await capturingErrors(() => f.registry.handleChanged('test', { added: [descriptor('x', true), descriptor('y')] })); From 7d859cb91aac4e06fd206a7e1e714186cc5d08e8 Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Fri, 9 Oct 2026 20:32:04 -0400 Subject: [PATCH 0085/1112] fix(mcpl): speech engages a channel decided closed as a reply does; a failed channel_open says the channel stays selected Basil-1007's review of e5cb38a (room-203 #65256) found two states where the resident's picture is "closed" while the code treated it otherwise. - A failed channel_close, then speech. routeSpeech opened only when `!entry.open`. After a failed close the transport is still known open and the decision is `closed`, so speech published, left the decision closed, and announced nothing, while a reply in the same state opened and announced (e5cb38a's test). routeSpeech now also engages a channel decided `closed`, so the two paths agree and `opened` announces it. This is the PR's own contract: a fresh send after a close has settled is a new engagement. It doesn't take the reply path's whole condition (anything not decided open or tuned out): a server without the lifecycle grant records no decision, its channels can be open by the server's choice, and speech opens BEFORE it sends, so engaging an undecided channel there would block speech that main delivers. A reply opens after its send, so its broader condition costs nothing there. - A failed channel_open. openChannelNow records `open` before the round-trip, and the decision outlives a failure (intent sticks; reconciliation retries). Under e5cb38a's rule a later delivery then opens the channel without a notice, which would be the resident's first sign that it was still wanted open. The failure text now says so when the channel isn't open and its decision is still `open`: "(it stays selected for opening: a later delivery or reconnect tries again, and channel_close withdraws that)". A failed backscroll on a live channel, and an open superseded by a newer decision, don't carry it. Also pins queue serialization, which the decision rule had left with no failing test: e5cb38a's `opened` no longer depends on transport order, so "speech opening reports the replacement registration label" stopped catching the queue's removal. The #185 test now asserts one RPC in flight per channel once the replacement has registered, which is the claim its fixture comment makes. Tests: the failed-close test now runs for speech as well as reply; a new "a failed explicit open says the channel stays selected for opening, and a later delivery opens it quietly"; doesNotMatch checks in the failed-backscroll and overlapping-intent tests. Against a6130d0's src the speech twin and the failed-open test fail, each on its own claim, and the file's other 40 pass. Locally (node 24.19) the two #210 files pass 52/52, and the 23 channel files one per process pass apart from conversation-routing's "Queue is closed" after-test rejection, the race #263 addresses. --- .../210-channel-lifecycle-order.fixed.md | 2 +- src/mcpl/channel-registry.ts | 20 ++++++- test/channel-lifecycle-order.test.ts | 55 +++++++++++++++---- 3 files changed, 61 insertions(+), 16 deletions(-) diff --git a/changelog.d/210-channel-lifecycle-order.fixed.md b/changelog.d/210-channel-lifecycle-order.fixed.md index fb6f6546f..24d85ad31 100644 --- a/changelog.d/210-channel-lifecycle-order.fixed.md +++ b/changelog.d/210-channel-lifecycle-order.fixed.md @@ -1,4 +1,4 @@ -- Failed backscroll requests preserve the registry's known-open state rather than claiming a close. Automatic delivery joins pending lifecycle work without reopening a channel closed during the wait. A delivery or reply is announced as opening a channel only when it turned a channel decided closed open, using the current registration label; reconfirming the transport of a channel already decided open or tuned out isn't announced. +- Failed backscroll requests preserve the registry's known-open state rather than claiming a close. Automatic delivery joins pending lifecycle work without reopening a channel closed during the wait. A delivery or reply is announced as opening a channel only when it turned a channel decided closed open, using the current registration label; reconfirming the transport of a channel already decided open or tuned out isn't announced. Speech engages a channel decided closed as a reply does, even while a failed close has left its transport open. A failed `channel_open` says the channel stays selected for opening, since that decision outlives the failed round-trip and a later delivery or reconnect tries again. - Channel lifecycle operations now run in order per channel and reconcile the latest desired state and registration after an in-flight operation completes. Successful reconciliation confirms the current target and desired state at its receipt boundary; a superseded tool request reports that its requested state no longer applies. Automatic delivery preserves active tune-out attention state while opening or awaiting transport. diff --git a/src/mcpl/channel-registry.ts b/src/mcpl/channel-registry.ts index e0911c70c..4f62d9d6d 100644 --- a/src/mcpl/channel-registry.ts +++ b/src/mcpl/channel-registry.ts @@ -2774,9 +2774,18 @@ export class ChannelRegistry { }, }; } catch (err) { + // The decision was recorded before the round-trip and outlives its + // failure. Say so: otherwise a later delivery that opens the channel + // would be the resident's first sign that it was still wanted open. + const current = this.channels.get(`${entry.serverId}:${input.channelId}`); + const stillSelected = current !== undefined && !current.open && + this.getDesiredState(entry.serverId, input.channelId) === 'open'; return { success: false, - error: `Failed to open channel: ${(err as Error).message}`, + error: `Failed to open channel: ${(err as Error).message}` + + (stillSelected + ? ' (it stays selected for opening: a later delivery or reconnect tries again, and channel_close withdraws that)' + : ''), isError: true, }; } @@ -3183,8 +3192,13 @@ export class ChannelRegistry { // that woke the agent — DMs register closed) OPENS the channel first, // so typing indicators, reaction machinery, and inbound forwarding come // alive with the reply. If the open fails, the speech does NOT go out: - // a half-alive delivery is worse than a loud marker. - if (!entry.open) { + // a half-alive delivery is worse than a loud marker. A channel decided + // closed is engaged even while its transport is still open (a close that + // failed), as a reply would engage it. A channel with no decision at all + // (a server without the lifecycle grant records none) keeps publishing + // while its transport is open: speech opens before it sends, and that + // open could never succeed there. + if (!entry.open || this.getDesiredState(entry.serverId, entry.descriptor.id) === 'closed') { try { const { opened } = await this.openChannelNow(entry, 'opened-by-delivery'); entry = await this.waitForOpenChannel(entry); diff --git a/test/channel-lifecycle-order.test.ts b/test/channel-lifecycle-order.test.ts index fbc9b180e..17f4f5e7e 100644 --- a/test/channel-lifecycle-order.test.ts +++ b/test/channel-lifecycle-order.test.ts @@ -88,6 +88,8 @@ test('remove/re-add during reconcile converges on the replacement desired state, removed: ['x'], added: [descriptor('x', true, 'replacement')], }, { respond: () => { acknowledged = true; } }); assert.equal(acknowledged, true, 'registration ACK must not wait for the lifecycle queue'); + await tick(); + assert.equal(f.pending.length, 1, 'the replacement waits for the operation in flight: one RPC per channel at a time'); await f.drain(first, replacement); assert.equal(f.registry.getDesiredState('test', 'x'), 'open'); assert.equal(f.isOpen(), true); @@ -120,6 +122,7 @@ for (const firstKind of ['open', 'close'] as const) { await f.drain(first, last); assert.equal((await first).success, false); assert.match(String((await first).error), /superseded by a newer lifecycle decision/); + assert.doesNotMatch(String((await first).error), /stays selected/, 'a newer decision replaced it'); assert.equal((await last).success, true); const desired = lastKind === 'open'; assert.equal(f.registry.getDesiredState('test', 'x'), lastKind === 'open' ? 'open' : 'closed'); @@ -252,6 +255,7 @@ test('a failed backscroll open preserves an already-live subscription', async () await tick(); f.pending.shift()!.fail(); assert.equal((await backscroll).success, false); + assert.doesNotMatch(String((await backscroll).error), /stays selected/, 'the channel is still open'); assert.equal(f.isOpen(), true, 'a failed redundant open is not a confirmed close'); const speech = f.registry.routeSpeech('resident', 'hello', 'x'); await f.drain(speech); @@ -650,24 +654,51 @@ test('an explicit open waits for a close in flight rather than answering already assert.equal(f.isOpen(), true); }); -test('a reply into a channel whose close failed opens it again, and says so', async () => { +for (const delivery of ['reply', 'speech'] as const) { + test('a ' + delivery + ' into a channel whose close failed opens it again, and says so', async () => { + const f = fixture(); + await f.registry.handleChanged('test', { added: [descriptor('x', true)] }); + f.calls.length = 0; + f.held.add('x'); + const closing = f.tool('close'); + await tick(); + f.pending.shift()!.fail(); + assert.equal((await closing).success, false); + await tick(); + assert.equal(f.registry.getDesiredState('test', 'x'), 'closed'); + assert.equal(f.isOpen(), true, 'a failed same-target close leaves the transport known open'); + f.held.delete('x'); + if (delivery === 'reply') { + const reply = await f.registry.openIfClosedForSend('x', 'test'); + assert.equal(reply.status, 'opened', 'the reply turned a channel decided closed open again'); + } else { + assert.equal((await f.registry.routeSpeech('resident', 'hello', 'x'))?.delivered, true); + assert.equal(f.notices.length, 1, 'the speech turned a channel decided closed open again'); + assert.deepEqual(f.publishedWhileOpen, [true]); + } + assert.equal(f.registry.getDesiredState('test', 'x'), 'open'); + assert.equal(f.traces.filter((e) => e.type === 'mcpl:channel-opened-by-send').length, 1); + assert.deepEqual(f.calls.map((c) => c.kind), ['close', 'open'], 'the transport is confirmed again after the failed close'); + }); +} + +test('a failed explicit open says the channel stays selected for opening, and a later delivery opens it quietly', async () => { const f = fixture(); - await f.registry.handleChanged('test', { added: [descriptor('x', true)] }); + await f.registry.handleChanged('test', { added: [descriptor()] }); f.calls.length = 0; f.held.add('x'); - const closing = f.tool('close'); + const opening = f.tool('open'); await tick(); f.pending.shift()!.fail(); - assert.equal((await closing).success, false); - await tick(); - assert.equal(f.registry.getDesiredState('test', 'x'), 'closed'); - assert.equal(f.isOpen(), true, 'a failed same-target close leaves the transport known open'); + const result = await opening; + assert.equal(result.success, false); + assert.match(String(result.error), /stays selected for opening/); + assert.equal(f.registry.getDesiredState('test', 'x'), 'open', 'the decision outlives a failed round-trip'); f.held.delete('x'); - const reply = await f.registry.openIfClosedForSend('x', 'test'); - assert.equal(reply.status, 'opened', 'the reply turned a channel decided closed open again'); - assert.equal(f.registry.getDesiredState('test', 'x'), 'open'); - assert.equal(f.traces.filter((e) => e.type === 'mcpl:channel-opened-by-send').length, 1); - assert.deepEqual(f.calls.map((c) => c.kind), ['close', 'open'], 'the transport is confirmed again after the failed close'); + await tick(); + assert.equal((await f.registry.routeSpeech('resident', 'hello', 'x'))?.delivered, true); + assert.equal(f.notices.length, 0, 'the resident was already told the channel stays selected for opening'); + assert.deepEqual(f.calls.map((c) => c.kind), ['open', 'open']); }); for (const delivery of ['speech', 'reply'] as const) { From fd5fc52964b06f512764241dee5ae114a1b74f20 Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Fri, 9 Oct 2026 20:35:16 -0400 Subject: [PATCH 0086/1112] fix(mcpl): speech engages a decided-closed channel only where the host can open it Basil-1007's review of 7d859cb (room-203 #65264): a server without the channels.lifecycle grant can still hold a `closed` decision. handleToolClose records `closed` before its grant check, so on such a server channel_close returns the grant error with the decision recorded (as on main), and a server whose grant was withdrawn keeps its old decisions. Its transport can still be open: accepted inbound traffic marks the entry open, lifecycle grant or not. 7d859cb then sent speech into that locus through openChannelNow, which fails at the grant check, and blocked the speech ("locus channel is closed and open failed"), where main and a6130d0 deliver it. routeSpeech now engages a channel decided closed, with its transport still open, only where the server's grant includes channels.lifecycle. Elsewhere such a channel keeps publishing while its transport is open, as on main. A channel whose transport is closed still opens first, as before; that open can fail visibly on a grant-less server, as on main. Test: "speech into a channel decided closed keeps publishing where no open can succeed, without the lifecycle grant". The fixture gains setGrant. The channel opens under the full grant; the grant then loses channels.lifecycle; channel_close is refused for it with the decision recorded; speech is delivered, with no open attempted and no notice. Against 7d859cb's src it fails on `delivered` (undefined: the speech was blocked), and the file's other 42 pass. Locally (node 24.19) the two #210 files pass 53/53. --- .../210-channel-lifecycle-order.fixed.md | 2 +- src/mcpl/channel-registry.ts | 11 ++++++----- test/channel-lifecycle-order.test.ts | 19 ++++++++++++++++++- 3 files changed, 25 insertions(+), 7 deletions(-) diff --git a/changelog.d/210-channel-lifecycle-order.fixed.md b/changelog.d/210-channel-lifecycle-order.fixed.md index 24d85ad31..eee7abf91 100644 --- a/changelog.d/210-channel-lifecycle-order.fixed.md +++ b/changelog.d/210-channel-lifecycle-order.fixed.md @@ -1,4 +1,4 @@ -- Failed backscroll requests preserve the registry's known-open state rather than claiming a close. Automatic delivery joins pending lifecycle work without reopening a channel closed during the wait. A delivery or reply is announced as opening a channel only when it turned a channel decided closed open, using the current registration label; reconfirming the transport of a channel already decided open or tuned out isn't announced. Speech engages a channel decided closed as a reply does, even while a failed close has left its transport open. A failed `channel_open` says the channel stays selected for opening, since that decision outlives the failed round-trip and a later delivery or reconnect tries again. +- Failed backscroll requests preserve the registry's known-open state rather than claiming a close. Automatic delivery joins pending lifecycle work without reopening a channel closed during the wait. A delivery or reply is announced as opening a channel only when it turned a channel decided closed open, using the current registration label; reconfirming the transport of a channel already decided open or tuned out isn't announced. Speech engages a channel decided closed as a reply does, even while a failed close has left its transport open, wherever the server's grant lets the host open it. A failed `channel_open` says the channel stays selected for opening, since that decision outlives the failed round-trip and a later delivery or reconnect tries again. - Channel lifecycle operations now run in order per channel and reconcile the latest desired state and registration after an in-flight operation completes. Successful reconciliation confirms the current target and desired state at its receipt boundary; a superseded tool request reports that its requested state no longer applies. Automatic delivery preserves active tune-out attention state while opening or awaiting transport. diff --git a/src/mcpl/channel-registry.ts b/src/mcpl/channel-registry.ts index 4f62d9d6d..9aecdf63d 100644 --- a/src/mcpl/channel-registry.ts +++ b/src/mcpl/channel-registry.ts @@ -3194,11 +3194,12 @@ export class ChannelRegistry { // alive with the reply. If the open fails, the speech does NOT go out: // a half-alive delivery is worse than a loud marker. A channel decided // closed is engaged even while its transport is still open (a close that - // failed), as a reply would engage it. A channel with no decision at all - // (a server without the lifecycle grant records none) keeps publishing - // while its transport is open: speech opens before it sends, and that - // open could never succeed there. - if (!entry.open || this.getDesiredState(entry.serverId, entry.descriptor.id) === 'closed') { + // failed), as a reply would engage it, wherever an open can succeed. + // Without the lifecycle grant it can't, and speech opens before it + // sends, so there a channel whose transport is open keeps publishing. + const engagesDecidedClosed = this.getDesiredState(entry.serverId, entry.descriptor.id) === 'closed' && + CapabilityGrant.of(this.serverRegistry.getServer(entry.serverId)).has('channels.lifecycle'); + if (!entry.open || engagesDecidedClosed) { try { const { opened } = await this.openChannelNow(entry, 'opened-by-delivery'); entry = await this.waitForOpenChannel(entry); diff --git a/test/channel-lifecycle-order.test.ts b/test/channel-lifecycle-order.test.ts index 17f4f5e7e..2bf48d2c6 100644 --- a/test/channel-lifecycle-order.test.ts +++ b/test/channel-lifecycle-order.test.ts @@ -53,6 +53,7 @@ function fixture(onNotice?: NoticeHandler, store?: JsStore) { }; let currentServer = server; const replaceServer = () => { currentServer = { ...server }; }; + const setGrant = (paths: readonly string[]) => { currentServer.grant = new CapabilityGrant(new Set(paths), []); }; const traces: Array<{ type: string; [key: string]: unknown }> = []; const registry = new ChannelRegistry( { getServer: () => currentServer } as unknown as McplServerRegistry, @@ -74,7 +75,7 @@ function fixture(onNotice?: NoticeHandler, store?: JsStore) { assert.equal(settled, true, 'lifecycle operations must settle'); await result; }; - return { registry, actual, calls, pending, held, traces, isOpen, tool, drain, publishedWhileOpen, notices, routeFailures, replaceServer, answeredFor }; + return { registry, actual, calls, pending, held, traces, isOpen, tool, drain, publishedWhileOpen, notices, routeFailures, replaceServer, answeredFor, setGrant }; } test('remove/re-add during reconcile converges on the replacement desired state, with ACK first (#185)', async () => { @@ -682,6 +683,22 @@ for (const delivery of ['reply', 'speech'] as const) { }); } +test('speech into a channel decided closed keeps publishing where no open can succeed, without the lifecycle grant', async () => { + const f = fixture(); + await f.registry.handleChanged('test', { added: [descriptor('x', true)] }); + assert.equal(f.isOpen(), true); + f.setGrant(ALL_CAPABILITY_PATHS.filter((path) => path !== 'channels.lifecycle')); + const closing = await f.tool('close'); + assert.equal(closing.success, false); + assert.match(String(closing.error), /channels\.lifecycle/); + assert.equal(f.registry.getDesiredState('test', 'x'), 'closed', 'the refused close is still recorded as a decision'); + f.calls.length = 0; + assert.equal((await f.registry.routeSpeech('resident', 'hello', 'x'))?.delivered, true); + assert.deepEqual(f.publishedWhileOpen, [true]); + assert.deepEqual(f.calls, [], 'no open is attempted where it could not succeed'); + assert.equal(f.notices.length, 0); +}); + test('a failed explicit open says the channel stays selected for opening, and a later delivery opens it quietly', async () => { const f = fixture(); await f.registry.handleChanged('test', { added: [descriptor()] }); From 5633ca2a27df71d9b8403ac8e6fc4b318752da79 Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Fri, 9 Oct 2026 20:37:54 -0400 Subject: [PATCH 0087/1112] test(mcpl): state the open decision the in-flight-close test depends on Basil-1007's read of a6130d0 (room-203 #65271): 'an explicit open waits for a close in flight rather than answering already open' reaches channel_open's already-open shortcut only if the decision is already 'open' when the second open runs; otherwise the shortcut is skipped whatever its pending-work condition says, and the test would be blind. It holds because openChannelNow records the decision before its first await and the first open runs first. The test now asserts it. --- test/channel-lifecycle-order.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/test/channel-lifecycle-order.test.ts b/test/channel-lifecycle-order.test.ts index 2bf48d2c6..86cb11694 100644 --- a/test/channel-lifecycle-order.test.ts +++ b/test/channel-lifecycle-order.test.ts @@ -645,6 +645,7 @@ test('an explicit open waits for a close in flight rather than answering already const closing = f.tool('close'); await tick(); const reopening = f.tool('open'); + assert.equal(f.registry.getDesiredState('test', 'x'), 'open', 'the next open meets an open decision and a pending close'); let settled = false; const again = f.tool('open').then((result) => { settled = true; return result; }); await tick(); From f5402db2b091c059f04328ccf85860570553fa51 Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Fri, 9 Oct 2026 20:47:51 -0400 Subject: [PATCH 0088/1112] test(mcpl): pin five more lifecycle pieces the removal run found untested A second removal pass over pieces the first one hadn't named, at 5633ca2, left six with every test passing. Five do real work, and each now has a test that fails without it: - waitForOpenChannel swallowing a joined operation's ordinary failure: "speech joined to a failed backscroll still reaches a channel that stayed open". Without it, a failed redundant backscroll failed every delivery that had joined it, although the channel stayed open. - The reply path's tuned-out clause in its already-open answer: "a reply into a tuned-out channel whose transport is open needs no lifecycle RPC". Without it the reply sends a redundant open. - A failed reply open reporting the current registration's label: "a failed reply open reports the label of the registration that replaced it". Without it the result names the replaced registration's label. - reconcileChannels skipping a channel removed earlier in its batch: "a channel removed earlier in its batch does not hold up the rest of the batch". Without it the batch queues behind the removed channel's operation still in flight, then fails it as removed. - The admission loop reading only registered entries: "a descriptor the registration rejected is neither admitted nor announced". channels/ register reconciles every submitted descriptor, the rejected ones too. Without the check, a descriptor rejected for a missing id was admitted by an 'auto' policy and listed in its notice. The test registers twice, because the first registration migrates the legacy policy instead. The sixth, markAccepted clearing the unconfirmed flag, can't be reached: the flag is read only while the entry isn't open, and the entry leaves the open state again only through a receipt, which clears the flag, or an invalidation, which sets it. Locally (node 24.19) the two #210 files pass 58/58. --- test/channel-lifecycle-order.test.ts | 71 ++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/test/channel-lifecycle-order.test.ts b/test/channel-lifecycle-order.test.ts index 86cb11694..e916e73d5 100644 --- a/test/channel-lifecycle-order.test.ts +++ b/test/channel-lifecycle-order.test.ts @@ -787,3 +787,74 @@ test('a corrective close that cannot converge reports its exhaustion once', asyn while (f.pending.length > 0) f.pending.shift()!.finish(); } }); + +test('speech joined to a failed backscroll still reaches a channel that stayed open', async () => { + const f = fixture(); + await f.registry.handleChanged('test', { added: [descriptor('x', true)] }); + f.held.add('x'); + const backscroll = f.registry.handleChannelToolCall('channel_open', { serverId: 'test', channelId: 'x', backscroll: 5 }); + await tick(); + const speech = f.registry.routeSpeech('resident', 'joined', 'x'); + f.pending.shift()!.fail(); + assert.equal((await backscroll).success, false); + assert.equal((await speech)?.delivered, true, 'a failed redundant open leaves the channel usable for a joined delivery'); + assert.deepEqual(f.publishedWhileOpen, [true]); +}); + +test('a reply into a tuned-out channel whose transport is open needs no lifecycle RPC', async () => { + const f = fixture(); + await f.registry.handleChanged('test', { added: [descriptor('x', true)] }); + f.registry.enterTuneOut('test', 'x', { + epochId: 'quiet', cadenceSeconds: 60, backlogCap: 20, maxWakes: 3, startedAtSequence: 1, + }, 'agent-tool'); + f.calls.length = 0; + assert.equal((await f.registry.openIfClosedForSend('x', 'test')).status, 'already-open'); + assert.deepEqual(f.calls, []); + assert.equal(f.registry.getTuneOutState('test', 'x')?.params.epochId, 'quiet'); +}); + +test('a failed reply open reports the label of the registration that replaced it', async () => { + const f = fixture(); + await f.registry.handleChanged('test', { added: [descriptor()] }); + f.held.add('x'); + const reply = f.registry.openIfClosedForSend('x', 'test'); + await tick(); + const replacement = f.registry.handleChanged('test', { + removed: ['x'], added: [{ ...descriptor('x', false, 'replacement'), label: 'Replacement label' }], + }); + f.pending.shift()!.fail(); + const result = await reply; + assert.equal(result.status, 'open-failed'); + assert.equal(result.label, 'Replacement label'); + await f.drain(replacement); +}); + +test('a channel removed earlier in its batch does not hold up the rest of the batch', async () => { + const f = fixture(); + f.held.add('x'); + const first = f.registry.handleChanged('test', { added: [descriptor('x')] }); + await tick(); + assert.equal(f.pending.length, 1, "x's first operation is in flight"); + f.held.add('y'); + let batchSettled = false; + const batch = f.registry.handleChanged('test', { added: [descriptor('y'), descriptor('x')] }) + .then(() => { batchSettled = true; }); + await tick(); + assert.equal(f.pending.length, 2, "y's operation is in flight too"); + await f.registry.handleChanged('test', { removed: ['x'] }); + f.pending.pop()!.finish(); // y's receipt; x's first operation stays in flight + for (let i = 0; i < 5; i++) await tick(); + assert.equal(batchSettled, true, "the batch skips x rather than queueing behind x's operation in flight"); + await f.drain(first, batch); + assert.equal(f.traces.filter((e) => e.type === 'mcpl:channel-reconcile-failed').length, 0); +}); + +test('a descriptor the registration rejected is neither admitted nor announced', async () => { + const f = fixture(); + f.registry.setSubscriptionPolicy('test', 'auto'); + // The first registration migrates the legacy policy; the second is admitted by it. + await f.registry.handleRegister('test', { channels: [descriptor('x')] }); + await f.registry.handleRegister('test', { channels: [descriptor('y'), { ...descriptor('rejected'), id: '' }] }); + assert.deepEqual(f.notices.map((n) => n.channels.map((c) => c.channelId)), [['y']]); + assert.equal(f.registry.getDesiredState('test', ''), undefined); +}); From c67d06c798ea3ac23319802fb2d0c383aedc3c83 Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Fri, 9 Oct 2026 20:52:35 -0400 Subject: [PATCH 0089/1112] docs(changelog): an announced open is one the channel wasn't already decided for The fixed fragment said a send is announced only when it 'turned a channel decided closed open'. openChannelNow's rule is that the channel wasn't already decided open or tuned out, which includes a channel with no decision at all, such as a lazily registered DM. Say that. --- changelog.d/210-channel-lifecycle-order.fixed.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.d/210-channel-lifecycle-order.fixed.md b/changelog.d/210-channel-lifecycle-order.fixed.md index eee7abf91..e5de6a05e 100644 --- a/changelog.d/210-channel-lifecycle-order.fixed.md +++ b/changelog.d/210-channel-lifecycle-order.fixed.md @@ -1,4 +1,4 @@ -- Failed backscroll requests preserve the registry's known-open state rather than claiming a close. Automatic delivery joins pending lifecycle work without reopening a channel closed during the wait. A delivery or reply is announced as opening a channel only when it turned a channel decided closed open, using the current registration label; reconfirming the transport of a channel already decided open or tuned out isn't announced. Speech engages a channel decided closed as a reply does, even while a failed close has left its transport open, wherever the server's grant lets the host open it. A failed `channel_open` says the channel stays selected for opening, since that decision outlives the failed round-trip and a later delivery or reconnect tries again. +- Failed backscroll requests preserve the registry's known-open state rather than claiming a close. Automatic delivery joins pending lifecycle work without reopening a channel closed during the wait. A delivery or reply is announced as opening a channel, with the current registration label, only when the channel wasn't already decided open or tuned out; reconfirming the transport of a channel that was isn't announced. Speech engages a channel decided closed as a reply does, even while a failed close has left its transport open, wherever the server's grant lets the host open it. A failed `channel_open` says the channel stays selected for opening, since that decision outlives the failed round-trip and a later delivery or reconnect tries again. - Channel lifecycle operations now run in order per channel and reconcile the latest desired state and registration after an in-flight operation completes. Successful reconciliation confirms the current target and desired state at its receipt boundary; a superseded tool request reports that its requested state no longer applies. Automatic delivery preserves active tune-out attention state while opening or awaiting transport. From 27fdd5bc302e719a512b638530e156ec824fbd22 Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Fri, 9 Oct 2026 20:59:54 -0400 Subject: [PATCH 0090/1112] docs(changelog): state the delivery-notice rule the same way in both fragments Basil-1007 (room-203 #65286): since c67d06c the fixed fragment states the rule as 'wasn't already decided open or tuned out', while the breaking fragment still said 'turned a closed channel open'. Not wrong, since a channel with no decision is closed by default, but one wording leaves a reader nothing to reconcile. --- changelog.d/210-policy-admission-notice.breaking.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.d/210-policy-admission-notice.breaking.md b/changelog.d/210-policy-admission-notice.breaking.md index a324c147a..eb209f3ff 100644 --- a/changelog.d/210-policy-admission-notice.breaking.md +++ b/changelog.d/210-policy-admission-notice.breaking.md @@ -1 +1 @@ -- `onChannelAutoOpened` with `source: 'subscription-policy'` now reports a durable admission decision before transport reconciliation. Treat it as intent; await the registration call and inspect `getOpenChannels()` when confirmed live state is needed. `opened-by-delivery` reports a delivery that turned a closed channel open, once its transport is confirmed; a delivery that only reconfirms the transport of a channel already decided open or tuned out isn't reported. The built-in host notice now distinguishes selected-for-opening from currently open. +- `onChannelAutoOpened` with `source: 'subscription-policy'` now reports a durable admission decision before transport reconciliation. Treat it as intent; await the registration call and inspect `getOpenChannels()` when confirmed live state is needed. `opened-by-delivery` reports a delivery that opened a channel not already decided open or tuned out, once its transport is confirmed; a delivery that only reconfirms the transport of a channel that was isn't reported. The built-in host notice now distinguishes selected-for-opening from currently open. From 28dbcbac72e64396aa33b5203a1e000854f60b0b Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Fri, 9 Oct 2026 21:13:43 -0400 Subject: [PATCH 0091/1112] chore(changelog): move #188's entry into a changelog.d fragment The branch edited CHANGELOG.md's Unreleased section directly, which conflicts with main (02adf27) now that main's own entries live in changelog.d/ (CONTRIBUTING.md#changelog). Same text, moved verbatim into changelog.d/fix-188-route-failure-owner.fixed.md; CHANGELOG.md is back to the merge base's version, so the branch no longer touches it. --- CHANGELOG.md | 4 ---- changelog.d/fix-188-route-failure-owner.fixed.md | 1 + 2 files changed, 1 insertion(+), 4 deletions(-) create mode 100644 changelog.d/fix-188-route-failure-owner.fixed.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 155762653..6b9a30b4c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,10 +12,6 @@ Releases up to and including 0.7.3 predate this file; for their contents see ## Unreleased -### Fixed - -- Speech-route failure notices now target the speaking agent, including conversation forks, and respect that agent's mid-turn deferral. Unregistered conversation IDs retain the primary-agent fallback; notices remain context-only and do not trigger inference. A fork's final closure notice is flushed before disposal, and deferred writes retain their target through quiesce until they can be stored (#188). - ## 0.19.0 — 2026-09-28 ### Added diff --git a/changelog.d/fix-188-route-failure-owner.fixed.md b/changelog.d/fix-188-route-failure-owner.fixed.md new file mode 100644 index 000000000..4cd786d7b --- /dev/null +++ b/changelog.d/fix-188-route-failure-owner.fixed.md @@ -0,0 +1 @@ +- Speech-route failure notices now target the speaking agent, including conversation forks, and respect that agent's mid-turn deferral. Unregistered conversation IDs retain the primary-agent fallback; notices remain context-only and do not trigger inference. A fork's final closure notice is flushed before disposal, and deferred writes retain their target through quiesce until they can be stored (#188). From 5fed304403bb8b2c835f2168b5c4da5009b62848 Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Fri, 9 Oct 2026 21:28:39 -0400 Subject: [PATCH 0092/1112] fix(routing): address an agent's notices by the slot it reads MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Skill pass on #204 (preparing-upstream-pull-requests), reading main 02adf27. Three findings changed the design; probes in ~/.local/state/ada-1017/tmp/chk/pr204/route-failure-probes.test.ts ran on the trial merge and on main. 1. Residents share ONE message slot (sharedSlotAgents; forks and ephemeral runs are isolated). Addressing a resident (forAgent) only changed which context manager wrote the same slot, and swapped addMessage's guard from "primary's turn + any agent's tool cycle" to "the resident's own turn + cycle". Probes: with the primary mid-turn, or mid tool cycle, a secondary's notice was written at once (main defers it); with a secondary mid cycle, the primary's own notice was written at once (main defers it). The speaker already reads the slot, so residents stay on the default path. 2. Ephemeral runs (connectome-host subagents inherit the parent's proseRouting, so their prose routes) are isolated speakers too. On the old head their last notice was deferred against their live turn, then dropped by the teardown flush after runEphemeralToCompletion's finally had deregistered them ("not a registered agent — message dropped"); main gave it to the primary. 3. The same "a notice about X lands in the primary's window" defect sits in bounceProse (notice to the primary, wake and clipboard to the speaker) and injectScriptWake (envelope to the primary, wake to the script's agent). recordChannelAutoOpenNotice is left: a channel's open state is host-wide, and later traffic can reach the primary. Changes: - noticeTargetFor(name): forAgent unless the agent is a shared-slot resident; used by onRouteFailure, bounceProse, injectScriptWake. An unregistered name is addressed too, so addMessage drops it, logged, instead of giving the primary a notice about something it didn't do (was: primary fallback). - landQueuedWritesBeforeRelease(agent): when a fork is disposed or an ephemeral run deregistered, write what is still queued for it into its own context manager (Chronicle keeps it), then ack. Skipped while quiesced (a write could land mid-surgery); the next flush drops them, logged. Replaces 0b41769's mechanism (disposal guard with five conditions, disposal moved after the flush, per-agent flush gate), which covered forks only and whose per-agent teardown gate let a resident's forAgent writes (EventGate) land while another resident sharing the slot was mid tool cycle. Fork disposal is back at main's position. - Context: main c49f394 deprecated per-channel conversation routing (forks); ephemeral runs are the isolated speakers that stay. Tests (15, rewritten): residents x5 (default path and its guard), fork reply, fork own-turn deferral, unregistered drop, ephemeral last notice, ephemeral bounce, fork script wake, closure x4 (ordinary, other agent's tool cycle, quiesced: dropped and logged, refused write: logged and still disposed). Local Node 24.19: 15/15. --- .../fix-188-route-failure-owner.fixed.md | 6 +- src/framework.ts | 78 +++- test/route-failure-targeting.test.ts | 421 +++++++++++------- 3 files changed, 328 insertions(+), 177 deletions(-) diff --git a/changelog.d/fix-188-route-failure-owner.fixed.md b/changelog.d/fix-188-route-failure-owner.fixed.md index 4cd786d7b..c2376aed9 100644 --- a/changelog.d/fix-188-route-failure-owner.fixed.md +++ b/changelog.d/fix-188-route-failure-owner.fixed.md @@ -1 +1,5 @@ -- Speech-route failure notices now target the speaking agent, including conversation forks, and respect that agent's mid-turn deferral. Unregistered conversation IDs retain the primary-agent fallback; notices remain context-only and do not trigger inference. A fork's final closure notice is flushed before disposal, and deferred writes retain their target through quiesce until they can be stored (#188). +- Notices about an agent's own action now reach the window that agent reads (#188): + - A conversation fork's or an ephemeral run's failed-reply notice (`[discord-send-failed]`), bounced-prose notice and background-script wake land in its own window, deferred against its own turn, instead of the primary's. In connectome-host this moves a subagent's notices from its parent to the subagent. + - Residents share one message slot, so a resident's notices stay on the default path, unchanged. + - A notice for a speaker that is no longer registered is dropped and logged, rather than given to the primary. + - Writes still queued for a fork or an ephemeral run when it is released land in its own window first, which Chronicle keeps. While the host is quiesced they are dropped instead, logged. diff --git a/src/framework.ts b/src/framework.ts index c761b2cb9..27e6b385e 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -4137,6 +4137,7 @@ export class AgentFramework { // The late stream finalizer intentionally refuses name-keyed cleanup once // deregistered, so disposal must release EventGate liveness itself. this.eventGate?.onInferenceEnded(agent.name); + this.landQueuedWritesBeforeRelease(agent); this.agents.delete(agent.name); } this.toolImageLedgers.delete(agent.name); @@ -7169,21 +7170,10 @@ export class AgentFramework { * forever. The fork's context stays in Chronicle for investigation. */ private disposeConversationAgent(agentName: string): void { - // A closure can finish while writes are still deferred (e.g. quiesce), - // or while a successor owns the turn token. Keep the target registered - // until those writes land; the fallback reaper can dispose it afterward. - const belongsToAgent = (msg: { forAgent?: string }) => - (msg.forAgent ?? this.primaryAgentName) === agentName; - if ( - this.activeTurnTokens.has(agentName) || - this.activeStreams.has(agentName) || - this.pendingAssistantBlocks.has(agentName) || - this.deferredMessages.some(belongsToAgent) || - this.unackedDeferredWrites.some(belongsToAgent) - ) return; this.closingConversationAgents.delete(agentName); const channelId = this.conversationAgentHomes.get(agentName); const agent = this.agents.get(agentName); + if (agent) this.landQueuedWritesBeforeRelease(agent); this.agents.delete(agentName); this.toolImageLedgers.delete(agentName); this.agentConfigs.delete(agentName); @@ -8568,6 +8558,7 @@ export class AgentFramework { 'user', [{ type: 'text', text: notice }], { system: true, kind: 'prose-bounce' }, + this.noticeTargetFor(name), ); if (id) this.emitTrace({ type: 'message:added', messageId: id, source: 'prose-bounce' }); else this.emitTrace({ type: 'message:added', messageId: 'deferred', source: 'prose-bounce' }); @@ -10378,6 +10369,12 @@ export class AgentFramework { this.pendingAssistantBlocks.delete(agent.name); } + // A conversation fork whose TTL closure turn just finished is done for + // good — dispose it so the agent map doesn't grow monotonically. + if (ownsPhysicalStream && this.closingConversationAgents.has(agent.name)) { + this.disposeConversationAgent(agent.name); + } + // The turn is torn down — release the turn-alive marker BEFORE the // deferred flush below, so the flush appends at the settled tail. // Token-matched: if a successor turn already owns the agent (endTurn @@ -10394,7 +10391,7 @@ export class AgentFramework { // pending). Only THIS agent's messages: other targets' entries wait // for their own boundaries (re-adding via addMessage re-defers if the // target has meanwhile started a turn). - if (frameReachedTerminal && this.deferredMessages.length > 0 && !this.pendingAssistantBlocks.has(agent.name)) { + if (frameReachedTerminal && this.deferredMessages.length > 0 && this.pendingAssistantBlocks.size === 0) { const deferred = this.drainDeferredFor(agent.name); for (const msg of deferred) { this.addMessage(msg.participant, msg.content, msg.metadata, { @@ -10404,13 +10401,6 @@ export class AgentFramework { } this.ackDeferredWrites(); } - - // Flush the closure turn's notices while its owner is still registered. - // If quiesce or a successor re-deferred them, disposal waits for the - // fallback reaper after their eventual safe write boundary. - if (frameReachedTerminal && ownsPhysicalStream && this.closingConversationAgents.has(agent.name)) { - this.disposeConversationAgent(agent.name); - } } } @@ -11046,7 +11036,7 @@ export class AgentFramework { source: 'background-script', scriptId: record.id, tags: ['script:wake'], - }); + }, this.noticeTargetFor(record.agentName)); this.pendingRequests.push({ agentName: record.agentName, reason: 'script:wake', @@ -11964,6 +11954,48 @@ export class AgentFramework { return mine; } + /** + * Where a notice about an agent's own action belongs: its failed reply, its + * bounced prose, its background script's wake. A conversation fork or an + * ephemeral run reads only its own isolated slot, so the notice is addressed + * to it. Residents share one message slot, which the default path already + * writes, under the guard that waits for any agent's tool cycle and the + * primary's turn; addressing a resident would narrow that guard to its own + * turn and cycle. A name no longer registered (a fork or run released before + * its failure was known) is addressed too, so addMessage drops the notice, + * logged, rather than give it to the primary, which did nothing. + */ + private noticeTargetFor(agentName: string): { forAgent: string } | undefined { + const agent = this.agents.get(agentName); + return agent && this.sharedSlotAgents.has(agent) ? undefined : { forAgent: agentName }; + } + + /** + * A conversation fork or an ephemeral run is about to be deregistered. + * Writes still queued for its window would wait for a boundary it never + * reaches, and the flush that finds it gone would drop them; land them in + * its own window now, which stays in Chronicle for investigation. Its + * stream is over or cancelled, so nothing it compiles can diverge from + * them. While quiesced a write could land mid-surgery, so they stay + * queued, and the next flush to find their target gone drops them, logged. + */ + private landQueuedWritesBeforeRelease(agent: Agent): void { + if (this.quiesced) return; + const cm = agent.getContextManager(); + for (const msg of this.drainDeferredFor(agent.name)) { + try { + cm.addMessage(msg.participant, msg.content, withDeferredWriteId(msg.metadata, msg.id)); + } catch (err) { + console.error( + `[deferred-flush] ${agent.name}: failed to store a queued write before release ` + + `(participant=${msg.participant}):`, + err, + ); + } + } + this.ackDeferredWrites(); + } + private editMessage(id: MessageId, content: ContentBlock[]): void { const agent = this.primaryAgentName ? this.agents.get(this.primaryAgentName) @@ -12576,9 +12608,7 @@ export class AgentFramework { text: `[discord-send-failed] Your previous reply (${textLen} chars) could not be delivered to ${where} (${reason}). It was saved to your archive but the human did not receive it.`, }], { system: true, kind: 'discord-send-failed', channelId: channelId ?? '', reason }, - // Keep the notice with its speaker, including mid-turn deferral. - // Unknown conversation IDs retain the primary-agent fallback. - this.agents.has(conversationId) ? { forAgent: conversationId } : undefined, + this.noticeTargetFor(conversationId), ); } catch (err) { console.error('onRouteFailure: failed to record send-failure marker:', err); diff --git a/test/route-failure-targeting.test.ts b/test/route-failure-targeting.test.ts index 2b8890de3..351dd4114 100644 --- a/test/route-failure-targeting.test.ts +++ b/test/route-failure-targeting.test.ts @@ -23,11 +23,19 @@ function internals(framework: AgentFramework) { lastConversationSweep: number; sweepExpiredConversations(): void; quiesced: boolean; + deferredWritesPersisted: boolean; + unackedDeferredWrites: unknown[]; flushDeferredWrites(label: string): Promise; + injectScriptWake(record: { id: string; agentName: string }, envelope: string): void; }; } -describe('speech route failure notice targeting', () => { +const SEND_FAILED = '[discord-send-failed]'; +const BOUNCED = 'was not delivered —'; +const count = (messages: Array<{ content: unknown }>, needle: string) => + messages.filter((m) => JSON.stringify(m.content).includes(needle)).length; + +describe('notices about an agent\'s own action', () => { let tempDir: string; let framework: AgentFramework; let membrane: MockMembrane; @@ -57,144 +65,8 @@ describe('speech route failure notice targeting', () => { rmSync(tempDir, { recursive: true, force: true }); }); - it('records a secondary agent failure through that agent\'s context manager', async (t) => { - // Ordinary residents share a Chronicle message slot. Check the receiving - // manager here; the real conversation fork below checks window isolation. - const primary = t.mock.method(framework.getAgent('primary')!.getContextManager(), 'addMessage'); - const secondary = t.mock.method(framework.getAgent('secondary')!.getContextManager(), 'addMessage'); - const result = await internals(framework).channelRegistry.routeSpeech('secondary', 'hello', 'missing-channel'); - - assert.equal(result, null); - assert.equal(primary.mock.callCount(), 0); - assert.equal(secondary.mock.callCount(), 1); - const [participant, content, metadata] = secondary.mock.calls[0]!.arguments; - assert.equal(participant, 'user'); - assert.match(JSON.stringify(content), /\[discord-send-failed\].*5 chars.*missing-channel/); - assert.equal(metadata?.system, true); - assert.equal(metadata?.kind, 'discord-send-failed'); - assert.equal(metadata?.channelId, 'missing-channel'); - assert.match(String(metadata?.reason), /no registered channel/); - assert.equal(internals(framework).pendingRequests.length, 0, 'a failure notice must not request inference'); - assert.equal(membrane.calls.length, 0); - }); - - it('keeps primary failures in the primary context manager', async (t) => { - const primary = t.mock.method(framework.getAgent('primary')!.getContextManager(), 'addMessage'); - const secondary = t.mock.method(framework.getAgent('secondary')!.getContextManager(), 'addMessage'); - - await internals(framework).channelRegistry.routeSpeech('primary', 'hello', null); - - assert.equal(primary.mock.callCount(), 1); - assert.equal(secondary.mock.callCount(), 0); - assert.equal(primary.mock.calls[0]!.arguments[2]?.kind, 'discord-send-failed'); - assert.equal(internals(framework).pendingRequests.length, 0); - }); - - it('falls back to the primary for an unregistered conversation', async (t) => { - const primary = t.mock.method(framework.getAgent('primary')!.getContextManager(), 'addMessage'); - const secondary = t.mock.method(framework.getAgent('secondary')!.getContextManager(), 'addMessage'); - - await internals(framework).channelRegistry.routeSpeech('unknown-conversation', 'hello', null); - - assert.equal(primary.mock.callCount(), 1); - assert.equal(secondary.mock.callCount(), 0); - assert.equal(primary.mock.calls[0]!.arguments[2]?.kind, 'discord-send-failed'); - assert.equal(internals(framework).pendingRequests.length, 0); - }); - - it('defers a failure notice against the speaking agent\'s live turn', async (t) => { - const primary = t.mock.method(framework.getAgent('primary')!.getContextManager(), 'addMessage'); - const secondary = t.mock.method(framework.getAgent('secondary')!.getContextManager(), 'addMessage'); - const state = internals(framework); - state.activeTurnTokens.set('secondary', 1); - - await state.channelRegistry.routeSpeech('secondary', 'hello', null); - - assert.equal(primary.mock.callCount(), 0); - assert.equal(secondary.mock.callCount(), 0, 'the notice waits for the owner\'s safe boundary'); - assert.equal(state.deferredMessages.length, 1); - assert.equal(state.deferredMessages[0]!.forAgent, 'secondary'); - assert.equal(state.deferredMessages[0]!.metadata?.kind, 'discord-send-failed'); - assert.equal(state.pendingRequests.length, 0); - }); - - for (const condition of ['ordinary', 'other-tool-cycle', 'quiesced'] as const) { - it('retains a failed TTL-closure reply notice before disposal: ' + condition, async (t) => { - const state = internals(framework); - const channelId = 'closure-channel'; - let failPublishing = false; - const publisher = { - grant: new CapabilityGrant(new Set(['channels.publish']), []), - sendChannelsTyping() {}, - async sendChannelsPublish() { - if (failPublishing && condition === 'quiesced') state.quiesced = true; - return { delivered: !failPublishing }; - }, - }; - t.mock.method(state.mcplServerRegistry, 'getServer', () => publisher as never); - (state.channelRegistry as unknown as { channels: Map }).channels.set('srv:' + channelId, { - serverId: 'srv', descriptor: { id: channelId, type: 'srv', label: 'closure test' }, open: true, - }); - membrane.pushResponse(createMockResponse([{ type: 'text', text: 'Initial reply.' }])); - framework.pushEvent({ - type: 'mcpl:channel-incoming', serverId: 'srv', channelId, messageId: 'closure-incoming', - author: { id: 'user', name: 'User' }, - content: [{ type: 'text', text: 'Please help.' }], - timestamp: new Date().toISOString(), - metadata: { mentioned: true }, - triggerInference: true, - } as unknown as ProcessEvent); - await framework.runUntilIdle(); - const binding = framework.getConversationRouter()!.getBinding(channelId)!; - assert.ok(binding); - const fork = framework.getAgent(binding.agentName)!; - - failPublishing = true; - const finalReply = 'Final reply that the server cannot deliver.'; - membrane.pushResponse(createMockResponse([{ type: 'text', text: finalReply }])); - if (condition === 'other-tool-cycle') state.pendingAssistantBlocks.set('secondary', []); - binding.lastActivity = 0; - state.lastConversationSweep = 0; - state.sweepExpiredConversations(); - await framework.runUntilIdle(); - state.pendingAssistantBlocks.clear(); - - if (condition === 'quiesced') { - assert.ok(framework.getAgent(fork.name) === fork, 'a deferred write keeps its owner registered while quiesced'); - assert.equal(state.deferredMessages.length, 1); - state.lastConversationSweep = 0; - state.sweepExpiredConversations(); - assert.ok(framework.getAgent(fork.name) === fork, 'the fallback reaper must also retain the pending write target'); - state.quiesced = false; - await state.flushDeferredWrites('closure test resume'); - state.lastConversationSweep = 0; - state.sweepExpiredConversations(); - } - - assert.equal(framework.getAgent(fork.name), null, 'the settled fork is eventually disposed'); - assert.equal(state.deferredMessages.length, 0); - // Open a fresh manager over the retained fork namespace, rather than - // relying only on the disposed Agent's in-memory context manager. - const archive = await ContextManager.open({ - store: framework.getStore(), namespace: 'conversations/' + fork.name, - isolate: true, strategy: new PassthroughStrategy(), - }); - const { messages } = await archive.compile(); - const noticeIndex = messages.findIndex((m) => JSON.stringify(m.content).includes('[discord-send-failed]')); - const replyIndex = messages.findIndex((m) => m.participant === fork.name && JSON.stringify(m.content).includes(finalReply)); - assert.ok(replyIndex >= 0, 'the failed closure reply remains archived'); - assert.ok(noticeIndex > replyIndex, 'the archived failure notice follows the closure reply'); - assert.equal(messages.filter((m) => JSON.stringify(m.content).includes('[discord-send-failed]')).length, 1); - assert.match(JSON.stringify(messages[noticeIndex]!.content), /delivered:false/); - const primary = await framework.getAgent('primary')!.getContextManager().compile(); - assert.ok(!primary.messages.some((m) => JSON.stringify(m.content).includes('[discord-send-failed]'))); - assert.equal(membrane.calls.length, 2, 'only the initial and closure turns ran'); - assert.equal(state.pendingRequests.length, 0, 'a failed closure reply does not wake another turn'); - }); - } - - it('delivers a failed fork reply notice to the fork after its turn, without waking it again', async () => { - const reply = 'Reply from the fork.'; + /** A real conversation fork: an incoming DM binds one, and its reply fails (no such channel). */ + async function spawnFork(reply = 'Reply from the fork.') { membrane.pushResponse(createMockResponse([{ type: 'text', text: reply }])); framework.pushEvent({ type: 'mcpl:channel-incoming', @@ -208,20 +80,265 @@ describe('speech route failure notice targeting', () => { triggerInference: true, } as unknown as ProcessEvent); await framework.runUntilIdle(); - const binding = framework.getConversationRouter()!.getBinding('missing-channel'); assert.ok(binding, 'the incoming DM must spawn a real conversation fork'); - const fork = framework.getAgent(binding.agentName)!; - const { messages: forkMessages } = await fork.getContextManager().compile(); - const { messages: primaryMessages } = await framework.getAgent('primary')!.getContextManager().compile(); - const notices = forkMessages.filter((m) => JSON.stringify(m.content).includes('[discord-send-failed]')); - assert.equal(notices.length, 1, 'the fork must see its own failure notice exactly once'); - assert.ok(!primaryMessages.some((m) => JSON.stringify(m.content).includes('[discord-send-failed]'))); - const replyIndex = forkMessages.findIndex((m) => m.participant === binding.agentName && JSON.stringify(m.content).includes(reply)); - assert.ok(replyIndex >= 0, 'the attempted reply is still archived'); - assert.ok(forkMessages.indexOf(notices[0]!) > replyIndex, 'failure notice follows the attempted reply'); - assert.equal(internals(framework).deferredMessages.length, 0, 'the completed turn flushes its notice'); - assert.equal(internals(framework).pendingRequests.length, 0); - assert.equal(membrane.calls.length, 1, 'failure notice is context only, not another wake'); + return framework.getAgent(binding.agentName)!; + } + + async function runEphemeral(proseRouting: 'locus' | 'explicit', speech: string) { + const created = await framework.createEphemeralAgent({ + name: 'worker', model: 'test-model', systemPrompt: 'Do the task.', allowedTools: 'all', proseRouting, + }); + created.contextManager.addMessage('user', [{ type: 'text', text: 'Run once.' }]); + membrane.pushResponse(createMockResponse([{ type: 'text', text: speech }])); + const run = framework.runEphemeralToCompletion(created.agent, created.contextManager); + framework.start(); + await run; + await framework.runUntilIdle(); + // The run is released; read what it left in Chronicle. + const archive = await ContextManager.open({ + store: framework.getStore(), namespace: 'subagent/worker', isolate: true, strategy: new PassthroughStrategy(), + }); + return (await archive.compile()).messages; + } + + describe('residents share one message slot', () => { + it('keeps a secondary resident\'s notice on the default path, where it reads it', async (t) => { + const primary = t.mock.method(framework.getAgent('primary')!.getContextManager(), 'addMessage'); + const secondary = t.mock.method(framework.getAgent('secondary')!.getContextManager(), 'addMessage'); + const result = await internals(framework).channelRegistry.routeSpeech('secondary', 'hello', 'missing-channel'); + + assert.equal(result, null); + assert.equal(primary.mock.callCount(), 1, 'the default path writes the shared slot'); + assert.equal(secondary.mock.callCount(), 0); + const [participant, content, metadata] = primary.mock.calls[0]!.arguments; + assert.equal(participant, 'user'); + assert.match(JSON.stringify(content), /\[discord-send-failed\].*5 chars.*missing-channel/); + assert.equal(metadata?.system, true); + assert.equal(metadata?.kind, 'discord-send-failed'); + assert.equal(metadata?.channelId, 'missing-channel'); + assert.match(String(metadata?.reason), /no registered channel/); + t.mock.restoreAll(); + const { messages } = await framework.getAgent('secondary')!.getContextManager().compile(); + assert.equal(count(messages, SEND_FAILED), 1, 'the speaking resident reads the shared slot'); + assert.equal(internals(framework).pendingRequests.length, 0, 'a failure notice must not request inference'); + assert.equal(membrane.calls.length, 0); + }); + + it('keeps primary failures in the primary context manager', async (t) => { + const primary = t.mock.method(framework.getAgent('primary')!.getContextManager(), 'addMessage'); + const secondary = t.mock.method(framework.getAgent('secondary')!.getContextManager(), 'addMessage'); + + await internals(framework).channelRegistry.routeSpeech('primary', 'hello', null); + + assert.equal(primary.mock.callCount(), 1); + assert.equal(secondary.mock.callCount(), 0); + assert.equal(primary.mock.calls[0]!.arguments[2]?.kind, 'discord-send-failed'); + assert.equal(internals(framework).pendingRequests.length, 0); + }); + + it('waits for the primary\'s live turn before writing a secondary resident\'s notice', async () => { + const state = internals(framework); + state.activeTurnTokens.set('primary', 1); + + await state.channelRegistry.routeSpeech('secondary', 'hello', null); + + assert.equal(state.deferredMessages.length, 1, 'the shared slot is mid-turn for the primary'); + assert.equal(state.deferredMessages[0]!.forAgent, undefined); + const { messages } = await framework.getAgent('primary')!.getContextManager().compile(); + assert.equal(count(messages, SEND_FAILED), 0); + }); + + for (const [speaker, cycling] of [['secondary', 'primary'], ['primary', 'secondary']] as const) { + it(`waits for ${cycling}'s tool cycle before writing ${speaker}'s notice`, async () => { + const state = internals(framework); + state.pendingAssistantBlocks.set(cycling, []); + + await state.channelRegistry.routeSpeech(speaker, 'hello', null); + + assert.equal(state.deferredMessages.length, 1, 'a write now would split a tool_use from its result'); + const { messages } = await framework.getAgent('primary')!.getContextManager().compile(); + assert.equal(count(messages, SEND_FAILED), 0); + }); + } + }); + + describe('a fork or an ephemeral run reads only its own window', () => { + it('delivers a failed fork reply notice to the fork after its turn, without waking it again', async () => { + const reply = 'Reply from the fork.'; + const fork = await spawnFork(reply); + const { messages: forkMessages } = await fork.getContextManager().compile(); + const { messages: primaryMessages } = await framework.getAgent('primary')!.getContextManager().compile(); + const notices = forkMessages.filter((m) => JSON.stringify(m.content).includes(SEND_FAILED)); + assert.equal(notices.length, 1, 'the fork must see its own failure notice exactly once'); + assert.equal(count(primaryMessages, SEND_FAILED), 0); + const replyIndex = forkMessages.findIndex((m) => m.participant === fork.name && JSON.stringify(m.content).includes(reply)); + assert.ok(replyIndex >= 0, 'the attempted reply is still archived'); + assert.ok(forkMessages.indexOf(notices[0]!) > replyIndex, 'failure notice follows the attempted reply'); + assert.equal(internals(framework).deferredMessages.length, 0, 'the completed turn flushes its notice'); + assert.equal(internals(framework).pendingRequests.length, 0); + assert.equal(membrane.calls.length, 1, 'failure notice is context only, not another wake'); + }); + + it('defers a fork\'s notice against the fork\'s own live turn', async (t) => { + const fork = await spawnFork(); + const state = internals(framework); + const primary = t.mock.method(framework.getAgent('primary')!.getContextManager(), 'addMessage'); + const forkCm = t.mock.method(fork.getContextManager(), 'addMessage'); + state.activeTurnTokens.set(fork.name, 99); + + await state.channelRegistry.routeSpeech(fork.name, 'hello', null); + + assert.equal(primary.mock.callCount(), 0); + assert.equal(forkCm.mock.callCount(), 0, 'the notice waits for the fork\'s safe boundary'); + assert.equal(state.deferredMessages.length, 1); + assert.equal(state.deferredMessages[0]!.forAgent, fork.name); + assert.equal(state.deferredMessages[0]!.metadata?.kind, 'discord-send-failed'); + assert.equal(state.pendingRequests.length, 0); + }); + + it('drops a notice for a speaker that is no longer registered, logged, rather than give it to the primary', async (t) => { + const primary = t.mock.method(framework.getAgent('primary')!.getContextManager(), 'addMessage'); + const secondary = t.mock.method(framework.getAgent('secondary')!.getContextManager(), 'addMessage'); + const errors: string[] = []; + t.mock.method(console, 'error', (...args: unknown[]) => { errors.push(args.map(String).join(' ')); }); + + await internals(framework).channelRegistry.routeSpeech('released-conversation', 'hello', null); + + assert.equal(primary.mock.callCount(), 0); + assert.equal(secondary.mock.callCount(), 0); + assert.ok(errors.some((e) => /"released-conversation" is not a registered agent — message dropped/.test(e))); + assert.equal(internals(framework).deferredMessages.length, 0); + assert.equal(internals(framework).pendingRequests.length, 0); + }); + + it('lands an ephemeral run\'s last failure notice in its own window before releasing it', async (t) => { + const errors: string[] = []; + t.mock.method(console, 'error', (...args: unknown[]) => { errors.push(args.map(String).join(' ')); }); + const speech = 'Worker speech with no channel to go to.'; + + const archived = await runEphemeral('locus', speech); + + const replyIndex = archived.findIndex((m) => m.participant === 'worker' && JSON.stringify(m.content).includes(speech)); + const noticeIndex = archived.findIndex((m) => JSON.stringify(m.content).includes(SEND_FAILED)); + assert.ok(replyIndex >= 0, 'the run\'s reply is archived'); + assert.ok(noticeIndex > replyIndex, 'its failure notice follows the reply in its own window'); + assert.equal(count(archived, SEND_FAILED), 1); + const { messages } = await framework.getAgent('primary')!.getContextManager().compile(); + assert.equal(count(messages, SEND_FAILED), 0, 'the primary did not write that reply'); + assert.ok(!errors.some((e) => /not a registered agent/.test(e))); + assert.equal(internals(framework).deferredMessages.length, 0); + }); + + it('sends an ephemeral run\'s bounced prose notice to its own window, beside its clipboard', async () => { + const archived = await runEphemeral('explicit', 'Worker prose with no destination.'); + + assert.equal(count(archived, BOUNCED), 1, 'the speaker reads why its text did not go out'); + const { messages } = await framework.getAgent('primary')!.getContextManager().compile(); + assert.equal(count(messages, BOUNCED), 0, 'the primary wrote no such text'); + assert.equal(internals(framework).deferredMessages.length, 0); + }); + + it('wakes a fork\'s background script into the fork\'s own window', async () => { + const fork = await spawnFork(); + const envelope = '[background script s-1] Your background script DIED 0m after start.'; + + internals(framework).injectScriptWake({ id: 's-1', agentName: fork.name }, envelope); + + const { messages: forkMessages } = await fork.getContextManager().compile(); + const { messages: primaryMessages } = await framework.getAgent('primary')!.getContextManager().compile(); + assert.equal(count(forkMessages, envelope), 1); + assert.equal(count(primaryMessages, envelope), 0); + }); + + for (const condition of ['ordinary', 'other-tool-cycle', 'quiesced', 'write-fails'] as const) { + it('handles a failed TTL-closure reply notice at disposal: ' + condition, async (t) => { + const state = internals(framework); + const channelId = 'closure-channel'; + let failPublishing = false; + const publisher = { + grant: new CapabilityGrant(new Set(['channels.publish']), []), + sendChannelsTyping() {}, + async sendChannelsPublish() { + if (failPublishing && condition === 'quiesced') state.quiesced = true; + return { delivered: !failPublishing }; + }, + }; + t.mock.method(state.mcplServerRegistry, 'getServer', () => publisher as never); + (state.channelRegistry as unknown as { channels: Map }).channels.set('srv:' + channelId, { + serverId: 'srv', descriptor: { id: channelId, type: 'srv', label: 'closure test' }, open: true, + }); + membrane.pushResponse(createMockResponse([{ type: 'text', text: 'Initial reply.' }])); + framework.pushEvent({ + type: 'mcpl:channel-incoming', serverId: 'srv', channelId, messageId: 'closure-incoming', + author: { id: 'user', name: 'User' }, + content: [{ type: 'text', text: 'Please help.' }], + timestamp: new Date().toISOString(), + metadata: { mentioned: true }, + triggerInference: true, + } as unknown as ProcessEvent); + await framework.runUntilIdle(); + const binding = framework.getConversationRouter()!.getBinding(channelId)!; + assert.ok(binding); + const fork = framework.getAgent(binding.agentName)!; + + failPublishing = true; + const finalReply = 'Final reply that the server cannot deliver.'; + membrane.pushResponse(createMockResponse([{ type: 'text', text: finalReply }])); + if (condition === 'other-tool-cycle') state.pendingAssistantBlocks.set('secondary', []); + // A host with a recovery queue: what lands at release is acked out of it. + if (condition === 'ordinary') state.deferredWritesPersisted = true; + if (condition === 'write-fails') { + const cm = fork.getContextManager(); + const store = cm.addMessage.bind(cm); + t.mock.method(cm, 'addMessage', (...args: Parameters) => { + if (JSON.stringify(args[1]).includes(SEND_FAILED)) throw new Error('store refused the write'); + return store(...args); + }); + } + const errors: string[] = []; + t.mock.method(console, 'error', (...args: unknown[]) => { errors.push(args.map(String).join(' ')); }); + binding.lastActivity = 0; + state.lastConversationSweep = 0; + state.sweepExpiredConversations(); + await framework.runUntilIdle(); + state.pendingAssistantBlocks.clear(); + + assert.equal(framework.getAgent(fork.name), null, 'the fork is disposed when its closure turn ends'); + assert.equal(state.deferredMessages.length, 0, 'nothing is left queued for a disposed fork'); + assert.equal(state.unackedDeferredWrites.length, 0, 'nor waiting in the recovery queue'); + // Open a fresh manager over the retained fork namespace, rather than + // relying only on the disposed Agent's in-memory context manager. + const archive = await ContextManager.open({ + store: framework.getStore(), namespace: 'conversations/' + fork.name, + isolate: true, strategy: new PassthroughStrategy(), + }); + const { messages } = await archive.compile(); + const replyIndex = messages.findIndex((m) => m.participant === fork.name && JSON.stringify(m.content).includes(finalReply)); + assert.ok(replyIndex >= 0, 'the failed closure reply remains archived'); + if (condition === 'quiesced') { + // A write while quiesced could land mid-surgery: the notice is not + // written, and the flush that finds its target gone says so. + assert.equal(count(messages, SEND_FAILED), 0); + assert.ok(errors.some((e) => new RegExp(`"${fork.name}" is not a registered agent — message dropped`).test(e))); + } else if (condition === 'write-fails') { + // A write the store refuses is logged; disposal still completes. + assert.equal(count(messages, SEND_FAILED), 0); + assert.ok(errors.some((e) => e.includes(`[deferred-flush] ${fork.name}: failed to store a queued write before release`))); + } else { + const noticeIndex = messages.findIndex((m) => JSON.stringify(m.content).includes(SEND_FAILED)); + assert.ok(noticeIndex > replyIndex, 'the archived failure notice follows the closure reply'); + assert.equal(count(messages, SEND_FAILED), 1); + assert.match(JSON.stringify(messages[noticeIndex]!.content), /delivered:false/); + // Every write of a drained entry carries its durable id (drainDeferredFor). + const stored = archive.getAllMessages().find((m) => JSON.stringify(m.content).includes(SEND_FAILED)); + assert.equal(typeof (stored?.metadata as { deferredWriteId?: unknown } | undefined)?.deferredWriteId, 'string'); + } + const primary = await framework.getAgent('primary')!.getContextManager().compile(); + assert.equal(count(primary.messages, SEND_FAILED), 0); + assert.equal(membrane.calls.length, 2, 'only the initial and closure turns ran'); + assert.equal(state.pendingRequests.length, 0, 'a failed closure reply does not wake another turn'); + }); + } }); }); From ce0c3b29d0ca4f652bab9f8def1c8f4dcb24203b Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Fri, 9 Oct 2026 21:42:28 -0400 Subject: [PATCH 0093/1112] fix(routing): pass an ended speaker's failed reply to the primary, by name Basil-1007's review of 5fed304 (room-203 #65318): a notice that a reply never reached the human is only useful to an agent that can still act on it. Landing an ended speaker's notice in its archive, or dropping it when the speaker was already unregistered, told no agent. - passFailedReplyToPrimary(speaker, metadata): for a discord-send-failed notice whose delivery named a channel, add to the primary (default path, so the shared-slot guard and quiesce defer it as usual) "[discord-send- failed] A reply by (N chars) could not be delivered to (). has ended, and the human did not receive it.", metadata { system, kind, channelId, reason, textLen, speaker }. With no channel (no locus) no human was waiting: it logs "[route-failure] has ended, and its reply had no channel to reach: no one else is told" and writes nothing. Not applied to bounces or script wakes: an ended run's typical bounce is a subagent's final answer, which its parent already has as the tool result, and a released agent's script wake has no one to wake. - onRouteFailure: an unregistered speaker (released before its delivery failed, e.g. an async delivered:false) goes to passFailedReplyToPrimary. The notice metadata gains textLen, so a queued notice can be re-told. - landQueuedWritesBeforeRelease: drain first and return when nothing is queued, before touching the context manager (tool-wrapper-prose-guard- integration's "conversation disposal reclaims logical-turn state" disposes a fake fork with no context manager; dropping this early return in 5fed304 failed it on the trial merge's Node 24 run). While quiesced the archive write is dropped, logged, instead of leaving the queue untouched, so the failed reply can still be passed on; ack always, as the teardown flush right after does anyway. Each queued failed reply is also passed to the primary. - channelForNotice(channelId): the label rendering moved out of onRouteFailure, shared with the by-name notice. Two tests added for pieces no test reached: a script wake for an agent no longer registered is dropped, logged, not given to the primary (5fed304's noticeTargetFor); a bounce at release is not passed on (no "[route-failure]" log). Tests 18 (local 18/18 on the PR base): unregistered speaker with a channel -> primary told by name (not woken); without a channel -> nobody, logged; ephemeral with a failing default channel -> its archive plus the primary by name; ephemeral with no locus -> archive only, logged; closure x4 -> the primary is told by name (after the other agent's tool cycle / after resume where deferred). --- .../fix-188-route-failure-owner.fixed.md | 3 +- src/framework.ts | 107 +++++++++++++----- test/route-failure-targeting.test.ts | 81 +++++++++++-- 3 files changed, 147 insertions(+), 44 deletions(-) diff --git a/changelog.d/fix-188-route-failure-owner.fixed.md b/changelog.d/fix-188-route-failure-owner.fixed.md index c2376aed9..e1d289d61 100644 --- a/changelog.d/fix-188-route-failure-owner.fixed.md +++ b/changelog.d/fix-188-route-failure-owner.fixed.md @@ -1,5 +1,6 @@ - Notices about an agent's own action now reach the window that agent reads (#188): - A conversation fork's or an ephemeral run's failed-reply notice (`[discord-send-failed]`), bounced-prose notice and background-script wake land in its own window, deferred against its own turn, instead of the primary's. In connectome-host this moves a subagent's notices from its parent to the subagent. - Residents share one message slot, so a resident's notices stay on the default path, unchanged. - - A notice for a speaker that is no longer registered is dropped and logged, rather than given to the primary. + - When a fork or an ephemeral run has ended without reading that its reply to a channel failed, the primary is told by name instead: "A reply by `` (N chars) could not be delivered to … (…). `` has ended, and the human did not receive it." That notice's metadata carries `speaker`, and every failed-reply notice's metadata now carries `textLen`. - Writes still queued for a fork or an ephemeral run when it is released land in its own window first, which Chronicle keeps. While the host is quiesced they are dropped instead, logged. + - A background-script wake for an agent that is no longer registered is dropped, logged, rather than given to the primary. diff --git a/src/framework.ts b/src/framework.ts index 27e6b385e..a5349e4ef 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -11961,9 +11961,10 @@ export class AgentFramework { * to it. Residents share one message slot, which the default path already * writes, under the guard that waits for any agent's tool cycle and the * primary's turn; addressing a resident would narrow that guard to its own - * turn and cycle. A name no longer registered (a fork or run released before - * its failure was known) is addressed too, so addMessage drops the notice, - * logged, rather than give it to the primary, which did nothing. + * turn and cycle. A name no longer registered (a fork or run already + * released) is addressed too, so addMessage drops the notice, logged, rather + * than give it to the primary as if it were the primary's; a failed reply is + * passed on by name instead (passFailedReplyToPrimary). */ private noticeTargetFor(agentName: string): { forAgent: string } | undefined { const agent = this.agents.get(agentName); @@ -11973,29 +11974,78 @@ export class AgentFramework { /** * A conversation fork or an ephemeral run is about to be deregistered. * Writes still queued for its window would wait for a boundary it never - * reaches, and the flush that finds it gone would drop them; land them in - * its own window now, which stays in Chronicle for investigation. Its - * stream is over or cancelled, so nothing it compiles can diverge from - * them. While quiesced a write could land mid-surgery, so they stay - * queued, and the next flush to find their target gone drops them, logged. + * reaches, and the flush that finds it gone would drop them. Land them in + * its own window now, which Chronicle keeps for investigation; its stream + * is over or cancelled, so nothing it compiles can diverge from them. A + * failed reply among them is also passed to the primary, because the + * speaker will never read it. While quiesced, a write could land + * mid-surgery, so the writes are dropped instead, logged. */ private landQueuedWritesBeforeRelease(agent: Agent): void { - if (this.quiesced) return; + const queued = this.drainDeferredFor(agent.name); + if (queued.length === 0) return; const cm = agent.getContextManager(); - for (const msg of this.drainDeferredFor(agent.name)) { - try { - cm.addMessage(msg.participant, msg.content, withDeferredWriteId(msg.metadata, msg.id)); - } catch (err) { + for (const msg of queued) { + if (this.quiesced) { console.error( - `[deferred-flush] ${agent.name}: failed to store a queued write before release ` + - `(participant=${msg.participant}):`, - err, + `[deferred-flush] ${agent.name}: released while quiesced, so a queued write for its window ` + + `is dropped (participant=${msg.participant})`, ); + } else { + try { + cm.addMessage(msg.participant, msg.content, withDeferredWriteId(msg.metadata, msg.id)); + } catch (err) { + console.error( + `[deferred-flush] ${agent.name}: failed to store a queued write before release ` + + `(participant=${msg.participant}):`, + err, + ); + } } + this.passFailedReplyToPrimary(agent.name, msg.metadata); } this.ackDeferredWrites(); } + /** + * A fork or an ephemeral run ended before reading that its reply never + * reached the human: it was released with the notice still queued, or its + * delivery failed after release. Tell the primary instead, naming the + * speaker, so an agent who can still act learns of it: to resend, or to + * tell the human. Only for a delivery to a channel; a reply that had no + * channel to go to (no locus) had no human waiting for it. + */ + private passFailedReplyToPrimary(speaker: string, metadata: MessageMetadata | undefined): void { + const failure = metadata as { kind?: unknown; channelId?: unknown; reason?: unknown; textLen?: unknown } | undefined; + if (failure?.kind !== 'discord-send-failed') return; + if (typeof failure.channelId !== 'string' || failure.channelId === '') { + console.error(`[route-failure] ${speaker} has ended, and its reply had no channel to reach: no one else is told`); + return; + } + this.addMessage( + 'user', + [{ + type: 'text', + text: `[discord-send-failed] A reply by ${speaker} (${String(failure.textLen)} chars) could not be delivered to ` + + `${this.channelForNotice(failure.channelId)} (${String(failure.reason)}). ${speaker} has ended, and the human did not receive it.`, + }], + { system: true, kind: 'discord-send-failed', channelId: failure.channelId, reason: failure.reason, textLen: failure.textLen, speaker } as MessageMetadata, + ); + } + + /** + * A channel as a notice names it: the label when the registry has one, since + * a bare snowflake is unresolvable for the agent (the 2026-07-21 incident + * read as "a stale artifact", not a live failure). + */ + private channelForNotice(channelId: string | null): string { + if (!channelId) return 'the channel'; + const label = this.channelRegistry?.getDescriptor(channelId)?.label; + return label && label !== channelId + ? `${label.startsWith('#') ? label : `#${label}`} (${channelId})` + : channelId; + } + private editMessage(id: MessageId, content: ContentBlock[]): void { const agent = this.primaryAgentName ? this.agents.get(this.primaryAgentName) @@ -12588,26 +12638,21 @@ export class AgentFramework { // yes, wake no). onRouteFailure: ({ conversationId, channelId, reason, textLen }) => { try { - // Render a human-readable channel name when we can — a bare - // snowflake in the marker is unresolvable for the agent (the - // 2026-07-21 incident read as "a stale artifact", not a live - // failure). The marker is `system: true`, so it is never - // conversational and never influences routing. - const label = channelId - ? this.channelRegistry?.getDescriptor(channelId)?.label - : undefined; - const where = channelId - ? label && label !== channelId - ? `${label.startsWith('#') ? label : `#${label}`} (${channelId})` - : channelId - : 'the channel'; + // The marker is `system: true`, so it is never conversational and + // never influences routing. + const metadata: MessageMetadata = { system: true, kind: 'discord-send-failed', channelId: channelId ?? '', reason, textLen }; + if (!this.agents.has(conversationId)) { + // Released before its delivery failed: no window left to read it. + this.passFailedReplyToPrimary(conversationId, metadata); + return; + } this.addMessage( 'user', [{ type: 'text', - text: `[discord-send-failed] Your previous reply (${textLen} chars) could not be delivered to ${where} (${reason}). It was saved to your archive but the human did not receive it.`, + text: `[discord-send-failed] Your previous reply (${textLen} chars) could not be delivered to ${this.channelForNotice(channelId)} (${reason}). It was saved to your archive but the human did not receive it.`, }], - { system: true, kind: 'discord-send-failed', channelId: channelId ?? '', reason }, + metadata, this.noticeTargetFor(conversationId), ); } catch (err) { diff --git a/test/route-failure-targeting.test.ts b/test/route-failure-targeting.test.ts index 351dd4114..c31daadbc 100644 --- a/test/route-failure-targeting.test.ts +++ b/test/route-failure-targeting.test.ts @@ -197,19 +197,32 @@ describe('notices about an agent\'s own action', () => { assert.equal(state.pendingRequests.length, 0); }); - it('drops a notice for a speaker that is no longer registered, logged, rather than give it to the primary', async (t) => { + it('tells the primary, by name, when the speaker ended before its delivery failed', async (t) => { const primary = t.mock.method(framework.getAgent('primary')!.getContextManager(), 'addMessage'); const secondary = t.mock.method(framework.getAgent('secondary')!.getContextManager(), 'addMessage'); + + await internals(framework).channelRegistry.routeSpeech('released-conversation', 'hello', 'missing-channel'); + + assert.equal(secondary.mock.callCount(), 0); + assert.equal(primary.mock.callCount(), 1); + const [, content, metadata] = primary.mock.calls[0]!.arguments; + assert.match(JSON.stringify(content), + /\[discord-send-failed\] A reply by released-conversation \(5 chars\) could not be delivered to missing-channel \(no registered channel[^)]*\)\. released-conversation has ended, and the human did not receive it\./); + assert.equal(metadata?.speaker, 'released-conversation'); + assert.equal(metadata?.textLen, 5); + assert.equal(internals(framework).pendingRequests.length, 0, 'context only: the primary is not woken'); + }); + + it('tells no one, logged, when an ended speaker\'s reply had no channel to reach', async (t) => { + const primary = t.mock.method(framework.getAgent('primary')!.getContextManager(), 'addMessage'); const errors: string[] = []; t.mock.method(console, 'error', (...args: unknown[]) => { errors.push(args.map(String).join(' ')); }); await internals(framework).channelRegistry.routeSpeech('released-conversation', 'hello', null); assert.equal(primary.mock.callCount(), 0); - assert.equal(secondary.mock.callCount(), 0); - assert.ok(errors.some((e) => /"released-conversation" is not a registered agent — message dropped/.test(e))); assert.equal(internals(framework).deferredMessages.length, 0); - assert.equal(internals(framework).pendingRequests.length, 0); + assert.ok(errors.some((e) => e.includes('[route-failure] released-conversation has ended, and its reply had no channel to reach'))); }); it('lands an ephemeral run\'s last failure notice in its own window before releasing it', async (t) => { @@ -227,18 +240,48 @@ describe('notices about an agent\'s own action', () => { const { messages } = await framework.getAgent('primary')!.getContextManager().compile(); assert.equal(count(messages, SEND_FAILED), 0, 'the primary did not write that reply'); assert.ok(!errors.some((e) => /not a registered agent/.test(e))); + assert.ok(errors.some((e) => e.includes('[route-failure] worker has ended, and its reply had no channel to reach'))); assert.equal(internals(framework).deferredMessages.length, 0); }); - it('sends an ephemeral run\'s bounced prose notice to its own window, beside its clipboard', async () => { + it('passes an ephemeral run\'s failed delivery to a channel on to the primary, by name', async () => { + (internals(framework).channelRegistry as unknown as { defaultPublishChannel: string | null }).defaultPublishChannel = 'missing-channel'; + const speech = 'Worker speech for a channel that is not registered.'; + + const archived = await runEphemeral('locus', speech); + + assert.equal(count(archived, '[discord-send-failed] Your previous reply'), 1, 'the run keeps its own record'); + const { messages } = await framework.getAgent('primary')!.getContextManager().compile(); + const named = messages.filter((m) => JSON.stringify(m.content).includes(SEND_FAILED)); + assert.equal(named.length, 1); + assert.match(JSON.stringify(named[0]!.content), /A reply by worker \(\d+ chars\) could not be delivered to missing-channel .*worker has ended/); + assert.equal(internals(framework).deferredMessages.length, 0); + }); + + it('sends an ephemeral run\'s bounced prose notice to its own window, beside its clipboard', async (t) => { + const errors: string[] = []; + t.mock.method(console, 'error', (...args: unknown[]) => { errors.push(args.map(String).join(' ')); }); + const archived = await runEphemeral('explicit', 'Worker prose with no destination.'); assert.equal(count(archived, BOUNCED), 1, 'the speaker reads why its text did not go out'); const { messages } = await framework.getAgent('primary')!.getContextManager().compile(); - assert.equal(count(messages, BOUNCED), 0, 'the primary wrote no such text'); + assert.equal(count(messages, BOUNCED), 0, 'the primary wrote no such text, and is not told of it'); + assert.ok(!errors.some((e) => e.includes('[route-failure]')), 'a bounce is not a failed reply'); assert.equal(internals(framework).deferredMessages.length, 0); }); + it('drops, logged, a background-script wake for an agent no longer registered', async (t) => { + const primary = t.mock.method(framework.getAgent('primary')!.getContextManager(), 'addMessage'); + const errors: string[] = []; + t.mock.method(console, 'error', (...args: unknown[]) => { errors.push(args.map(String).join(' ')); }); + + internals(framework).injectScriptWake({ id: 's-2', agentName: 'released-run' }, '[background script s-2] Your background script DIED.'); + + assert.equal(primary.mock.callCount(), 0, 'the primary started no such script'); + assert.ok(errors.some((e) => /"released-run" is not a registered agent — message dropped/.test(e))); + }); + it('wakes a fork\'s background script into the fork\'s own window', async () => { const fork = await spawnFork(); const envelope = '[background script s-1] Your background script DIED 0m after start.'; @@ -305,7 +348,20 @@ describe('notices about an agent\'s own action', () => { state.pendingAssistantBlocks.clear(); assert.equal(framework.getAgent(fork.name), null, 'the fork is disposed when its closure turn ends'); - assert.equal(state.deferredMessages.length, 0, 'nothing is left queued for a disposed fork'); + if (condition === 'quiesced') { + // A write while quiesced could land mid-surgery: the archive write + // is dropped, logged, and the primary's notice waits for resume. + assert.ok(errors.some((e) => e.includes(`[deferred-flush] ${fork.name}: released while quiesced, so a queued write for its window is dropped`))); + assert.equal(state.deferredMessages.length, 1); + state.quiesced = false; + await state.flushDeferredWrites('closure test resume'); + } + if (condition === 'other-tool-cycle') { + // The primary's notice waits for the shared slot's tool cycle. + assert.equal(state.deferredMessages.length, 1); + await state.flushDeferredWrites('the primary\'s next boundary'); + } + assert.equal(state.deferredMessages.length, 0, 'nothing is left queued'); assert.equal(state.unackedDeferredWrites.length, 0, 'nor waiting in the recovery queue'); // Open a fresh manager over the retained fork namespace, rather than // relying only on the disposed Agent's in-memory context manager. @@ -317,10 +373,7 @@ describe('notices about an agent\'s own action', () => { const replyIndex = messages.findIndex((m) => m.participant === fork.name && JSON.stringify(m.content).includes(finalReply)); assert.ok(replyIndex >= 0, 'the failed closure reply remains archived'); if (condition === 'quiesced') { - // A write while quiesced could land mid-surgery: the notice is not - // written, and the flush that finds its target gone says so. assert.equal(count(messages, SEND_FAILED), 0); - assert.ok(errors.some((e) => new RegExp(`"${fork.name}" is not a registered agent — message dropped`).test(e))); } else if (condition === 'write-fails') { // A write the store refuses is logged; disposal still completes. assert.equal(count(messages, SEND_FAILED), 0); @@ -329,13 +382,17 @@ describe('notices about an agent\'s own action', () => { const noticeIndex = messages.findIndex((m) => JSON.stringify(m.content).includes(SEND_FAILED)); assert.ok(noticeIndex > replyIndex, 'the archived failure notice follows the closure reply'); assert.equal(count(messages, SEND_FAILED), 1); - assert.match(JSON.stringify(messages[noticeIndex]!.content), /delivered:false/); + assert.match(JSON.stringify(messages[noticeIndex]!.content), /Your previous reply.*delivered:false/); // Every write of a drained entry carries its durable id (drainDeferredFor). const stored = archive.getAllMessages().find((m) => JSON.stringify(m.content).includes(SEND_FAILED)); assert.equal(typeof (stored?.metadata as { deferredWriteId?: unknown } | undefined)?.deferredWriteId, 'string'); } + // The fork never reads its notice again, so the primary is told, by name. const primary = await framework.getAgent('primary')!.getContextManager().compile(); - assert.equal(count(primary.messages, SEND_FAILED), 0); + const told = primary.messages.filter((m) => JSON.stringify(m.content).includes(SEND_FAILED)); + assert.equal(told.length, 1); + assert.match(JSON.stringify(told[0]!.content), + new RegExp(`A reply by ${fork.name} \\(${finalReply.length} chars\\) could not be delivered to #closure test \\(closure-channel\\) \\(.*delivered:false.*\\)\\. ${fork.name} has ended, and the human did not receive it\\.`)); assert.equal(membrane.calls.length, 2, 'only the initial and closure turns ran'); assert.equal(state.pendingRequests.length, 0, 'a failed closure reply does not wake another turn'); }); From 43dd1c7e4cb657a9f6fc33649bbb6fc00f6fcfe0 Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Fri, 9 Oct 2026 21:47:46 -0400 Subject: [PATCH 0094/1112] fix(routing): a release never fails on the primary's by-name notice Basil-1007's approval of ce0c3b2 (room-203 #65323) asked for one guard: passFailedReplyToPrimary runs inside a release, before agents.delete, and called addMessage bare. A throw ("No agents configured", or a store error from the CM write) would leave a fork registered, or escape runEphemeralToCompletion's finally before deregistration. - try/catch around the write, logged: "[route-failure] failed to tell the primary that 's reply to failed:". - Found while writing its test: with no primary (agents: []), the default path falls back to the first registered agent, which can be the very run being released, so it would have been told about itself. Check for a registered primary first; without one, log "[route-failure] has ended, and there is no primary to tell that its reply to failed" and write nothing. Tests (20, local 20/20): a store error on the primary's notice -> the run is still released, its archive keeps its own notice, logged; no primary -> the run's archive has no by-name notice about itself, logged. Removing the primary check fails the second (checked locally). --- src/framework.ts | 31 +++++++++++------ test/route-failure-targeting.test.ts | 51 ++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 10 deletions(-) diff --git a/src/framework.ts b/src/framework.ts index a5349e4ef..664578c3b 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -12013,7 +12013,8 @@ export class AgentFramework { * delivery failed after release. Tell the primary instead, naming the * speaker, so an agent who can still act learns of it: to resend, or to * tell the human. Only for a delivery to a channel; a reply that had no - * channel to go to (no locus) had no human waiting for it. + * channel to go to (no locus) had no human waiting for it. Never throws: + * it runs inside a release, which must complete whatever this write meets. */ private passFailedReplyToPrimary(speaker: string, metadata: MessageMetadata | undefined): void { const failure = metadata as { kind?: unknown; channelId?: unknown; reason?: unknown; textLen?: unknown } | undefined; @@ -12022,15 +12023,25 @@ export class AgentFramework { console.error(`[route-failure] ${speaker} has ended, and its reply had no channel to reach: no one else is told`); return; } - this.addMessage( - 'user', - [{ - type: 'text', - text: `[discord-send-failed] A reply by ${speaker} (${String(failure.textLen)} chars) could not be delivered to ` + - `${this.channelForNotice(failure.channelId)} (${String(failure.reason)}). ${speaker} has ended, and the human did not receive it.`, - }], - { system: true, kind: 'discord-send-failed', channelId: failure.channelId, reason: failure.reason, textLen: failure.textLen, speaker } as MessageMetadata, - ); + // The default path falls back to the first registered agent when there + // is no primary, which could be the speaker being released. + if (!this.primaryAgentName || !this.agents.has(this.primaryAgentName)) { + console.error(`[route-failure] ${speaker} has ended, and there is no primary to tell that its reply to ${failure.channelId} failed`); + return; + } + try { + this.addMessage( + 'user', + [{ + type: 'text', + text: `[discord-send-failed] A reply by ${speaker} (${String(failure.textLen)} chars) could not be delivered to ` + + `${this.channelForNotice(failure.channelId)} (${String(failure.reason)}). ${speaker} has ended, and the human did not receive it.`, + }], + { system: true, kind: 'discord-send-failed', channelId: failure.channelId, reason: failure.reason, textLen: failure.textLen, speaker } as MessageMetadata, + ); + } catch (err) { + console.error(`[route-failure] failed to tell the primary that ${speaker}'s reply to ${failure.channelId} failed:`, err); + } } /** diff --git a/test/route-failure-targeting.test.ts b/test/route-failure-targeting.test.ts index c31daadbc..a28c3db10 100644 --- a/test/route-failure-targeting.test.ts +++ b/test/route-failure-targeting.test.ts @@ -258,6 +258,57 @@ describe('notices about an agent\'s own action', () => { assert.equal(internals(framework).deferredMessages.length, 0); }); + it('still releases an ephemeral run when the primary cannot be told, logged', async (t) => { + (internals(framework).channelRegistry as unknown as { defaultPublishChannel: string | null }).defaultPublishChannel = 'missing-channel'; + const primaryCm = framework.getAgent('primary')!.getContextManager(); + const store = primaryCm.addMessage.bind(primaryCm); + t.mock.method(primaryCm, 'addMessage', (...args: Parameters) => { + if (JSON.stringify(args[1]).includes('A reply by worker')) throw new Error('store refused the write'); + return store(...args); + }); + const errors: string[] = []; + t.mock.method(console, 'error', (...args: unknown[]) => { errors.push(args.map(String).join(' ')); }); + + const archived = await runEphemeral('locus', 'Worker speech for a channel that is not registered.'); + + assert.equal(framework.getAgent('worker'), null, 'the run is released'); + assert.equal(count(archived, '[discord-send-failed] Your previous reply'), 1); + assert.ok(errors.some((e) => e.includes('[route-failure] failed to tell the primary that worker\'s reply to missing-channel failed'))); + }); + + it('tells no one, logged, when there is no primary to tell', async (t) => { + const dir = mkdtempSync(join(tmpdir(), 'route-failure-no-primary-')); + const bare = await AgentFramework.create({ + storePath: join(dir, 'test.chronicle'), membrane: membrane.asMembrane(), agents: [], modules: [], + }); + try { + await internals(bare).initializeMcpl([]); + (internals(bare).channelRegistry as unknown as { defaultPublishChannel: string | null }).defaultPublishChannel = 'missing-channel'; + const errors: string[] = []; + t.mock.method(console, 'error', (...args: unknown[]) => { errors.push(args.map(String).join(' ')); }); + const created = await bare.createEphemeralAgent({ + name: 'worker', model: 'test-model', systemPrompt: 'Do the task.', allowedTools: 'all', proseRouting: 'locus', + }); + created.contextManager.addMessage('user', [{ type: 'text', text: 'Run once.' }]); + membrane.pushResponse(createMockResponse([{ type: 'text', text: 'Worker speech with nobody to tell.' }])); + const run = bare.runEphemeralToCompletion(created.agent, created.contextManager); + bare.start(); + await run; + await bare.runUntilIdle(); + + const archive = await ContextManager.open({ + store: bare.getStore(), namespace: 'subagent/worker', isolate: true, strategy: new PassthroughStrategy(), + }); + const { messages } = await archive.compile(); + assert.equal(count(messages, 'A reply by worker'), 0, 'the run is not told about itself'); + assert.equal(count(messages, '[discord-send-failed] Your previous reply'), 1); + assert.ok(errors.some((e) => e.includes('[route-failure] worker has ended, and there is no primary to tell that its reply to missing-channel failed'))); + } finally { + await bare.stop(); + rmSync(dir, { recursive: true, force: true }); + } + }); + it('sends an ephemeral run\'s bounced prose notice to its own window, beside its clipboard', async (t) => { const errors: string[] = []; t.mock.method(console, 'error', (...args: unknown[]) => { errors.push(args.map(String).join(' ')); }); From 099efc9d4aba13655b456c98b970cd12aef4b636 Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Fri, 9 Oct 2026 22:34:48 -0400 Subject: [PATCH 0095/1112] chore(changelog): move #67's entry into a changelog.d fragment The branch edited CHANGELOG.md's Unreleased section directly, which conflicts with main (02adf27) now that main's own entries live in changelog.d/ (CONTRIBUTING.md#changelog). Same text, moved verbatim into changelog.d/fix-67-end-turn-accounting.fixed.md; CHANGELOG.md is back to the merge base's version, so the branch no longer touches it. --- CHANGELOG.md | 4 ---- changelog.d/fix-67-end-turn-accounting.fixed.md | 1 + 2 files changed, 1 insertion(+), 4 deletions(-) create mode 100644 changelog.d/fix-67-end-turn-accounting.fixed.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 66f48f1a4..6b9a30b4c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,10 +12,6 @@ Releases up to and including 0.7.3 predate this file; for their contents see ## Unreleased -### Fixed - -- Tool-ended turns now emit `inference:completed`, record a successful inference-log entry, and record cumulative token, cache, and estimated-cost usage exactly once, including turns ended by `skip_reply` or other `endTurn` tools. Log-backed health totals now include these turns. - ## 0.19.0 — 2026-09-28 ### Added diff --git a/changelog.d/fix-67-end-turn-accounting.fixed.md b/changelog.d/fix-67-end-turn-accounting.fixed.md new file mode 100644 index 000000000..58a9b2d58 --- /dev/null +++ b/changelog.d/fix-67-end-turn-accounting.fixed.md @@ -0,0 +1 @@ +- Tool-ended turns now emit `inference:completed`, record a successful inference-log entry, and record cumulative token, cache, and estimated-cost usage exactly once, including turns ended by `skip_reply` or other `endTurn` tools. Log-backed health totals now include these turns. From 344770608442cd4e7c12d602786ac85a3554a4f8 Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Fri, 9 Oct 2026 23:59:37 -0400 Subject: [PATCH 0096/1112] fix(usage): count a stream's usage once however it ends; a tool-ended turn ends as a completion does Skill pass on #207 (house review of Jasper-314's 6ac4743/b715877, now on current main via 4484548). The rule #207 set at one site (a tool-ended stream's usage reaches the session tracker) holds at its siblings now: - A context-budget or physical-window restart cancels its stream at the same tool boundary as endTurn, after rounds that were billed. It is recorded where the restart is decided: counted once and logged as a success with the restart's reason (stopReason 'context_budget' or 'physical_window'), with no inference:completed (the turn goes on; a completed would end typing, TTS activations and piped waits mid-turn). - Every other end (a stream error, an abort, a throw, a provider cooldown, a superseded generation) counts the rounds it finished in driveStream's finally, once (countUsage's flag; the natural completion's and the guard's authoritative usage set it first). Every inference-log entry the stream writes carries that usage unless it names its own (logStream), so log-backed health totals agree with the tracker. - main's #159 guard path counts its abandoned stream through the same countUsage, so the backstop can't count it twice. A tool-ended turn is a successful turn end, so it ends what a natural completion ends: besides inference:completed (whose emitTrace intercept already resets the hard-down streak and the poison-history rewind budget), the provider-cooldown recovery receipt and the refusal-rewind episode (a forced /unstick reports "the model responded"). endRefusalEpisode() is the natural path's else-branch, moved into a helper both call. Growth: logInference stores a compiled request over 10 KB as an uncompressed, fsync'd, content-addressed chronicle blob, one per entry, with no retention. Jasper's tool-ended entry kept the request body, adding one whole-context blob per tool-ended turn for turns that wrote nothing on main. Boundary-ended entries (tool-ended, restarts) record a note instead (BOUNDARY_END_REQUEST); natural completions and failures keep theirs. Cut as not paying rent: the copy of each usage event (membrane >=0.5.78 emits a fresh snapshot each round, and AF never mutates it); the completionAccounted flag and both WeakMap deletes (no path calls a stream's boundary end twice or after its driveStream ends: every end resets the agent or leaves its state to a successor; the map is weak). Docs: InferenceLogEntry's request/tokenUsage/stopReason say what a boundary end records; the inference:completed trace says a tool-ended turn emits it before inference:turn_ended. Tests: end-turn-usage gains restarts (both reasons: counted when the restart is decided, logged, not completed), failure ends after a finished round (error event, abort, throw), and a tool-ended turn ending the failure streak, the refusal episode with a forced /unstick, a provider cooldown (recovery receipt) and a cooldown that cuts a turn after a round; the first test checks the request note and that usage is counted by turn_ended. tool-result-guard's abandoned-round test asserts 1007 exactly. --- src/framework.ts | 212 ++++++++++++--------- src/types/framework.ts | 10 +- src/types/trace.ts | 6 + test/end-turn-usage.test.ts | 337 ++++++++++++++++++++++++++++++++- test/tool-result-guard.test.ts | 2 +- 5 files changed, 475 insertions(+), 92 deletions(-) diff --git a/src/framework.ts b/src/framework.ts index b15d0d988..e1770e483 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -490,6 +490,16 @@ const PROCESS_LOG_ID = 'framework/process-log'; const TURN_CHECKPOINTS_ID = 'framework/turn-checkpoints'; // legacy single-map layout, read-only fallback const TURN_CHECKPOINTS_TREE_ID = 'framework/turn-checkpoints/tree'; +/** Why the framework ended a stream at a tool boundary: an endTurn tool + * result, or a context-budget restart (the `inference:stream_restarted` + * reasons). It is the stream's inference-log stop reason. */ +type StreamBoundaryEnd = 'turn_ended' | 'context_budget' | 'physical_window'; + +/** The request a boundary-ended stream's inference-log entry records in + * place of its body. A body is the whole compiled context, one blob per + * entry; these ends happen every tool-ended turn and budget restart. */ +const BOUNDARY_END_REQUEST = { note: 'request body not kept for a stream ended at a tool boundary' }; + /** Maximum number of turn checkpoints to keep per agent. */ const MAX_TURN_CHECKPOINTS = 20; const DEFAULT_DISCORD_AWARENESS_DEADLINE_MS = 10_000; @@ -1015,10 +1025,14 @@ export class AgentFramework { private processLoggingPersist: boolean; private processLoggingBroadcast: boolean; private activeStreams: Map> = new Map(); - // Tool results settle outside driveStream. Key accounting by the physical - // stream so an endTurn result can finalize it before cancellation teardown - // or ephemeral disposal, without touching a successor's usage. - private streamCompletionAccounting = new WeakMap void>(); + // The framework ends a stream at a tool boundary from handleProcessEvent: + // an endTurn result, or a context-budget restart. Each live stream + // registers here how to record that end (its usage, its log entry and, for + // endTurn, the completed turn), so the record is made where the decision + // is taken, before the cancellation's asynchronous teardown or an + // ephemeral's disposal. Keyed by the physical stream, so a successor's + // record is never touched. + private streamBoundaryEnds = new WeakMap void>(); /** Per-agent output locus FROZEN for the CURRENT logical turn. Resolved * eagerly in startAgentStream (home → addressed trigger → global default) @@ -4968,6 +4982,23 @@ export class AgentFramework { } } + /** + * A turn that ended WITHOUT a refusal (a natural completion that wasn't + * one, or a tool-ended turn) ends the rewind episode: the model responded. + * Leave the consolidated marker in place as the durable record; just + * clear the per-episode counters, and report a forced `/unstick` done. + */ + private endRefusalEpisode(agentName: string): void { + if (this.forcedRewind.has(agentName)) { + this.finishUnstick(agentName, true); + } + if (this.refusalRewinds.get(agentName)) { + this.refusalRewinds.set(agentName, 0); + } + this.rewindEpisode.delete(agentName); + this.refusalStreak.delete(agentName); + } + /** * Conclude an active `/unstick` session: post the outcome (what was shed and * whether the model stopped refusing) to the channel the command came from, @@ -6871,8 +6902,10 @@ export class AgentFramework { // driveStream's finally is the backstop.) this.eventGate?.onInferenceEnded(agent.name); this.settleAgent(agent.name, { stopReason: 'turn_ended', speech: '' }); + // A successful turn end, recorded as a natural completion's is: + // inference:completed with the rounds' usage, then turn_ended. if (currentState.stream) { - this.streamCompletionAccounting.get(currentState.stream)?.(); + this.streamBoundaryEnds.get(currentState.stream)?.('turn_ended'); } this.emitTrace({ type: 'inference:turn_ended', agentName: agent.name }); } else if (overBudget || overPhysical) { @@ -6895,6 +6928,9 @@ export class AgentFramework { // overwrote it). Harmless by accident before turn-alive // tracking; a genuine teardown leak after it. currentState.stream.cancel(); + // Its rounds were billed; the restart opens a fresh stream + // with its own usage. + this.streamBoundaryEnds.get(currentState.stream)?.(overBudget ? 'context_budget' : 'physical_window'); } agent.cancelStream(); this.emitTrace({ @@ -9488,50 +9524,89 @@ export class AgentFramework { const startTime = Date.now(); const requestId = `${agent.name}-${startTime}-${Math.random().toString(36).slice(2, 8)}`; const myStreamId = agent.streamId; + // Membrane's usage events are cumulative across the stream's tool loop, + // so the latest is everything the stream has been billed for so far: + // the measure of a stream that ends without a final response. let latestUsage: DetailedUsage | undefined; - let completionAccounted = false; - const completeAccounting = ( - usage: DetailedUsage | undefined, - response?: NormalizedResponse, - durationMs = Date.now() - startTime, - ): void => { - if (completionAccounted) return; - completionAccounted = true; - this.streamCompletionAccounting.delete(stream); - const tokenUsage = usage ? { - input: usage.inputTokens, - output: usage.outputTokens, - cacheCreation: usage.cacheCreationTokens, - cacheRead: usage.cacheReadTokens, - } : undefined; + const tokenUsageOf = (usage: DetailedUsage | undefined): InferenceLogEntry['tokenUsage'] => usage + ? { + input: usage.inputTokens, + output: usage.outputTokens, + cacheCreation: usage.cacheCreationTokens, + cacheRead: usage.cacheReadTokens, + } + : undefined; + // The stream's billed usage reaches the session totals once, however the + // stream ends: a final response's usage where there is one (completion, + // or the guard's abandoned round), else the rounds it finished (the + // boundary ends below, or `finally` for every other end). + let usageCounted = false; + const countUsage = (usage: DetailedUsage | undefined): void => { + if (usageCounted) return; + usageCounted = true; + if (!usage) return; + this.usageTracker.onInferenceCompleted(agent.name, usage, usage.estimatedCost + ? { total: usage.estimatedCost.total, currency: usage.estimatedCost.currency } + : undefined); + this.persistUsageState(); + }; + // Every inference-log entry of this stream carries the usage of the + // rounds it finished, unless the entry names its own. + const logStream = (entry: Omit): void => { + this.logInference({ + timestamp: startTime, + agentName: agent.name, + requestId, + tokenUsage: tokenUsageOf(latestUsage), + ...entry, + }); + }; + // A turn that ended successfully: with its final response (natural + // completion), or without one, when a tool result with endTurn ended it. + const completeTurn = (usage: DetailedUsage | undefined, durationMs: number, response?: NormalizedResponse): void => { + const tokenUsage = tokenUsageOf(usage); this.emitTrace({ type: 'inference:completed', agentName: agent.name, durationMs, tokenUsage, }); - if (usage) { - this.usageTracker.onInferenceCompleted(agent.name, usage, usage.estimatedCost - ? { total: usage.estimatedCost.total, currency: usage.estimatedCost.currency } - : undefined); - this.persistUsageState(); - } - // Health snapshots and log queries read this same terminal accounting. - // A tool-ended stream has no final provider response; record that fact - // rather than fabricating response content or waiting for cancellation. - this.logInference({ - timestamp: startTime, - agentName: agent.name, - requestId, + countUsage(usage); + logStream(response + ? { + success: true, + request: compiledRequest ?? { note: 'streaming request' }, + response: response.raw ?? { note: 'streaming response' }, + durationMs, + tokenUsage, + stopReason: response.stopReason, + } + : { + success: true, + request: BOUNDARY_END_REQUEST, + response: { note: 'stream ended by tool result' }, + durationMs, + tokenUsage, + stopReason: 'turn_ended', + }); + }; + this.streamBoundaryEnds.set(stream, (end) => { + const durationMs = Date.now() - startTime; + if (end === 'turn_ended') { + this.recordProviderAccelerationRecovery(agent, compiledRequest, end); + completeTurn(latestUsage, durationMs); + this.endRefusalEpisode(agent.name); + return; + } + countUsage(latestUsage); + logStream({ success: true, - request: compiledRequest ?? { note: 'streaming request' }, - response: response?.raw ?? { note: response ? 'streaming response' : 'stream ended by tool result' }, + request: BOUNDARY_END_REQUEST, + response: { note: 'stream restarted to compress its context' }, durationMs, - tokenUsage, - stopReason: response?.stopReason ?? 'turn_ended', + stopReason: end, }); - }; - this.streamCompletionAccounting.set(stream, () => completeAccounting(latestUsage)); + }); // Membrane usage events are cumulative across the native/XML tool loop. // Keep the previous cumulative sample so consumers that operate at the // physical provider-call boundary (estimator calibration and kv receipt @@ -9979,20 +10054,8 @@ export class AgentFramework { // The abandoned stream was billed (membrane usage here is // cumulative across this physical tool loop): count it before // the retry opens a fresh stream with its own usage. - const abandonedUsage = response.details?.usage; - if (abandonedUsage) { - this.usageTracker.onInferenceCompleted(agent.name, { - inputTokens: abandonedUsage.inputTokens, - outputTokens: abandonedUsage.outputTokens, - cacheCreationTokens: abandonedUsage.cacheCreationTokens, - cacheReadTokens: abandonedUsage.cacheReadTokens, - }, abandonedUsage.estimatedCost - ? { total: abandonedUsage.estimatedCost.total, currency: abandonedUsage.estimatedCost.currency } - : undefined); - this.persistUsageState(); - } - this.logInference({ - timestamp: startTime, agentName: agent.name, requestId, + countUsage(response.details?.usage); + logStream({ success: false, error: 'Tool output withheld by the guard', request: compiledRequest ?? {}, response, durationMs, tokenUsage, stopReason: 'refusal', }); @@ -10185,7 +10248,7 @@ export class AgentFramework { .join('\n'), }); - completeAccounting(du, response, durationMs); + completeTurn(du, durationMs, response); // Surface refusals instead of going silently mute: stderr line // (headless inference failures are otherwise under-logged) + an @@ -10293,17 +10356,7 @@ export class AgentFramework { void this.reactToRefusal(agent.name, category); } } else { - // A turn that completed WITHOUT a refusal ends the rewind episode: - // the model responded. Leave the consolidated marker in place as - // the durable record; just clear the per-episode counters. - if (this.forcedRewind.has(agent.name)) { - this.finishUnstick(agent.name, true); - } - if (this.refusalRewinds.get(agent.name)) { - this.refusalRewinds.set(agent.name, 0); - } - this.rewindEpisode.delete(agent.name); - this.refusalStreak.delete(agent.name); + this.endRefusalEpisode(agent.name); } // Dispatch speech (and thoughts if any) @@ -10517,10 +10570,7 @@ export class AgentFramework { stack: err.stack, }); - this.logInference({ - timestamp: startTime, - agentName: agent.name, - requestId, + logStream({ success: false, error: err.message, request: compiledRequest ?? { note: 'streaming request failed' }, @@ -10684,10 +10734,7 @@ export class AgentFramework { // state. Without this, abort-terminated inferences are // invisible to forensic queries (only request-side telemetry // via llm-calls.jsonl shows them, and only by absence). - this.logInference({ - timestamp: startTime, - agentName: agent.name, - requestId, + logStream({ success: false, error: `Stream aborted: ${reason}`, request: compiledRequest ?? { note: 'streaming request aborted' }, @@ -10699,12 +10746,8 @@ export class AgentFramework { } case 'usage': { - // Already cumulative across rounds: replace, rather than sum, and - // retain Membrane's cost at the rates that served those rounds. - latestUsage = { - ...event.usage, - estimatedCost: event.usage.estimatedCost ? { ...event.usage.estimatedCost } : undefined, - }; + // Cumulative across the stream's rounds: replace, never sum. + latestUsage = event.usage; agent.lastStreamInputTokens = event.usage.inputTokens; agent.lastStreamRealInputTokens = (event.usage.inputTokens ?? 0) + @@ -10807,7 +10850,7 @@ export class AgentFramework { const durationMs = Date.now() - startTime; if (ownsProviderGate && this.holdProviderAcceleration(agent, err, trigger)) { this.emitTrace({ type: 'inference:failed', agentName: agent.name, error: err.message, stack: err.stack }); - this.logInference({ timestamp: startTime, agentName: agent.name, requestId, success: false, + logStream({ success: false, error: `Provider acceleration cooldown: ${err.message}`, request: compiledRequest ?? { note: 'streaming request rate-limited' }, durationMs }); this.abortAgentScript(agent.name, 'provider acceleration cooldown'); @@ -10834,10 +10877,7 @@ export class AgentFramework { // Postmortem 2026-05-28 P2 #7: catch-path failures (stream itself // threw) were previously only visible via in-memory trace listeners. // Persist to inference log for forensic attribution. - this.logInference({ - timestamp: startTime, - agentName: agent.name, - requestId, + logStream({ success: false, error: `Stream threw: ${err.message}`, request: compiledRequest ?? { note: 'streaming request threw' }, @@ -10848,6 +10888,9 @@ export class AgentFramework { agent.reset(); if (this.agents.get(agent.name) === agent && agent.streamId === myStreamId) this.eventGate?.onInferenceEnded(agent.name); } finally { + // Every end the cases above didn't count (a stream error or abort, a + // throw, a superseded generation) still billed the rounds it finished. + countUsage(latestUsage); const unsettledKvSubmissions = drainKvSubmissionIds?.() ?? []; if (unsettledKvSubmissions.length > 0) { try { @@ -10938,7 +10981,6 @@ export class AgentFramework { } } this.frameworkCancelledStreams.delete(`${agent.name}:${myStreamId}`); - this.streamCompletionAccounting.delete(stream); if (ownsPhysicalStream) { this.activeStreams.delete(agent.name); this.pendingAssistantBlocks.delete(agent.name); diff --git a/src/types/framework.ts b/src/types/framework.ts index a127866c2..87fb67491 100644 --- a/src/types/framework.ts +++ b/src/types/framework.ts @@ -403,18 +403,24 @@ export interface InferenceLogEntry { success: boolean; /** Error message if failed */ error?: string; - /** Request data or blob ID if large */ + /** Request data or blob ID if large. A stream the framework ended at a + * tool boundary (stopReason 'turn_ended', 'context_budget' or + * 'physical_window') records a note in place of the body. */ request: unknown | { blobId: string }; /** Response data or blob ID if large (only if successful) */ response?: unknown | { blobId: string }; durationMs: number; + /** The stream's usage: its final response's, or, for a stream that ended + * without one, the provider rounds it finished before it ended. */ tokenUsage?: { input: number; output: number; cacheCreation?: number; cacheRead?: number; }; - /** Stop reason from the model, or 'turn_ended' for a tool-ended stream. */ + /** Stop reason from the model, or why the framework ended the stream at a + * tool boundary: 'turn_ended' (a tool result with endTurn), or + * 'context_budget' / 'physical_window' (a context-budget restart). */ stopReason?: string; } diff --git a/src/types/trace.ts b/src/types/trace.ts index de8f41ea8..0adc00d68 100644 --- a/src/types/trace.ts +++ b/src/types/trace.ts @@ -52,6 +52,12 @@ export type TraceEvent = silent?: boolean; }) | (TraceEventBase & { + /** + * The model answered and the turn ended: with the stream's final + * response, or with a tool result carrying `endTurn`, which + * `inference:turn_ended` then follows. `tokenUsage` is the stream's + * usage, cumulative across its rounds. + */ type: 'inference:completed'; agentName: string; durationMs: number; diff --git a/test/end-turn-usage.test.ts b/test/end-turn-usage.test.ts index 276c6dd63..84946b564 100644 --- a/test/end-turn-usage.test.ts +++ b/test/end-turn-usage.test.ts @@ -3,11 +3,12 @@ import assert from 'node:assert/strict'; import { mkdtempSync, rmSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import type { DetailedUsage, NormalizedResponse } from '@animalabs/membrane'; +import { MembraneError } from '@animalabs/membrane'; +import type { DetailedUsage, NormalizedRequest, NormalizedResponse, StreamEvent, YieldingStream } from '@animalabs/membrane'; import { AgentFramework } from '../src/index.js'; import { HealthModule } from '../src/modules/health/index.js'; -import type { Module, ToolCall, TraceEvent } from '../src/index.js'; -import { createMockResponse, MockMembrane } from './helpers/mock-membrane.js'; +import type { EventResponse, Module, ProcessEvent, ToolCall, TraceEvent } from '../src/index.js'; +import { createMockResponse, MockMembrane, MockYieldingStream } from './helpers/mock-membrane.js'; class TurnTools implements Module { readonly name = 'test'; @@ -70,14 +71,19 @@ async function fixture() { }); const traces: TraceEvent[] = []; const completionStates: Array = []; + // What the session totals already hold when the turn's terminal traces fire. + const countedAt: Array<{ type: string; inferenceCount: number }> = []; framework.onTrace((event) => { traces.push(event); if (event.type === 'inference:completed') { completionStates.push(framework.getAgent(event.agentName)?.state.status); } + if (event.type === 'inference:turn_ended' || event.type === 'inference:stream_restarted') { + countedAt.push({ type: event.type, inferenceCount: framework.getSessionUsage().inferenceCount }); + } }); framework.start(); - return { dir, storePath, membrane, framework, traces, completionStates }; + return { dir, storePath, membrane, framework, traces, completionStates, countedAt }; } async function run(framework: AgentFramework, name: string) { @@ -115,6 +121,10 @@ describe('tool-ended turn usage accounting', () => { assert.equal(logs.entries[0].entry.stopReason, 'turn_ended'); assert.deepEqual(logs.entries[0].entry.tokenUsage, expectedTokens(totalUsage)); assert.deepEqual(logs.entries[0].entry.response, { note: 'stream ended by tool result' }); + assert.deepEqual(logs.entries[0].entry.request, { note: 'request body not kept for a stream ended at a tool boundary' }, + 'no compiled-context blob per tool-ended turn'); + assert.deepEqual(f.countedAt, [{ type: 'inference:turn_ended', inferenceCount: 1 }], + 'counted before turn_ended fires, as a completion is before its caller resumes'); assert.equal(logs.entries[0].entry.durationMs, completed[0].durationMs); assert.equal(f.traces.filter((e) => e.type === 'inference:turn_ended').length, 1); assert.equal(f.traces.filter((e) => e.type === 'usage:updated').length, 1); @@ -243,3 +253,322 @@ describe('tool-ended turn usage accounting', () => { } }); }); + +/** One queued response per stream: a restart opens a fresh stream. */ +class SequentialStreamMembrane extends MockMembrane { + private next = 0; + override streamYielding(request: NormalizedRequest): YieldingStream { + this.calls.push(request); + const stream = new MockYieldingStream(this.responses.slice(this.next, ++this.next)); + this.lastStream = stream; + return stream; + } +} + +/** A stream that finishes one tool round with `usage`, then ends as `ending` says. */ +class RoundThenEnd implements YieldingStream { + isWaitingForTools = false; + pendingToolCallIds: string[] = []; + toolDepth = 0; + isCancelled = false; + private resumed = false; + private resume: (() => void) | null = null; + constructor(private readonly usage: DetailedUsage, private readonly ending: StreamEvent | Error) {} + provideToolResults(): void { + this.isWaitingForTools = false; + this.resumed = true; + this.resume?.(); + } + cancel(): void { + this.isCancelled = true; + this.resumed = true; + this.resume?.(); + } + async *[Symbol.asyncIterator](): AsyncIterator { + yield { type: 'usage', usage: this.usage } as StreamEvent; + const call = { type: 'tool_use' as const, id: 'echo-1', name: 'test--echo', input: {} }; + this.isWaitingForTools = true; + this.pendingToolCallIds = [call.id]; + yield { + type: 'tool-calls', + calls: [{ id: call.id, name: call.name, input: {} }], + context: { rawText: '', preamble: '', depth: 0, previousResults: [], accumulated: '', roundContent: [call] }, + } as StreamEvent; + if (!this.resumed) await new Promise((r) => { this.resume = r; }); + if (this.isCancelled) { + yield { type: 'aborted', reason: 'user' } as StreamEvent; + return; + } + if (this.ending instanceof Error) throw this.ending; + yield this.ending; + } +} + +class StreamsMembrane extends MockMembrane { + constructor(private readonly streams: Array<() => YieldingStream>) { super(); } + override streamYielding(request: NormalizedRequest): YieldingStream { + this.calls.push(request); + const make = this.streams.shift(); + return make ? make() : new MockYieldingStream([createMockResponse([{ type: 'text', text: 'done' }])]); + } +} + +async function pollUntil(cond: () => boolean, timeoutMs = 5_000): Promise { + const deadline = Date.now() + timeoutMs; + while (!cond() && Date.now() < deadline) await new Promise((r) => setTimeout(r, 10)); + return cond(); +} + +function finalResponse(text: string, usage: DetailedUsage): NormalizedResponse { + const response = createMockResponse([{ type: 'text', text }]); + response.usage = usage; + response.details!.usage = usage; + return response; +} + +const secondUsage: DetailedUsage = { + inputTokens: 300, outputTokens: 7, cacheCreationTokens: 0, cacheReadTokens: 200, + estimatedCost: { input: 0.02, output: 0.01, total: 0.03, currency: 'USD' }, +}; + +function summed(a: DetailedUsage, b: DetailedUsage) { + return { + inputTokens: a.inputTokens + b.inputTokens, outputTokens: a.outputTokens + b.outputTokens, + cacheCreationTokens: (a.cacheCreationTokens ?? 0) + (b.cacheCreationTokens ?? 0), + cacheReadTokens: (a.cacheReadTokens ?? 0) + (b.cacheReadTokens ?? 0), + estimatedCost: { total: a.estimatedCost!.total + b.estimatedCost!.total, currency: 'USD' }, + }; +} + +describe('a stream counts its usage once, however it ends', () => { + const nearCap: DetailedUsage = { + inputTokens: 5_000, outputTokens: 5, cacheCreationTokens: 0, cacheReadTokens: 195_000, + estimatedCost: { input: 0.05, output: 0.01, total: 0.06, currency: 'USD' }, + }; + for (const [reason, limits, usage] of [ + ['context_budget', { maxStreamTokens: 50 }, firstUsage], + ['physical_window', { physicalWindowTokens: 200_000, maxTokens: 4_000 }, nearCap], + ] as const) { + it(`a ${reason} restart counts and logs the stream it cancels, without completing the turn`, async () => { + const dir = mkdtempSync(join(tmpdir(), 'af-end-turn-usage-')); + const membrane = new SequentialStreamMembrane(); + membrane.pushResponse(toolResponse('echo', 'echo-1', usage)); + membrane.pushResponse(finalResponse('Recovered.', secondUsage)); + const framework = await AgentFramework.create({ + storePath: join(dir, 'store.chronicle'), membrane: membrane.asMembrane(), agents: [], + modules: [new TurnTools()], syncIntervalMs: 0, + }); + const traces: TraceEvent[] = []; + const countedAtRestart: number[] = []; + framework.onTrace((event) => { + traces.push(event); + if (event.type === 'inference:stream_restarted') countedAtRestart.push(framework.getSessionUsage().inferenceCount); + }); + framework.start(); + try { + const { agent, contextManager } = await framework.createEphemeralAgent({ + name: 'worker', model: 'test-model', systemPrompt: 'Do the task.', allowedTools: 'all', + proseRouting: 'disabled', ...limits, + }); + contextManager.addMessage('user', [{ type: 'text', text: 'Run once.' }]); + await framework.runEphemeralToCompletion(agent, contextManager); + await new Promise((resolve) => setImmediate(resolve)); + + assert.equal(membrane.calls.length, 2, 'the restart opened a second stream'); + assert.equal((traces.find((e) => e.type === 'inference:stream_restarted') as { reason?: string })?.reason, reason); + assert.deepEqual(countedAtRestart, [1], 'the cancelled stream is counted where the restart is decided'); + assert.equal(traces.filter((e) => e.type === 'inference:completed').length, 1, 'only the turn\'s end completes'); + assert.equal(traces.filter((e) => e.type === 'usage:updated').length, 2); + const snapshot = framework.getSessionUsage(); + assert.equal(snapshot.inferenceCount, 2); + assert.deepEqual(snapshot.totals, summed(usage, secondUsage)); + const logs = framework.queryInferenceLogs({ agentName: 'worker' }).entries.map((e) => e.entry).reverse(); + assert.equal(logs.length, 2); + assert.deepEqual( + { success: logs[0].success, stopReason: logs[0].stopReason, tokenUsage: logs[0].tokenUsage, request: logs[0].request, response: logs[0].response }, + { + success: true, stopReason: reason, tokenUsage: expectedTokens(usage), + request: { note: 'request body not kept for a stream ended at a tool boundary' }, + response: { note: 'stream restarted to compress its context' }, + }, + ); + assert.equal(logs[1].stopReason, 'end_turn'); + assert.deepEqual(logs[1].tokenUsage, expectedTokens(secondUsage)); + } finally { + await framework.stop(); + rmSync(dir, { recursive: true, force: true }); + } + }); + } + + for (const [name, ending] of [ + ['a stream error', { type: 'error', error: new MembraneError({ type: 'auth', retryable: false, message: 'key revoked', rawError: { status: 401 } }) } as StreamEvent], + ['an abort', { type: 'aborted', reason: 'timeout' } as StreamEvent], + ['a thrown stream', new Error('socket hang up')], + ] as const) { + it(`${name} after a finished round counts that round once and logs it with the failure`, async () => { + const dir = mkdtempSync(join(tmpdir(), 'af-end-turn-usage-')); + const membrane = new StreamsMembrane([() => new RoundThenEnd(firstUsage, ending)]); + const framework = await AgentFramework.create({ + storePath: join(dir, 'store.chronicle'), membrane: membrane.asMembrane(), agents: [], + modules: [new TurnTools()], syncIntervalMs: 0, + }); + const traces: TraceEvent[] = []; + framework.onTrace((event) => traces.push(event)); + framework.start(); + try { + const { agent, contextManager } = await framework.createEphemeralAgent({ + name: 'worker', model: 'test-model', systemPrompt: 'Do the task.', allowedTools: 'all', proseRouting: 'disabled', + }); + contextManager.addMessage('user', [{ type: 'text', text: 'Run once.' }]); + await framework.runEphemeralToCompletion(agent, contextManager).catch(() => undefined); + assert.ok(await pollUntil(() => traces.some((e) => e.type === 'usage:updated')), 'the finished round was counted'); + await new Promise((r) => setTimeout(r, 50)); + + assert.equal(membrane.calls.length, 1, 'no retry'); + assert.equal(traces.filter((e) => e.type === 'inference:completed').length, 0); + assert.equal(traces.filter((e) => e.type === 'usage:updated').length, 1); + assert.deepEqual(framework.getSessionUsage().totals, expectedTotals(firstUsage)); + const logs = framework.queryInferenceLogs({ agentName: 'worker' }).entries.map((e) => e.entry); + assert.equal(logs.length, 1); + assert.equal(logs[0].success, false); + assert.deepEqual(logs[0].tokenUsage, expectedTokens(firstUsage), 'the failure entry carries the finished round'); + } finally { + await framework.stop(); + rmSync(dir, { recursive: true, force: true }); + } + }); + } +}); + +class InputModule implements Module { + readonly name = 'input'; + async start() {} + async stop() {} + getTools() { return []; } + async handleToolCall() { return { success: false, isError: true, error: 'none' }; } + async onProcess(event: ProcessEvent): Promise { + if (event.type !== 'external-message') return {}; + return { addMessages: [{ participant: 'User', content: [{ type: 'text', text: String(event.content) }] }], requestInference: true }; + } +} + +function accelerationLimit(request: unknown): MembraneError { + return new MembraneError({ + type: 'rate_limit', retryable: true, httpStatus: 429, + message: "This request would exceed your organization's maximum usage increase rate for input tokens per minute", + rawError: { status: 429 }, rawRequest: request, + }); +} + +class ThrowingStream implements YieldingStream { + isWaitingForTools = false; + pendingToolCallIds: string[] = []; + toolDepth = 0; + isCancelled = false; + constructor(private readonly error: Error) {} + provideToolResults(): void { throw new Error('not waiting'); } + cancel(): void { this.isCancelled = true; } + async *[Symbol.asyncIterator](): AsyncIterator { + throw this.error; + } +} + +/** A resident whose streams come from `streams`, one per inference, with a 1 s provider cooldown. */ +async function resident(streams: Array<(request: NormalizedRequest) => YieldingStream>) { + const dir = mkdtempSync(join(tmpdir(), 'af-end-turn-usage-')); + const membrane = new MockMembrane(); + membrane.streamYielding = (request: NormalizedRequest): YieldingStream => { + membrane.calls.push(request); + const make = streams.shift(); + return make ? make(request) : new MockYieldingStream([createMockResponse([{ type: 'text', text: 'done' }])]); + }; + const framework = await AgentFramework.create({ + storePath: join(dir, 'store.chronicle'), membrane: membrane.asMembrane(), + agents: [{ name: 'resident', model: 'test-model', systemPrompt: 'system', allowedTools: 'all' }], + modules: [new InputModule(), new TurnTools()], syncIntervalMs: 0, maintenanceIntervalMs: 0, + }); + const internal = framework as unknown as { + providerAccelerationDefaultCooldownMs: number; + providerAccelerationJitterMs: number; + providerAccelerationLastRecovery: Map; + }; + internal.providerAccelerationDefaultCooldownMs = 1_000; + internal.providerAccelerationJitterMs = 0; + const traces: TraceEvent[] = []; + framework.onTrace((event) => traces.push(event)); + return { dir, membrane, framework, internal, traces }; +} + +describe('a tool-ended turn ends what a successful turn ends', () => { + for (const [ending, response] of [ + ['a tool-ended turn', () => toolResponse('finish', 'finish-1', totalUsage)], + ['a natural completion', () => createMockResponse([{ type: 'text', text: 'Done.' }])], + ] as const) { + it(`the failure streak and the refusal-rewind episode, a forced /unstick included: ${ending}`, async () => { + const f = await fixture(); + try { + const internal = f.framework as unknown as Record< + 'consecutiveInferenceFailures' | 'exhaustionRewinds' | 'refusalRewinds' | 'refusalStreak' | 'rewindEpisode' | 'forcedRewind', + Map + >; + for (const key of ['consecutiveInferenceFailures', 'exhaustionRewinds', 'refusalRewinds', 'refusalStreak'] as const) { + internal[key].set('worker', 2); + } + internal.rewindEpisode.set('worker', { markerId: 'marker', count: 1, category: 'cyber' }); + internal.forcedRewind.set('worker', { remaining: 1, removed: [], serverId: 'none', channelId: '' }); + f.membrane.pushResponse(response()); + await run(f.framework, 'worker'); + assert.equal(internal.consecutiveInferenceFailures.get('worker'), 0, 'the hard-down streak'); + assert.equal(internal.exhaustionRewinds.get('worker'), 0, 'the poison-history rewind budget'); + assert.equal(internal.refusalRewinds.get('worker'), 0); + assert.equal(internal.refusalStreak.has('worker'), false); + assert.equal(internal.rewindEpisode.has('worker'), false); + assert.equal(internal.forcedRewind.has('worker'), false, 'the /unstick session reports the model responded'); + } finally { + await f.framework.stop(); + rmSync(f.dir, { recursive: true, force: true }); + } + }); + } + + it('a provider cooldown, which records its recovery', async () => { + const r = await resident([ + (request) => new ThrowingStream(accelerationLimit(request)), + () => new MockYieldingStream([toolResponse('finish', 'finish-1', totalUsage)]), + ]); + r.framework.start(); + try { + r.framework.pushEvent({ type: 'external-message', source: 'test', content: 'first', metadata: {} } as unknown as ProcessEvent); + await r.framework.runUntilIdle(); + assert.equal(r.membrane.calls.length, 1, 'held for the cooldown'); + await new Promise((resolve) => setTimeout(resolve, 1_100)); + await r.framework.runUntilIdle(); + assert.ok(await pollUntil(() => r.traces.some((e) => e.type === 'inference:turn_ended')), 'the retried turn ended by tool'); + assert.equal(r.membrane.calls.length, 2); + assert.equal(r.internal.providerAccelerationLastRecovery.get('resident')?.stopReason, 'turn_ended'); + } finally { + await r.framework.stop(); + rmSync(r.dir, { recursive: true, force: true }); + } + }); + + it('a cooldown that cuts a turn after a finished round logs and counts that round', async () => { + const r = await resident([(request) => new RoundThenEnd(firstUsage, accelerationLimit(request))]); + r.framework.start(); + try { + r.framework.pushEvent({ type: 'external-message', source: 'test', content: 'first', metadata: {} } as unknown as ProcessEvent); + await r.framework.runUntilIdle(); + assert.ok(await pollUntil(() => r.traces.some((e) => e.type === 'usage:updated')), 'the finished round was counted'); + const logs = r.framework.queryInferenceLogs({ agentName: 'resident' }).entries.map((e) => e.entry); + assert.equal(logs.length, 1); + assert.match(String(logs[0].error), /^Provider acceleration cooldown: /); + assert.deepEqual(logs[0].tokenUsage, expectedTokens(firstUsage)); + assert.deepEqual(r.framework.getSessionUsage().totals, expectedTotals(firstUsage)); + assert.equal(r.traces.filter((e) => e.type === 'usage:updated').length, 1); + } finally { + await r.framework.stop(); + rmSync(r.dir, { recursive: true, force: true }); + } + }); +}); diff --git a/test/tool-result-guard.test.ts b/test/tool-result-guard.test.ts index 71ad0787a..dd3474649 100644 --- a/test/tool-result-guard.test.ts +++ b/test/tool-result-guard.test.ts @@ -523,7 +523,7 @@ test('usage of an abandoned guarded round is counted in session totals', async ( try { await h.run(); const totals = h.framework.getSessionUsage().totals as unknown as Record; - assert.ok(totals.inputTokens >= 1_007, `refused round usage missing: ${JSON.stringify(totals)}`); + assert.equal(totals.inputTokens, 1_007, `the refused round is counted, once: ${JSON.stringify(totals)}`); } finally { await h.framework.stop(); } }); From dc2158931f8af80ef3c8c96a4411ebffb75ca2e0 Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Sat, 10 Oct 2026 00:13:48 -0400 Subject: [PATCH 0097/1112] docs(changelog): #67's fragment says what callers will notice The fragment named only the tool-ended turn's new trace, log entry and usage. It now says, in callers' terms: a tool-ended turn emits inference:completed then inference:turn_ended and ends what a completion ends (failure streak, poison-history rewind budget, refusal episode and a forced /unstick, provider-cooldown recovery), so consumers that end a turn's display on inference:completed end it on these turns too; every stream's billed rounds reach the session totals once however it ends (restarts counted and logged without inference:completed; failures carry their finished rounds on their log entries; inferenceCount and health totals include them); and boundary-ended entries record a note instead of the compiled request. --- .../fix-67-end-turn-accounting.fixed.md | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/changelog.d/fix-67-end-turn-accounting.fixed.md b/changelog.d/fix-67-end-turn-accounting.fixed.md index 58a9b2d58..41c6895fd 100644 --- a/changelog.d/fix-67-end-turn-accounting.fixed.md +++ b/changelog.d/fix-67-end-turn-accounting.fixed.md @@ -1 +1,17 @@ -- Tool-ended turns now emit `inference:completed`, record a successful inference-log entry, and record cumulative token, cache, and estimated-cost usage exactly once, including turns ended by `skip_reply` or other `endTurn` tools. Log-backed health totals now include these turns. +- **A turn ended by a tool result (`endTurn`, such as `skip_reply`) is recorded as a completed + turn** (#67). It emits `inference:completed` with the stream's cumulative usage, then + `inference:turn_ended`, writes a successful inference-log entry (`stopReason: 'turn_ended'`), + and adds its usage once to the session totals (`usage:updated`, `getSessionUsage()`). Like a + completion, it resets the consecutive-failure streak and the poison-history rewind budget, ends + a refusal-rewind episode (a forced `/unstick` reports that the model responded), and records a + provider-cooldown recovery. Consumers that end a turn's display on `inference:completed`, such as + typing indicators, voice relays and alerts that close on success, now end it on these turns too. +- **Every stream's billed rounds reach the session totals once, however the stream ends.** A + context-budget or physical-window restart is counted and logged where it is decided + (`stopReason: 'context_budget'` or `'physical_window'`), without `inference:completed`. A stream + that errors, aborts or throws after finishing rounds counts them, and its failure entry in the + inference log carries them. `inferenceCount` counts these streams too, and log-backed health + totals include them. +- An inference-log entry for a stream the framework ended at a tool boundary (a tool-ended turn or + a restart) records a note in place of the compiled request, so these frequent entries don't each + add a whole-context blob to the store. From d7ea2944870740e26ddefb50badf907359412a8f Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Sat, 10 Oct 2026 00:19:19 -0400 Subject: [PATCH 0098/1112] fix(usage): a final response without usage counts the rounds' samples Basil-1007's review of 3447706: countUsage(undefined) took the once-flag without counting, so a natural completion whose response carries no details.usage, or #159's guard abandoning such a response, lost the rounds already finished, and the finally backstop couldn't recover them. countUsage now falls back to the latest cumulative sample, and completeTurn uses the same value for inference:completed's tokenUsage and the log entry, so the trace, the totals and the log agree. Identical to main whenever the final response carries usage. Tests: a natural completion whose response has no details.usage counts the last sample in all three places; an abandoned guarded round whose response has none counts each stream once from its samples. --- src/framework.ts | 20 ++++++++++++-------- test/end-turn-usage.test.ts | 21 +++++++++++++++++++++ test/tool-result-guard.test.ts | 11 +++++++++++ 3 files changed, 44 insertions(+), 8 deletions(-) diff --git a/src/framework.ts b/src/framework.ts index e1770e483..7bb11917b 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -9537,16 +9537,18 @@ export class AgentFramework { } : undefined; // The stream's billed usage reaches the session totals once, however the - // stream ends: a final response's usage where there is one (completion, - // or the guard's abandoned round), else the rounds it finished (the - // boundary ends below, or `finally` for every other end). + // stream ends: a final response's usage where it carries one (completion, + // or the guard's abandoned round), else the rounds it finished (a final + // response without usage, the boundary ends below, or `finally` for + // every other end). let usageCounted = false; const countUsage = (usage: DetailedUsage | undefined): void => { if (usageCounted) return; usageCounted = true; - if (!usage) return; - this.usageTracker.onInferenceCompleted(agent.name, usage, usage.estimatedCost - ? { total: usage.estimatedCost.total, currency: usage.estimatedCost.currency } + const counted = usage ?? latestUsage; + if (!counted) return; + this.usageTracker.onInferenceCompleted(agent.name, counted, counted.estimatedCost + ? { total: counted.estimatedCost.total, currency: counted.estimatedCost.currency } : undefined); this.persistUsageState(); }; @@ -9564,14 +9566,16 @@ export class AgentFramework { // A turn that ended successfully: with its final response (natural // completion), or without one, when a tool result with endTurn ended it. const completeTurn = (usage: DetailedUsage | undefined, durationMs: number, response?: NormalizedResponse): void => { - const tokenUsage = tokenUsageOf(usage); + // The trace, the session totals and the log entry all say the same. + const used = usage ?? latestUsage; + const tokenUsage = tokenUsageOf(used); this.emitTrace({ type: 'inference:completed', agentName: agent.name, durationMs, tokenUsage, }); - countUsage(usage); + countUsage(used); logStream(response ? { success: true, diff --git a/test/end-turn-usage.test.ts b/test/end-turn-usage.test.ts index 84946b564..4636625e7 100644 --- a/test/end-turn-usage.test.ts +++ b/test/end-turn-usage.test.ts @@ -169,6 +169,27 @@ describe('tool-ended turn usage accounting', () => { } }); + it('counts the rounds\' samples when a final response carries no usage', async () => { + const f = await fixture(); + try { + f.membrane.pushResponse(toolResponse('echo', 'echo-1', firstUsage)); + const response = createMockResponse([{ type: 'text', text: 'Done.' }]); + response.usage = totalUsage; // the last round's cumulative sample; no details.usage + f.membrane.pushResponse(response); + await run(f.framework, 'ordinary'); + const completed = f.traces.filter((e) => e.type === 'inference:completed'); + assert.equal(completed.length, 1); + assert.deepEqual(completed[0].tokenUsage, expectedTokens(totalUsage)); + assert.equal(f.traces.filter((e) => e.type === 'usage:updated').length, 1); + assert.deepEqual(f.framework.getSessionUsage().totals, expectedTotals(totalUsage)); + const logs = f.framework.queryInferenceLogs({ agentName: 'ordinary' }); + assert.deepEqual(logs.entries[0].entry.tokenUsage, expectedTokens(totalUsage), 'the trace, the totals and the log agree'); + } finally { + await f.framework.stop(); + rmSync(f.dir, { recursive: true, force: true }); + } + }); + it('attributes successive streams separately and persists their totals through store reopen', async () => { const f = await fixture(); let stopped = false; diff --git a/test/tool-result-guard.test.ts b/test/tool-result-guard.test.ts index dd3474649..2dd9e2768 100644 --- a/test/tool-result-guard.test.ts +++ b/test/tool-result-guard.test.ts @@ -527,6 +527,17 @@ test('usage of an abandoned guarded round is counted in session totals', async ( } finally { await h.framework.stop(); } }); +test('an abandoned guarded round whose response carries no usage still counts its rounds', async () => { + // Each scripted response emits a usage sample of 10 input tokens; neither + // final response carries details.usage, so both streams count their samples. + const h = await harness([[calls('one'), refused()], [answer()]], { toolResultGuard: true }); + try { + await h.run(); + const totals = h.framework.getSessionUsage().totals as unknown as Record; + assert.equal(totals.inputTokens, 20, `each stream is counted once, from its samples: ${JSON.stringify(totals)}`); + } finally { await h.framework.stop(); } +}); + // --------------------------------------------------------------------------- // Greptile review regressions (PR #159, head 647f081). // --------------------------------------------------------------------------- From f7d45e5616c48bf27e8bcffb9c57746c1e737ce2 Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Sat, 10 Oct 2026 00:24:36 -0400 Subject: [PATCH 0099/1112] fix(usage): the guard's abandoned stream logs and notes what it counts Basil-1007's review of d7ea294: the guard path logged `response.details?.usage ?? response.usage` while countUsage counted `details.usage ?? latestUsage`, so with details.usage missing the log entry said response.usage while the totals had the samples, and with both missing logStream's default was overwritten by an explicit undefined. - The guard path computes one value, `details.usage ?? latestUsage`, for countUsage, its log entry and noteRefusal. - logStream defaults tokenUsage when an entry's is undefined, not only when the key is absent, so no caller can erase it. - countUsage no longer falls back itself (d7ea294 added that): every caller now passes the one value it also reports (completeTurn's `used`, the guard's `abandoned`, latestUsage at a restart and in finally), so the fallback there was redundant. Test: the guard test without final usage also checks that the withheld stream's and the retry's log entries carry what the totals counted. --- src/framework.ts | 25 +++++++++++-------------- test/tool-result-guard.test.ts | 4 ++++ 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/src/framework.ts b/src/framework.ts index 7bb11917b..50e26ca6d 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -9540,15 +9540,14 @@ export class AgentFramework { // stream ends: a final response's usage where it carries one (completion, // or the guard's abandoned round), else the rounds it finished (a final // response without usage, the boundary ends below, or `finally` for - // every other end). + // every other end). Each caller passes the one value it also reports. let usageCounted = false; const countUsage = (usage: DetailedUsage | undefined): void => { if (usageCounted) return; usageCounted = true; - const counted = usage ?? latestUsage; - if (!counted) return; - this.usageTracker.onInferenceCompleted(agent.name, counted, counted.estimatedCost - ? { total: counted.estimatedCost.total, currency: counted.estimatedCost.currency } + if (!usage) return; + this.usageTracker.onInferenceCompleted(agent.name, usage, usage.estimatedCost + ? { total: usage.estimatedCost.total, currency: usage.estimatedCost.currency } : undefined); this.persistUsageState(); }; @@ -9559,8 +9558,8 @@ export class AgentFramework { timestamp: startTime, agentName: agent.name, requestId, - tokenUsage: tokenUsageOf(latestUsage), ...entry, + tokenUsage: entry.tokenUsage ?? tokenUsageOf(latestUsage), }); }; // A turn that ended successfully: with its final response (natural @@ -10049,16 +10048,14 @@ export class AgentFramework { // after the single recovery retry. proseStream?.reset(); if (withheld) { - const usage = response.details?.usage ?? response.usage; - const tokenUsage = usage ? { - input: usage.inputTokens, output: usage.outputTokens, - cacheCreation: usage.cacheCreationTokens, cacheRead: usage.cacheReadTokens, - } : undefined; - this.noteRefusal(agent.name, category, tokenUsage); // The abandoned stream was billed (membrane usage here is // cumulative across this physical tool loop): count it before - // the retry opens a fresh stream with its own usage. - countUsage(response.details?.usage); + // the retry opens a fresh stream with its own usage. One value + // for the count, the log entry and the refusal note. + const abandoned = response.details?.usage ?? latestUsage; + const tokenUsage = tokenUsageOf(abandoned); + this.noteRefusal(agent.name, category, tokenUsage); + countUsage(abandoned); logStream({ success: false, error: 'Tool output withheld by the guard', request: compiledRequest ?? {}, response, durationMs, tokenUsage, stopReason: 'refusal', diff --git a/test/tool-result-guard.test.ts b/test/tool-result-guard.test.ts index 2dd9e2768..510ca08b9 100644 --- a/test/tool-result-guard.test.ts +++ b/test/tool-result-guard.test.ts @@ -535,6 +535,10 @@ test('an abandoned guarded round whose response carries no usage still counts it await h.run(); const totals = h.framework.getSessionUsage().totals as unknown as Record; assert.equal(totals.inputTokens, 20, `each stream is counted once, from its samples: ${JSON.stringify(totals)}`); + // The log says what the totals count: the withheld stream, then the retry. + const logs = h.framework.queryInferenceLogs({ agentName: 'assistant' }).entries.map((e) => e.entry).reverse(); + assert.deepEqual(logs.map((e) => [e.success, e.tokenUsage]), + [[false, { input: 10, output: 5 }], [true, { input: 10, output: 5 }]]); } finally { await h.framework.stop(); } }); From 378c2b33423ee5a227d9d53f413707cf5bf95c85 Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Sat, 10 Oct 2026 02:16:33 -0400 Subject: [PATCH 0100/1112] test(drafts): pin that a draft's sending mark ends with its own attempt, and with a publish that throws MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The skill pass on #247's post-approval commit 315d12c removed each of its pieces one at a time. Two survived: the resend loop's delete of the sending mark right after each publish, and the batch's finally backstop. Nothing tested either. - Without the per-publish delete, a draft delivered early in a batch stays "being sent right now" until the whole batch ends, so dismissing it is refused with the wrong reason instead of "already delivered — nothing to dismiss". - Without the backstop, a publish that throws leaves its draft marked as sending for the life of the process, and every later dismissal of it is refused. Both tests pass on 1d6d2bb's source too (no sending marks there): they pin that 315d12c's marks don't outlive their attempt. With them, all eight of 315d12c's pieces are caught when removed (D1 tool refusal, D2/D3 the loop's mark and unmark, D4 the backstop, D5 the {{unsent}} mark, D6 the bounce refusal, D7/D8 the receipt), on Node 24.21. Co-Authored-By: Claude Opus 5.5 --- test/prose-drafts.test.ts | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/test/prose-drafts.test.ts b/test/prose-drafts.test.ts index 09ec8f432..defe3a294 100644 --- a/test/prose-drafts.test.ts +++ b/test/prose-drafts.test.ts @@ -956,6 +956,36 @@ describe('drafts resend ownership', () => { } }); + it("a draft whose own result is in isn't being sent any more, though the rest of its batch is", async () => { + const t = await setup(); + try { + const batch = t.internals.handleDraftsTool('scout', { action: 'resend', draftIds: [t.a.id, t.b.id], destination: 'chan' }); + await t.tick(); + t.pending.shift()!.resolve({ status: 'delivered', messageId: 'm-a' }); + await t.tick(); + assert.deepEqual(t.pending.map((p) => p.text), ['B words'], 'A is in; B is out'); + assert.match(t.say(await t.internals.handleDraftsTool('scout', { action: 'dismiss', draftIds: [t.a.id] })), + new RegExp(`${t.a.id}: already delivered — nothing to dismiss`)); + t.pending.shift()!.resolve({ status: 'delivered', messageId: 'm-b' }); + await batch; + } finally { + await t.close(); + } + }); + + it('a publish that throws leaves its draft dismissable, not marked as being sent', async () => { + const t = await setup(); + try { + const registry = (t.framework as unknown as { channelRegistry: Record }).channelRegistry; + registry.publish = async () => { throw new Error('transport gone'); }; + await t.internals.handleDraftsTool('scout', { action: 'resend', draftIds: [t.a.id], destination: 'chan' }).catch(() => undefined); + assert.match(t.say(await t.internals.handleDraftsTool('scout', { action: 'dismiss', draftIds: [t.a.id] })), + new RegExp(`${t.a.id}: dismissed`)); + } finally { + await t.close(); + } + }); + it('>>skip_reply {{unsent}} leaves the latest bounce alone while it is being sent, and tells the resident', async () => { const t = await setup(); try { From ee43daf65614ac9f03d070a97e532ebe9a59c90d Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Sat, 10 Oct 2026 18:11:23 -0400 Subject: [PATCH 0101/1112] fix(requests): request-only turns stay where the reply's request had them MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Context-manager #155 (thinking binding): a provider that binds signed thinking to the prefix it was minted under refuses that thinking once a later request changes anything before it. Two agent-framework request-only constructs changed the prefix between the request that minted a reply and every later request. 1. Silent heartbeat: the tick's instruction rode an ephemeral system-position injection (InferenceRequest.ephemeralSystemPrompt -> ContextInjection 'framework:silent-heartbeat'), so every heartbeat request's system differed from every other request's. That request missed the prompt cache from the system on, every retained thinking block failed its binding there, and the tick reply's own thinking failed at the next ordinary turn. Now the instruction is the separator's text (SILENT_HEARTBEAT_SEPARATOR), which every compile already renders byte-identically before the tick's first surviving row and which the tick's own request ends on. Every past tick reads it too, telling the agent those turns were private self-checks. Steady-state cost: 199 characters where the separator was 16, at each tick whose rows survive in the window (roughly 50 tokens a tick). One-time cost: the first compile after upgrading renders past ticks with the new text, one prompt-cache rewrite (and no strip if this ships before agent-framework engages context-manager #160's pass). ephemeralSystemPrompt is removed from InferenceRequest (types/agent.ts), the push-event wake, the batched wake, the budget restart and the system injection: the heartbeat was its only producer; connectome-host's main neither sets nor reads it (git grep), nor does anything else in the anima checkouts here. It is an exported interface, so the removal has its own breaking fragment naming who would need to act; the behaviour is in the changed fragment. 2. [Continue]: a compile that ends on the agent's own message gets a trailing request-only [Continue] turn, and later compiles didn't render it, so the reply it prompted sat straight after the previous assistant message: the formatter merged them (signed thinking from two responses in one message, the "cannot be modified" 400 when it is the latest), and the reply's binding failed from the next request. Context-manager's chain never sees request-only turns (it chains its own compiled messages), so it would send that thinking. Now the reply's first stored row is stamped metadata.promptedBy: 'continue' and request builds render [Continue] immediately before it, found the way tick rows are found (tool_use ids, thinking signatures; text never). Same known limit as ticks: a reply with neither isn't found, and carries no thinking to fail. Mechanics: - silent-heartbeat.ts is now the request-only-rows module. indexRequestOnlyRows returns ticks (identity key -> eventId, this agent's tick rows) and continued (the identity keys of stamped rows), independently, so a row can be both. renderRequestOnlyRows puts a tick's separator before the tick's first surviving row and [Continue] immediately before each stamped row, after any separator, as the reply's request had them. A stamped row is one stored row of the agent's own reply, which a compile never splits, so [Continue] needs no once-per-group check. (Basil-1007 asked about a row carrying both stamps: no path found today opens a stream inside a tick on [Continue] — every same-turn restart or retry compiles past stored tool results or an unchanged opening — but the first version would have rendered only the separator for such a row.) Internal names only; the public exports (SILENT_HEARTBEAT_SEPARATOR, silentHeartbeatOf, the tick types) keep their names. - Agent.buildActivationRequest's body is now assembleActivationRequest (private), which also reports endsOnContinue; buildActivationRequest wraps it for previews. startStreamWithInjections sets continuePromptPending from it; addAssistantResponse stamps the first row it stores while that's set. Every stream start resets it (a budget restart, guard retry or error retry compiles afresh); the other addAssistantResponse callers store the stream's own rows. The direct Agent API stores through contextManager.addMessage and never adds [Continue]. Tests: silent-heartbeat-rows (+4: the tick's system is the agent's own; the reply to [Continue] is re-rendered byte-stable; only its first row is marked; a row with both stamps gets both turns, separator first, also when it is the tick's first surviving row) and silent-heartbeat (the instruction is the request's last turn and the system is unchanged). The separator literal in the rows test is now the exported constant. Red at 02adf27 with only these test files: 4 of the integration tests fail (the both-stamps unit test needs the new exports). Reviewed by Basil-1007. --- .../request-only-rows-stay.breaking.md | 8 + changelog.d/request-only-rows-stay.changed.md | 23 +++ src/agent.ts | 89 +++++++---- src/framework.ts | 19 +-- src/silent-heartbeat.ts | 150 ++++++++++++------ src/types/agent.ts | 6 +- test/silent-heartbeat-rows.test.ts | 132 +++++++++++++-- test/silent-heartbeat.test.ts | 6 +- 8 files changed, 318 insertions(+), 115 deletions(-) create mode 100644 changelog.d/request-only-rows-stay.breaking.md create mode 100644 changelog.d/request-only-rows-stay.changed.md diff --git a/changelog.d/request-only-rows-stay.breaking.md b/changelog.d/request-only-rows-stay.breaking.md new file mode 100644 index 000000000..4fd6cbd18 --- /dev/null +++ b/changelog.d/request-only-rows-stay.breaking.md @@ -0,0 +1,8 @@ +- **Code that sets or reads `InferenceRequest.ephemeralSystemPrompt`:** the + field is removed. It carried a system-position prompt for one turn; the + silent heartbeat was its only producer, and its instruction now rides the + tick's request-only separator. There is no replacement: a system prompt + that changes for one turn costs that request its prompt cache from the + system on, and changes what every retained signed-thinking block is bound + to. Nothing in this package or in connectome-host sets or reads it. + Unchanged: `suppressProse` and `silentHeartbeat` on `InferenceRequest`. diff --git a/changelog.d/request-only-rows-stay.changed.md b/changelog.d/request-only-rows-stay.changed.md new file mode 100644 index 000000000..067641c96 --- /dev/null +++ b/changelog.d/request-only-rows-stay.changed.md @@ -0,0 +1,23 @@ +- A silent heartbeat tick's instruction now rides the tick's request-only + separator instead of a system-position injection, so a tick's request + carries the agent's own system prompt, like every other turn. A system + prompt that changed for one turn cost the heartbeat's request its prompt + cache from the system on, and on a provider that binds signed thinking to + the prefix it was minted under (context-manager #155) it failed every + retained thinking block in that request, and the tick reply's own thinking + at the next ordinary turn. `SILENT_HEARTBEAT_SEPARATOR` is now that + instruction, so every past tick reads it too: 199 characters where it was + 16, at each tick whose rows survive in the window (roughly 50 tokens a + tick). The first compile after upgrading renders past ticks with the new + text, one prompt-cache rewrite. (`InferenceRequest.ephemeralSystemPrompt` + is removed; see the breaking entry.) +- When a compile ends on the agent's own message and the request gets a + trailing `[Continue]` turn, the reply it prompts now has its first stored + row marked `metadata.promptedBy: 'continue'`, and every later request + renders `[Continue]` again just before that row, the way a silent tick's + separator is rendered before its first row. Without it, later requests + merged the reply into the previous assistant message (one message carrying + signed thinking from two responses, a 400 when it is the latest), and + dropped a turn the reply's request held, which a provider that binds + thinking refuses. A reply with neither a tool call nor signed thinking + can't be found in a compiled window and merges as before. diff --git a/src/agent.ts b/src/agent.ts index 412707a33..7a01884b9 100644 --- a/src/agent.ts +++ b/src/agent.ts @@ -4,11 +4,13 @@ import { createHash } from 'node:crypto'; import type { CacheWireReceipt, KvUnifiedRequestHooks } from './kv-unified-wire.js'; import { ToolResultGuard, TOOL_RESULT_GUARD_NOTICE } from './tool-result-guard.js'; import { - indexSilentTickRows, - separateSilentHeartbeatTicks, + PROMPTED_BY_CONTINUE, + continueTurn, + indexRequestOnlyRows, + renderRequestOnlyRows, silentHeartbeatSeparatorTurn, + type RequestOnlyRowIndex, type SilentHeartbeatTick, - type SilentTickIndex, type StoredRowLike, } from './silent-heartbeat.js'; import { @@ -794,13 +796,13 @@ export class Agent { * `ContextManager.compile` is itself side-effect-free (compression runs in * the background). Safe to call regardless of the agent's current status. */ - async buildActivationRequest( + private async assembleActivationRequest( availableTools: ToolDefinition[], injections?: ContextInjection[], budget?: TokenBudget, compressionTools: ToolDefinition[] = availableTools, options: ActivationRequestOptions = {}, - ): Promise { + ): Promise<{ request: NormalizedRequest; endsOnContinue: boolean }> { // Compression may revisit hidden tools in recorded history. Keep its full // definition set separate from the resident's advertised tools: the // autobiographical strategy must declare the same tools on its @@ -854,31 +856,32 @@ export class Agent { })) .filter((m) => m.content.length > 0); - // Silent heartbeat ticks store no prompt row. Render a request-only - // separator before each tick's first surviving row so the wire formatter - // cannot merge the tick into the previous assistant message (see - // silent-heartbeat.ts for what can and cannot be matched). This agent's - // own first request for a tick ends on that same turn, so the prefix it - // caches is the one every later compile renders. "First" is per agent: a - // broadcast tick shares its eventId with every resident. - const ticks = this.indexSilentHeartbeatTicks(); - messages = separateSilentHeartbeatTicks(messages, ticks.rows, this.name); + // Silent heartbeat ticks and replies to a [Continue] turn store no + // prompt row. Render each request-only turn before its response's first + // surviving row, so the wire formatter cannot merge the response into the + // previous assistant message and every later request keeps the prefix + // the response was minted under (see silent-heartbeat.ts for what can + // and cannot be matched). The request that minted the response ended on + // that same turn, so the prefix it caches is the one every later compile + // renders. "First" is per agent: a broadcast tick shares its eventId with + // every resident. + const requestOnly = this.indexRequestOnlyRows(); + messages = renderRequestOnlyRows(messages, requestOnly, this.name); const openingTick = options.silentHeartbeat !== undefined - && !ticks.started.has(options.silentHeartbeat.eventId); + && !requestOnly.started.has(options.silentHeartbeat.eventId); + let endsOnContinue = false; if (openingTick) { messages = [...messages, silentHeartbeatSeparatorTurn()]; } else if (messages.length > 0 && messages[messages.length - 1]!.participant === this.name) { // Safety: ensure messages don't end with an assistant message. // Some models reject trailing assistant messages ("prefill not supported"), // and after context compression a stale assistant turn can end up last. - messages = [...messages, { - participant: 'user', - content: [{ type: 'text', text: '[Continue]' }], - }]; + messages = [...messages, continueTurn()]; + endsOnContinue = true; } - return { + const request: NormalizedRequest = { messages: this.toolResultGuard.prepareRequest(messages), system: this.buildSystemPrompt(systemInjections), config: { @@ -895,28 +898,47 @@ export class Agent { ...(this.providerParams && { providerParams: this.providerParams }), assistantParticipant: this.name, }; + return { request, endsOnContinue }; + } + + /** The request `assembleActivationRequest` builds (see there), for preview + * and debug tooling: what this agent's next activation would send. */ + async buildActivationRequest( + availableTools: ToolDefinition[], + injections?: ContextInjection[], + budget?: TokenBudget, + compressionTools: ToolDefinition[] = availableTools, + options: ActivationRequestOptions = {}, + ): Promise { + return (await this.assembleActivationRequest(availableTools, injections, budget, compressionTools, options)).request; } - /** Stored silent-tick rows, memoized on the store's cached message array - * (ContextManager returns the same array until the store changes). */ - private silentTickIndex: { source: readonly unknown[]; index: SilentTickIndex } | null = null; + /** Stored rows that follow a request-only turn, memoized on the store's + * cached message array (ContextManager returns the same array until the + * store changes). */ + private requestOnlyIndex: { source: readonly unknown[]; index: RequestOnlyRowIndex } | null = null; - private indexSilentHeartbeatTicks(): SilentTickIndex { - const empty: SilentTickIndex = { rows: new Map(), started: new Set() }; + private indexRequestOnlyRows(): RequestOnlyRowIndex { + const empty: RequestOnlyRowIndex = { ticks: new Map(), continued: new Set(), started: new Set() }; const cm = this.contextManager as Partial; if (typeof cm.getAllMessages !== 'function') return empty; try { const stored = cm.getAllMessages() as unknown as readonly StoredRowLike[]; - if (this.silentTickIndex?.source === stored) return this.silentTickIndex.index; - const index = indexSilentTickRows(stored, this.name); - this.silentTickIndex = { source: stored, index }; + if (this.requestOnlyIndex?.source === stored) return this.requestOnlyIndex.index; + const index = indexRequestOnlyRows(stored, this.name); + this.requestOnlyIndex = { source: stored, index }; return index; } catch (error) { - console.error(`[silent-heartbeat] ${this.name}: could not index tick rows; separators omitted:`, error); + console.error(`[request-only-rows] ${this.name}: could not index stored rows; request-only turns omitted:`, error); return empty; } } + /** The current stream's request ended on a [Continue] turn, and the reply + * it prompted hasn't stored a row yet: its first row is stamped + * `promptedBy: 'continue'`, so later requests render the turn before it. */ + private continuePromptPending = false; + /** * Start a yielding stream with context injections. * Same as startStream but passes injections through to compile. @@ -961,8 +983,11 @@ export class Agent { this.lastStreamRealInputTokens = 0; this.lastStreamOutputTokens = 0; - const request = await this.buildActivationRequest(availableTools, injections, budget, compressionTools, options); + const { request, endsOnContinue } = await this.assembleActivationRequest( + availableTools, injections, budget, compressionTools, options, + ); request.messages = this.toolResultGuard.prepareRequest(request.messages, true); + this.continuePromptPending = endsOnContinue; const receiptAware = (this.contextManager as unknown as { getStrategy?: () => unknown }) .getStrategy?.() as { @@ -1088,6 +1113,10 @@ export class Agent { ); return; } + if (this.continuePromptPending) { + this.continuePromptPending = false; + metadata = { ...metadata, promptedBy: PROMPTED_BY_CONTINUE } as MessageMetadata; + } this.contextManager.addMessage(this.name, content, metadata); } diff --git a/src/framework.ts b/src/framework.ts index 349bdd693..c8b211542 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -6906,7 +6906,6 @@ export class AgentFramework { source: 'framework', timestamp: Date.now(), suppressProse: this.activeTurnTriggers.get(agent.name)?.suppressProse, - ephemeralSystemPrompt: this.activeTurnTriggers.get(agent.name)?.ephemeralSystemPrompt, silentHeartbeat: this.activeTurnTriggers.get(agent.name)?.silentHeartbeat, }); } else if (currentState.stream) { @@ -8159,9 +8158,11 @@ export class AgentFramework { addressed: isAddressedMessage(event.tags, event.origin), ...(silentHeartbeat ? { suppressProse: true, - ephemeralSystemPrompt: - '[silent heartbeat] Scheduled private self-check. Review pending matters privately. ' + - 'Do not narrate or publish plain prose; use an explicit send tool only if you deliberately choose to contact someone.', + // The tick's instruction rides its request-only separator + // (SILENT_HEARTBEAT_SEPARATOR), never the system prompt: a + // per-turn system change would cost the heartbeat's request + // its prompt cache from the system on, and fail the binding of + // every retained thinking block (context-manager #155). silentHeartbeat: { eventId: event.eventId, serverId: event.serverId }, } : {}), ...(event.coalescingSubject ? { coalescingSubject: event.coalescingSubject } : {}), @@ -8498,7 +8499,6 @@ export class AgentFramework { ? [...new Set(requests.flatMap((r) => r.coalescingBatchSubjects ?? [r.coalescingSubject!]))] : undefined, suppressProse: silentOnly ? trigger?.suppressProse : undefined, - ephemeralSystemPrompt: silentOnly ? trigger?.ephemeralSystemPrompt : undefined, silentHeartbeat: silentOnly ? trigger?.silentHeartbeat : undefined, channelId: channelReq?.channelId, addressed: addressedReq !== undefined, @@ -9336,15 +9336,6 @@ export class AgentFramework { } } - if (trigger?.ephemeralSystemPrompt) { - const silentInjection: ContextInjection = { - namespace: 'framework:silent-heartbeat', - position: 'system', - content: [{ type: 'text', text: trigger.ephemeralSystemPrompt }], - }; - injections = injections ? [...injections, silentInjection] : [silentInjection]; - } - // An ephemeral watchdog may dispose this Agent while hooks/compile await. // Do not create a provider stream after its generation lost ownership. if (this.agents.get(agent.name) !== agent) { diff --git a/src/silent-heartbeat.ts b/src/silent-heartbeat.ts index 9d4601375..f9e4bd0d3 100644 --- a/src/silent-heartbeat.ts +++ b/src/silent-heartbeat.ts @@ -1,40 +1,62 @@ /** - * Stored-row handling for silent heartbeat ticks. + * Request-only rows: user turns a request carries before a stored response + * that the store never holds, rendered again by every later compile. * - * A silent tick stores no prompt row (#216): its reply is written straight - * after whatever the window held before it — often the previous assistant - * message, since a suppressed tick leaves no delivery receipt behind. The - * wire formatter merges consecutive same-role messages, so without a turn - * between them the tick would read as an unprompted continuation of the - * earlier response and one assistant message would carry signed thinking - * from two separate responses (a provider 400 when it is the latest - * assistant message; see `Agent.addAssistantResponse`). + * Two kinds exist, and both are anchored to the response they prompted: * - * Every row a tick writes is therefore stamped with `metadata.silentHeartbeat` - * and, at request-build time only, a terse user turn is rendered immediately - * before the first of the tick's rows that survives compilation. The - * separator is never stored and its position is a function of the stamped - * rows alone, so every compile renders it in the same place (the tick's own - * request ends on the same turn), which keeps the prompt-cache prefix - * stable — unlike a per-compile injection that re-anchors to the newest user - * message. + * - A silent heartbeat tick stores no prompt row (#216): its reply is written + * straight after whatever the window held before it — often the previous + * assistant message, since a suppressed tick leaves no delivery receipt + * behind. Every row a tick writes is stamped with `metadata.silentHeartbeat`, + * and a separator carrying the tick's instruction is rendered before the + * first of the tick's rows that survives compilation. + * - A compile that ends on the agent's own message gets a trailing + * `[Continue]` turn (some models reject a trailing assistant message). The + * reply it prompts is stored straight after that message, so the reply's + * first stored row is stamped `metadata.promptedBy: 'continue'`, and the + * `[Continue]` turn is rendered before it. + * + * Without the row, the wire formatter merges consecutive same-role messages: + * the reply would read as an unprompted continuation of the earlier response, + * and one assistant message would carry signed thinking from two separate + * responses (a provider 400 when it is the latest assistant message; see + * `Agent.addAssistantResponse`). And the request that minted the reply held + * the row: a provider that binds signed thinking to the prefix it was minted + * under (context-manager #155) refuses that thinking once a later request + * drops it. Each row's text is constant and its position is a function of the + * stamped rows alone, so every compile renders it in the same place (the + * request that minted the reply ends on that same turn), which keeps the + * prompt-cache prefix and the binding stable — unlike a per-compile injection + * that re-anchors to the newest user message, or a system prompt that changes + * for one turn. * * Known limit: compiled messages carry no store provenance, so a stored row * is recognised only by structural identity the provider made unique - * (tool_use ids, thinking signatures). A tick reply with neither — unsigned - * and tool-free — gets no separator and still merges as before. Without - * signed thinking that merge carries no provider-rejection risk; it is only - * a misreading. Reply text is deliberately never used as identity: identical - * replies ("all quiet") would collide, and an ordinary reply with the same - * text could take a tick's separator and rewrite cached history. Exact - * matching for every row would need the compile result to expose each - * message's source rows. + * (tool_use ids, thinking signatures). A reply with neither — unsigned and + * tool-free — gets no row and still merges as before. Without signed thinking + * that merge carries no provider-rejection risk; it is only a misreading. + * Reply text is deliberately never used as identity: identical replies ("all + * quiet") would collide, and an ordinary reply with the same text could take + * another reply's row and rewrite cached history. Exact matching for every + * row would need the compile result to expose each message's source rows. */ import type { ContentBlock, NormalizedMessage } from '@animalabs/membrane'; -/** Request-only separator text. Constant on purpose: it carries no time or - * id, so it is byte-identical on every compile. */ -export const SILENT_HEARTBEAT_SEPARATOR = '[heartbeat tick]'; +/** Request-only separator before a silent tick's first surviving row. It + * carries the tick's instruction, so the tick's own request reads it as its + * prompt and every later compile shows it at each past tick. Constant on + * purpose: it carries no time or id, so it is byte-identical on every + * compile. */ +export const SILENT_HEARTBEAT_SEPARATOR = + '[silent heartbeat] Scheduled private self-check. Review pending matters privately. ' + + 'Do not narrate or publish plain prose; use an explicit send tool only if you deliberately choose to contact someone.'; + +/** Request-only turn a compile that ends on the agent's own message gets. */ +export const CONTINUE_ROW = '[Continue]'; + +/** `metadata.promptedBy` on the first row a reply to a `[Continue]` turn + * stores. */ +export const PROMPTED_BY_CONTINUE = 'continue'; /** Identity of one authenticated silent heartbeat tick. */ export interface SilentHeartbeatTick { @@ -63,7 +85,20 @@ export function silentHeartbeatOf(metadata: unknown): SilentHeartbeatStamp | und } export function silentHeartbeatSeparatorTurn(): NormalizedMessage { - return { participant: 'user', content: [{ type: 'text', text: SILENT_HEARTBEAT_SEPARATOR }] }; + return requestOnlyTurn(SILENT_HEARTBEAT_SEPARATOR); +} + +export function continueTurn(): NormalizedMessage { + return requestOnlyTurn(CONTINUE_ROW); +} + +function requestOnlyTurn(text: string): NormalizedMessage { + return { participant: 'user', content: [{ type: 'text', text }] }; +} + +/** Whether a stored row is a reply's first row after a `[Continue]` turn. */ +export function isPromptedByContinue(metadata: unknown): boolean { + return (metadata as { promptedBy?: unknown } | undefined)?.promptedBy === PROMPTED_BY_CONTINUE; } /** @@ -92,48 +127,63 @@ export interface StoredRowLike { metadata?: unknown; } -export interface SilentTickIndex { - /** Identity key of every assistant row this agent stored during a tick → tick event id. */ - rows: Map; +export interface RequestOnlyRowIndex { + /** Identity key of every assistant row this agent stored during a silent + * tick → the tick's event id. */ + ticks: Map; + /** Identity keys of the first row of each reply to a `[Continue]` turn. */ + continued: Set; /** Ticks this agent has already stored rows for (its own, not another resident's). */ started: Set; } -/** Index the rows this agent stored during silent ticks. */ -export function indexSilentTickRows(stored: readonly StoredRowLike[], agentName: string): SilentTickIndex { - const rows = new Map(); +/** Index the rows that follow request-only turns: every row this agent stored + * during its silent ticks, and the first row of each reply to a `[Continue]`. + * The two are independent, so a row can be both. */ +export function indexRequestOnlyRows(stored: readonly StoredRowLike[], agentName: string): RequestOnlyRowIndex { + const ticks = new Map(); + const continued = new Set(); const started = new Set(); for (const row of stored) { const stamp = silentHeartbeatOf(row.metadata); - if (!stamp || stamp.agentName !== agentName) continue; - started.add(stamp.eventId); - if (row.participant !== agentName) continue; - for (const key of identityKeys(row.content)) rows.set(key, stamp.eventId); + if (stamp && stamp.agentName === agentName) { + started.add(stamp.eventId); + if (row.participant === agentName) { + for (const key of identityKeys(row.content)) ticks.set(key, stamp.eventId); + } + } + if (isPromptedByContinue(row.metadata)) { + for (const key of identityKeys(row.content)) continued.add(key); + } } - return { rows, started }; + return { ticks, continued, started }; } /** - * Insert the separator turn immediately before the first surviving row of - * each tick in a compiled window — the tick's first row when it is rendered, - * otherwise whichever later row a strategy kept. Rows folded into summaries - * simply do not match; ordinary messages pass through untouched. + * Insert the request-only turns into a compiled window: a tick's separator + * immediately before the first of the tick's rows that survives compilation + * (its first row when rendered, otherwise whichever later row a strategy + * kept), and `[Continue]` immediately before each reply it prompted, after + * any separator, as the reply's request had them. Rows folded into + * summaries simply do not match; ordinary messages pass through untouched. */ -export function separateSilentHeartbeatTicks( +export function renderRequestOnlyRows( messages: NormalizedMessage[], - rows: ReadonlyMap, + index: Pick, agentName: string, ): NormalizedMessage[] { - if (rows.size === 0) return messages; + if (index.ticks.size === 0 && index.continued.size === 0) return messages; const separated = new Set(); const out: NormalizedMessage[] = []; for (const message of messages) { if (message.participant === agentName) { - const eventId = identityKeys(message.content).map((key) => rows.get(key)).find((id) => id !== undefined); - if (eventId !== undefined && !separated.has(eventId)) { - separated.add(eventId); + const keys = identityKeys(message.content); + const tick = keys.map((key) => index.ticks.get(key)).find((id) => id !== undefined); + if (tick !== undefined && !separated.has(tick)) { + separated.add(tick); out.push(silentHeartbeatSeparatorTurn()); } + if (keys.some((key) => index.continued.has(key))) out.push(continueTurn()); } out.push(message); } diff --git a/src/types/agent.ts b/src/types/agent.ts index bbf4971ca..3fe5509e2 100644 --- a/src/types/agent.ts +++ b/src/types/agent.ts @@ -360,11 +360,9 @@ export interface InferenceRequest { /** Suppress every automatic plain-prose delivery for this logical turn. * Explicit tool calls remain available. Used by authenticated silent wakes. */ suppressProse?: boolean; - /** Ephemeral system-position prompt for this turn only. Never written to - * Chronicle; callers must supply bounded non-secret control text. */ - ephemeralSystemPrompt?: string; /** The authenticated silent heartbeat tick this turn answers. Every row the * turn stores is stamped `metadata.silentHeartbeat` with it, and request - * builds render a request-only separator before the tick's first row. */ + * builds render a request-only separator carrying the tick's instruction + * before the tick's first row (`SILENT_HEARTBEAT_SEPARATOR`). */ silentHeartbeat?: SilentHeartbeatTick; } diff --git a/test/silent-heartbeat-rows.test.ts b/test/silent-heartbeat-rows.test.ts index 618d45257..c9a8fff26 100644 --- a/test/silent-heartbeat-rows.test.ts +++ b/test/silent-heartbeat-rows.test.ts @@ -15,7 +15,8 @@ import assert from 'node:assert/strict'; import { mkdtempSync, rmSync } from 'node:fs'; import { join } from 'node:path'; import { tmpdir } from 'node:os'; -import { AgentFramework } from '../src/index.js'; +import { AgentFramework, SILENT_HEARTBEAT_SEPARATOR } from '../src/index.js'; +import { indexRequestOnlyRows, renderRequestOnlyRows } from '../src/silent-heartbeat.js'; import type { AgentConfig, Module, ModuleContext, ToolCall, ToolDefinition, ToolResult } from '../src/index.js'; import { TOOL_RESULT_GUARD_NOTICE } from '../src/tool-result-guard.js'; import { PassthroughStrategy } from '@animalabs/context-manager'; @@ -189,10 +190,10 @@ describe('silent heartbeat stored rows', () => { } const later = x.membrane.calls[2]!.messages; const tickAt = later.findIndex((m) => textOf(m) === 'tick note'); - assert.equal(textOf(later[tickAt - 1]!), '[heartbeat tick]'); + assert.equal(textOf(later[tickAt - 1]!), SILENT_HEARTBEAT_SEPARATOR); assert.equal(textOf(later[tickAt - 2]!), 'ordinary reply'); // The tick's own request saw the separator in place of '[Continue]'. - assert.equal(textOf(x.membrane.calls[1]!.messages.at(-1)!), '[heartbeat tick]'); + assert.equal(textOf(x.membrane.calls[1]!.messages.at(-1)!), SILENT_HEARTBEAT_SEPARATOR); assert.deepEqual(later.slice(0, x.membrane.calls[1]!.messages.length), x.membrane.calls[1]!.messages); } finally { await x.framework.stop(); rmSync(x.dir, { recursive: true, force: true }); } }); @@ -219,7 +220,7 @@ describe('silent heartbeat stored rows', () => { for (const row of all.slice(0, before)) { assert.equal(row.metadata?.silentHeartbeat, undefined, 'ordinary rows are not stamped'); } - assert.ok(!JSON.stringify(all).includes('[heartbeat tick]'), 'the separator is never stored'); + assert.ok(!JSON.stringify(all).includes(SILENT_HEARTBEAT_SEPARATOR), 'the separator is never stored'); } finally { await x.framework.stop(); rmSync(x.dir, { recursive: true, force: true }); } }); @@ -234,16 +235,16 @@ describe('silent heartbeat stored rows', () => { const tickRequest = x.membrane.calls[1]!.messages; const later = x.membrane.calls[2]!.messages; // The tick's own request ends on the separator turn, not '[Continue]'. - assert.equal(textOf(tickRequest.at(-1)!), '[heartbeat tick]'); + assert.equal(textOf(tickRequest.at(-1)!), SILENT_HEARTBEAT_SEPARATOR); assert.equal(tickRequest.at(-1)!.participant, 'user'); assert.ok(!tickRequest.some((m) => textOf(m) === '[Continue]')); // Every later compile renders the same turn in the same place, so the // tick request is a byte-exact prefix of the next request. assert.deepEqual(later.slice(0, tickRequest.length), tickRequest); - const separatorAt = later.findIndex((m) => textOf(m) === '[heartbeat tick]'); + const separatorAt = later.findIndex((m) => textOf(m) === SILENT_HEARTBEAT_SEPARATOR); assert.equal(separatorAt, tickRequest.length - 1); assert.equal(textOf(later[separatorAt + 1]!), 'checking'); - assert.equal(later.filter((m) => textOf(m) === '[heartbeat tick]').length, 1, + assert.equal(later.filter((m) => textOf(m) === SILENT_HEARTBEAT_SEPARATOR).length, 1, 'one separator per tick, before its first row only'); const provider = wire(x.membrane.calls[2]!); @@ -279,7 +280,7 @@ describe('silent heartbeat stored rows', () => { const opening = membrane.calls[1]!.messages; const restarted = membrane.calls[2]!.messages; assert.deepEqual(restarted.slice(0, opening.length), opening, 'restart compiles the same prefix'); - assert.equal(restarted.filter((m) => textOf(m) === '[heartbeat tick]').length, 1); + assert.equal(restarted.filter((m) => textOf(m) === SILENT_HEARTBEAT_SEPARATOR).length, 1); assert.ok(!restarted.some((m) => textOf(m) === '[Continue]'), 'the restart window ends on the tool_result'); assert.match(JSON.stringify(membrane.calls[2]), /Scheduled private self-check/, 'still a silent turn'); } finally { await x.framework.stop(); rmSync(x.dir, { recursive: true, force: true }); } @@ -299,8 +300,8 @@ describe('silent heartbeat stored rows', () => { const later = x.membrane.calls[2]!.messages; assert.ok(!later.some((m) => textOf(m) === 'checking'), 'the strategy dropped the tool round'); const finalAt = later.findIndex((m) => textOf(m) === 'all quiet'); - assert.equal(textOf(later[finalAt - 1]!), '[heartbeat tick]'); - assert.equal(later.filter((m) => textOf(m) === '[heartbeat tick]').length, 1); + assert.equal(textOf(later[finalAt - 1]!), SILENT_HEARTBEAT_SEPARATOR); + assert.equal(later.filter((m) => textOf(m) === SILENT_HEARTBEAT_SEPARATOR).length, 1); assert.deepEqual(later.slice(0, x.membrane.calls[1]!.messages.length), x.membrane.calls[1]!.messages); } finally { await x.framework.stop(); rmSync(x.dir, { recursive: true, force: true }); } }); @@ -315,7 +316,7 @@ describe('silent heartbeat stored rows', () => { await wake(x, silentEvent(), signed('sig-b', 'all quiet')); await wake(x, ordinaryEvent('later'), signed('sig-later', 'later reply')); const later = x.membrane.calls[4]!.messages; - const separators = later.flatMap((m, i) => textOf(m) === '[heartbeat tick]' ? [i] : []); + const separators = later.flatMap((m, i) => textOf(m) === SILENT_HEARTBEAT_SEPARATOR ? [i] : []); // Only the two signed ticks are separated, each directly before its // own reply. The unsigned, tool-free tick has no structural identity // and merges as it did before (no signed thinking, so no 400 risk); @@ -340,7 +341,7 @@ describe('silent heartbeat stored rows', () => { await wake(x, ordinaryEvent('later'), signed('sig-later', 'later reply')); const later = x.membrane.calls[2]!.messages; const tickAt = later.findIndex((m) => textOf(m) === 'quiet'); - assert.equal(textOf(later[tickAt - 1]!), '[heartbeat tick]'); + assert.equal(textOf(later[tickAt - 1]!), SILENT_HEARTBEAT_SEPARATOR); for (const message of wire(x.membrane.calls[2]!)) assert.ok(signatures(message).length <= 1); assert.deepEqual(later.slice(0, x.membrane.calls[1]!.messages.length), x.membrane.calls[1]!.messages); } finally { await x.framework.stop(); rmSync(x.dir, { recursive: true, force: true }); } @@ -362,11 +363,11 @@ describe('silent heartbeat stored rows', () => { const betaCalls = x.membrane.calls.filter((c) => c.assistantParticipant === 'beta'); assert.equal(betaCalls.length, 3); const [, betaTick, betaLater] = betaCalls; - assert.equal(textOf(betaTick!.messages.at(-1)!), '[heartbeat tick]', "beta opens its own tick"); + assert.equal(textOf(betaTick!.messages.at(-1)!), SILENT_HEARTBEAT_SEPARATOR, "beta opens its own tick"); assert.deepEqual(betaLater!.messages.slice(0, betaTick!.messages.length), betaTick!.messages, "beta's tick request stays a byte-exact prefix of its next request"); const betaTickAt = betaLater!.messages.findIndex((m) => textOf(m) === 'beta tick'); - assert.equal(textOf(betaLater!.messages[betaTickAt - 1]!), '[heartbeat tick]'); + assert.equal(textOf(betaLater!.messages[betaTickAt - 1]!), SILENT_HEARTBEAT_SEPARATOR); const owners = stored(x.framework, 'beta') .filter((m) => m.metadata?.silentHeartbeat?.eventId === tick.eventId) .map((m) => [m.participant, m.metadata!.silentHeartbeat.agentName]); @@ -417,7 +418,7 @@ describe('silent heartbeat stored rows', () => { const opening = membrane.calls[1]!.messages; const retry = membrane.calls[2]!.messages; assert.deepEqual(retry.slice(0, opening.length), opening, 'the retry compiles the same prefix'); - assert.equal(retry.filter((m) => textOf(m) === '[heartbeat tick]').length, 1); + assert.equal(retry.filter((m) => textOf(m) === SILENT_HEARTBEAT_SEPARATOR).length, 1); assert.match(JSON.stringify(membrane.calls[2]), /Scheduled private self-check/, 'still a silent turn'); } finally { await x.framework.stop(); rmSync(x.dir, { recursive: true, force: true }); } }); @@ -437,10 +438,109 @@ describe('silent heartbeat stored rows', () => { await x.framework.runUntilIdle(); assert.equal(x.membrane.calls.length, 3); for (const call of x.membrane.calls) { - assert.ok(!call.messages.some((m) => textOf(m) === '[heartbeat tick]')); + assert.ok(!call.messages.some((m) => textOf(m) === SILENT_HEARTBEAT_SEPARATOR)); } assert.equal(textOf(x.membrane.calls[2]!.messages.at(-1)!), '[Continue]'); for (const row of stored(x.framework)) assert.equal(row.metadata?.silentHeartbeat, undefined); } finally { await x.framework.stop(); rmSync(x.dir, { recursive: true, force: true }); } }); + + it('sends the agent\'s own system prompt on a tick: a heartbeat changes nothing before the messages', async () => { + const x = await make(); + try { + await wake(x, ordinaryEvent('hello'), signed('sig-ordinary', 'ordinary reply')); + await wake(x, silentEvent(), signed('sig-tick', 'tick note')); + await wake(x, ordinaryEvent('later'), signed('sig-later', 'later reply')); + assert.deepEqual(x.membrane.calls.map((call) => call.system), ['test', 'test', 'test'], + 'the instruction rides the separator, not a one-turn system prompt'); + assert.match(textOf(x.membrane.calls[1]!.messages.at(-1)!), /Scheduled private self-check/); + } finally { await x.framework.stop(); rmSync(x.dir, { recursive: true, force: true }); } + }); +}); + +/** A wake with no stored row (a module or timer wake), answered with `content`. */ +async function timerWake(x: { membrane: MockMembrane; framework: AgentFramework }, ...responses: ContentBlock[][]) { + for (const content of responses) { + x.membrane.pushResponse(createMockResponse(content, content.some((b) => b.type === 'tool_use') ? 'tool_use' : 'end_turn')); + } + (x.framework as any).pendingRequests.push({ + agentName: 'assistant', reason: 'timer', source: 'framework', timestamp: Date.now(), + }); + await x.framework.runUntilIdle(); +} + +describe('replies to a [Continue] turn', () => { + it('renders [Continue] again before the reply it prompted, byte-stable from that request on', async () => { + // Undelivered prose leaves no receipt, so the window ends on the reply. + const x = await make({ deliver: false }); + try { + await wake(x, ordinaryEvent('hello'), signed('sig-1', 'first reply')); + await timerWake(x, signed('sig-c', 'continued reply')); + await wake(x, ordinaryEvent('later'), signed('sig-later', 'later reply')); + + const continued = x.membrane.calls[1]!.messages; + const later = x.membrane.calls[2]!.messages; + assert.equal(textOf(continued.at(-1)!), '[Continue]'); + // The request that minted the reply is a byte-exact prefix of the next. + assert.deepEqual(later.slice(0, continued.length), continued); + const replyAt = later.findIndex((m) => textOf(m) === 'continued reply'); + assert.equal(textOf(later[replyAt - 1]!), '[Continue]'); + + const provider = wire(x.membrane.calls[2]!); + for (const message of provider) { + assert.ok(signatures(message).length <= 1, + `one ${message.role} message carries signed thinking from: ${signatures(message).join(', ')}`); + } + + const rows = stored(x.framework); + const reply = rows.find((m) => m.content.some((b) => (b as { text?: string }).text === 'continued reply'))!; + assert.equal(reply.metadata?.promptedBy, 'continue'); + for (const row of rows.filter((m) => m !== reply)) assert.equal(row.metadata?.promptedBy, undefined); + assert.ok(!rows.some((m) => m.content.some((b) => (b as { text?: string }).text === '[Continue]')), + 'the [Continue] turn is never stored'); + } finally { await x.framework.stop(); rmSync(x.dir, { recursive: true, force: true }); } + }); + + it('a row that is both a tick row and a reply to [Continue] gets both turns, separator first', () => { + const tick = { eventId: 'hb-x', serverId: 'heartbeat', agentName: 'assistant' }; + const stored = [ + { participant: 'assistant', content: signed('sig-a', 'earlier reply') }, + { participant: 'assistant', content: signed('sig-1', 'tick round'), metadata: { silentHeartbeat: tick } }, + { participant: 'assistant', content: signed('sig-2', 'tick continued'), + metadata: { silentHeartbeat: tick, promptedBy: 'continue' } }, + ]; + const index = indexRequestOnlyRows(stored, 'assistant'); + const rendered = renderRequestOnlyRows(stored.map(({ participant, content }) => ({ participant, content })), index, 'assistant'); + assert.deepEqual(rendered.map(textOf), [ + 'earlier reply', SILENT_HEARTBEAT_SEPARATOR, 'tick round', '[Continue]', 'tick continued', + ]); + // With the tick's first row folded away, the stamped row is the tick's + // first surviving row: both turns precede it, the separator first. + const folded = renderRequestOnlyRows( + [stored[0]!, stored[2]!].map(({ participant, content }) => ({ participant, content })), index, 'assistant'); + assert.deepEqual(folded.map(textOf), ['earlier reply', SILENT_HEARTBEAT_SEPARATOR, '[Continue]', 'tick continued']); + }); + + it('marks only the first row the reply stores, and renders one [Continue] before it', async () => { + const x = await make({ deliver: false }); + try { + await wake(x, ordinaryEvent('hello'), signed('sig-1', 'first reply')); + await timerWake(x, + signed('sig-c', 'checking', [{ type: 'tool_use', id: 'c-t1', name: 'tools--ping', input: {} } as ContentBlock]), + signed('sig-c2', 'all done')); + assert.equal(x.mod.calls.length, 1); + await wake(x, ordinaryEvent('later'), signed('sig-later', 'later reply')); + + const rows = stored(x.framework); + const marked = rows.filter((m) => m.metadata?.promptedBy === 'continue'); + assert.equal(marked.length, 1); + assert.ok(marked[0]!.content.some((b) => b.type === 'tool_use'), 'the first row: the tool round'); + + const later = x.membrane.calls[2]!.messages; + assert.equal(later.filter((m) => textOf(m) === '[Continue]').length, 1); + const roundAt = later.findIndex((m) => textOf(m) === 'checking'); + assert.equal(textOf(later[roundAt - 1]!), '[Continue]'); + assert.deepEqual(later.slice(0, x.membrane.calls[1]!.messages.length), x.membrane.calls[1]!.messages); + } finally { await x.framework.stop(); rmSync(x.dir, { recursive: true, force: true }); } + }); }); diff --git a/test/silent-heartbeat.test.ts b/test/silent-heartbeat.test.ts index 328f60de8..eae5301ef 100644 --- a/test/silent-heartbeat.test.ts +++ b/test/silent-heartbeat.test.ts @@ -55,7 +55,7 @@ async function make() { } describe('silent heartbeat', () => { - it('wakes once with ephemeral control context, stores no push message, and routes no prose', async () => { + it('wakes once with its instruction in a request-only turn, stores no push message, and routes no prose', async () => { const x = await make(); try { x.membrane.pushResponse(createMockResponse([{ type: 'text', text: 'private self-check prose' }] as ContentBlock[])); @@ -72,6 +72,10 @@ describe('silent heartbeat', () => { assert.match(wire, /silent heartbeat/); assert.match(wire, /Scheduled private self-check/); assert.doesNotMatch(wire, /private self-check prose/); + // The instruction is the request's last turn; the system prompt is the + // agent's own, as on every other turn. + assert.equal(x.membrane.calls[0]!.system, 'test'); + assert.match(JSON.stringify(x.membrane.calls[0]!.messages.at(-1)), /Scheduled private self-check/); } finally { await x.framework.stop(); rmSync(x.dir, { recursive: true, force: true }); } }); From 3d83ad0cb229c6efbaac37510d89347d16898e77 Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Sat, 10 Oct 2026 17:52:54 -0400 Subject: [PATCH 0102/1112] fix(deferral): hold window-only messages to the turn's end; injected messages are read Context-manager #155 (thinking binding): a provider that binds signed thinking to the prefix it was minted under refuses a reply whose stored prefix holds a message its request didn't. Context-manager #160 stamps each reply from the store as it stands when the reply is stored, and the host's side of that contract (acceptRound's doc) is that what's stored after a compile, up to a reply, is what the round's request carried. Three agent-framework paths broke it. 1. Window-only deferred messages. At a tool boundary every message deferred for the agent was stored, and only the injectable ones were injected: one carrying tool blocks (the membrane's tool cycle) or named as the agent (a prefill) was stored between the turn's rounds without being injected. Sources: the built-in Discord module's backfill stores the bot's own messages under the agent's name; any module's addMessage as the agent or with tool blocks; the puppet pair's defensive queueing. Oscar-1005 showed the direction (room-203 #67671): the reply stored after such a message is stamped with it, later compiles render it, the stamp matches, and the thinking goes out bound to a request without it. Fix: drainDeferredFor takes an optional stopAt; the tool boundary drains the agent's queue only up to its first message that isInjectableMidTurn rejects. That message and everything queued behind it stay queued for the turn-end flush (driveStream's finally), where no reply follows them in the stream. A cut rather than a skip: the queue's order is kept everywhere else, the Discord backfill depends on chronology, and a pair must stay adjacent. Cost: messages queued behind a held one wait for the turn's end. A budget restart doesn't drain at its turn start (continuingTurn), so they wait through it too. 2. Re-deferral split a pair. addMessage's deferral skipped any write carrying tool_result. A turn-end flush can run after a successor turn has taken the agent (the 2026-07-31 interleave) and re-adds drained entries through addMessage, so a queued pair's tool_use re-deferred while its tool_result was stored alone. (1) sends every mid-turn pair through that flush. Now only a fresh tool_result write keeps the bypass; one coming back from the queue (deferredWriteId) re-defers like anything else. resume() clears quiesced before its flush and boot recovery runs with no turn alive, so neither changes. This also covers a flush while another agent's tool cycle is pending (the default path's global midToolCycle). The post-surgery flush (the surgery hold's release) was the one re-add from the queue that passed no deferredWriteId, so its re-adds counted as fresh, and boot recovery couldn't dedupe its writes after a crash; it now passes the id like the others (Petra-1012 found it reading the PR text). 3. RFC-006 watermark. The consumed watermark moved only at a compile (and boot, branch switch, a new fork). A message injected at a tool boundary is stored above the turn compile's watermark, so isUnreadStoredMessage called it unread and a coalesced replacement for its subject could remove it (push-coalescer acceptPlain/acceptDeferred/retract -> removePlacement -> cm.removeMessage), mid-turn or before the next compile: a message the agent heard and answered, and, under #160, either a stripped reply or a refused one depending on whether the removal came after or before the reply was stored (Oscar-1005, room-475 #67755). Fix: after provideToolResults hands injected messages to the stream, agent.markContextConsumed(). Store head, not the last injected sequence: conservative for other slots' messages stored since the compile (they read as read, so a replacement appends). Only when something was injected; endTurn, restart and non-streaming boundaries inject nothing and leave the watermark alone. Tests: test/deferred-window-only.test.ts, 7. Red at 02adf27: 6 fail; the control (a message stored at an endTurn boundary stays unread) passes before and after. operator-surgery, +1: a write deferred behind a surgery lands with its durable id (fails with the post-surgery flush passing none). --- .../mid-turn-window-only-held.fixed.md | 18 ++ src/agent.ts | 6 +- src/framework.ts | 87 ++++-- test/deferred-window-only.test.ts | 251 ++++++++++++++++++ test/operator-surgery.test.ts | 29 ++ 5 files changed, 360 insertions(+), 31 deletions(-) create mode 100644 changelog.d/mid-turn-window-only-held.fixed.md create mode 100644 test/deferred-window-only.test.ts diff --git a/changelog.d/mid-turn-window-only-held.fixed.md b/changelog.d/mid-turn-window-only-held.fixed.md new file mode 100644 index 000000000..50cbaf296 --- /dev/null +++ b/changelog.d/mid-turn-window-only-held.fixed.md @@ -0,0 +1,18 @@ +- A message deferred during a turn that can't join the live stream at a tool + boundary — one carrying tool blocks, or named as the agent itself — now + waits for the turn's end, after its last round, instead of being stored at + the boundary between rounds the stream never showed it. Messages queued + behind it wait with it, so the queue keeps its order; messages queued + before it are still heard at the boundary. Stored between rounds, it sat + before the turn's later replies in every later compile although their + requests never held it: a prompt-cache divergence, and on a provider that + binds signed thinking to the prefix it was minted under (context-manager + #155), a reply whose thinking is refused. +- A message injected into the live stream at a tool boundary now counts as + read for RFC-006 coalescing: the consumed watermark moves when messages + are injected, not only at a compile. Before, an injected message sat above + the watermark of the turn's compile, so a coalesced replacement for its + subject could remove a message the agent had heard and answered. +- A deferred write that goes back into the queue because a turn is alive now + does so whatever its content. A `tool_result` used to land regardless, + apart from the `tool_use` it answers, which went back into the queue. diff --git a/src/agent.ts b/src/agent.ts index 412707a33..4b3ad9f2c 100644 --- a/src/agent.ts +++ b/src/agent.ts @@ -555,7 +555,8 @@ export class Agent { /** * RFC-006 consumed watermark: the newest stored message at the last compile - * of a model request, per branch. Everything above it has never been in a + * of a model request, or at the last tool boundary that injected messages + * into a live one, per branch. Everything above it has never been in a * request and may still be replaced or withdrawn in place by its sender. */ private consumedWatermark: { branch: string; sequence: number } | null = null; @@ -564,7 +565,8 @@ export class Agent { return this.consumedWatermark; } - /** Mark everything currently stored as consumed (boot, branch switch). */ + /** Mark everything currently stored as consumed (boot, branch switch, and a + * tool boundary that injected messages into the live stream). */ markContextConsumed(): void { // Tolerates partial context-manager doubles (tests build Agents over // stubs): without a watermark nothing is ever treated as unread. diff --git a/src/framework.ts b/src/framework.ts index 349bdd693..63b84efd8 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -480,6 +480,14 @@ function withDeferredWriteId(metadata: MessageMetadata | undefined, id: string): return { ...(metadata ?? {}), deferredWriteId: id } as MessageMetadata; } +/** Whether a deferred message can join a live stream at a tool boundary: + * no tool blocks (the membrane's tool cycle) and not named as the agent + * itself (it would render as an assistant turn, a prefill). */ +function isInjectableMidTurn(msg: { participant: string; content: ContentBlock[] }, agentName: string): boolean { + return msg.participant !== agentName + && !msg.content.some((b) => b.type === 'tool_use' || b.type === 'tool_result'); +} + function isTurnContinuation(reason: string): boolean { return reason === 'context_budget_restart' || reason === 'tool_results_ready' || reason === 'tool_result_guard_retry'; @@ -5925,8 +5933,10 @@ export class AgentFramework { if (this.activeTurnTokens.has(name)) continue; for (const msg of this.drainDeferredFor(name)) { try { - this.addMessage(msg.participant, msg.content, msg.metadata, - msg.forAgent ? { forAgent: msg.forAgent } : undefined); + this.addMessage(msg.participant, msg.content, msg.metadata, { + deferredWriteId: msg.id, + ...(msg.forAgent ? { forAgent: msg.forAgent } : {}), + }); } catch (error) { console.error(`[operator] post-surgery deferred flush failed for ${name}: ${error instanceof Error ? error.message : String(error)}`); } @@ -6694,28 +6704,29 @@ export class AgentFramework { // target's live stream (deaf agent + window/live divergence). Left // queued, they flush at the target's own next boundary — with // injection — or in driveStream's finally when its turn ends. + // + // Only what can be injected is drained here. Tool blocks would + // corrupt the tool-cycle structure the membrane enforces, and a + // message named as the agent itself would render as an ASSISTANT + // turn on the wire (an unintended prefill the model would + // continue). Stored here without being injected, such a message + // would sit in the window before the turn's later replies although + // their requests never held it: the divergence above, and a reply + // whose signed thinking the provider then refuses, since it binds + // thinking to the prefix it was minted under (context-manager + // #155). So the drain stops at the first such message, which waits + // with everything queued behind it — the queue keeps its order — + // for the turn-end flush, where no reply follows it in this stream. const midTurnInjections: Array<{ participant: string; content: ContentBlock[]; metadata?: MessageMetadata }> = []; { - const deferred = this.drainDeferredFor(agent.name); - if (deferred.length > 0) { - for (const msg of deferred) { - agent.getContextManager().addMessage(msg.participant, msg.content, withDeferredWriteId(msg.metadata, msg.id)); - // Injection guards: tool blocks would corrupt the tool-cycle - // structure the membrane enforces, and a message named as the - // agent itself would render as an ASSISTANT turn on the wire - // (an unintended prefill the model would continue). Such - // messages stay window-only — visible next turn. - const hasToolBlocks = msg.content.some( - (b) => b.type === 'tool_use' || b.type === 'tool_result' - ); - if (!hasToolBlocks && msg.participant !== agent.name) { - midTurnInjections.push({ - participant: msg.participant, - content: msg.content, - ...(msg.metadata ? { metadata: msg.metadata } : {}), - }); - } - } + const deferred = this.drainDeferredFor(agent.name, (msg) => !isInjectableMidTurn(msg, agent.name)); + for (const msg of deferred) { + agent.getContextManager().addMessage(msg.participant, msg.content, withDeferredWriteId(msg.metadata, msg.id)); + midTurnInjections.push({ + participant: msg.participant, + content: msg.content, + ...(msg.metadata ? { metadata: msg.metadata } : {}), + }); } } @@ -6918,6 +6929,13 @@ export class AgentFramework { agent.toolResultGuard.submissionResults(membraneResults), midTurnInjections.length > 0 ? { injectedMessages: midTurnInjections } : undefined, ); + // RFC-006 §3.3: consumption is determined at assembly, and this + // is assembly too — the injected messages are in the live + // request. Without this they would sit above the watermark of + // the turn's compile and read as unread, so a coalesced + // replacement could remove a message the agent heard and + // answered (and that its later replies were minted after). + if (midTurnInjections.length > 0) agent.markContextConsumed(); agent.setStreaming(currentState.stream); this.emitTrace({ type: 'inference:stream_resumed', @@ -12549,7 +12567,13 @@ export class AgentFramework { // against a FROZEN context — module events and api message.send must not // append mid-surgery. Deferred here, flushed by resume(). tool_result // still lands (a draining turn's continuation depends on it). - const hasToolResult = content.some(b => b.type === 'tool_result'); + // + // A write coming back from the deferred queue (deferredWriteId) is + // re-deferred whatever its content: its tool_use went back into the + // queue just before it, in the same drain, and a tool_result stored + // without it would split the pair. (A turn-end flush can run after a + // successor turn has taken the agent: the 2026-07-31 interleave.) + const freshToolResult = opts?.deferredWriteId === undefined && content.some(b => b.type === 'tool_result'); // Explicitly-targeted deliveries scope the tool-cycle check to the // target (pendingAssistantBlocks is keyed by agent); the default path // keeps the historical GLOBAL check byte-for-byte (conservative: any @@ -12558,7 +12582,7 @@ export class AgentFramework { ? this.pendingAssistantBlocks.has(agent.name) : this.pendingAssistantBlocks.size > 0; if ( - !hasToolResult && + !freshToolResult && opts?.bypassDeferralFor !== agent.name && (this.quiesced || midToolCycle || @@ -12592,9 +12616,12 @@ export class AgentFramework { /** * Splice out the deferred messages addressed to `agentName` (explicitly * via forAgent, or implicitly when it is the primary), leaving other - * targets' messages queued for their own boundaries. + * targets' messages queued for their own boundaries. With `stopAt`, the + * drain ends at the first of them (in queue order) that it matches: that + * message and every later one stay queued, so what is drained is always a + * prefix of the agent's queue. */ - private drainDeferredFor(agentName: string): Array<{ + private drainDeferredFor(agentName: string, stopAt?: (msg: DeferredWrite) => boolean): Array<{ /** Durable identity — every write of this entry must carry it * (`withDeferredWriteId` / `opts.deferredWriteId`). */ id: string; @@ -12610,14 +12637,16 @@ export class AgentFramework { const target = msg.forAgent ?? this.primaryAgentName; (target === agentName ? mine : rest).push(msg); } - this.deferredMessages = rest; mine.sort(bySeq); + const stop = stopAt ? mine.findIndex(stopAt) : -1; + const drained = stop === -1 ? mine : mine.slice(0, stop); + this.deferredMessages = stop === -1 ? rest : [...rest, ...mine.slice(stop)]; // The drained messages are about to be written by the caller; they stay // in the DURABLE queue (as un-acked) until the caller's // ackDeferredWrites() has synced the chronicle. Removing them here would // acknowledge before the write is durable. - this.handOffDeferredWrites(mine); - return mine; + this.handOffDeferredWrites(drained); + return drained; } private editMessage(id: MessageId, content: ContentBlock[]): void { diff --git a/test/deferred-window-only.test.ts b/test/deferred-window-only.test.ts new file mode 100644 index 000000000..6bfa3302d --- /dev/null +++ b/test/deferred-window-only.test.ts @@ -0,0 +1,251 @@ +/** + * What the window stores mid-turn is what the live stream carried. + * + * A message deferred during a turn is stored at the agent's next tool + * boundary and injected into the resumed stream (hear-while-acting), so the + * next compile's prefix is the request the turn's later replies were minted + * under. Two kinds of message can't be injected: one carrying tool blocks + * (it would break the tool cycle membrane enforces) and one named as the + * agent itself (a prefill). Those wait for the turn's end, and so does + * everything queued behind them, so the queue's order holds. A provider that + * binds signed thinking to its prefix (context-manager #155) refuses a reply + * whose stored prefix holds a message its request didn't. + * + * A message the stream carried has been read: RFC-006's consumed watermark + * covers it, so a coalesced replacement can no longer remove it. + */ + +import { describe, it, beforeEach, afterEach } from 'node:test'; +import assert from 'node:assert/strict'; +import { tmpdir } from 'node:os'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { join } from 'node:path'; +import type { + Module, + ModuleContext, + ProcessState, + ProcessEvent, + EventResponse, + ToolDefinition, + ToolCall, + ToolResult, +} from '../src/index.js'; +import { AgentFramework } from '../src/index.js'; +import { MockMembrane, createMockResponse } from './helpers/mock-membrane.js'; +import type { ContentBlock } from '@animalabs/membrane'; + +interface Interjection { + participant: string; + content: ContentBlock[]; +} + +/** Tools that deliver messages while they run, the way a module or a channel + * would while the agent is busy: `move` continues the turn, `halt` ends it. */ +class InterjectingModule implements Module { + readonly name = 'robot'; + framework: AgentFramework | null = null; + /** Delivered, in order, by the next tool call before it returns. */ + interjections: Interjection[] = []; + + async start(_ctx: ModuleContext): Promise {} + async stop(): Promise {} + + getTools(): ToolDefinition[] { + return [ + { name: 'move', description: 'Move', inputSchema: { type: 'object', properties: {} } }, + { name: 'halt', description: 'End the turn', inputSchema: { type: 'object', properties: {} } }, + ]; + } + + async handleToolCall(call: ToolCall): Promise { + const pending = this.interjections; + this.interjections = []; + for (const interjection of pending) { + this.framework!.pushEvent({ type: 'interjection', source: 'test', interjection } as unknown as ProcessEvent); + } + // Let the run loop handle them while this round's tools are pending, so + // they land in the deferred queue before the tool-result boundary. + if (pending.length > 0) await new Promise((r) => setTimeout(r, 30)); + return call.name.endsWith('halt') + ? { success: true, data: { halted: true }, endTurn: true } + : { success: true, data: { ok: true } }; + } + + async onProcess(event: ProcessEvent, _state: ProcessState): Promise { + if (event.type === 'external-message') { + return { + addMessages: [{ participant: 'Antra', content: [{ type: 'text', text: 'go' }] }], + requestInference: true, + }; + } + if ((event as { type: string }).type === 'interjection') { + const { participant, content } = (event as unknown as { interjection: Interjection }).interjection; + // No wake: this suite is about where the message lands in this turn. + return { addMessages: [{ participant, content }] }; + } + return {}; + } +} + +const text = (t: string): ContentBlock[] => [{ type: 'text', text: t } as ContentBlock]; + +/** One line per stored message: participant, then its text or block types. */ +function windowOf(framework: AgentFramework): string[] { + return framework.getAgent('assistant')!.getContextManager().getAllMessages().map((m) => { + const parts = m.content.map((b) => (b.type === 'text' ? (b as { text: string }).text : `<${b.type}>`)); + return `${m.participant}: ${parts.join(' ')}`; + }); +} + +describe('mid-turn deliveries keep the window what the stream carried', () => { + let tempDir: string; + let membrane: MockMembrane; + let module: InterjectingModule; + + beforeEach(() => { + tempDir = mkdtempSync(join(tmpdir(), 'dwo-test-')); + membrane = new MockMembrane(); + module = new InterjectingModule(); + }); + + afterEach(() => { + rmSync(tempDir, { recursive: true, force: true }); + }); + + async function createFramework(): Promise { + const framework = await AgentFramework.create({ + storePath: join(tempDir, 'test.chronicle'), + membrane: membrane.asMembrane(), + agents: [{ name: 'assistant', model: 'test-model', systemPrompt: 'You are a robot pilot.' }], + modules: [module], + }); + module.framework = framework; + return framework; + } + + /** A turn of two rounds: `tool` with the interjections, then a reply. */ + async function runTurn(tool: 'move' | 'halt', interjections: Interjection[]): Promise { + membrane.pushResponse(createMockResponse([ + { type: 'text', text: 'Moving.' }, + { type: 'tool_use', id: 'c1', name: `robot--${tool}`, input: {} }, + ] as ContentBlock[], 'tool_use')); + membrane.pushResponse(createMockResponse(text('Done.'))); + const framework = await createFramework(); + module.interjections = interjections; + framework.pushEvent({ type: 'external-message', source: 'test', content: 'go' } as unknown as ProcessEvent); + await framework.runUntilIdle(); + return framework; + } + + function injected(): string[] { + const options = membrane.lastStream!.receivedToolResultOptions[0]; + return (options?.injectedMessages ?? []).map((m) => + `${m.participant}: ${(m.content as Array<{ text?: string }>).map((b) => b.text).join(' ')}`); + } + + it('a message named as the agent waits for the turn to end, after its last round', async () => { + const framework = await runTurn('move', [{ participant: 'assistant', content: text('my own words') }]); + assert.deepEqual(injected(), []); + assert.deepEqual(windowOf(framework), [ + 'Antra: go', + 'assistant: Moving. ', + 'user: ', + 'assistant: Done.', + 'assistant: my own words', + ]); + await framework.stop(); + }); + + it('a message carrying tool blocks waits too', async () => { + const framework = await runTurn('move', [{ + participant: 'Antra', + content: [{ type: 'tool_use', id: 'x1', name: 'robot--move', input: {} } as ContentBlock], + }]); + assert.deepEqual(injected(), []); + assert.deepEqual(windowOf(framework).slice(2), ['user: ', 'assistant: Done.', 'Antra: ']); + await framework.stop(); + }); + + it('what is queued behind a held message waits with it, in order', async () => { + const framework = await runTurn('move', [ + { participant: 'assistant', content: text('my own words') }, + { participant: 'Antra', content: text('look left') }, + ]); + assert.deepEqual(injected(), []); + assert.deepEqual(windowOf(framework).slice(2), [ + 'user: ', + 'assistant: Done.', + 'assistant: my own words', + 'Antra: look left', + ]); + await framework.stop(); + }); + + it('what is queued before a held message is still heard at the boundary', async () => { + const framework = await runTurn('move', [ + { participant: 'Antra', content: text('look left') }, + { participant: 'assistant', content: text('my own words') }, + ]); + assert.deepEqual(injected(), ['Antra: look left']); + assert.deepEqual(windowOf(framework).slice(2), [ + 'user: ', + 'Antra: look left', + 'assistant: Done.', + 'assistant: my own words', + ]); + await framework.stop(); + }); + + it('a message the stream carried is read: the consumed watermark covers it', async () => { + const framework = await runTurn('move', [{ participant: 'Antra', content: text('look left') }]); + assert.deepEqual(injected(), ['Antra: look left']); + const agent = framework.getAgent('assistant')!; + const heard = agent.getContextManager().getAllMessages() + .find((m) => m.content.some((b) => (b as { text?: string }).text === 'look left'))!; + const isUnread = (framework as unknown as { + isUnreadStoredMessage: (a: typeof agent, id: string) => boolean; + }).isUnreadStoredMessage.bind(framework); + assert.equal(isUnread(agent, heard.id), false, 'a coalesced replacement must not remove what the agent heard'); + await framework.stop(); + }); + + it('a message stored at a boundary that ends the turn was never carried, and stays unread', async () => { + const framework = await runTurn('halt', [{ participant: 'Antra', content: text('too late') }]); + const agent = framework.getAgent('assistant')!; + const late = agent.getContextManager().getAllMessages() + .find((m) => m.content.some((b) => (b as { text?: string }).text === 'too late'))!; + const isUnread = (framework as unknown as { + isUnreadStoredMessage: (a: typeof agent, id: string) => boolean; + }).isUnreadStoredMessage.bind(framework); + assert.equal(isUnread(agent, late.id), true); + await framework.stop(); + }); + + it('a write coming back from the deferred queue re-defers while a turn is alive, tool_result or not', async () => { + const framework = await createFramework(); + const internals = framework as unknown as { + activeTurnTokens: Map; + deferredMessages: Array<{ id: string }>; + addMessage: (p: string, c: ContentBlock[], m?: unknown, o?: { deferredWriteId?: string }) => string; + }; + internals.activeTurnTokens.set('assistant', 1); + try { + // A queued pair's tool_result must follow its tool_use back into the + // queue, or the pair splits (the turn-end flush can run after a + // successor turn has started). + const requeued = internals.addMessage('user', [ + { type: 'tool_result', toolUseId: 'p1', content: 'ok' } as ContentBlock, + ], undefined, { deferredWriteId: 'w1' }); + assert.equal(requeued, ''); + assert.ok(internals.deferredMessages.some((m) => m.id === 'w1')); + // A fresh tool_result write keeps landing as before. + const fresh = internals.addMessage('user', [ + { type: 'tool_result', toolUseId: 'p2', content: 'ok' } as ContentBlock, + ]); + assert.notEqual(fresh, ''); + } finally { + internals.activeTurnTokens.delete('assistant'); + } + await framework.stop(); + }); +}); diff --git a/test/operator-surgery.test.ts b/test/operator-surgery.test.ts index aeb162b59..2be1ce269 100644 --- a/test/operator-surgery.test.ts +++ b/test/operator-surgery.test.ts @@ -228,6 +228,35 @@ describe('live operator surgery', () => { } }); + it('a write deferred behind the surgery lands with its durable id, as every flush from the queue does', async () => { + const c = cm() as unknown as { switchBranch: (name: string) => Promise }; + const originalSwitch = c.switchBranch; + let releaseSwitch!: () => void; + const gate = new Promise((resolve) => { releaseSwitch = resolve; }); + c.switchBranch = async (name: string) => { await gate; return originalSwitch.call(c, name); }; + const internals = framework as unknown as { + surgeryHold: unknown; + deferredMessages: Array<{ id: string }>; + addMessage(participant: string, content: unknown[], metadata?: Record): string; + }; + try { + const rollback = framework.rollbackToMessage('scout', { messageId: ids[2] }); + await new Promise((resolve) => setImmediate(resolve)); + assert.ok(internals.surgeryHold, 'store hold is up while the switch is awaited'); + assert.equal(internals.addMessage('user', [{ type: 'text', text: 'arrived mid-surgery' }]), '', 'deferred'); + const queuedId = internals.deferredMessages.at(-1)!.id; + releaseSwitch(); + await rollback; + const landed = cm().getAllMessages().find((m) => (m.content[0] as { text?: string }).text === 'arrived mid-surgery'); + assert.ok(landed, 'flushed when the hold released'); + // The id makes a queued pair's tool_result re-defer with its tool_use + // rather than land alone, and lets boot recovery skip a replay. + assert.equal((landed!.metadata as { deferredWriteId?: string } | undefined)?.deferredWriteId, queuedId); + } finally { + c.switchBranch = originalSwitch; + } + }); + // --- strategy-initialization failure ----------------------------------------- // switchBranch()/fork() move the chronicle branch BEFORE awaiting strategy // initialization, so a rejection there leaves the store on the new branch. From 903beb1e9afe4aa19c6f3893bf1b41d8abc76a7f Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Sat, 10 Oct 2026 19:19:54 -0400 Subject: [PATCH 0103/1112] fix(deferral): a turn's end delivers despite another agent's tool cycle; stop() stores what is still deferred Greptile's P1 on #286 (discussion_r4239537209): the cut leaves held messages depending on the turn-end flush, and that flush was gated on EVERY agent's pending tool cycle (pendingAssistantBlocks.size === 0, from the single-agent era, efc7e7a). If agent A's turn ended while agent B waited on tools, A's flush was skipped; B's boundary and turn end drain only B's entries, so A's messages stayed queued until A's next turn, and stop() never drained the queue, so a memory-only queue lost them. - driveStream's turn-end flush is now gated on this agent's own cycle (!pendingAssistantBlocks.has(agent.name)), matching addMessage's scoped check for targeted writes. Each drained entry still goes through addMessage: a targeted write lands; a primary-bound (default-path) write keeps the historical global rule and re-defers under its id and seq while any agent's cycle is pending, for the target's next turn start or stop(). - stop() now drains every agent's remaining entries and stores them (deferredWriteId stamped), then acks, after every turn is over and before the final sync and close. A quiesced framework re-defers them into the persisted queue, as before. Tests (deferred-window-only, +2): a message held for the turn's end lands then while another agent's cycle is pending (simulated through pendingAssistantBlocks); a message still deferred at stop() is in the store when it's reopened. Both fail at 3d83ad0's source. Neighbours (deferred-*, present-while-acting, operator-surgery, host-quiesce, mcpl-coalescing, puppet-*, offline-recovery-branch): 168/168. --- .../mid-turn-window-only-held.fixed.md | 8 +++ src/framework.ts | 32 +++++++++++- test/deferred-window-only.test.ts | 52 +++++++++++++++++++ 3 files changed, 90 insertions(+), 2 deletions(-) diff --git a/changelog.d/mid-turn-window-only-held.fixed.md b/changelog.d/mid-turn-window-only-held.fixed.md index 50cbaf296..36641ad8e 100644 --- a/changelog.d/mid-turn-window-only-held.fixed.md +++ b/changelog.d/mid-turn-window-only-held.fixed.md @@ -16,3 +16,11 @@ - A deferred write that goes back into the queue because a turn is alive now does so whatever its content. A `tool_result` used to land regardless, apart from the `tool_use` it answers, which went back into the queue. +- A turn's end now delivers the agent's deferred messages even while another + agent's tool cycle is pending; only the agent's own cycle holds them. A + write still deferred then (a primary-bound write waits out any agent's + cycle) lands at the agent's next turn start, and `stop()` now stores + whatever is still deferred before the store closes. Before, a message + deferred for an agent whose turn ended during another agent's tool cycle + stayed queued until that agent's next turn, and a memory-only queue lost it + at `stop()`. diff --git a/src/framework.ts b/src/framework.ts index 63b84efd8..9f6a45760 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -2043,6 +2043,29 @@ export class AgentFramework { } } + // Every turn is over, so nothing else will deliver what is still + // deferred: a message held for a turn's end whose flush found the write + // still deferred (a primary-bound write waits out any agent's tool + // cycle), or one that arrived for an idle agent during another's cycle. + // Land each at its target now, while the store is open; a memory-only + // queue would otherwise lose it. One addMessage still defers (quiesced) + // stays in the persisted queue for boot recovery. + if (this.deferredMessages.length > 0) { + for (const name of this.agents.keys()) { + for (const msg of this.drainDeferredFor(name)) { + try { + this.addMessage(msg.participant, msg.content, msg.metadata, { + deferredWriteId: msg.id, + ...(msg.forAgent ? { forAgent: msg.forAgent } : {}), + }); + } catch (error) { + console.error(`[stop] deferred write for ${name} could not be stored: ${error instanceof Error ? error.message : String(error)}`); + } + } + } + this.ackDeferredWrites(); + } + // Final sync before closing try { this.store.sync(); @@ -10953,8 +10976,13 @@ export class AgentFramework { // Flush any deferred messages (e.g. if stream failed while tools were // pending). Only THIS agent's messages: other targets' entries wait // for their own boundaries (re-adding via addMessage re-defers if the - // target has meanwhile started a turn). - if (frameReachedTerminal && this.deferredMessages.length > 0 && this.pendingAssistantBlocks.size === 0) { + // target has meanwhile started a turn). Gated on this agent's own tool + // cycle, not every agent's: messages held for this turn's end depend on + // this flush, and another agent's pending tools say nothing about this + // window. A write addMessage still defers (a primary-bound write waits + // out any agent's cycle) goes back into the queue under its id, for the + // target's next turn start or stop(). + if (frameReachedTerminal && this.deferredMessages.length > 0 && !this.pendingAssistantBlocks.has(agent.name)) { const deferred = this.drainDeferredFor(agent.name); for (const msg of deferred) { this.addMessage(msg.participant, msg.content, msg.metadata, { diff --git a/test/deferred-window-only.test.ts b/test/deferred-window-only.test.ts index 6bfa3302d..246293027 100644 --- a/test/deferred-window-only.test.ts +++ b/test/deferred-window-only.test.ts @@ -46,6 +46,8 @@ class InterjectingModule implements Module { framework: AgentFramework | null = null; /** Delivered, in order, by the next tool call before it returns. */ interjections: Interjection[] = []; + /** Run inside the next tool call, after the interjections are delivered. */ + duringTool: (() => void) | null = null; async start(_ctx: ModuleContext): Promise {} async stop(): Promise {} @@ -66,6 +68,9 @@ class InterjectingModule implements Module { // Let the run loop handle them while this round's tools are pending, so // they land in the deferred queue before the tool-result boundary. if (pending.length > 0) await new Promise((r) => setTimeout(r, 30)); + const during = this.duringTool; + this.duringTool = null; + during?.(); return call.name.endsWith('halt') ? { success: true, data: { halted: true }, endTurn: true } : { success: true, data: { ok: true } }; @@ -221,6 +226,53 @@ describe('mid-turn deliveries keep the window what the stream carried', () => { await framework.stop(); }); + /** The framework's own queue and its record of pending tool cycles. */ + function internalsOf(framework: AgentFramework) { + return framework as unknown as { + pendingAssistantBlocks: Map; + deferredMessages: Array<{ id: string }>; + addMessage(participant: string, content: ContentBlock[], metadata?: undefined, opts?: { forAgent?: string }): string; + }; + } + + it('a message held for the turn\'s end lands then, while another agent\'s tool cycle is pending', async () => { + membrane.pushResponse(createMockResponse([ + { type: 'text', text: 'Moving.' }, + { type: 'tool_use', id: 'c1', name: 'robot--move', input: {} }, + ] as ContentBlock[], 'tool_use')); + membrane.pushResponse(createMockResponse(text('Done.'))); + const framework = await createFramework(); + const internals = internalsOf(framework); + module.duringTool = () => { + // Another agent is mid tool cycle when this turn ends, and a write + // named as this agent (held: it can't join the live stream) waits. + internals.pendingAssistantBlocks.set('builder', [{ type: 'tool_use', id: 'b1', name: 'robot--move', input: {} } as ContentBlock]); + assert.equal(internals.addMessage('assistant', text('my own words'), undefined, { forAgent: 'assistant' }), '', 'deferred'); + }; + framework.pushEvent({ type: 'external-message', source: 'test', content: 'go' } as unknown as ProcessEvent); + await framework.runUntilIdle(); + assert.ok(internals.pendingAssistantBlocks.has('builder'), 'the other cycle was still pending at the flush'); + assert.deepEqual(windowOf(framework).slice(-2), ['assistant: Done.', 'assistant: my own words']); + internals.pendingAssistantBlocks.delete('builder'); + await framework.stop(); + }); + + it('a message still deferred when the framework stops lands before the store closes', async () => { + const framework = await createFramework(); + const internals = internalsOf(framework); + // A primary-bound write waits out any agent's tool cycle, even with this + // agent idle; that cycle's stream then ends without delivering it. + internals.pendingAssistantBlocks.set('builder', [{ type: 'tool_use', id: 'b1', name: 'robot--move', input: {} } as ContentBlock]); + assert.equal(internals.addMessage('Antra', text('late word')), '', 'deferred'); + internals.pendingAssistantBlocks.delete('builder'); + assert.equal(internals.deferredMessages.length, 1); + await framework.stop(); + + const reopened = await createFramework(); + assert.ok(windowOf(reopened).includes('Antra: late word'), 'stored before the store closed'); + await reopened.stop(); + }); + it('a write coming back from the deferred queue re-defers while a turn is alive, tool_result or not', async () => { const framework = await createFramework(); const internals = framework as unknown as { From 6f08783c663eb60495e0a7f893489ecaca807d5d Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Sat, 10 Oct 2026 19:25:14 -0400 Subject: [PATCH 0104/1112] fix(requests): request-only turns render the text they were sent with, for every row their reply stores Greptile's three P1 findings on #284 (discussion_r4239514606, _611, _615), all valid: 1. Dropped rounds lost [Continue]. Only the reply's FIRST row was stamped, so when a strategy dropped that tool exchange but kept the signed final reply (the shape OmitToolExchange tests for ticks), the surviving reply sat straight after the earlier assistant message: the merge this PR set out to stop. Now the [Continue] prompt is turn-level, as a tick is. At each stream start the Agent decides the prompt the stream's reply follows (StartStreamResult.continuePrompt, { turn: uuid, text }, optional on the public type): a new one when the request ends on [Continue]; else the turn's carried prompt (ActivationRequestOptions.continuePrompt) only if this request renders it (renderRequestOnlyRows reports promptedTurns, as it does separatedTicks); else none. The framework sets the turn's trigger (InferenceRequest.continuePrompt) to that at each stream start, carries it into the budget restart's request (the guard and error retries carry it on their trigger), and silentTurnRowMetadata (the per-turn row metadata, named for its first use) stamps every row the turn stores with it as metadata.promptedBy. Request builds render the prompt's text before the first of its rows that survives. The keep-only-if-rendered rule is Basil-1007's, from their review of this commit: a trigger the provider cooldown held (holdProviderAcceleration) carried the prompt its first stream start set, so a resumed compile that ended on a message stored meanwhile rendered no [Continue], yet the reply's rows were stamped, and later compiles put [Continue] before a reply whose request never had it. The Agent no longer stamps rows (continuePromptPending is gone); a direct runInference caller never adds [Continue]. 2. A tick's restart could rewrite earlier turns. Whether a tick "opens" (its request ends on the separator) was decided by whether the tick had ever stored a row (`started`). A restart whose compile folded away all the tick's rows therefore got no separator, ended on [Continue] when the window ended on the agent's message, and later compiles put the separator before a reply whose request never held it. Now renderRequestOnlyRows reports which ticks it separated, and a tick opens wherever its separator wasn't rendered. `started` is gone. 3. Upgrades would have invalidated saved thinking. Changing SILENT_HEARTBEAT_SEPARATOR's text re-rendered every past tick with it, after compile, where context-manager can't see it; replies stored after a past tick would have their thinking refused (or 400 without drop_block). Now each tick's rows record the separator their request rendered (SilentHeartbeatStamp.separator, set by silentTurnRowMetadata), and rendering uses the recorded text; rows stamped without it render the legacy '[heartbeat tick]' byte for byte. The [Continue] stamp records its text the same way. So no stored reply's prefix changes on upgrade, and a future change to either constant changes only rows stored after it. Mechanics: indexRequestOnlyRows now indexes only the agent's own rows with identity keys: ticks (key -> { eventId, text }) and prompts (key -> { turn, text }); renderRequestOnlyRows returns { messages, separatedTicks }. silentTurnRowMetadata keeps its name (renaming it would touch seven call sites other open PRs change). RequestOnlyPrompt is exported as a type. Out of scope, noted: the non-streaming fallback's continuation request ('tool_results_ready') carries none of the turn's properties (suppressProse, silentHeartbeat, and now continuePrompt), as before. Tests (silent-heartbeat-rows): the tick stamp carries its separator and no row stores the separator; recorded texts render as recorded and legacy tick stamps render '[heartbeat tick]'; a restart whose compile left none of the tick's rows ends on the separator and the next request keeps it as a prefix; every row of a [Continue] reply's turn is marked, one [Continue] before the first; with the reply's tool exchange dropped, [Continue] precedes the surviving final reply and no wire message carries two signatures; a reply to [Continue] that restarts at its tool boundary carries the prompt into the restart's rows, so a window that later folds the first round renders [Continue] before the restart's reply; a carried prompt whose request doesn't render it is dropped (Basil-1007's probe, failing at 5188d31); the both-stamps unit test on the new shapes. At ee43daf's source 7 of them fail (the restart and dropped-round tests on behaviour, the rest on the new stamp shapes). Neighbours touching ticks, [Continue] or stream starts: 236/236, one file at a time. --- changelog.d/request-only-rows-stay.changed.md | 39 ++-- src/agent.ts | 53 +++--- src/framework.ts | 42 +++-- src/index.ts | 1 + src/silent-heartbeat.ts | 154 +++++++++------- src/types/agent.ts | 9 +- test/silent-heartbeat-rows.test.ts | 169 ++++++++++++++++-- 7 files changed, 343 insertions(+), 124 deletions(-) diff --git a/changelog.d/request-only-rows-stay.changed.md b/changelog.d/request-only-rows-stay.changed.md index 067641c96..5365f9f38 100644 --- a/changelog.d/request-only-rows-stay.changed.md +++ b/changelog.d/request-only-rows-stay.changed.md @@ -6,18 +6,29 @@ the prefix it was minted under (context-manager #155) it failed every retained thinking block in that request, and the tick reply's own thinking at the next ordinary turn. `SILENT_HEARTBEAT_SEPARATOR` is now that - instruction, so every past tick reads it too: 199 characters where it was - 16, at each tick whose rows survive in the window (roughly 50 tokens a - tick). The first compile after upgrading renders past ticks with the new - text, one prompt-cache rewrite. (`InferenceRequest.ephemeralSystemPrompt` - is removed; see the breaking entry.) + instruction (199 characters where it was 16, roughly 50 tokens at each + tick whose rows survive in the window). Each tick's rows now record the + separator their request rendered (`SilentHeartbeatStamp.separator`), and + later compiles render that text, so ticks stored before upgrading keep + their `[heartbeat tick]` marker byte for byte: no stored reply's prefix + changes. (`InferenceRequest.ephemeralSystemPrompt` is removed; see the + breaking entry.) +- A tick whose restart compiles none of the tick's rows (a strategy folded + them all) opens again: the restart's request ends on the separator, where + later compiles render it before the restart's reply. It used to end on + `[Continue]`, and later compiles put the separator before a reply whose + request never held it. - When a compile ends on the agent's own message and the request gets a - trailing `[Continue]` turn, the reply it prompts now has its first stored - row marked `metadata.promptedBy: 'continue'`, and every later request - renders `[Continue]` again just before that row, the way a silent tick's - separator is rendered before its first row. Without it, later requests - merged the reply into the previous assistant message (one message carrying - signed thinking from two responses, a 400 when it is the latest), and - dropped a turn the reply's request held, which a provider that binds - thinking refuses. A reply with neither a tool call nor signed thinking - can't be found in a compiled window and merges as before. + trailing `[Continue]` turn, every row the reply's turn stores now records + that prompt (`metadata.promptedBy`, a `{ turn, text }` pair), and every + later request renders `[Continue]` again just before the first of them + that survives, the way a silent tick's separator is rendered. A restart or + retry of the turn keeps the prompt only while its own request renders it. + Without it, later requests merged the reply into the previous assistant + message (one message carrying signed thinking from two responses, a 400 + when it is the latest), and dropped a turn the reply's request held, which + a provider that binds thinking refuses. A reply with neither a tool call + nor signed thinking can't be found in a compiled window and merges as + before. `StartStreamResult.continuePrompt` and + `InferenceRequest.continuePrompt` carry it (new, optional); the agent and + the framework set them. diff --git a/src/agent.ts b/src/agent.ts index 7a01884b9..24200ba12 100644 --- a/src/agent.ts +++ b/src/agent.ts @@ -4,15 +4,17 @@ import { createHash } from 'node:crypto'; import type { CacheWireReceipt, KvUnifiedRequestHooks } from './kv-unified-wire.js'; import { ToolResultGuard, TOOL_RESULT_GUARD_NOTICE } from './tool-result-guard.js'; import { - PROMPTED_BY_CONTINUE, + CONTINUE_ROW, continueTurn, indexRequestOnlyRows, renderRequestOnlyRows, silentHeartbeatSeparatorTurn, + type RequestOnlyPrompt, type RequestOnlyRowIndex, type SilentHeartbeatTick, type StoredRowLike, } from './silent-heartbeat.js'; +import { randomUUID } from 'node:crypto'; import { toolResultDataToHistoryString, truncateForHistory, @@ -20,6 +22,9 @@ import { } from './tool-result-history.js'; export interface ActivationRequestOptions { + /** The `[Continue]` prompt this turn's earlier streams replied to, if any: + * kept for this request's reply only while this request renders it. */ + continuePrompt?: RequestOnlyPrompt; /** The silent heartbeat tick this request answers, if any. */ silentHeartbeat?: SilentHeartbeatTick; } @@ -27,6 +32,12 @@ export interface ActivationRequestOptions { export interface StartStreamResult { stream: YieldingStream; request: NormalizedRequest; + /** The request-only `[Continue]` prompt this request's reply follows, if + * any: a new one when the request ends on `[Continue]`, else the turn's + * prompt (`ActivationRequestOptions.continuePrompt`) only if this request + * renders it. The framework stamps the reply's rows with it + * (`metadata.promptedBy`), for later compiles to render it before them. */ + continuePrompt?: RequestOnlyPrompt; takeKvSubmission?: () => { submissionId: string; wireReceipt: CacheWireReceipt } | undefined; drainKvSubmissionIds?: () => string[]; } @@ -802,7 +813,7 @@ export class Agent { budget?: TokenBudget, compressionTools: ToolDefinition[] = availableTools, options: ActivationRequestOptions = {}, - ): Promise<{ request: NormalizedRequest; endsOnContinue: boolean }> { + ): Promise<{ request: NormalizedRequest; continuePrompt?: RequestOnlyPrompt }> { // Compression may revisit hidden tools in recorded history. Keep its full // definition set separate from the resident's advertised tools: the // autobiographical strategy must declare the same tools on its @@ -865,12 +876,20 @@ export class Agent { // that same turn, so the prefix it caches is the one every later compile // renders. "First" is per agent: a broadcast tick shares its eventId with // every resident. - const requestOnly = this.indexRequestOnlyRows(); - messages = renderRequestOnlyRows(messages, requestOnly, this.name); + const rendered = renderRequestOnlyRows(messages, this.indexRequestOnlyRows(), this.name); + messages = rendered.messages; + // A tick opens wherever its separator isn't already rendered: on its first + // request, and on a restart whose compile left none of the tick's rows, + // so the restart's reply follows the separator, where every later compile + // renders it. const openingTick = options.silentHeartbeat !== undefined - && !requestOnly.started.has(options.silentHeartbeat.eventId); + && !rendered.separatedTicks.has(options.silentHeartbeat.eventId); - let endsOnContinue = false; + // The [Continue] prompt this request's reply follows: a new one when it + // ends on [Continue]; else the turn's prompt only where this request + // renders it (a stream resumed after a cooldown, say, whose compile now + // ends on a message that arrived meanwhile, renders none); else none. + let continuePrompt: RequestOnlyPrompt | undefined; if (openingTick) { messages = [...messages, silentHeartbeatSeparatorTurn()]; } else if (messages.length > 0 && messages[messages.length - 1]!.participant === this.name) { @@ -878,7 +897,10 @@ export class Agent { // Some models reject trailing assistant messages ("prefill not supported"), // and after context compression a stale assistant turn can end up last. messages = [...messages, continueTurn()]; - endsOnContinue = true; + continuePrompt = { turn: randomUUID(), text: CONTINUE_ROW }; + } + if (!continuePrompt && options.continuePrompt && rendered.promptedTurns.has(options.continuePrompt.turn)) { + continuePrompt = options.continuePrompt; } const request: NormalizedRequest = { @@ -898,7 +920,7 @@ export class Agent { ...(this.providerParams && { providerParams: this.providerParams }), assistantParticipant: this.name, }; - return { request, endsOnContinue }; + return { request, ...(continuePrompt ? { continuePrompt } : {}) }; } /** The request `assembleActivationRequest` builds (see there), for preview @@ -919,7 +941,7 @@ export class Agent { private requestOnlyIndex: { source: readonly unknown[]; index: RequestOnlyRowIndex } | null = null; private indexRequestOnlyRows(): RequestOnlyRowIndex { - const empty: RequestOnlyRowIndex = { ticks: new Map(), continued: new Set(), started: new Set() }; + const empty: RequestOnlyRowIndex = { ticks: new Map(), prompts: new Map() }; const cm = this.contextManager as Partial; if (typeof cm.getAllMessages !== 'function') return empty; try { @@ -934,11 +956,6 @@ export class Agent { } } - /** The current stream's request ended on a [Continue] turn, and the reply - * it prompted hasn't stored a row yet: its first row is stamped - * `promptedBy: 'continue'`, so later requests render the turn before it. */ - private continuePromptPending = false; - /** * Start a yielding stream with context injections. * Same as startStream but passes injections through to compile. @@ -983,11 +1000,10 @@ export class Agent { this.lastStreamRealInputTokens = 0; this.lastStreamOutputTokens = 0; - const { request, endsOnContinue } = await this.assembleActivationRequest( + const { request, continuePrompt } = await this.assembleActivationRequest( availableTools, injections, budget, compressionTools, options, ); request.messages = this.toolResultGuard.prepareRequest(request.messages, true); - this.continuePromptPending = endsOnContinue; const receiptAware = (this.contextManager as unknown as { getStrategy?: () => unknown }) .getStrategy?.() as { @@ -1035,6 +1051,7 @@ export class Agent { return { stream, request, + ...(continuePrompt ? { continuePrompt } : {}), ...(kvEnabled ? { takeKvSubmission: () => kvQueue.shift(), @@ -1113,10 +1130,6 @@ export class Agent { ); return; } - if (this.continuePromptPending) { - this.continuePromptPending = false; - metadata = { ...metadata, promptedBy: PROMPTED_BY_CONTINUE } as MessageMetadata; - } this.contextManager.addMessage(this.name, content, metadata); } diff --git a/src/framework.ts b/src/framework.ts index c8b211542..8c9a0d891 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -8,7 +8,7 @@ import type { Membrane, ContentBlock, NormalizedRequest, YieldingStream, ToolRes import { MembraneError } from '@animalabs/membrane'; import { ContextManager, PassthroughStrategy, WindowedPassthroughStrategy, OverBudgetError, UncoveredDropError } from '@animalabs/context-manager'; import type { CacheWireReceipt } from './kv-unified-wire.js'; -import type { SilentHeartbeatStamp } from './silent-heartbeat.js'; +import { SILENT_HEARTBEAT_SEPARATOR, type SilentHeartbeatStamp } from './silent-heartbeat.js'; import { SUBCONSCIOUS_TOOLS, SUBCONSCIOUS_TOOL_NAMES, type SubconsciousConfig } from './tune-out/tools.js'; import { TuneOutCoordinator, TUNE_OUT_DEFAULTS } from './tune-out/coordinator.js'; import type { @@ -6905,6 +6905,7 @@ export class AgentFramework { reason: 'context_budget_restart', source: 'framework', timestamp: Date.now(), + continuePrompt: this.activeTurnTriggers.get(agent.name)?.continuePrompt, suppressProse: this.activeTurnTriggers.get(agent.name)?.suppressProse, silentHeartbeat: this.activeTurnTriggers.get(agent.name)?.silentHeartbeat, }); @@ -8514,20 +8515,32 @@ export class AgentFramework { } /** - * Metadata for rows the agent's current turn stores: a silent heartbeat - * tick stamps each of them (assistant responses, tool_result rows) so they - * stay identifiable after the turn — request builds key the tick separator - * on it, and a later retention policy can find or collapse them. The stamp - * names the agent: residents share one message slot and a broadcast tick - * reaches each of them with the same eventId. Undefined for ordinary - * turns, whose rows are stored exactly as before. + * Metadata for rows the agent's current turn stores (assistant responses, + * tool_result rows), for the request-only turns that came before them: + * - a silent heartbeat tick stamps each row, with the separator text its + * request rendered, so they stay identifiable after the turn — request + * builds render the separator before the first that survives, and a later + * retention policy can find or collapse them. The stamp names the agent: + * residents share one message slot and a broadcast tick reaches each of + * them with the same eventId; + * - a turn whose stream opened on `[Continue]` stamps each row from then on + * with that prompt (`promptedBy`), so request builds render it before the + * first that survives. + * Undefined for ordinary turns, whose rows are stored exactly as before. + * (Named for its first use, the silent tick.) */ private silentTurnRowMetadata(agentName: string): MessageMetadata | undefined { - const tick = this.activeTurnTriggers.get(agentName)?.silentHeartbeat; + const trigger = this.activeTurnTriggers.get(agentName); + const tick = trigger?.silentHeartbeat; const stamp: SilentHeartbeatStamp | undefined = tick - ? { eventId: tick.eventId, serverId: tick.serverId, agentName } + ? { eventId: tick.eventId, serverId: tick.serverId, agentName, separator: SILENT_HEARTBEAT_SEPARATOR } : undefined; - return stamp ? { silentHeartbeat: stamp } : undefined; + const prompt = trigger?.continuePrompt; + if (!stamp && !prompt) return undefined; + return { + ...(stamp ? { silentHeartbeat: stamp } : {}), + ...(prompt ? { promptedBy: prompt } : {}), + } as MessageMetadata; } /** Record prose segments suppressed by explicit-send silencing. */ @@ -9360,9 +9373,16 @@ export class AgentFramework { request: compiledRequest, takeKvSubmission, drainKvSubmissionIds, + continuePrompt, } = await agent.startStreamWithInjections(tools, injections, undefined, compressionTools, { silentHeartbeat: trigger?.silentHeartbeat, + continuePrompt: trigger?.continuePrompt, }); + // The [Continue] prompt this stream's reply follows, if any: every row + // the turn stores from here on carries it (silentTurnRowMetadata), and the + // turn's restarts and retries carry it in the trigger to the next + // stream start, which keeps it only if its request renders it. + if (trigger) trigger.continuePrompt = continuePrompt; if (this.agents.get(agent.name) !== agent) { stream.cancel(); agent.cancelStream(); diff --git a/src/index.ts b/src/index.ts index adfab80b2..0696aad78 100644 --- a/src/index.ts +++ b/src/index.ts @@ -153,6 +153,7 @@ export * from "./tool-presentation.js"; export { SILENT_HEARTBEAT_SEPARATOR, silentHeartbeatOf, + type RequestOnlyPrompt, type SilentHeartbeatStamp, type SilentHeartbeatTick, } from './silent-heartbeat.js'; diff --git a/src/silent-heartbeat.ts b/src/silent-heartbeat.ts index f9e4bd0d3..1a2211c31 100644 --- a/src/silent-heartbeat.ts +++ b/src/silent-heartbeat.ts @@ -12,9 +12,9 @@ * first of the tick's rows that survives compilation. * - A compile that ends on the agent's own message gets a trailing * `[Continue]` turn (some models reject a trailing assistant message). The - * reply it prompts is stored straight after that message, so the reply's - * first stored row is stamped `metadata.promptedBy: 'continue'`, and the - * `[Continue]` turn is rendered before it. + * reply it prompts is stored straight after that message, so every row the + * reply's turn stores is stamped `metadata.promptedBy` (`RequestOnlyPrompt`), + * and the turn is rendered before the first of them that survives. * * Without the row, the wire formatter merges consecutive same-role messages: * the reply would read as an unprompted continuation of the earlier response, @@ -23,12 +23,14 @@ * `Agent.addAssistantResponse`). And the request that minted the reply held * the row: a provider that binds signed thinking to the prefix it was minted * under (context-manager #155) refuses that thinking once a later request - * drops it. Each row's text is constant and its position is a function of the - * stamped rows alone, so every compile renders it in the same place (the - * request that minted the reply ends on that same turn), which keeps the - * prompt-cache prefix and the binding stable — unlike a per-compile injection - * that re-anchors to the newest user message, or a system prompt that changes - * for one turn. + * drops it. Each row's text is read from the stamp (what the request that + * minted the reply rendered, so a later change to the constant changes no + * stored reply's prefix) and its position is a function of the stamped rows + * alone, so every compile renders it in the same place (the request that + * minted the reply ends on that same turn), which keeps the prompt-cache + * prefix and the binding stable — unlike a per-compile injection that + * re-anchors to the newest user message, or a system prompt that changes for + * one turn. * * Known limit: compiled messages carry no store provenance, so a stored row * is recognised only by structural identity the provider made unique @@ -42,22 +44,23 @@ */ import type { ContentBlock, NormalizedMessage } from '@animalabs/membrane'; -/** Request-only separator before a silent tick's first surviving row. It - * carries the tick's instruction, so the tick's own request reads it as its - * prompt and every later compile shows it at each past tick. Constant on - * purpose: it carries no time or id, so it is byte-identical on every - * compile. */ +/** Request-only separator a silent tick's request ends on. It carries the + * tick's instruction, so the tick's own request reads it as its prompt, and + * every later compile renders it again before the tick's first surviving row. + * Constant on purpose: it carries no time or id. Each tick's rows record the + * text their request rendered (`SilentHeartbeatStamp.separator`), so changing + * this constant changes only ticks that open afterwards. */ export const SILENT_HEARTBEAT_SEPARATOR = '[silent heartbeat] Scheduled private self-check. Review pending matters privately. ' + 'Do not narrate or publish plain prose; use an explicit send tool only if you deliberately choose to contact someone.'; +/** The separator ticks rendered before their rows recorded its text: rows + * stamped without `separator` still render it, byte for byte. */ +const LEGACY_SILENT_HEARTBEAT_SEPARATOR = '[heartbeat tick]'; + /** Request-only turn a compile that ends on the agent's own message gets. */ export const CONTINUE_ROW = '[Continue]'; -/** `metadata.promptedBy` on the first row a reply to a `[Continue]` turn - * stores. */ -export const PROMPTED_BY_CONTINUE = 'continue'; - /** Identity of one authenticated silent heartbeat tick. */ export interface SilentHeartbeatTick { /** The MCPL push event that woke the tick. A broadcast tick reaches every @@ -72,18 +75,38 @@ export interface SilentHeartbeatTick { * (tool_result rows are stored as `user`). */ export interface SilentHeartbeatStamp extends SilentHeartbeatTick { agentName: string; + /** The separator text the tick's request rendered, which later compiles + * render again. Absent on rows stamped before it was recorded; they render + * the bare marker those requests carried. */ + separator?: string; } /** The tick stamp on a stored row's metadata, if any. */ export function silentHeartbeatOf(metadata: unknown): SilentHeartbeatStamp | undefined { const stamp = (metadata as { silentHeartbeat?: unknown } | undefined)?.silentHeartbeat; if (!stamp || typeof stamp !== 'object') return undefined; - const { eventId, serverId, agentName } = stamp as Record; + const { eventId, serverId, agentName, separator } = stamp as Record; return typeof eventId === 'string' && typeof serverId === 'string' && typeof agentName === 'string' - ? { eventId, serverId, agentName } + ? { eventId, serverId, agentName, ...(typeof separator === 'string' ? { separator } : {}) } : undefined; } +/** A request-only turn a request ended on before the agent's reply, as + * `metadata.promptedBy` on every row the reply's turn stores: `turn` names + * the one prompt, and `text` is what the request rendered. */ +export interface RequestOnlyPrompt { + turn: string; + text: string; +} + +/** The request-only prompt a stored row replies to, if any. */ +export function promptedByOf(metadata: unknown): RequestOnlyPrompt | undefined { + const prompt = (metadata as { promptedBy?: unknown } | undefined)?.promptedBy; + if (!prompt || typeof prompt !== 'object') return undefined; + const { turn, text } = prompt as Record; + return typeof turn === 'string' && typeof text === 'string' ? { turn, text } : undefined; +} + export function silentHeartbeatSeparatorTurn(): NormalizedMessage { return requestOnlyTurn(SILENT_HEARTBEAT_SEPARATOR); } @@ -96,11 +119,6 @@ function requestOnlyTurn(text: string): NormalizedMessage { return { participant: 'user', content: [{ type: 'text', text }] }; } -/** Whether a stored row is a reply's first row after a `[Continue]` turn. */ -export function isPromptedByContinue(metadata: unknown): boolean { - return (metadata as { promptedBy?: unknown } | undefined)?.promptedBy === PROMPTED_BY_CONTINUE; -} - /** * Structural identity of one stored response wherever a compile renders it: * tool_use ids and thinking signatures, which the provider made unique and @@ -129,63 +147,77 @@ export interface StoredRowLike { export interface RequestOnlyRowIndex { /** Identity key of every assistant row this agent stored during a silent - * tick → the tick's event id. */ - ticks: Map; - /** Identity keys of the first row of each reply to a `[Continue]` turn. */ - continued: Set; - /** Ticks this agent has already stored rows for (its own, not another resident's). */ - started: Set; + * tick → the tick, and the separator text to render before its first + * surviving row. */ + ticks: Map; + /** Identity key of every assistant row this agent stored in reply to a + * request-only prompt → that prompt. */ + prompts: Map; } -/** Index the rows that follow request-only turns: every row this agent stored - * during its silent ticks, and the first row of each reply to a `[Continue]`. - * The two are independent, so a row can be both. */ +/** Index the agent's own rows that follow request-only turns: the rows of its + * silent ticks, and the rows of each reply to a `[Continue]`. The two are + * independent, so a row can be both. */ export function indexRequestOnlyRows(stored: readonly StoredRowLike[], agentName: string): RequestOnlyRowIndex { - const ticks = new Map(); - const continued = new Set(); - const started = new Set(); + const ticks = new Map(); + const prompts = new Map(); for (const row of stored) { + if (row.participant !== agentName) continue; + const keys = identityKeys(row.content); + if (keys.length === 0) continue; const stamp = silentHeartbeatOf(row.metadata); if (stamp && stamp.agentName === agentName) { - started.add(stamp.eventId); - if (row.participant === agentName) { - for (const key of identityKeys(row.content)) ticks.set(key, stamp.eventId); - } - } - if (isPromptedByContinue(row.metadata)) { - for (const key of identityKeys(row.content)) continued.add(key); + const tick = { eventId: stamp.eventId, text: stamp.separator ?? LEGACY_SILENT_HEARTBEAT_SEPARATOR }; + for (const key of keys) ticks.set(key, tick); } + const prompt = promptedByOf(row.metadata); + if (prompt) for (const key of keys) prompts.set(key, prompt); } - return { ticks, continued, started }; + return { ticks, prompts }; +} + +export interface RenderedRequestOnlyRows { + messages: NormalizedMessage[]; + /** The ticks whose separator this window renders. A tick none of whose + * rows survive compilation renders none. */ + separatedTicks: Set; + /** The prompts (`RequestOnlyPrompt.turn`) this window renders, the same + * way. */ + promptedTurns: Set; } /** - * Insert the request-only turns into a compiled window: a tick's separator - * immediately before the first of the tick's rows that survives compilation - * (its first row when rendered, otherwise whichever later row a strategy - * kept), and `[Continue]` immediately before each reply it prompted, after - * any separator, as the reply's request had them. Rows folded into - * summaries simply do not match; ordinary messages pass through untouched. + * Insert the request-only turns into a compiled window, each immediately + * before the first of its rows that survives compilation (the first row when + * it is rendered, otherwise whichever later row a strategy kept): a tick's + * separator, and the prompt each reply to `[Continue]` followed, after any + * separator, as the reply's request had them. Rows folded into summaries + * simply do not match; ordinary messages pass through untouched. */ export function renderRequestOnlyRows( messages: NormalizedMessage[], - index: Pick, + index: RequestOnlyRowIndex, agentName: string, -): NormalizedMessage[] { - if (index.ticks.size === 0 && index.continued.size === 0) return messages; - const separated = new Set(); +): RenderedRequestOnlyRows { + const separatedTicks = new Set(); + const promptedTurns = new Set(); + if (index.ticks.size === 0 && index.prompts.size === 0) return { messages, separatedTicks, promptedTurns }; const out: NormalizedMessage[] = []; for (const message of messages) { if (message.participant === agentName) { const keys = identityKeys(message.content); - const tick = keys.map((key) => index.ticks.get(key)).find((id) => id !== undefined); - if (tick !== undefined && !separated.has(tick)) { - separated.add(tick); - out.push(silentHeartbeatSeparatorTurn()); + const tick = keys.map((key) => index.ticks.get(key)).find((t) => t !== undefined); + if (tick !== undefined && !separatedTicks.has(tick.eventId)) { + separatedTicks.add(tick.eventId); + out.push(requestOnlyTurn(tick.text)); + } + const prompt = keys.map((key) => index.prompts.get(key)).find((p) => p !== undefined); + if (prompt !== undefined && !promptedTurns.has(prompt.turn)) { + promptedTurns.add(prompt.turn); + out.push(requestOnlyTurn(prompt.text)); } - if (keys.some((key) => index.continued.has(key))) out.push(continueTurn()); } out.push(message); } - return out; + return { messages: out, separatedTicks, promptedTurns }; } diff --git a/src/types/agent.ts b/src/types/agent.ts index 3fe5509e2..a4fc4b9e1 100644 --- a/src/types/agent.ts +++ b/src/types/agent.ts @@ -1,7 +1,7 @@ import type { ContentBlock, YieldingStream } from '@animalabs/membrane'; import type { ContextStrategy } from '@animalabs/context-manager'; import type { ToolCallId, ToolResult, ToolCall } from './events.js'; -import type { SilentHeartbeatTick } from '../silent-heartbeat.js'; +import type { RequestOnlyPrompt, SilentHeartbeatTick } from '../silent-heartbeat.js'; export type SameRoundThinkTextPolicy = 'public' | 'private'; export type SameRoundThinkTextPolicySource = @@ -360,6 +360,13 @@ export interface InferenceRequest { /** Suppress every automatic plain-prose delivery for this logical turn. * Explicit tool calls remain available. Used by authenticated silent wakes. */ suppressProse?: boolean; + /** Set by the framework at each stream start of this turn: the request-only + * `[Continue]` prompt the stream's reply follows (`StartStreamResult. + * continuePrompt`), or none. Every row the turn stores is stamped + * `metadata.promptedBy` with it, so request builds render the turn before + * the first that survives. Carried into the turn's restarts and retries, + * whose stream start keeps it only if its own request renders it. */ + continuePrompt?: RequestOnlyPrompt; /** The authenticated silent heartbeat tick this turn answers. Every row the * turn stores is stamped `metadata.silentHeartbeat` with it, and request * builds render a request-only separator carrying the tick's instruction diff --git a/test/silent-heartbeat-rows.test.ts b/test/silent-heartbeat-rows.test.ts index c9a8fff26..a0fd91fae 100644 --- a/test/silent-heartbeat-rows.test.ts +++ b/test/silent-heartbeat-rows.test.ts @@ -76,11 +76,13 @@ class QueuedStreamsMembrane extends MockMembrane { } } -/** A strategy that leaves every tool exchange out of the compiled window. */ +/** A strategy that leaves every tool exchange out of the compiled window + * (from the moment `omit` is set, for a window that folds them later). */ class OmitToolExchange extends PassthroughStrategy { + omit = true; select(...args: Parameters) { - return super.select(...args).filter((entry) => - !entry.content.some((block) => block.type === 'tool_use' || block.type === 'tool_result')); + return super.select(...args).filter((entry) => !this.omit + || !entry.content.some((block) => block.type === 'tool_use' || block.type === 'tool_result')); } } @@ -213,14 +215,15 @@ describe('silent heartbeat stored rows', () => { 'tool round assistant row, its tool_result row, trailing assistant row — nothing else'); for (const row of tickRows) { assert.deepEqual(row.metadata?.silentHeartbeat, - { eventId: tick.eventId, serverId: 'heartbeat', agentName: 'assistant' }, + { eventId: tick.eventId, serverId: 'heartbeat', agentName: 'assistant', separator: SILENT_HEARTBEAT_SEPARATOR }, `row stamped: ${JSON.stringify(row.content)}`); } assert.ok(tickRows[1]!.content.every((b) => b.type === 'tool_result'), 'the only user row is the tool_result'); for (const row of all.slice(0, before)) { assert.equal(row.metadata?.silentHeartbeat, undefined, 'ordinary rows are not stamped'); } - assert.ok(!JSON.stringify(all).includes(SILENT_HEARTBEAT_SEPARATOR), 'the separator is never stored'); + assert.ok(!all.some((m) => JSON.stringify(m.content).includes(SILENT_HEARTBEAT_SEPARATOR)), + 'the separator is never stored as a row'); } finally { await x.framework.stop(); rmSync(x.dir, { recursive: true, force: true }); } }); @@ -286,6 +289,34 @@ describe('silent heartbeat stored rows', () => { } finally { await x.framework.stop(); rmSync(x.dir, { recursive: true, force: true }); } }); + it('opens the tick again on a restart whose compile left none of its rows', async () => { + const membrane = new QueuedStreamsMembrane(); + const x = await make({ membrane, physicalWindowTokens: 200_000, deliver: false, agent: { strategy: new OmitToolExchange() } }); + try { + membrane.queues.push([createMockResponse(signed('sig-ordinary', 'ordinary reply'))]); + await wake(x, ordinaryEvent('hello')); + const nearCap = createMockResponse(TICK_TOOL_ROUND, 'tool_use'); + (nearCap as { usage: unknown }).usage = { inputTokens: 5_000, outputTokens: 5, cacheReadTokens: 195_000 }; + membrane.queues.push([nearCap], [createMockResponse(signed('sig-tick-final', 'all quiet'))]); + await wake(x, silentEvent()); + membrane.queues.push([createMockResponse(signed('sig-later', 'later reply'))]); + await wake(x, ordinaryEvent('later')); + assert.equal(membrane.calls.length, 4, 'ordinary, tick, its restart, later'); + + // The strategy dropped the tick's only rows (its tool exchange), so the + // restart's compile ends on the ordinary reply: the tick opens again. + const restarted = membrane.calls[2]!.messages; + assert.ok(!restarted.some((m) => textOf(m) === 'checking')); + assert.equal(textOf(restarted.at(-1)!), SILENT_HEARTBEAT_SEPARATOR); + assert.ok(!restarted.some((m) => textOf(m) === '[Continue]')); + // The reply it minted follows the separator in every later request. + const later = membrane.calls[3]!.messages; + assert.deepEqual(later.slice(0, restarted.length), restarted); + assert.equal(later.filter((m) => textOf(m) === SILENT_HEARTBEAT_SEPARATOR).length, 1); + assert.equal(textOf(later[later.findIndex((m) => textOf(m) === 'all quiet') - 1]!), SILENT_HEARTBEAT_SEPARATOR); + } finally { await x.framework.stop(); rmSync(x.dir, { recursive: true, force: true }); } + }); + it('separates the first SURVIVING tick row when a strategy drops the tick\'s tool exchange', async () => { const x = await make({ deliver: false, agent: { strategy: new OmitToolExchange() } }); try { @@ -494,7 +525,8 @@ describe('replies to a [Continue] turn', () => { const rows = stored(x.framework); const reply = rows.find((m) => m.content.some((b) => (b as { text?: string }).text === 'continued reply'))!; - assert.equal(reply.metadata?.promptedBy, 'continue'); + assert.equal(reply.metadata?.promptedBy?.text, '[Continue]'); + assert.equal(typeof reply.metadata?.promptedBy?.turn, 'string'); for (const row of rows.filter((m) => m !== reply)) assert.equal(row.metadata?.promptedBy, undefined); assert.ok(!rows.some((m) => m.content.some((b) => (b as { text?: string }).text === '[Continue]')), 'the [Continue] turn is never stored'); @@ -502,26 +534,43 @@ describe('replies to a [Continue] turn', () => { }); it('a row that is both a tick row and a reply to [Continue] gets both turns, separator first', () => { - const tick = { eventId: 'hb-x', serverId: 'heartbeat', agentName: 'assistant' }; + const tick = { eventId: 'hb-x', serverId: 'heartbeat', agentName: 'assistant', separator: SILENT_HEARTBEAT_SEPARATOR }; + const prompt = { turn: 'p-1', text: '[Continue]' }; const stored = [ { participant: 'assistant', content: signed('sig-a', 'earlier reply') }, { participant: 'assistant', content: signed('sig-1', 'tick round'), metadata: { silentHeartbeat: tick } }, { participant: 'assistant', content: signed('sig-2', 'tick continued'), - metadata: { silentHeartbeat: tick, promptedBy: 'continue' } }, + metadata: { silentHeartbeat: tick, promptedBy: prompt } }, ]; const index = indexRequestOnlyRows(stored, 'assistant'); - const rendered = renderRequestOnlyRows(stored.map(({ participant, content }) => ({ participant, content })), index, 'assistant'); - assert.deepEqual(rendered.map(textOf), [ + const bare = (rows: typeof stored) => rows.map(({ participant, content }) => ({ participant, content })); + assert.deepEqual(renderRequestOnlyRows(bare(stored), index, 'assistant').messages.map(textOf), [ 'earlier reply', SILENT_HEARTBEAT_SEPARATOR, 'tick round', '[Continue]', 'tick continued', ]); // With the tick's first row folded away, the stamped row is the tick's // first surviving row: both turns precede it, the separator first. - const folded = renderRequestOnlyRows( - [stored[0]!, stored[2]!].map(({ participant, content }) => ({ participant, content })), index, 'assistant'); - assert.deepEqual(folded.map(textOf), ['earlier reply', SILENT_HEARTBEAT_SEPARATOR, '[Continue]', 'tick continued']); + assert.deepEqual(renderRequestOnlyRows(bare([stored[0]!, stored[2]!]), index, 'assistant').messages.map(textOf), + ['earlier reply', SILENT_HEARTBEAT_SEPARATOR, '[Continue]', 'tick continued']); }); - it('marks only the first row the reply stores, and renders one [Continue] before it', async () => { + it('renders the text each stamp recorded, and the bare marker for ticks stamped before it was recorded', () => { + const stored = [ + { participant: 'assistant', content: signed('sig-old', 'old tick reply'), + metadata: { silentHeartbeat: { eventId: 'hb-old', serverId: 'heartbeat', agentName: 'assistant' } } }, + { participant: 'assistant', content: signed('sig-new', 'new tick reply'), + metadata: { silentHeartbeat: { eventId: 'hb-new', serverId: 'heartbeat', agentName: 'assistant', separator: 'recorded separator' } } }, + { participant: 'assistant', content: signed('sig-c', 'continued reply'), + metadata: { promptedBy: { turn: 'p-1', text: 'recorded prompt' } } }, + ]; + const index = indexRequestOnlyRows(stored, 'assistant'); + const rendered = renderRequestOnlyRows(stored.map(({ participant, content }) => ({ participant, content })), index, 'assistant'); + assert.deepEqual(rendered.messages.map(textOf), [ + '[heartbeat tick]', 'old tick reply', 'recorded separator', 'new tick reply', 'recorded prompt', 'continued reply', + ]); + assert.deepEqual([...rendered.separatedTicks].sort(), ['hb-new', 'hb-old']); + }); + + it('marks every row the reply\'s turn stores, and renders one [Continue] before the first', async () => { const x = await make({ deliver: false }); try { await wake(x, ordinaryEvent('hello'), signed('sig-1', 'first reply')); @@ -532,9 +581,10 @@ describe('replies to a [Continue] turn', () => { await wake(x, ordinaryEvent('later'), signed('sig-later', 'later reply')); const rows = stored(x.framework); - const marked = rows.filter((m) => m.metadata?.promptedBy === 'continue'); - assert.equal(marked.length, 1); - assert.ok(marked[0]!.content.some((b) => b.type === 'tool_use'), 'the first row: the tool round'); + const marked = rows.filter((m) => m.metadata?.promptedBy !== undefined); + assert.deepEqual(marked.map((m) => m.participant), ['assistant', 'user', 'assistant'], + 'the tool round, its tool_result row and the final reply'); + assert.equal(new Set(marked.map((m) => m.metadata!.promptedBy.turn)).size, 1, 'one prompt'); const later = x.membrane.calls[2]!.messages; assert.equal(later.filter((m) => textOf(m) === '[Continue]').length, 1); @@ -543,4 +593,89 @@ describe('replies to a [Continue] turn', () => { assert.deepEqual(later.slice(0, x.membrane.calls[1]!.messages.length), x.membrane.calls[1]!.messages); } finally { await x.framework.stop(); rmSync(x.dir, { recursive: true, force: true }); } }); + + it('carries the prompt into the reply\'s budget restart, for a window that later folds the first round', async () => { + const membrane = new QueuedStreamsMembrane(); + const strategy = new OmitToolExchange(); + strategy.omit = false; + const x = await make({ membrane, physicalWindowTokens: 200_000, deliver: false, agent: { strategy } }); + try { + membrane.queues.push([createMockResponse(signed('sig-1', 'first reply'))]); + await wake(x, ordinaryEvent('hello')); + // The reply to [Continue] restarts at its tool boundary; the restart's + // compile ends on the tool_result, so it opens on no prompt of its own. + const nearCap = createMockResponse( + signed('sig-c', 'checking', [{ type: 'tool_use', id: 'c-t1', name: 'tools--ping', input: {} } as ContentBlock]), + 'tool_use'); + (nearCap as { usage: unknown }).usage = { inputTokens: 5_000, outputTokens: 5, cacheReadTokens: 195_000 }; + membrane.queues.push([nearCap], [createMockResponse(signed('sig-c2', 'all done'))]); + (x.framework as any).pendingRequests.push({ agentName: 'assistant', reason: 'timer', source: 'framework', timestamp: Date.now() }); + await x.framework.runUntilIdle(); + assert.equal(membrane.calls.length, 3, 'first, the continued reply, its restart'); + assert.equal(textOf(membrane.calls[1]!.messages.at(-1)!), '[Continue]'); + assert.notEqual(textOf(membrane.calls[2]!.messages.at(-1)!), '[Continue]'); + + const marked = stored(x.framework).filter((m) => m.metadata?.promptedBy !== undefined); + assert.equal(marked.length, 3, 'the round, its result and the restart\'s reply'); + assert.equal(new Set(marked.map((m) => m.metadata!.promptedBy.turn)).size, 1); + + // Later, the window folds the first round: [Continue] moves to the + // restart's reply, the first of the prompt's rows that survives. + strategy.omit = true; + membrane.queues.push([createMockResponse(signed('sig-later', 'later reply'))]); + await wake(x, ordinaryEvent('later')); + const later = membrane.calls[3]!.messages; + assert.ok(!later.some((m) => textOf(m) === 'checking')); + assert.equal(textOf(later[later.findIndex((m) => textOf(m) === 'all done') - 1]!), '[Continue]'); + } finally { await x.framework.stop(); rmSync(x.dir, { recursive: true, force: true }); } + }); + + it('drops a carried prompt its request doesn\'t render (a stream resumed after a cooldown)', async () => { + // The shape a provider cooldown leaves (Basil-1007's probe): the held + // trigger carries the prompt its first stream start set, and a message + // lands meanwhile, so the resumed compile ends on that message and + // renders no [Continue]. + const x = await make({ deliver: false }); + try { + await wake(x, ordinaryEvent('hello'), signed('sig-1', 'first reply')); + x.framework.getAgent('assistant')!.getContextManager() + .addMessage('nissa', [{ type: 'text', text: 'arrived during the cooldown' }] as ContentBlock[]); + x.membrane.pushResponse(createMockResponse(signed('sig-r', 'resumed reply'), 'end_turn')); + (x.framework as any).pendingRequests.push({ + agentName: 'assistant', reason: 'timer', source: 'framework', timestamp: Date.now(), + continuePrompt: { turn: 'stale-p1', text: '[Continue]' }, + }); + await x.framework.runUntilIdle(); + await wake(x, ordinaryEvent('later'), signed('sig-later', 'later reply')); + + const minting = x.membrane.calls[1]!.messages; + assert.equal(textOf(minting.at(-1)!), 'arrived during the cooldown'); + const reply = stored(x.framework).find((m) => m.content.some((b) => (b as { text?: string }).text === 'resumed reply'))!; + assert.equal(reply.metadata?.promptedBy, undefined); + const later = x.membrane.calls[2]!.messages; + const replyAt = later.findIndex((m) => textOf(m) === 'resumed reply'); + assert.deepEqual(later.slice(0, replyAt), minting, 'what precedes the reply is what its request held'); + } finally { await x.framework.stop(); rmSync(x.dir, { recursive: true, force: true }); } + }); + + it('renders [Continue] before the first SURVIVING row when a strategy drops the reply\'s tool exchange', async () => { + const x = await make({ deliver: false, agent: { strategy: new OmitToolExchange() } }); + try { + await wake(x, ordinaryEvent('hello'), signed('sig-1', 'first reply')); + await timerWake(x, + signed('sig-c', 'checking', [{ type: 'tool_use', id: 'c-t1', name: 'tools--ping', input: {} } as ContentBlock]), + signed('sig-c2', 'all done')); + await wake(x, ordinaryEvent('later'), signed('sig-later', 'later reply')); + + const later = x.membrane.calls[2]!.messages; + assert.ok(!later.some((m) => textOf(m) === 'checking'), 'the strategy dropped the tool round'); + const finalAt = later.findIndex((m) => textOf(m) === 'all done'); + assert.equal(textOf(later[finalAt - 1]!), '[Continue]'); + assert.equal(later.filter((m) => textOf(m) === '[Continue]').length, 1); + for (const message of wire(x.membrane.calls[2]!)) { + assert.ok(signatures(message).length <= 1, + `one ${message.role} message carries signed thinking from: ${signatures(message).join(', ')}`); + } + } finally { await x.framework.stop(); rmSync(x.dir, { recursive: true, force: true }); } + }); }); From 2c198b5df51875834a25cbe1298d0b9678ea6543 Mon Sep 17 00:00:00 2001 From: Petra-1012 Date: Sun, 11 Oct 2026 14:43:58 -0400 Subject: [PATCH 0105/1112] fix(drafts): notices count on the branch they're on, the catch-up names every draft it marks, and the store keeps what it needs sw-af247 (the house's haiku review of 02adf27..378c2b3), part 1: the drafts store and its notices. - I2: a draft counts as named while a message that named it is on the current branch. The 'noticed' record now carries the naming message's id (the catch-up's own message, or the one settleTurnDrafts finds), and the turn-start catch-up names every open draft none of whose naming messages is present (cm.getMessage), so a rollback or branch move past its notice names it again, once. A record written without a message id names nothing, so its draft is named once more. - I4 and I29: the catch-up names every draft it marks, newest first: the newest eight with state and preview, the rest by id ("and N more: ..."). unnoticed() returns newest first, as open() does. - I27: settleTurnDrafts reads only the window it scans (the last 500 messages, DRAFT_NOTICE_SCAN), not a copy of the whole history, and returns each draft's naming message. - I6: the store keeps each resident's 100 newest drafts and, beyond them, any draft something happened to (held, named, attempted, answered, dismissed) in the last 7 days; at a checkpoint every 500 entries it lets the rest go, and opening replays only the checkpoint and what follows. The latest bounce is tracked by id, so {{unsent}} means nothing once it is let go, even if an older bounce is kept; newId never reuses that id. The drafts tool's description and its unknown-id refusal say so. Retention is a constructor option, for tests. - I10: a draft whose held record reached the store before its sync failed is in the reconciled journal, so hold() counts it as held (and logs the failure); only the segments after it are reported not held. Tests (prose-drafts): the hold-failure test now expects 'x' held and one not held; new: notices counted per branch (store), the checkpoint and retention across a reopen (store, mock Date), the catch-up naming ten drafts, and a rollback past a receipt naming its draft again, once. --- src/framework.ts | 92 +++++++++++++------- src/prose-drafts.ts | 173 +++++++++++++++++++++++++++++++------- test/prose-drafts.test.ts | 142 ++++++++++++++++++++++++++++--- 3 files changed, 337 insertions(+), 70 deletions(-) diff --git a/src/framework.ts b/src/framework.ts index 8534ce4c6..052ddd180 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -85,7 +85,7 @@ import { ProseStreamRouter } from './mcpl/prose-stream-router.js'; import { detectKnownToolWrapperProse } from './tool-wrapper-prose-guard.js'; import { InferenceRouter } from './mcpl/inference-router.js'; import { ChannelRegistry, type ChannelToolOrigin, type PublishDestination, type PublishOutcome } from './mcpl/channel-registry.js'; -import { ProseDraftStore, draftState, uncertainAttempt, type Draft, type DraftReason, type DraftSource, type DraftState, type InheritedRisk } from './prose-drafts.js'; +import { KEEP_IDLE_MS, KEEP_NEWEST, ProseDraftStore, draftState, uncertainAttempt, type Draft, type DraftReason, type DraftSource, type DraftState, type InheritedRisk } from './prose-drafts.js'; import { ConversationRouter, CONVERSATION_ROUTING_DEPRECATION_NOTICE } from './mcpl/conversation-router.js'; import { INBOUND_SOURCE_KEY, type InboundAcceptanceObserver, type InboundChannelSource, type InboundLane, type InboundSource, type InboundUnscopedSource } from './mcpl/inbound-source.js'; import { safeSlice } from './safe-slice.js'; @@ -4676,10 +4676,13 @@ export class AgentFramework { inputSchema: { type: 'object' }, }; + /** How long, in days, a draft beyond a resident's newest is kept (KEEP_IDLE_MS). */ + private static readonly DRAFT_KEEP_DAYS = Math.round(KEEP_IDLE_MS / 86_400_000); + /** Synthesized drafts tool — present whenever MCPL channels are * configured. A resident's held drafts (src/prose-drafts.ts): plain speech - * that was not sent, never published except by its resident's resend and - * kept until resent or dismissed. Nullable + * that was not sent, never published except by its resident's resend, and + * kept as that module's retention says. Nullable * fields, cast like save_recent_image's: a caller whose provider presents * every property as required can still say "not in use". */ private static readonly DRAFTS_TOOL = { @@ -4687,8 +4690,9 @@ export class AgentFramework { description: 'Your held drafts: plain speech of yours that was NOT sent — held back because an explicit send in the ' + 'same round holds plain speech back, because a routing prefix bounced, or because it had no destination. ' + - 'Drafts are never published except by your own resend, only you can list or act on them, and they stay ' + - 'until you resend or dismiss them. list: your open drafts, newest first. read: drafts in full, with every delivery attempt. ' + + `Drafts are never published except by your own resend, and only you can list or act on them. Your ${KEEP_NEWEST} newest ` + + `drafts are always kept; an older one is let go after ${AgentFramework.DRAFT_KEEP_DAYS} days in which nothing happened to it. ` + + 'list: your open drafts, newest first. read: drafts in full, with every delivery attempt. ' + 'resend: publish drafts verbatim, in the order given, to one destination ("#channel", "@person" or a ' + 'channel id) — an explicit send, like send_message; it stops at the first draft not confirmed delivered. ' + 'A draft already delivered returns its receipt instead of sending again. If an earlier attempt may ' + @@ -9136,38 +9140,51 @@ export class AgentFramework { } /** - * Turn start (or after a crash): name any open drafts no notice has named - * yet, privately, before the turn's compile. + * Turn start (or after a crash): name, privately and before the turn's + * compile, any open drafts no notice on the current branch names: none + * reached the history (a crash, an aborted turn), or a rollback or branch + * move took the notices away. Every draft it marks is named by id, newest + * first: the newest eight with their state and a preview, the rest by id. */ private noticeUnnoticedDrafts(agent: Agent): void { - const pending = this.proseDrafts.unnoticed(agent.name); + const cm = agent.getContextManager(); + const pending = this.proseDrafts.unnoticed(agent.name, (id) => cm.getMessage(id as MessageId) !== null); if (pending.length === 0) return; const shown = pending.slice(0, 8).map((d) => `${d.id} (${this.draftStateText(d)}; held ${this.draftTime(d.heldAt)}, ` + `${AgentFramework.DRAFT_REASON_SHORT[d.reason]}, ${d.text.length} chars: ${AgentFramework.draftPreview(d.text, 60)})`); - const more = pending.length > shown.length ? ` · and ${pending.length - shown.length} more` : ''; + const rest = pending.slice(shown.length).map((d) => d.id); + const more = rest.length > 0 ? ` · and ${rest.length} more: ${rest.join(', ')}` : ''; const text = `[drafts] ${agent.name}: ${pending.length === 1 ? 'a held draft' : `${pending.length} held drafts`} of yours ` + - `${pending.length === 1 ? 'has' : 'have'} not been named to you yet: ${shown.join(' · ')}${more}. ` + + `${pending.length === 1 ? 'is' : 'are'} unnamed on this branch of your history: ${shown.join(' · ')}${more}. ` + 'drafts(action: "list") shows your open drafts; resend or dismiss them by id.'; + let mid: string; try { - const mid = agent.getContextManager().addMessage( + mid = cm.addMessage( 'user', [{ type: 'text', text }], { system: true, kind: 'prose-drafts', draftOwner: agent.name, draftIds: pending.map((d) => d.id) } as MessageMetadata, ); this.emitTrace({ type: 'message:added', messageId: mid, source: 'prose-drafts' }); - this.proseDrafts.markNoticed(agent.name, pending.map((d) => d.id)); } catch (err) { console.error('[drafts] catch-up notice failed:', err); + return; + } + try { + this.proseDrafts.markNoticed(agent.name, pending.map((d) => d.id), mid); + } catch (err) { + // Named again by the next turn's catch-up. + console.error('[drafts] could not record the catch-up notice:', err); } } /** * Logical turn end, after the turn's deferred notices are flushed: record * as noticed exactly the drafts a stored message names (the receipt, a - * held-draft notice, a bounce notice). One whose notice never reached the - * history stays unnoticed, for the next turn's catch-up. + * held-draft notice, a bounce notice), each with the message that names + * it. One whose notice never reached the history stays unnoticed, for the + * next turn's catch-up. */ private settleTurnDrafts(agent: Agent): void { const held = this.turnDrafts.get(agent.name); @@ -9175,29 +9192,43 @@ export class AgentFramework { this.turnDraftFailures.delete(agent.name); this.turnProsePrivate.delete(agent.name); if (!held?.length) return; - const named = this.draftsNamedInHistory(agent, held.map((d) => d.id)); - if (named.length === 0) return; - try { - this.proseDrafts.markNoticed(agent.name, named); - } catch (err) { - // Unmarked drafts are named again by the next turn's catch-up. - console.error('[drafts] could not record notice:', err); + const byMessage = new Map(); + for (const [draftId, messageId] of this.draftsNamedInHistory(agent, held.map((d) => d.id))) { + byMessage.set(messageId, [...(byMessage.get(messageId) ?? []), draftId]); + } + for (const [messageId, ids] of byMessage) { + try { + this.proseDrafts.markNoticed(agent.name, ids, messageId); + } catch (err) { + // Unmarked drafts are named again by the next turn's catch-up. + console.error('[drafts] could not record notice:', err); + } } } + /** How far back draftsNamedInHistory looks: this turn's notices are near + * the end. One beyond it leaves its draft for the next turn's catch-up. */ + private static readonly DRAFT_NOTICE_SCAN = 500; + /** Which of these drafts a stored notice or receipt OF THIS RESIDENT names * (draft ids are unique only per resident, and residents can share one - * history), scanning the recent history, where this turn's notices are. */ - private draftsNamedInHistory(agent: Agent, ids: string[]): string[] { + * history), each with the newest message that names it, reading only the + * recent history, where this turn's notices are. */ + private draftsNamedInHistory(agent: Agent, ids: string[]): Map { const wanted = new Set(ids); - const named = new Set(); - const messages = agent.getContextManager().getAllMessages(); - for (let i = messages.length - 1, scanned = 0; i >= 0 && scanned < 500 && named.size < wanted.size; i--, scanned++) { + const named = new Map(); + const cm = agent.getContextManager(); + const total = cm.getMessageCount(); + const start = Math.max(0, total - AgentFramework.DRAFT_NOTICE_SCAN); + const { messages } = cm.getMessageWindow(start, total - start, { resolveBlobs: false }); + for (let i = messages.length - 1; i >= 0 && named.size < wanted.size; i--) { const meta = messages[i]!.metadata as { draftIds?: unknown; draftOwner?: unknown } | undefined; if (meta?.draftOwner !== agent.name || !Array.isArray(meta.draftIds)) continue; - for (const id of meta.draftIds) if (typeof id === 'string' && wanted.has(id)) named.add(id); + for (const id of meta.draftIds) { + if (typeof id === 'string' && wanted.has(id) && !named.has(id)) named.set(id, String(messages[i]!.id)); + } } - return ids.filter((id) => named.has(id)); + return new Map(ids.filter((id) => named.has(id)).map((id) => [id, named.get(id)!])); } /** Page size for drafts(action: "list"). */ @@ -9277,7 +9308,10 @@ export class AgentFramework { else missing.push(id); } if (missing.length > 0) { - return refuse(`No draft of yours with id ${missing.join(', ')}. drafts(action: "list") shows your open drafts.`); + return refuse( + `No draft of yours with id ${missing.join(', ')}: there was none, or it was let go (a draft beyond your ${KEEP_NEWEST} newest ` + + `is kept for ${AgentFramework.DRAFT_KEEP_DAYS} days after anything last happened to it). drafts(action: "list") shows your open drafts.`, + ); } if (action === 'read') { diff --git a/src/prose-drafts.ts b/src/prose-drafts.ts index 7001db0fc..c290a749b 100644 --- a/src/prose-drafts.ts +++ b/src/prose-drafts.ts @@ -22,7 +22,17 @@ * rollback, suppression and branch moves never touch them. A draft written * in a turn that is later undone stays usable (the words were genuinely * written and held back), and a delivered draft stays delivered. Its - * originating branch is recorded as provenance only. + * originating branch is recorded as provenance only. The notices that name + * a draft are conversation, so they do follow the branch: a draft counts as + * named while a message naming it is on the current branch, and one a + * rollback or branch move has left unnamed is named again. + * + * Drafts are kept, not hoarded. Each resident's KEEP_NEWEST newest drafts + * are always kept; an older one is let go once nothing has happened to it + * (held, named, attempted, answered, dismissed) for KEEP_IDLE_MS. The words + * stay in the resident's history either way. The journal checkpoints what + * it keeps every CHECKPOINT_EVERY entries, so opening it replays only what + * came after. * * A delivery attempt is journaled durably BEFORE its request leaves the * host, and its outcome after: an attempt with no recorded outcome (a crash @@ -119,7 +129,10 @@ export interface Draft { source: DraftSource; attempts: DraftAttempt[]; dismissedAt?: number; - /** When an in-band notice named this draft. */ + /** The messages whose notices named this draft, by id. It counts as named + * while one of them is on the current branch. */ + noticedIn?: string[]; + /** When a notice last named it. */ noticedAt?: number; } @@ -152,41 +165,126 @@ export function uncertainAttempt(draft: Draft): DraftAttempt | undefined { } type DraftEntry = - | { op: 'held'; draft: Omit } + | { op: 'held'; draft: Omit } | { op: 'attempt'; agent: string; id: string; attemptId: string; at: number; destination: PublishDestination; via: DraftAttempt['via']; confirmedDuplicate?: true } | { op: 'outcome'; agent: string; id: string; attemptId: string; outcome: DraftAttemptOutcome } | { op: 'dismissed'; agent: string; id: string; at: number } - | { op: 'noticed'; agent: string; ids: string[]; at: number }; + /** A notice naming these drafts is in the history as message `messageId`. + * (A record without one names nowhere the drafts can be found again, so + * they are named once more.) */ + | { op: 'noticed'; agent: string; ids: string[]; at: number; messageId?: string }; + +/** What a checkpoint keeps: every kept draft, in hold order, and each + * resident's latest bounce, which `{{unsent}}` means whatever became of it. */ +interface DraftSnapshot { + v: 1; + drafts: Draft[]; + latestBounces: Record; +} /** Unambiguous lowercase alphabet for short ids (no i/l/o/0/1). */ const ID_ALPHABET = 'abcdefghjkmnpqrstuvwxyz23456789'; +/** Each resident's newest drafts, by hold order, that are always kept. */ +export const KEEP_NEWEST = 100; +/** How long an older draft is kept after the last thing that happened to it. */ +export const KEEP_IDLE_MS = 7 * 24 * 60 * 60 * 1000; +/** Entries the journal takes between checkpoints. */ +export const CHECKPOINT_EVERY = 500; + +/** The store's retention, KEEP_NEWEST, KEEP_IDLE_MS and CHECKPOINT_EVERY + * unless given (tests give smaller ones). */ +export interface DraftRetention { + keepNewest: number; + keepIdleMs: number; + checkpointEvery: number; +} + +/** The last time anything happened to a draft: held, named, attempted, + * answered or dismissed. */ +function lastActivity(draft: Draft): number { + let last = Math.max(draft.heldAt, draft.noticedAt ?? 0, draft.dismissedAt ?? 0); + for (const attempt of draft.attempts) last = Math.max(last, attempt.at, attempt.outcome?.at ?? 0); + return last; +} + export class ProseDraftStore { - private readonly journal: RecordJournal; + private readonly journal: RecordJournal; /** agent → draft id → draft, in journal (hold) order. */ private drafts = new Map>(); + /** agent → the id of its latest bounce, kept or not. */ + private latestBounces = new Map(); + private readonly retention: DraftRetention; - constructor(store: JsStore) { - this.journal = new RecordJournal(store, { type: PROSE_DRAFT_RECORD_TYPE }); + constructor(store: JsStore, retention: Partial = {}) { + this.journal = new RecordJournal(store, { type: PROSE_DRAFT_RECORD_TYPE }); + this.retention = { + keepNewest: retention.keepNewest ?? KEEP_NEWEST, + keepIdleMs: retention.keepIdleMs ?? KEEP_IDLE_MS, + checkpointEvery: retention.checkpointEvery ?? CHECKPOINT_EVERY, + }; this.reload(); + this.checkpointIfDue(); } - /** Rebuild the projection from the journal (at open, and to reconcile). */ + /** Rebuild the projection from the journal (at open, and to reconcile): + * the latest checkpoint's drafts, then the entries after it. */ reload(): void { - const { entries } = this.journal.load(); + const { snapshot, entries } = this.journal.load(); this.drafts = new Map(); + this.latestBounces = new Map(Object.entries(snapshot?.latestBounces ?? {})); + for (const draft of snapshot?.drafts ?? []) this.agentDrafts(draft.agent).set(draft.id, draft); for (const { entry } of entries) this.reduce(entry); } + /** + * Write a checkpoint once CHECKPOINT_EVERY entries have been appended since + * the last one, letting go of the drafts that aren't kept (see the module + * header). The projection drops them only once the checkpoint is written, + * so a failed one changes nothing, and the next write tries again. + */ + private checkpointIfDue(): void { + const { keepNewest, keepIdleMs, checkpointEvery } = this.retention; + if (this.journal.entriesSinceCheckpoint < checkpointEvery) return; + const now = Date.now(); + const kept = new Map>(); + for (const [agent, drafts] of this.drafts) { + const all = [...drafts.values()]; + const keep = new Map(); + all.forEach((draft, i) => { + if (i >= all.length - keepNewest || now - lastActivity(draft) < keepIdleMs) keep.set(draft.id, draft); + }); + if (keep.size > 0) kept.set(agent, keep); + } + const snapshot: DraftSnapshot = { + v: 1, + drafts: [...kept.values()].flatMap((m) => [...m.values()]), + latestBounces: Object.fromEntries(this.latestBounces), + }; + try { + this.journal.checkpoint(snapshot); + } catch (err) { + console.error('[drafts] checkpoint failed; the next write tries again:', err); + if (this.journal.needsReconcile) this.reload(); + return; + } + this.drafts = kept; + } + private reduce(entry: DraftEntry): void { if (entry.op === 'held') { this.agentDrafts(entry.draft.agent).set(entry.draft.id, { ...entry.draft, attempts: [] }); + if (entry.draft.reason === 'bounced') this.latestBounces.set(entry.draft.agent, entry.draft.id); return; } if (entry.op === 'noticed') { + if (entry.messageId === undefined) return; for (const id of entry.ids) { const draft = this.drafts.get(entry.agent)?.get(id); - if (draft && draft.noticedAt === undefined) draft.noticedAt = entry.at; + if (!draft) continue; + draft.noticedIn ??= []; + if (!draft.noticedIn.includes(entry.messageId)) draft.noticedIn.push(entry.messageId); + draft.noticedAt = Math.max(draft.noticedAt ?? 0, entry.at); } return; } @@ -221,6 +319,7 @@ export class ProseDraftStore { throw err; } this.reduce(entry); + this.checkpointIfDue(); } private agentDrafts(agent: string): Map { @@ -232,12 +331,16 @@ export class ProseDraftStore { return map; } + /** A new id, unique among the resident's kept drafts and never its latest + * bounce's. One let go can be given again: the notices that named it are + * by then at least KEEP_IDLE_MS old and KEEP_NEWEST drafts back. */ private newId(agent: string): string { const existing = this.agentDrafts(agent); + const latestBounce = this.latestBounces.get(agent); for (;;) { let id = 'd-'; for (let i = 0; i < 5; i++) id += ID_ALPHABET[randomInt(ID_ALPHABET.length)]; - if (!existing.has(id)) return id; + if (!existing.has(id) && id !== latestBounce) return id; } } @@ -245,7 +348,9 @@ export class ProseDraftStore { * Hold words as drafts, in order, each journaled durably before this * returns. If the journal refuses a write (after reconciling), the drafts * already written stand and the rest are reported as not held: their words - * remain in the resident's history only. + * remain in the resident's history only. A draft whose record reached the + * store before its write failed (its sync did) is in the reconciled + * journal, so it is held, and counted so. */ hold( agent: string, @@ -270,7 +375,14 @@ export class ProseDraftStore { try { this.write({ op: 'held', draft }, true); } catch (err) { - return { held, notHeld: { count: segments.length - i, error: err instanceof Error ? err.message : String(err) } }; + const error = err instanceof Error ? err.message : String(err); + const landed = this.get(agent, draft.id); + if (landed) { + console.error(`[drafts] ${agent}: ${landed.id} reached the journal, but its write then failed (${error}); it is held`); + held.push(landed); + } + const count = segments.length - i - (landed ? 1 : 0); + return { held, ...(count > 0 ? { notHeld: { count, error } } : {}) }; } held.push(this.get(agent, draft.id)!); } @@ -289,21 +401,26 @@ export class ProseDraftStore { .reverse(); } - /** Open drafts no in-band notice has named yet (a crash before the notice). */ - unnoticed(agent: string): Draft[] { - return this.open(agent).filter((d) => d.noticedAt === undefined).reverse(); + /** + * Open drafts, newest first, that no notice on the current branch names: + * none was recorded (a crash before the notice), or a rollback or branch + * move took away every message that named them. `onBranch` says whether a + * message, by id, is on the current branch. + */ + unnoticed(agent: string, onBranch: (messageId: string) => boolean): Draft[] { + return this.open(agent).filter((d) => !d.noticedIn?.some(onBranch)); } /** - * Record that a notice naming these drafts is in the resident's history. - * The notice is conversation state, buffered until the store syncs, while - * this record reaches the OS at once: so the store is synced FIRST - * (afterCommittedState), and a notice that never became durable can't - * leave a record that suppresses the crash catch-up. + * Record that message `messageId`, a notice naming these drafts, is in the + * resident's history. The notice is conversation state, buffered until the + * store syncs, while this record reaches the OS at once: so the store is + * synced FIRST (afterCommittedState), and a notice that never became + * durable can't leave a record that suppresses the crash catch-up. */ - markNoticed(agent: string, ids: string[]): void { + markNoticed(agent: string, ids: string[], messageId: string): void { if (ids.length === 0) return; - this.write({ op: 'noticed', agent, ids, at: Date.now() }, true, true); + this.write({ op: 'noticed', agent, ids, at: Date.now(), messageId }, true, true); } dismiss(agent: string, id: string): void { @@ -349,14 +466,12 @@ export class ProseDraftStore { * that draft is still open (held or unconfirmed — the caller must not * resend an unconfirmed one without the resident's confirmation). * Latest-wins, like the clipboard it replaces: once the latest bounce is - * delivered or dismissed there is nothing to substitute, even if older - * bounces are still held (they stay in the collection, by id). + * delivered, dismissed or let go there is nothing to substitute, even if + * older bounces are still held (they stay in the collection, by id). */ latestBounce(agent: string): Draft | undefined { - let latest: Draft | undefined; - for (const draft of this.drafts.get(agent)?.values() ?? []) { - if (draft.reason === 'bounced') latest = draft; - } + const id = this.latestBounces.get(agent); + const latest = id === undefined ? undefined : this.get(agent, id); if (!latest) return undefined; const state = draftState(latest); return state === 'held' || state === 'unconfirmed' ? latest : undefined; diff --git a/test/prose-drafts.test.ts b/test/prose-drafts.test.ts index defe3a294..cf175e9c8 100644 --- a/test/prose-drafts.test.ts +++ b/test/prose-drafts.test.ts @@ -90,17 +90,19 @@ describe('ProseDraftStore', () => { } }); - it('reports words it could not hold, and stays consistent with its journal', () => { + it('reports words it could not hold, and stays consistent with its journal', (t) => { const { dir, store, drafts } = open(); try { const real = store.sync.bind(store); let fail = true; (store as unknown as { sync: () => void }).sync = () => { if (fail) throw new Error('EIO'); real(); }; + t.mock.method(console, 'error', () => {}); const result = drafts.hold('a', [{ text: 'x', source }, { text: 'y', source }], 'explicit-send'); - assert.equal(result.held.length, 0); - assert.deepEqual(result.notHeld, { count: 2, error: 'EIO' }); // The first record reached the store before its barrier failed: the - // journal was reconciled, so the projection shows what it holds. + // journal was reconciled, so the projection holds it, and so does the + // answer. Only the second is reported not held. + assert.deepEqual(result.held.map((d) => d.text), ['x']); + assert.deepEqual(result.notHeld, { count: 1, error: 'EIO' }); assert.deepEqual(drafts.open('a').map((d) => d.text), ['x']); fail = false; assert.equal(drafts.hold('a', [{ text: 'z', source }], 'explicit-send').held.length, 1); @@ -110,6 +112,74 @@ describe('ProseDraftStore', () => { } }); + it('a draft counts as named while a message that named it is on the branch, newest first', () => { + const { dir, store, drafts } = open(); + try { + const { held } = drafts.hold('a', [{ text: 'one', source }, { text: 'two', source }, { text: 'three', source }], 'explicit-send'); + const [one, two, three] = held; + drafts.markNoticed('a', [one!.id, two!.id], 'm-1'); + drafts.markNoticed('a', [two!.id], 'm-2'); + const unnamed = (present: string[]) => drafts.unnoticed('a', (id) => present.includes(id)).map((d) => d.text); + assert.deepEqual(unnamed(['m-1', 'm-2']), ['three'], 'only the draft no notice named'); + assert.deepEqual(unnamed(['m-2']), ['three', 'one'], 'm-1 rolled away: one is unnamed, two is still named by m-2; newest first'); + assert.deepEqual(unnamed([]), ['three', 'two', 'one']); + assert.deepEqual(drafts.get('a', three!.id)!.noticedIn, undefined); + // The record outlives a reopen; one written without a message id (an + // older build's) names nothing, so its draft is named once more. + store.appendJson(PROSE_DRAFT_RECORD_TYPE, { op: 'noticed', agent: 'a', ids: [three!.id], at: 1 }); + const again = new ProseDraftStore(store); + assert.deepEqual(again.unnoticed('a', (id) => id === 'm-2').map((d) => d.text), ['three', 'one']); + } finally { + store.close(); + rmSync(dir, { recursive: true, force: true }); + } + }); + + it('checkpoints what it keeps: the newest drafts, recent ones, and the latest bounce as {{unsent}} means it', (t) => { + const { dir, store } = open(); + const path = join(dir, 'store'); + t.mock.timers.enable({ apis: ['Date'], now: 1_000_000 }); + const hour = 60 * 60 * 1000; + const retention = { keepNewest: 2, keepIdleMs: hour, checkpointEvery: 9 }; + let drafts = new ProseDraftStore(store, retention); + try { + const [older] = drafts.hold('a', [{ text: 'an older bounce', source }], 'bounced').held; + const [latest] = drafts.hold('a', [{ text: 'the latest bounce', source }], 'bounced').held; + const [settled] = drafts.hold('a', [{ text: 'resent long ago', source }], 'explicit-send').held; + const attempt = drafts.beginAttempt('a', settled!.id, dest(), 'resend', false); + drafts.recordOutcome('a', settled!.id, attempt, { status: 'delivered', messageId: 'm-1', at: Date.now(), destination: dest() }); + drafts.hold('b', [{ text: "another resident's", source }], 'explicit-send'); + t.mock.timers.tick(2 * hour); + // Something happens to the older bounce now: it isn't idle. + drafts.beginAttempt('a', older!.id, dest(), 'resend', false); + const newest = drafts.hold('a', [{ text: 'new one', source }, { text: 'new two', source }], 'explicit-send').held; + // That was the ninth entry: the checkpoint let go of 'a''s idle + // drafts outside its two newest, the latest bounce among them. + const kept = () => drafts.open('a').map((d) => d.text); + assert.deepEqual(kept(), ['new two', 'new one', 'an older bounce']); + assert.equal(drafts.get('a', settled!.id), undefined, 'a settled, idle draft is let go'); + assert.equal(drafts.get('a', latest!.id), undefined); + assert.equal(drafts.latestBounce('a'), undefined, 'the latest bounce was let go: {{unsent}} has nothing, though an older bounce is kept'); + assert.deepEqual(drafts.open('b').map((d) => d.text), ["another resident's"], 'within its resident\'s newest'); + assert.equal(store.getRecordIdsByType(`${PROSE_DRAFT_RECORD_TYPE}/checkpoint`).length, 1); + // A reopen reads the checkpoint and the entries after it, and gets the same. + drafts.markNoticed('a', [newest[0]!.id], 'm-9'); + store.close(); + const reopened = JsStore.openOrCreate({ path }); + try { + drafts = new ProseDraftStore(reopened, retention); + assert.deepEqual(kept(), ['new two', 'new one', 'an older bounce']); + assert.equal(drafts.latestBounce('a'), undefined); + assert.deepEqual(drafts.get('a', newest[0]!.id)!.noticedIn, ['m-9']); + assert.equal(draftState(drafts.get('a', older!.id)!), 'unconfirmed', 'its attempt survives the checkpoint'); + } finally { + reopened.close(); + } + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + it('records a notice only after the history holding it is synced (afterCommittedState)', () => { const { dir, store, drafts } = open(); try { @@ -118,9 +188,9 @@ describe('ProseDraftStore', () => { const real = { sync: store.sync.bind(store), appendJson: store.appendJson.bind(store) }; (store as unknown as { sync: () => void }).sync = () => { calls.push('sync'); real.sync(); }; (store as unknown as { appendJson: (t: string, p: unknown) => unknown }).appendJson = (t, p) => { calls.push(`append:${t}`); return real.appendJson(t, p); }; - drafts.markNoticed('a', [held[0]!.id]); + drafts.markNoticed('a', [held[0]!.id], 'm-1'); assert.deepEqual(calls, ['sync', `append:${PROSE_DRAFT_RECORD_TYPE}`, 'sync']); - assert.ok(drafts.get('a', held[0]!.id)!.noticedAt); + assert.deepEqual(drafts.get('a', held[0]!.id)!.noticedIn, ['m-1']); } finally { store.close(); rmSync(dir, { recursive: true, force: true }); @@ -570,7 +640,7 @@ describe('held prose drafts, end to end', () => { h.store().beginAttempt('scout', midSend.id, dest(), 'resend', false); await h.turn([createMockResponse([])]); const notice = h.texts().find((t) => t.startsWith('[drafts]'))!; - assert.match(notice, /^\[drafts\] scout: 2 held drafts of yours have not been named to you yet: /); + assert.match(notice, /^\[drafts\] scout: 2 held drafts of yours are unnamed on this branch of your history: /); assert.match(notice, /d-[a-z2-9]{5} \(held \(not sent\); held .*"orphaned words"\)/); assert.match(notice, new RegExp(`${midSend.id} \\(UNCONFIRMED — an attempt to #room \\(Guild One\\) \\(${ROOM}\\) at .* may have been posted; .*"half-sent words"`)); assert.doesNotMatch(notice, /They were not sent/); @@ -581,6 +651,52 @@ describe('held prose drafts, end to end', () => { } }); + it('the catch-up names every draft it marks: the newest eight with previews, the rest by id', async () => { + const h = await harness(); + try { + const held = Array.from({ length: 10 }, (_, i) => h.store().hold('scout', [{ text: `orphan ${i}`, source }], 'explicit-send').held[0]!); + await h.turn([createMockResponse([])]); + const notice = h.texts().find((t) => t.startsWith('[drafts]'))!; + assert.match(notice, /^\[drafts\] scout: 10 held drafts of yours are unnamed on this branch of your history: /); + // Newest first: orphan 9 down to orphan 2 with their previews, then the + // two oldest by id alone. + const previews = [...notice.matchAll(/"(orphan \d)"/g)].map((m) => m[1]); + assert.deepEqual(previews, [9, 8, 7, 6, 5, 4, 3, 2].map((i) => `orphan ${i}`)); + assert.ok(notice.includes(` · and 2 more: ${held[1]!.id}, ${held[0]!.id}.`), notice); + await h.turn([createMockResponse([])]); + assert.equal(h.texts().filter((t) => t.startsWith('[drafts]')).length, 1, 'all ten named, so none is named again'); + } finally { + await h.close(); + } + }); + + it('a rollback past the notice that named a draft names it again at the next turn, once', async () => { + const h = await harness(); + try { + await h.turn([createMockResponse([text('hello')])]); + const anchor = h.messages().at(-1)!.id; + await h.turn([ + createMockResponse([text('words a send held back'), explicitSend('s1')], 'tool_use'), + createMockResponse([]), + ]); + const [draft] = h.drafts(); + assert.ok(h.texts().some((t) => t.includes(`held as draft ${draft!.id}`)), 'the turn-end receipt names it'); + const catchUps = () => h.texts().filter((t) => t.includes('unnamed on this branch of your history')); + assert.deepEqual(catchUps(), []); + // The rollback takes the turn, and its notices, off the branch; the + // draft stays, as drafts do. + await h.framework.rollbackToMessage('scout', { messageId: anchor }); + assert.ok(!h.texts().some((t) => t.includes(draft!.id)), 'nothing on the branch names it now'); + await h.turn([createMockResponse([])]); + assert.equal(catchUps().length, 1); + assert.match(catchUps()[0]!, new RegExp(`a held draft of yours is unnamed on this branch of your history: ${draft!.id} \\(held \\(not sent\\);.*"words a send held back"`)); + await h.turn([createMockResponse([])]); + assert.equal(catchUps().length, 1, 'named once on this branch'); + } finally { + await h.close(); + } + }); + it('explicit mode: a bounce is a draft that survives a restart, and {{unsent}} delivers and settles it', async () => { const h = await harness({ agents: [{ name: 'scout', proseRouting: 'explicit' }] }); try { @@ -762,12 +878,14 @@ describe('draft notices and turn-end settlement', () => { const agent = framework.getAgent('scout')!; const [named, lost] = internals.proseDrafts.hold('scout', [{ text: 'named', source }, { text: 'lost', source }], 'explicit-send').held; // Only the first draft's notice reached the history. - agent.getContextManager().addMessage('user', [{ type: 'text', text: '[drafts] …' }], { system: true, kind: 'prose-drafts', draftOwner: 'scout', draftIds: [named!.id] } as never); + const cm = agent.getContextManager(); + const notice = cm.addMessage('user', [{ type: 'text', text: '[drafts] …' }], { system: true, kind: 'prose-drafts', draftOwner: 'scout', draftIds: [named!.id] } as never); internals.turnDrafts.set('scout', [named, lost]); internals.settleTurnDrafts(agent); - assert.ok(internals.proseDrafts.get('scout', named!.id)!.noticedAt); - assert.equal(internals.proseDrafts.get('scout', lost!.id)!.noticedAt, undefined, 'left for the catch-up'); - assert.deepEqual(internals.proseDrafts.unnoticed('scout').map((d) => d.id), [lost!.id]); + assert.deepEqual(internals.proseDrafts.get('scout', named!.id)!.noticedIn, [String(notice)], 'recorded with the message that names it'); + assert.equal(internals.proseDrafts.get('scout', lost!.id)!.noticedIn, undefined, 'left for the catch-up'); + const onBranch = (id: string) => cm.getMessage(id as never) !== null; + assert.deepEqual(internals.proseDrafts.unnoticed('scout', onBranch).map((d) => d.id), [lost!.id]); } finally { await framework.stop(); rmSync(dir, { recursive: true, force: true }); @@ -805,7 +923,7 @@ describe('draft identity and risk evidence', () => { { system: true, kind: 'prose-drafts', draftOwner: 'other', draftIds: ['d-same2'] } as never); t.internals.turnDrafts.set('scout', [mine]); t.internals.settleTurnDrafts(t.framework.getAgent('scout')!); - assert.equal(t.internals.proseDrafts.get('scout', 'd-same2')!.noticedAt, undefined); + assert.equal(t.internals.proseDrafts.get('scout', 'd-same2')!.noticedIn, undefined); } finally { await t.close(); } From 548071d19ca1510acddf962a4befdd3fd35fc791 Mon Sep 17 00:00:00 2001 From: Petra-1012 Date: Sun, 11 Oct 2026 14:49:29 -0400 Subject: [PATCH 0106/1112] fix(publish): a publish always has a deadline, a refused send opens nothing, and a failed outcome record says what the draft reads sw-af247, part 2: the publish executor and recording its outcomes. - I11: a channels/publish request always has a deadline: the server's requestTimeoutMs, or 60 s (publishDeadlineMs) when that is 0. A publish's outcome is told to the resident and recorded for its draft, so one that is never answered held both open for good. Its expiry is a no-response, which publish() reads as unknown. McplServerConfig.requestTimeoutMs's doc says so. - I35: publish() checks the server and its channels.publish grant before opening a closed channel, so a send that would be refused opens nothing and reports no opened-by-delivery. - I37: when recording an {{unsent}} delivery's outcome fails, the resident is told privately what the draft reads as now. - I38: the resend's line after a failed outcome record reports what the draft reads as now (draftStateNow): a record that failed after reaching the journal was reconciled into it, so it may read delivered, not the "will read as unconfirmed" it promised. - I3 (text): the comment above publish() says channel_publish moves onto it with shelf-355 (#256), not shelf-356. Tests: - mcpl-request-timeout: with requestTimeoutMs 0, channels/publish still times out (publishDeadlineMs shortened), as a no-response. - publish-outcome (I17): a closed destination opens first and sends; a failed open sends nothing; a send the grant forbids opens nothing. - prose-drafts (I5), through the fixture and the real connection: error mode reads unknown with the connector's words, for speech (the marker) and for a resend (the attempt); hang mode, with requestTimeoutMs 1000, reads unknown once the request times out. - prose-drafts (I37, I38): a sync that fails right after an outcome record lands, for a resend and for an {{unsent}} delivery. I32 is re-judged not-worth in the ledger: resolveDestination and publish's lookup run in the same tick, so a destination that doesn't resolve can't be sent to. --- src/framework.ts | 19 +++++- src/mcpl/channel-registry.ts | 39 ++++++----- src/mcpl/server-connection.ts | 12 +++- src/mcpl/types.ts | 4 +- test/mcpl-request-timeout.test.ts | 20 +++++- test/prose-drafts.test.ts | 105 ++++++++++++++++++++++++++++++ test/publish-outcome.test.ts | 43 ++++++++++-- 7 files changed, 215 insertions(+), 27 deletions(-) diff --git a/src/framework.ts b/src/framework.ts index 052ddd180..3cba84f6c 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -9478,7 +9478,9 @@ export class AgentFramework { ...(outcome.messageId ? { messageId: outcome.messageId } : {}), }); const where = AgentFramework.destinationText(outcome.destination ?? destination); - const unrecorded = recorded ? '' : ' (this outcome could not be recorded; the draft will read as unconfirmed)'; + // A record that failed after reaching the journal was reconciled into + // it, so say what the draft reads as now rather than guess. + const unrecorded = recorded ? '' : ` (recording this outcome failed; the draft reads as ${this.draftStateNow(agentName, d.id)} now)`; if (outcome.status === 'delivered') { lines.push(`${d.id}: delivered to ${where}${outcome.messageId ? `, message ${outcome.messageId}` : ''}${unrecorded}.`); let engaged = this.turnEngagedChannels.get(agentName); @@ -9846,11 +9848,26 @@ export class AgentFramework { this.proseDrafts.recordOutcome(agent.name, unsent.id, attemptId, outcome); } catch (err) { console.error(`[drafts] ${agent.name}: outcome of ${unsent.id} not recorded:`, err); + const text = + `[prose-routing] {{unsent}} was draft ${unsent.id}; its send came back ${outcome.status}, but recording that ` + + `failed, so the draft reads as ${this.draftStateNow(agent.name, unsent.id)} now. drafts(action: "read") shows it.`; + try { + this.addMessage('user', [{ type: 'text', text }], { system: true, kind: 'prose-bounce' }, { forAgent: agent.name }); + } catch (noticeErr) { + console.error('[drafts] unrecorded-outcome notice failed:', noticeErr); + } } } return outcome; } + /** A draft's state as the journal reads it now: after a write that failed + * and was reconciled, whatever reached the journal counts. */ + private draftStateNow(agentName: string, id: string): DraftState | 'let go' { + const draft = this.proseDrafts.get(agentName, id); + return draft ? draftState(draft) : 'let go'; + } + /** Locus-preserving explicit publication. Source remains byte-identical in Chronicle. */ private async deliverHybridProse( agent: Agent, diff --git a/src/mcpl/channel-registry.ts b/src/mcpl/channel-registry.ts index 3e00ff168..222fb2d03 100644 --- a/src/mcpl/channel-registry.ts +++ b/src/mcpl/channel-registry.ts @@ -3077,7 +3077,7 @@ export class ChannelRegistry { * established (PublishOutcome). The delivery executor behind plain speech * (deliverSpeech / routeSpeech) and resident resends of held drafts; * channel_publish (and publishForAgent, which shares its handler) moves - * onto it with shelf-356 — until then that tool reads its own receipt. + * onto it with shelf-355 — until then that tool reads its own receipt. * * Resolution is exact (resolveDestination): `serverId` plus `channelId` * when the caller has the server, otherwise the channel id must be @@ -3086,8 +3086,9 @@ export class ChannelRegistry { * in use"; a supplied one must be a non-empty server id, and an empty or * non-string selector is refused rather than read as omission. It is not * possible to send into a closed channel: a closed but registered - * destination is opened first (`openSource` names why), and a failed open - * sends nothing. + * destination is opened first (`openSource` names why), once its server + * and that server's channels.publish grant are known to take the send, so + * a refused send opens nothing; and a failed open sends nothing. * * Only the connector's `delivered: true` confirms a post. `failed` means * nothing was posted: the attempt stopped before the request was written, @@ -3115,6 +3116,23 @@ export class ChannelRegistry { ...(entry.descriptor.label ? { label: entry.descriptor.label } : {}), }; + // A send that would be refused is refused before anything opens: a + // closed channel opens only for words that can go out. + const server = this.serverRegistry.getServer(entry.serverId); + if (!server) { + return { status: 'failed', destination, reason: `server "${entry.serverId}" not found`, at: at() }; + } + // §14.1: channels/publish requires channels.publish in the grant. The + // host not sending is the enforcement. + if (!CapabilityGrant.of(server).has('channels.publish')) { + return { + status: 'failed', + destination, + reason: `channels.publish not in "${entry.serverId}"'s effective grant (§14.1)`, + at: at(), + }; + } + if (!entry.open) { try { await this.openChannelNow(entry, openSource); @@ -3145,21 +3163,6 @@ export class ChannelRegistry { } } - const server = this.serverRegistry.getServer(entry.serverId); - if (!server) { - return { status: 'failed', destination, reason: `server "${entry.serverId}" not found`, at: at() }; - } - // §14.1: channels/publish requires channels.publish in the grant. The - // host not sending is the enforcement. - if (!CapabilityGrant.of(server).has('channels.publish')) { - return { - status: 'failed', - destination, - reason: `channels.publish not in "${entry.serverId}"'s effective grant (§14.1)`, - at: at(), - }; - } - let result: unknown; try { result = await server.sendChannelsPublish({ diff --git a/src/mcpl/server-connection.ts b/src/mcpl/server-connection.ts index 18f82d096..0a0025fca 100644 --- a/src/mcpl/server-connection.ts +++ b/src/mcpl/server-connection.ts @@ -240,6 +240,9 @@ export class McplServerConnection extends EventEmitter { /** Per-request timeout in ms (0 disables). See McplServerConfig.requestTimeoutMs. */ private requestTimeoutMs = DEFAULT_REQUEST_TIMEOUT_MS; + /** The deadline a channels/publish request keeps when requestTimeoutMs is 0 + * (see sendChannelsPublish). */ + private publishDeadlineMs = DEFAULT_REQUEST_TIMEOUT_MS; // Event buffering: control-plane traffic can be released independently from // inference-bearing data-plane traffic. This lets a server finish its own @@ -775,13 +778,20 @@ export class McplServerConnection extends EventEmitter { * Send `channels/publish`. * May be sent as a request (if an ACK is desired) or notification. * When `params.stream` is true, sends as a notification (no ACK). + * + * A request always has a deadline: the server's requestTimeoutMs, or + * publishDeadlineMs when that is 0 (disabled). A publish is a side effect + * whose outcome the resident is told and a held draft's attempt records, + * so one that is never answered would hold both open for good. Its expiry + * is a no-response, which the caller reads as unknown. */ sendChannelsPublish(params: ChannelsPublishParams): Promise { if (params.stream) { this.sendNotification(McplMethod.ChannelsPublish, params as unknown as Record); return Promise.resolve(); } - return this.sendRequest(McplMethod.ChannelsPublish, params as unknown as Record) as Promise; + const timeoutMs = this.requestTimeoutMs > 0 ? this.requestTimeoutMs : this.publishDeadlineMs; + return this.sendRequest(McplMethod.ChannelsPublish, params as unknown as Record, { timeoutMs }) as Promise; } /** Send `channels/outgoing/chunk` notification. */ diff --git a/src/mcpl/types.ts b/src/mcpl/types.ts index 0f913ea5a..fbc848860 100644 --- a/src/mcpl/types.ts +++ b/src/mcpl/types.ts @@ -374,7 +374,9 @@ export interface McplServerConfig { * accepts a request and never responds would otherwise freeze the awaiting * agent turn forever; with the timeout the pending request rejects with a * descriptive error, which the framework surfaces as a normal isError - * tool_result. Set 0 to disable. Default: 60000 (60 seconds). + * tool_result. Set 0 to disable, except for a channels/publish request, + * which keeps the 60 s default: its outcome is reported to the resident and + * recorded for its draft, so it must end. Default: 60000 (60 seconds). */ requestTimeoutMs?: number; diff --git a/test/mcpl-request-timeout.test.ts b/test/mcpl-request-timeout.test.ts index 2778469e3..8ebf050a6 100644 --- a/test/mcpl-request-timeout.test.ts +++ b/test/mcpl-request-timeout.test.ts @@ -8,7 +8,7 @@ import { test, afterEach } from 'node:test'; import assert from 'node:assert/strict'; -import { McplServerConnection } from '../src/mcpl/server-connection.js'; +import { McplRequestError, McplServerConnection } from '../src/mcpl/server-connection.js'; import type { McplHostCapabilities, McplServerConfig } from '../src/mcpl/types.js'; // Stdio server: answers initialize and tools/list, but swallows tools/call — @@ -100,6 +100,24 @@ test('requestTimeoutMs: 0 disables the per-request watchdog', async () => { assert.equal(settled, true); }); +test('requestTimeoutMs: 0 leaves channels/publish its own deadline, whose expiry is a no-response', async () => { + connection = await McplServerConnection.connect(config({ requestTimeoutMs: 0 }), HOST_CAPS); + connection.ready(); + // The default is 60 s; the test shortens it rather than wait. + (connection as unknown as { publishDeadlineMs: number }).publishDeadlineMs = 150; + const started = Date.now(); + await assert.rejects( + connection.sendChannelsPublish({ conversationId: 'c', channelId: 'x', content: [{ type: 'text', text: 'hi' }] }), + (err: Error) => { + assert.ok(err instanceof McplRequestError); + assert.equal(err.outcome, 'no-response'); + assert.match(err.message, /did not respond to channels\/publish .*within 150ms/); + return true; + }, + ); + assert.ok(Date.now() - started < 5000); +}); + test('the timeout error warns the model the tool may have completed server-side', async () => { connection = await McplServerConnection.connect(config({ requestTimeoutMs: 150 }), HOST_CAPS); connection.ready(); diff --git a/test/prose-drafts.test.ts b/test/prose-drafts.test.ts index cf175e9c8..b264fda27 100644 --- a/test/prose-drafts.test.ts +++ b/test/prose-drafts.test.ts @@ -272,6 +272,8 @@ function xmlMode(inner: MockMembrane): MockMembrane { async function harness(opts: { agents?: Array<{ name: string; proseRouting?: 'locus' | 'explicit' | 'hybrid' | 'disabled'; sameRoundThinkTextPolicy?: 'public' | 'private' }>; xml?: boolean; + /** The fixture server's requestTimeoutMs (the framework's default when unset). */ + requestTimeoutMs?: number; } = {}) { const dir = mkdtempSync(join(tmpdir(), 'prose-drafts-e2e-')); const commandPath = join(dir, 'commands.jsonl'); @@ -301,6 +303,7 @@ async function harness(opts: { command: process.execPath, args: [join(import.meta.dirname, 'fixtures/speech-route-mcpl-server.mjs')], env: { STATUS_PATH: statusPath, COMMAND_PATH: commandPath }, + ...(opts.requestTimeoutMs !== undefined ? { requestTimeoutMs: opts.requestTimeoutMs } : {}), }], modules: [], }); @@ -549,6 +552,48 @@ describe('held prose drafts, end to end', () => { } }); + it('through the real connection, an error response is unconfirmed, with the connector\'s words', async () => { + const h = await harness(); + try { + h.command({ op: 'publish-mode', mode: 'error' }); + await h.turn([createMockResponse([text('words into a failing connector')])]); + assert.equal(h.publishes().filter((p) => p.event === 'publish-error').length, 1, 'the request reached the server'); + await h.until(() => h.texts().some((t) => t.startsWith('[discord-send-failed]')), 'the marker'); + const marker = h.texts().find((t) => t.startsWith('[discord-send-failed]'))!; + assert.match(marker, /was not confirmed \(.*connector exploded.*\(delivery uncertain\)\)\. It may or may not have been posted; check the channel/); + // A resend of a held draft records the same outcome. + await h.turn([ + createMockResponse([text('held beside a send'), explicitSend('s1')], 'tool_use'), + createMockResponse([]), + ]); + const [draft] = h.drafts(); + await h.turn([ + createMockResponse([drafts('r1', { action: 'resend', draftIds: [draft!.id], destination: ROOM })], 'tool_use'), + createMockResponse([]), + ]); + const [attempt] = h.store().get('scout', draft!.id)!.attempts; + assert.equal(attempt!.outcome?.status, 'unknown'); + assert.match(attempt!.outcome!.reason!, /connector exploded/); + assert.equal(draftState(h.store().get('scout', draft!.id)!), 'unconfirmed'); + } finally { + await h.close(); + } + }); + + it('through the real connection, a server that never answers is unconfirmed once the request times out', async () => { + const h = await harness({ requestTimeoutMs: 1_000 }); + try { + h.command({ op: 'publish-mode', mode: 'hang' }); + await h.turn([createMockResponse([text('words into silence')])]); + await h.until(() => h.texts().some((t) => t.startsWith('[discord-send-failed]')), 'the marker'); + assert.equal(h.publishes().length, 0); + const marker = h.texts().find((t) => t.startsWith('[discord-send-failed]'))!; + assert.match(marker, /was not confirmed \(MCPL server "discord" did not respond to channels\/publish .*within 1000ms.*\(delivery uncertain\)\)\. It may or may not have been posted/); + } finally { + await h.close(); + } + }); + it('an unknown outcome is not cleared by a later failure: the draft stays unconfirmed', async () => { const h = await harness(); try { @@ -697,6 +742,66 @@ describe('held prose drafts, end to end', () => { } }); + /** Make the store's sync fail once, right after a draft outcome record + * reaches it: the record lands, its write fails, and the journal is + * reconciled with it in. */ + const failOutcomeSyncOnce = (h: Awaited>) => { + const store = (h.framework as unknown as { store: JsStore }).store; + const real = { appendJson: store.appendJson.bind(store), sync: store.sync.bind(store) }; + let armed = false; + let spent = false; + (store as unknown as { appendJson: (t: string, p: unknown) => unknown }).appendJson = (t, p) => { + const ref = real.appendJson(t, p); + if (!spent && t === PROSE_DRAFT_RECORD_TYPE && (p as { op?: string }).op === 'outcome') armed = true; + return ref; + }; + (store as unknown as { sync: () => void }).sync = () => { + if (armed && !spent) { + spent = true; + throw new Error('EIO after the outcome landed'); + } + real.sync(); + }; + }; + + it('a resend whose outcome record fails after landing reports what the draft reads as now', async (t) => { + const h = await harness(); + try { + await h.turn([ + createMockResponse([text('words to resend'), explicitSend('s1')], 'tool_use'), + createMockResponse([]), + ]); + const [draft] = h.drafts(); + failOutcomeSyncOnce(h); + t.mock.method(console, 'error', () => {}); + await h.turn([ + createMockResponse([drafts('r1', { action: 'resend', draftIds: [draft!.id], destination: ROOM })], 'tool_use'), + createMockResponse([]), + ]); + assert.match(h.toolResults()[0]!, new RegExp(`${draft!.id}: delivered to .*\\(recording this outcome failed; the draft reads as delivered now\\)\\.`)); + assert.equal(draftState(h.store().get('scout', draft!.id)!), 'delivered'); + } finally { + await h.close(); + } + }); + + it('an {{unsent}} delivery whose outcome record fails tells the resident what the draft reads as now', async (t) => { + const h = await harness({ agents: [{ name: 'scout', proseRouting: 'explicit' }] }); + try { + await h.turn([createMockResponse([text('words with nowhere to go')])]); + const [bounced] = h.drafts(); + failOutcomeSyncOnce(h); + t.mock.method(console, 'error', () => {}); + await h.turn([createMockResponse([text(`>>${ROOM} {{unsent}}`)])]); + assert.deepEqual(h.publishes().map((p) => p.text), ['words with nowhere to go']); + await h.until(() => h.texts().some((x) => x.includes('but recording that failed')), 'the notice'); + assert.ok(h.texts().some((x) => x === `[prose-routing] {{unsent}} was draft ${bounced!.id}; its send came back delivered, but recording that ` + + 'failed, so the draft reads as delivered now. drafts(action: "read") shows it.'), JSON.stringify(h.texts().filter((x) => x.startsWith('[prose-routing]')))); + } finally { + await h.close(); + } + }); + it('explicit mode: a bounce is a draft that survives a restart, and {{unsent}} delivers and settles it', async () => { const h = await harness({ agents: [{ name: 'scout', proseRouting: 'explicit' }] }); try { diff --git a/test/publish-outcome.test.ts b/test/publish-outcome.test.ts index d9d2d79d3..13f298add 100644 --- a/test/publish-outcome.test.ts +++ b/test/publish-outcome.test.ts @@ -24,8 +24,12 @@ import { MockMembrane, createMockResponse } from './helpers/mock-membrane.js'; type Publish = (params: { channelId: string }) => Promise; -function registryWith(servers: Record, failures: unknown[] = []) { +function registryWith( + servers: Record Promise }>, + failures: unknown[] = [], +) { const published: Array<{ serverId: string; channelId: string }> = []; + const opened: Array<{ serverId: string; channelId: string }> = []; const mocks = new Map(); for (const [id, s] of Object.entries(servers)) { mocks.set(id, { @@ -35,7 +39,10 @@ function registryWith(servers: Record ({}), + sendChannelsOpen: async (params: { channelId: string }) => { + opened.push({ serverId: id, channelId: params.channelId }); + return s.open ? s.open() : {}; + }, }); } const registry = new ChannelRegistry( @@ -48,9 +55,9 @@ function registryWith(servers: Record; }).channels; - const seed = (serverId: string, id: string, label: string) => - channels.set(`${serverId}:${id}`, { serverId, descriptor: { id, type: 'discord', label }, open: true }); - return { registry, published, seed }; + const seed = (serverId: string, id: string, label: string, open = true) => + channels.set(`${serverId}:${id}`, { serverId, descriptor: { id, type: 'discord', label }, open }); + return { registry, published, opened, seed }; } const ok: Publish = async () => ({ delivered: true, messageId: 'posted-1' }); @@ -66,6 +73,32 @@ describe('ChannelRegistry.publish outcomes', () => { assert.equal(typeof outcome.at, 'number'); }); + it('opens a closed destination first, and a failed open sends nothing', async () => { + const opens = registryWith({ discord: { publish: ok } }); + opens.seed('discord', 'c', '#c', false); + assert.equal((await opens.registry.publish('agent', 'x', { channelId: 'c' })).status, 'delivered'); + assert.deepEqual(opens.opened, [{ serverId: 'discord', channelId: 'c' }]); + assert.deepEqual(opens.published, [{ serverId: 'discord', channelId: 'c' }]); + + const fails = registryWith({ discord: { publish: ok, open: async () => { throw new Error('no such channel'); } } }); + fails.seed('discord', 'c', '#c', false); + const outcome = await fails.registry.publish('agent', 'x', { channelId: 'c' }); + assert.equal(outcome.status, 'failed'); + assert.equal(outcome.reason, 'channel is closed and open failed: no such channel'); + assert.deepEqual(fails.opened, [{ serverId: 'discord', channelId: 'c' }]); + assert.deepEqual(fails.published, [], 'nothing dispatched after a failed open'); + }); + + it('refuses a send the grant forbids before opening anything', async () => { + const { registry, seed, opened, published } = registryWith({ discord: { publish: ok, grant: false } }); + seed('discord', 'c', '#c', false); + const outcome = await registry.publish('agent', 'x', { channelId: 'c' }); + assert.equal(outcome.status, 'failed'); + assert.match(outcome.reason!, /channels\.publish not in "discord"'s effective grant/); + assert.deepEqual(opened, [], 'a refused send opens nothing'); + assert.deepEqual(published, []); + }); + it('treats delivered:false with no posted message as failed', async () => { const { registry, seed } = registryWith({ discord: { publish: async () => ({ delivered: false }) } }); seed('discord', 'c', '#c'); From d80210e460936e3fa2dceea9ff0b25896bd0f092 Mon Sep 17 00:00:00 2001 From: Petra-1012 Date: Sun, 11 Oct 2026 14:56:13 -0400 Subject: [PATCH 0107/1112] fix(drafts): a refused {{unsent}} holds its envelope, dismissals name their risk, and every stop and wait says what it is sw-af247, part 3: {{unsent}}, dismissals, and what the resident is told. - I19: a deliverable envelope whose {{unsent}} is refused (its draft is being sent, queued in a resend, or unconfirmed) is held as a bounced envelope would be: one bounce draft with the whole text, {{unsent}} written out under the re-bounce rule and its risk carried, named in the refusal notice. A bare {{unsent}} holds nothing new: its words already are that draft. takeUnsent takes the envelope's body and hold. - I26 and I43: {{unsent}} says "being sent right now" only while the draft's attempt is out (draftsSending), and "queued in a resend in progress" for a draft a batch has only claimed. The risk a queued draft's words carry says it was queued. Each draft a resend batch doesn't attempt gives the stop's own reason: an earlier draft not confirmed delivered, dismissed or changed while it waited, no longer found, or its attempt not recorded. - I23: dismissing a draft that may already have been posted names that risk, in the drafts tool's line and, for >>skip_reply {{unsent}}, in a private notice. - I31: when >>skip_reply {{unsent}} can't record the dismissal, the resident is told the bounce is still held and {{unsent}} still names it. - I44: an attempt still out (its draft's latest, while being sent) is not called one whose outcome "was never recorded": riskText, drafts read and a copied draft's inherited risk say it is being sent, its outcome not known yet (attemptIsOut). Tests (prose-drafts): - I8: the resend-ownership tests wait until a publish is pending, bounded at 10 s, instead of a fixed 5 ms. - I14: the publish-throws test checks that publish was entered, that the resend rejects with the stub's error, and that the draft reads unconfirmed before the dismissal, which now names the risk. - New: stop reasons for a dismissed and an unconfirmed draft; drafts read of an attempt still out; {{unsent}} of a bounce queued in a resend, holding the envelope with the risk; >>skip_reply {{unsent}} when the dismissal can't be recorded, then for an unconfirmed bounce. - The re-bounce test's refusal wording is updated. --- src/framework.ts | 140 +++++++++++++++++++++++++++----------- test/prose-drafts.test.ts | 136 ++++++++++++++++++++++++++++++++---- 2 files changed, 222 insertions(+), 54 deletions(-) diff --git a/src/framework.ts b/src/framework.ts index 3cba84f6c..783d02b31 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -85,7 +85,7 @@ import { ProseStreamRouter } from './mcpl/prose-stream-router.js'; import { detectKnownToolWrapperProse } from './tool-wrapper-prose-guard.js'; import { InferenceRouter } from './mcpl/inference-router.js'; import { ChannelRegistry, type ChannelToolOrigin, type PublishDestination, type PublishOutcome } from './mcpl/channel-registry.js'; -import { KEEP_IDLE_MS, KEEP_NEWEST, ProseDraftStore, draftState, uncertainAttempt, type Draft, type DraftReason, type DraftSource, type DraftState, type InheritedRisk } from './prose-drafts.js'; +import { KEEP_IDLE_MS, KEEP_NEWEST, ProseDraftStore, draftState, uncertainAttempt, type Draft, type DraftAttempt, type DraftReason, type DraftSource, type DraftState, type InheritedRisk } from './prose-drafts.js'; import { ConversationRouter, CONVERSATION_ROUTING_DEPRECATION_NOTICE } from './mcpl/conversation-router.js'; import { INBOUND_SOURCE_KEY, type InboundAcceptanceObserver, type InboundChannelSource, type InboundLane, type InboundSource, type InboundUnscopedSource } from './mcpl/inbound-source.js'; import { safeSlice } from './safe-slice.js'; @@ -9117,7 +9117,7 @@ export class AgentFramework { const risky = uncertainAttempt(draft); if (risky) { return `${draft.id}'s attempt to ${AgentFramework.destinationText(risky.destination)} at ` + - `${this.draftTime(risky.at)} may already have been posted: ${risky.outcome?.reason ?? 'its outcome was never recorded'}`; + `${this.draftTime(risky.at)} may already have been posted: ${risky.outcome?.reason ?? this.noOutcomeText(draft, risky)}`; } const inherited = draft.inheritedRisk!; const owner = inherited.sourceDraftId ?? inherited.draftId; @@ -9129,6 +9129,18 @@ export class AgentFramework { return `${draft.id} includes the words of ${inherited.draftId}${via}, and ${where} may already have been posted: ${inherited.reason}`; } + /** Whether an attempt with no outcome is still out: its draft's latest, + * while that draft is being sent. Otherwise it never got one (a crash). */ + private attemptIsOut(draft: Draft, attempt: DraftAttempt): boolean { + return !attempt.outcome && this.draftsSending.has(`${draft.agent}\u0000${draft.id}`) && + draft.attempts.at(-1)?.attemptId === attempt.attemptId; + } + + /** Why an attempt has no outcome, in a risk's words. */ + private noOutcomeText(draft: Draft, attempt: DraftAttempt): string { + return this.attemptIsOut(draft, attempt) ? "it is being sent right now, so its outcome isn't known yet" : 'its outcome was never recorded'; + } + /** A delivered draft's historical receipt. */ private draftReceipt(draft: Draft): string { const delivered = draft.attempts.find((a) => a.outcome?.status === 'delivered'); @@ -9323,7 +9335,7 @@ export class AgentFramework { ...d.attempts.map((a) => { const o = a.outcome; const status = !o - ? 'outcome never recorded — it may have been posted' + ? (this.attemptIsOut(d, a) ? 'being sent right now — no outcome yet' : 'outcome never recorded — it may have been posted') : o.status === 'delivered' ? `delivered${o.messageId ? `, message ${o.messageId}` : ''}` : o.status === 'unknown' @@ -9360,6 +9372,9 @@ export class AgentFramework { lines.push(`${d.id}: already dismissed.`); continue; } + // A dismissal sets the words aside; it can't take back an attempt + // that may already have been posted, so that is said with it. + const risk = state === 'unconfirmed' ? ` Note that ${this.riskText(d)}.` : ''; try { this.proseDrafts.dismiss(agentName, d.id); } catch (err) { @@ -9368,7 +9383,7 @@ export class AgentFramework { `${err instanceof Error ? err.message : String(err)}. It is still open.`, ); } - lines.push(`${d.id}: dismissed (your history keeps the words).`); + lines.push(`${d.id}: dismissed (your history keeps the words).${risk}`); } this.emitTrace({ type: 'prose:drafts-dismissed', agentName, draftIds: drafts.map((d) => d.id) }); return ok(lines.join('\n')); @@ -9424,11 +9439,13 @@ export class AgentFramework { const claims = drafts.map((d) => `${agentName}\u0000${d.id}`); for (const claim of claims) this.draftsInFlight.add(claim); const lines: string[] = []; - let stopped = false; + // Why the batch stopped, said for each draft after it: the resend sends + // in order, so one that can't go stops the rest. + let stopped: string | undefined; try { for (const queued of drafts) { if (stopped) { - lines.push(`${queued.id}: not attempted (an earlier draft was not confirmed delivered).`); + lines.push(`${queued.id}: not attempted (${stopped}).`); continue; } // The current projection, not the batch's snapshot (it may have been @@ -9436,7 +9453,7 @@ export class AgentFramework { const d = this.proseDrafts.get(agentName, queued.id); if (!d) { lines.push(`${queued.id}: no longer found among your drafts — not sent.`); - stopped = true; + stopped = `${queued.id} was no longer found`; continue; } if (draftState(d) === 'delivered') { @@ -9446,7 +9463,7 @@ export class AgentFramework { const refusal = this.resendRefusal(d, confirmDuplicate); if (refusal) { lines.push(`${refusal} It changed while this resend was waiting — not sent.`); - stopped = true; + stopped = draftState(d) === 'dismissed' ? `${d.id} was dismissed while it waited` : `${d.id} changed while it waited`; continue; } let attemptId: string; @@ -9454,7 +9471,7 @@ export class AgentFramework { attemptId = this.proseDrafts.beginAttempt(agentName, d.id, destination, 'resend', draftState(d) === 'unconfirmed'); } catch (err) { lines.push(`${d.id}: not sent — the attempt could not be recorded durably (${err instanceof Error ? err.message : String(err)}).`); - stopped = true; + stopped = `${d.id}'s attempt could not be recorded`; continue; } const sendingKey = `${agentName}\u0000${d.id}`; @@ -9491,7 +9508,7 @@ export class AgentFramework { engaged.add(destination.channelId); continue; } - stopped = true; + stopped = `${d.id} was not confirmed delivered`; const after = this.proseDrafts.get(agentName, d.id); const stillAtRisk = after !== undefined && draftState(after) === 'unconfirmed'; lines.push(outcome.status === 'unknown' @@ -9775,31 +9792,60 @@ export class AgentFramework { } /** - * `{{unsent}}` in an envelope body: the latest bounce draft's words. An - * unconfirmed one (its last attempt may have been posted) is never - * substituted silently — the resident gets the facts and the explicit - * route (drafts resend with confirmDuplicate) instead. + * `{{unsent}}` in an envelope body: the latest bounce draft's words. One + * that is out (being sent, or queued in a resend) or unconfirmed (its last + * attempt may have been posted) is never substituted — nothing is sent, + * and the resident gets the facts and the explicit route (drafts resend + * with confirmDuplicate) instead. The envelope's own words are held, as a + * bounced envelope's would be (unsentWords): one bounce draft with the + * whole text, `{{unsent}}` written out and its risk carried, so nothing + * the resident wrote has to be typed again. A bare `{{unsent}}` holds + * nothing new: its words already are that draft. */ - private takeUnsent(agent: Agent, prefix: '>>' | '>>>'): { draft?: Draft; refused?: true } { + private takeUnsent( + agent: Agent, + body: string, + hold: { round?: number; notice: 'now' | 'later' }, + ): { draft?: Draft; refused?: true } { const draft = this.proseDrafts.latestBounce(agent.name); if (!draft) return {}; - if (this.draftsInFlight.has(`${agent.name}\u0000${draft.id}`)) { - const busy = `[prose-routing] {{unsent}} is draft ${draft.id}, which is being sent right now. Nothing was sent; wait for that result.`; - try { - this.addMessage('user', [{ type: 'text', text: busy }], { system: true, kind: 'prose-bounce' }, { forAgent: agent.name }); - } catch (err) { - console.error('[drafts] in-flight-unsent notice failed:', err); - } - return { draft, refused: true }; + const key = `${agent.name}\u0000${draft.id}`; + let why: string; + let next: string; + if (this.draftsInFlight.has(key)) { + why = this.draftsSending.has(key) ? 'which is being sent right now' : 'which is queued in a resend in progress'; + next = 'wait for that result'; + } else if (draftState(draft) === 'unconfirmed') { + why = this.riskText(draft); + next = 'check that channel; to send that draft again anyway use ' + + `drafts(action: "resend", draftIds: ["${draft.id}"], destination: "#channel", confirmDuplicate: true)`; + } else { + return { draft }; } - if (draftState(draft) !== 'unconfirmed') return { draft }; - const text = - `[prose-routing] {{unsent}} is draft ${draft.id}: ${this.riskText(draft)}. Nothing was sent. Check that channel; ` + - `to send it again anyway use drafts(action: "resend", draftIds: ["${draft.id}"], destination: "#channel", confirmDuplicate: true).`; + const words = this.unsentWords(agent, body); + const held = 'draft' in words + ? undefined + : this.holdProseDrafts( + agent, + [{ text: words.text, ...(words.inheritedRisk ? { inheritedRisk: words.inheritedRisk } : {}) }], + 'bounced', + { round: hold.round, note: `{{unsent}} was draft ${draft.id}, which could not be sent`, notice: 'later' }, + )[0]; + const kept = held + ? ` Your message is held as draft ${held.id}, with {{unsent}} written out; since it includes ${draft.id}'s words, it ` + + 'can be resent only with confirmDuplicate: true, after that.' + : ''; + const text = `[prose-routing] {{unsent}} is draft ${draft.id}, ${why.startsWith('which') ? why : `and ${why}`}. ` + + `Nothing was sent; ${next}.${kept}`; try { - this.addMessage('user', [{ type: 'text', text }], { system: true, kind: 'prose-bounce' }, { forAgent: agent.name }); + this.addMessage( + 'user', + [{ type: 'text', text }], + { system: true, kind: 'prose-bounce', ...(held ? { draftOwner: agent.name, draftIds: [held.id] } : {}) } as MessageMetadata, + { forAgent: agent.name }, + ); } catch (err) { - console.error('[drafts] unconfirmed-unsent notice failed:', err); + console.error('[drafts] refused-unsent notice failed:', err); } return { draft, refused: true }; } @@ -9899,7 +9945,7 @@ export class AgentFramework { } let body = parsed.body; const usedClipboard = body.includes('{{unsent}}'); - const unsent = usedClipboard ? this.takeUnsent(agent, '>>>') : {}; + const unsent = usedClipboard ? this.takeUnsent(agent, body, hold) : {}; if (unsent.refused) { this.proseTargetPins.delete(agent.name); this.proseHybridSuppressed.set(agent.name, 'failed'); @@ -9966,26 +10012,35 @@ export class AgentFramework { /** `>>skip_reply {{unsent}}` and its hybrid form: set the latest bounce * draft aside — unless it is being sent right now, which a dismissal - * can't call back: the resident is told instead. */ + * can't call back: the resident is told instead. So are a dismissal that + * can't be recorded (the draft stays, and `{{unsent}}` still names it) + * and one whose draft may already have been posted. */ private dismissLatestBounce(agent: Agent): void { const draft = this.proseDrafts.latestBounce(agent.name); if (!draft) return; - if (this.draftsSending.has(`${agent.name}\u0000${draft.id}`)) { - const busy = `[prose-routing] {{unsent}} is draft ${draft.id}, which is being sent right now. It was not set aside, ` + - 'since that can\'t call back words already on their way; wait for that result.'; + const tell = (text: string, what: string): void => { try { - this.addMessage('user', [{ type: 'text', text: busy }], { system: true, kind: 'prose-bounce' }, { forAgent: agent.name }); + this.addMessage('user', [{ type: 'text', text }], { system: true, kind: 'prose-bounce' }, { forAgent: agent.name }); } catch (err) { - console.error('[drafts] in-flight-dismiss notice failed:', err); + console.error(`[drafts] ${what} notice failed:`, err); } + }; + if (this.draftsSending.has(`${agent.name}\u0000${draft.id}`)) { + tell(`[prose-routing] {{unsent}} is draft ${draft.id}, which is being sent right now. It was not set aside, ` + + 'since that can\'t call back words already on their way; wait for that result.', 'in-flight-dismiss'); return; } + const risk = draftState(draft) === 'unconfirmed' ? this.riskText(draft) : undefined; try { this.proseDrafts.dismiss(agent.name, draft.id); this.emitTrace({ type: 'prose:drafts-dismissed', agentName: agent.name, draftIds: [draft.id] }); } catch (err) { console.error(`[drafts] ${agent.name}: could not dismiss ${draft.id}:`, err); + tell(`[prose-routing] {{unsent}} is draft ${draft.id}, which could not be set aside ` + + `(${err instanceof Error ? err.message : String(err)}): it is still held, and {{unsent}} still names it.`, 'failed-dismiss'); + return; } + if (risk) tell(`[prose-routing] Draft ${draft.id} is set aside. Note that ${risk}.`, 'at-risk-dismiss'); } private async deliverProseEnvelope(agent: Agent, rawText: string, hold: { round?: number; notice: 'now' | 'later' }): Promise { @@ -10025,7 +10080,7 @@ export class AgentFramework { } const usedClipboard = body.includes('{{unsent}}'); - const unsent = usedClipboard ? this.takeUnsent(agent, '>>') : {}; + const unsent = usedClipboard ? this.takeUnsent(agent, body, hold) : {}; if (unsent.refused) return; if (usedClipboard) { body = body.replaceAll('{{unsent}}', unsent.draft?.text ?? ''); @@ -10074,7 +10129,9 @@ export class AgentFramework { draftId: previous.id, destination: risky.destination, at: risky.at, - reason: risky.outcome?.reason ?? 'its outcome was never recorded', + reason: risky.outcome?.reason ?? (this.attemptIsOut(previous, risky) + ? "it was being sent when these words were held, so its outcome wasn't known yet" + : 'its outcome was never recorded'), } : previous.inheritedRisk ? { @@ -10084,7 +10141,12 @@ export class AgentFramework { ...(previous.inheritedRisk.at !== undefined ? { at: previous.inheritedRisk.at } : {}), reason: previous.inheritedRisk.reason, } - : { draftId: previous.id, reason: 'it was being sent when these words were held' }; + : { + draftId: previous.id, + reason: this.draftsSending.has(`${name}\u0000${previous.id}`) + ? 'it was being sent when these words were held' + : 'it was queued in a resend when these words were held', + }; return { text: expanded, inheritedRisk }; } diff --git a/test/prose-drafts.test.ts b/test/prose-drafts.test.ts index b264fda27..9635ad821 100644 --- a/test/prose-drafts.test.ts +++ b/test/prose-drafts.test.ts @@ -841,7 +841,7 @@ describe('held prose drafts, end to end', () => { const before = h.publishes().length; await h.turn([createMockResponse([text(`>>${ROOM} {{unsent}}`)])]); assert.equal(h.publishes().length, before, 'neither draft goes out without confirmation'); - assert.ok(h.texts().some((t) => t.startsWith(`[prose-routing] {{unsent}} is draft ${fresh.id}: ${fresh.id} includes the words of ${risky!.id}`))); + assert.ok(h.texts().some((t) => t.startsWith(`[prose-routing] {{unsent}} is draft ${fresh.id}, and ${fresh.id} includes the words of ${risky!.id}`))); await h.turn([createMockResponse([drafts('r1', { action: 'resend', draftIds: [fresh.id], destination: ROOM, confirmDuplicate: true })], 'tool_use'), createMockResponse([])]); assert.deepEqual(h.publishes().slice(before).map((p) => p.text), ['risky words and a postscript']); assert.equal(draftState(h.store().get('scout', risky!.id)!), 'unconfirmed', 'the original still needs its own confirmation'); @@ -1091,10 +1091,18 @@ describe('drafts resend ownership', () => { }; const say = (r: { success: boolean; data?: Array<{ text: string }>; error?: string }) => r.success ? r.data!.map((b) => b.text).join('') : r.error!; - const tick = () => new Promise((r) => setTimeout(r, 5)); + /** Until a publish is waiting on the test, bounded: what the next step + * needs, rather than a fixed pause that a loaded machine can outrun. */ + const untilPublishing = async () => { + const deadline = Date.now() + 10_000; + while (pending.length === 0) { + if (Date.now() > deadline) throw new Error('no publish reached the stub within 10 s'); + await new Promise((r) => setTimeout(r, 2)); + } + }; const [a, b] = internals.proseDrafts.hold('scout', [{ text: 'A words', source }, { text: 'B words', source }], 'explicit-send').held; return { - dir, framework, internals, pending, say, tick, a: a!, b: b!, + dir, framework, internals, pending, say, untilPublishing, a: a!, b: b!, close: async () => { await framework.stop(); rmSync(dir, { recursive: true, force: true }); }, }; }; @@ -1103,7 +1111,7 @@ describe('drafts resend ownership', () => { const t = await setup(); try { const sending = t.internals.handleDraftsTool('scout', { action: 'resend', draftIds: [t.a.id], destination: 'chan' }); - await t.tick(); + await t.untilPublishing(); t.pending.shift()!.resolve({ status: 'delivered', messageId: 'm-a' }); assert.match(t.say(await sending), /delivered to \(chan\), message m-a/); // The channel is gone: destination resolution would now fail. @@ -1125,12 +1133,12 @@ describe('drafts resend ownership', () => { const t = await setup(); try { const batch = t.internals.handleDraftsTool('scout', { action: 'resend', draftIds: [t.a.id, t.b.id], destination: 'chan' }); - await t.tick(); + await t.untilPublishing(); assert.deepEqual(t.pending.map((p) => p.text), ['A words'], 'the batch is waiting on A'); const concurrent = t.say(await t.internals.handleDraftsTool('scout', { action: 'resend', draftIds: [t.b.id], destination: 'chan' })); assert.match(concurrent, new RegExp(`${t.b.id} is being sent right now \\(or is queued in a resend in progress\\)`)); t.pending.shift()!.resolve({ status: 'delivered', messageId: 'm-a' }); - await t.tick(); + await t.untilPublishing(); assert.deepEqual(t.pending.map((p) => p.text), ['B words']); t.pending.shift()!.resolve({ status: 'delivered', messageId: 'm-b' }); const result = t.say(await batch); @@ -1146,7 +1154,7 @@ describe('drafts resend ownership', () => { const t = await setup(); try { const batch = t.internals.handleDraftsTool('scout', { action: 'resend', draftIds: [t.a.id, t.b.id], destination: 'chan' }); - await t.tick(); + await t.untilPublishing(); const dismissed = t.say(await t.internals.handleDraftsTool('scout', { action: 'dismiss', draftIds: [t.b.id] })); assert.match(dismissed, new RegExp(`${t.b.id}: dismissed`)); t.pending.shift()!.resolve({ status: 'delivered', messageId: 'm-a' }); @@ -1163,7 +1171,7 @@ describe('drafts resend ownership', () => { const t = await setup(); try { const batch = t.internals.handleDraftsTool('scout', { action: 'resend', draftIds: [t.a.id, t.b.id], destination: 'chan' }); - await t.tick(); + await t.untilPublishing(); assert.deepEqual(t.pending.map((p) => p.text), ['A words'], 'A is out; B is queued'); const refused = t.say(await t.internals.handleDraftsTool('scout', { action: 'dismiss', draftIds: [t.b.id, t.a.id] })); assert.match(refused, new RegExp(`${t.a.id} is being sent right now, and dismissing it can't call those words back; .*Nothing was dismissed\\.`)); @@ -1183,9 +1191,9 @@ describe('drafts resend ownership', () => { const t = await setup(); try { const batch = t.internals.handleDraftsTool('scout', { action: 'resend', draftIds: [t.a.id, t.b.id], destination: 'chan' }); - await t.tick(); + await t.untilPublishing(); t.pending.shift()!.resolve({ status: 'delivered', messageId: 'm-a' }); - await t.tick(); + await t.untilPublishing(); assert.deepEqual(t.pending.map((p) => p.text), ['B words'], 'A is in; B is out'); assert.match(t.say(await t.internals.handleDraftsTool('scout', { action: 'dismiss', draftIds: [t.a.id] })), new RegExp(`${t.a.id}: already delivered — nothing to dismiss`)); @@ -1200,10 +1208,108 @@ describe('drafts resend ownership', () => { const t = await setup(); try { const registry = (t.framework as unknown as { channelRegistry: Record }).channelRegistry; - registry.publish = async () => { throw new Error('transport gone'); }; - await t.internals.handleDraftsTool('scout', { action: 'resend', draftIds: [t.a.id], destination: 'chan' }).catch(() => undefined); + let entered = 0; + registry.publish = async () => { + entered++; + throw new Error('transport gone'); + }; + await assert.rejects( + t.internals.handleDraftsTool('scout', { action: 'resend', draftIds: [t.a.id], destination: 'chan' }), + /transport gone/, + ); + assert.equal(entered, 1, 'the resend reached publish'); + assert.equal(draftState(t.internals.proseDrafts.get('scout', t.a.id)!), 'unconfirmed', 'its attempt has no outcome: it may have been posted'); assert.match(t.say(await t.internals.handleDraftsTool('scout', { action: 'dismiss', draftIds: [t.a.id] })), - new RegExp(`${t.a.id}: dismissed`)); + new RegExp(`${t.a.id}: dismissed \\(your history keeps the words\\)\\. Note that ${t.a.id}'s attempt to .* may already have been posted: its outcome was never recorded\\.`)); + } finally { + await t.close(); + } + }); + + it('a batch that stops says why for each draft after the stop', async () => { + const t = await setup(); + try { + const [c] = t.internals.proseDrafts.hold('scout', [{ text: 'C words', source }], 'explicit-send').held; + const batch = t.internals.handleDraftsTool('scout', { action: 'resend', draftIds: [t.a.id, t.b.id, c!.id], destination: 'chan' }); + await t.untilPublishing(); + t.say(await t.internals.handleDraftsTool('scout', { action: 'dismiss', draftIds: [t.b.id] })); + t.pending.shift()!.resolve({ status: 'delivered', messageId: 'm-a' }); + const result = t.say(await batch); + assert.match(result, new RegExp(`${c!.id}: not attempted \\(${t.b.id} was dismissed while it waited\\)\\.`)); + + const [d] = t.internals.proseDrafts.hold('scout', [{ text: 'D words', source }], 'explicit-send').held; + const again = t.internals.handleDraftsTool('scout', { action: 'resend', draftIds: [c!.id, d!.id], destination: 'chan' }); + await t.untilPublishing(); + t.pending.shift()!.resolve({ status: 'unknown', reason: 'no receipt' }); + assert.match(t.say(await again), new RegExp(`${d!.id}: not attempted \\(${c!.id} was not confirmed delivered\\)\\.`)); + } finally { + await t.close(); + } + }); + + it('an attempt still out reads as being sent, not as one whose outcome was never recorded', async () => { + const t = await setup(); + try { + const sending = t.internals.handleDraftsTool('scout', { action: 'resend', draftIds: [t.a.id], destination: 'chan' }); + await t.untilPublishing(); + const read = t.say(await t.internals.handleDraftsTool('scout', { action: 'read', draftIds: [t.a.id] })); + assert.match(read, /attempt .* → \(chan\): being sent right now — no outcome yet/); + t.pending.shift()!.resolve({ status: 'delivered', messageId: 'm-a' }); + await sending; + } finally { + await t.close(); + } + }); + + it('{{unsent}} of a bounce queued in a resend says so, and holds the envelope\'s words with the risk', async () => { + const t = await setup(); + try { + const [bounce] = t.internals.proseDrafts.hold('scout', [{ text: 'bounced words', source }], 'bounced').held; + const batch = t.internals.handleDraftsTool('scout', { action: 'resend', draftIds: [t.a.id, bounce!.id], destination: 'chan' }); + await t.untilPublishing(); + const agent = t.framework.getAgent('scout')!; + await t.internals.deliverProseEnvelope(agent, '>>chan {{unsent}} and a postscript', { notice: 'later' }); + assert.deepEqual(t.pending.map((p) => p.text), ['A words'], 'nothing more was sent'); + const fresh = t.internals.proseDrafts.latestBounce('scout')!; + assert.notEqual(fresh.id, bounce!.id); + assert.equal(fresh.text, 'bounced words and a postscript'); + assert.equal(fresh.inheritedRisk?.draftId, bounce!.id); + assert.match(fresh.inheritedRisk!.reason, /queued in a resend when these words were held/); + const words = agent.getContextManager().getAllMessages().flatMap((m) => m.content).map((b) => (b as { text?: string }).text ?? ''); + assert.ok(words.includes(`[prose-routing] {{unsent}} is draft ${bounce!.id}, which is queued in a resend in progress. Nothing was sent; ` + + `wait for that result. Your message is held as draft ${fresh.id}, with {{unsent}} written out; since it includes ` + + `${bounce!.id}'s words, it can be resent only with confirmDuplicate: true, after that.`), words.join('\n')); + t.pending.shift()!.resolve({ status: 'delivered', messageId: 'm-a' }); + await t.untilPublishing(); + t.pending.shift()!.resolve({ status: 'delivered', messageId: 'm-b' }); + await batch; + } finally { + await t.close(); + } + }); + + it('>>skip_reply {{unsent}} says when the bounce may already have been posted, or could not be set aside', async () => { + const t = await setup(); + try { + const agent = t.framework.getAgent('scout')!; + const words = () => agent.getContextManager().getAllMessages().flatMap((m) => m.content).map((b) => (b as { text?: string }).text ?? ''); + const [bounce] = t.internals.proseDrafts.hold('scout', [{ text: 'bounced words', source }], 'bounced').held; + const sending = t.internals.handleDraftsTool('scout', { action: 'resend', draftIds: [bounce!.id], destination: 'chan' }); + await t.untilPublishing(); + t.pending.shift()!.resolve({ status: 'unknown', reason: 'no receipt' }); + await sending; + // The journal refuses the dismissal: the bounce stays, and the resident is told. + const dismiss = t.internals.proseDrafts.dismiss.bind(t.internals.proseDrafts); + t.internals.proseDrafts.dismiss = () => { throw new Error('EIO'); }; + await t.internals.deliverProseEnvelope(agent, '>>skip_reply {{unsent}}', { notice: 'later' }); + assert.ok(words().includes(`[prose-routing] {{unsent}} is draft ${bounce!.id}, which could not be set aside (EIO): ` + + 'it is still held, and {{unsent}} still names it.'), words().join('\n')); + assert.equal(t.internals.proseDrafts.latestBounce('scout')?.id, bounce!.id); + t.internals.proseDrafts.dismiss = dismiss; + await t.internals.deliverProseEnvelope(agent, '>>skip_reply {{unsent}}', { notice: 'later' }); + assert.equal(draftState(t.internals.proseDrafts.get('scout', bounce!.id)!), 'dismissed'); + assert.ok(words().some((w) => w.startsWith(`[prose-routing] Draft ${bounce!.id} is set aside. Note that ${bounce!.id}'s attempt to (chan) at `) && + w.endsWith('may already have been posted: no receipt.')), words().join('\n')); } finally { await t.close(); } @@ -1214,7 +1320,7 @@ describe('drafts resend ownership', () => { try { const [bounce] = t.internals.proseDrafts.hold('scout', [{ text: 'bounced words', source }], 'bounced').held; const sending = t.internals.handleDraftsTool('scout', { action: 'resend', draftIds: [bounce!.id], destination: 'chan' }); - await t.tick(); + await t.untilPublishing(); const agent = t.framework.getAgent('scout')!; const envelope = (text: string) => t.internals.deliverProseEnvelope(agent, text, { notice: 'later' }); await envelope('>>skip_reply {{unsent}}'); @@ -1237,7 +1343,7 @@ describe('drafts resend ownership', () => { const [bounce] = t.internals.proseDrafts.hold('scout', [{ text: 'bounced words', source }], 'bounced').held; const agent = t.framework.getAgent('scout')!; const delivering = t.internals.deliverProseEnvelope(agent, '>>chan {{unsent}}', { notice: 'later' }); - await t.tick(); + await t.untilPublishing(); assert.deepEqual(t.pending.map((p) => p.text), ['bounced words'], 'the bounce is out through {{unsent}}'); const refused = t.say(await t.internals.handleDraftsTool('scout', { action: 'dismiss', draftIds: [bounce!.id] })); assert.match(refused, new RegExp(`${bounce!.id} is being sent right now, and dismissing it can't call those words back; .*Nothing was dismissed\\.`)); From 9feee54ceb6842b04a49ae52d2cfba934ba23706 Mon Sep 17 00:00:00 2001 From: Petra-1012 Date: Sun, 11 Oct 2026 15:00:52 -0400 Subject: [PATCH 0108/1112] fix(envelope): only the framework writes the envelope, conversation keys can't collide, and the texts say what the code does sw-af247, part 4: the inbound envelope, and the texts. - I12: a module's addMessages metadata loses any inboundSource it carries before it is stored (one echoed from a server's event, say); a console or API surface event's messages carry the framework's own, as before. - I36: conversationKey encodes its server-chosen parts as a JSON array, so two distinct (kind, server, channel, thread) never share a key, and an empty thread is not no thread. It is an in-memory comparison key, never stored, so its format can change. - I33: splitProseSegments is gone; the framework trims splitProseRuns' runs inline, and the tests trim them the same way. - I1: the inbound-source header says what the envelope is and that consumers read it through readInboundSource and conversationKey, without naming readers this change doesn't have. - I41: the acceptance observer's comment says it is an internal seam the receipt work (shelf-354) assigns inside the framework; unset here. - I9, I18, I28, I42, and the round's own changes, in the fragments and the routing doc: routeSpeech now returns null unless delivery was confirmed; what {{unsent}} does with a draft that may have been posted, on each route; retention and the let-go refusal; notices counted per branch; the drafts tool's one-time prompt-cache miss; a publish's deadline and a refused send opening nothing; coalesced: true present only when it is; a module's envelope dropped. Tests: - inbound-source (I15): the strict reader refuses a missing, empty or non-string serverId, a bad binding, and non-string threadId, replyTo and eventId, for channel and unscoped envelopes. - inbound-source (I36): keys built from parts holding NUL differ, and an empty thread differs from none; the surface key's new form. - inbound-source (I12): a module relaying a forged envelope stores none; a console message carries the surface stamp, whatever its metadata said. - prose-drafts (I16): a silent turn's prose beside a send is counted private, never held or named. Red check: with suppressProse removed from both turnProseRouting and silenceCause, it fails. With only one removed it passes, since either keeps the prose private. I34 is re-judged not-real in the ledger: Agent.proseRouting is readonly, so the live value is the turn's. --- changelog.d/inbound-source-envelope.added.md | 13 +++--- changelog.d/prose-drafts.added.md | 42 ++++++++++++++------ changelog.d/publish-outcomes.fixed.md | 6 ++- docs/explicit-prose-routing.md | 25 +++++++++--- src/framework.ts | 16 ++++++-- src/mcpl/inbound-source.ts | 28 +++++++------ src/prose-segments.ts | 7 +--- test/inbound-source.test.ts | 42 +++++++++++++++++++- test/prose-drafts.test.ts | 20 +++++++++- test/prose-segments.test.ts | 6 ++- 10 files changed, 154 insertions(+), 51 deletions(-) diff --git a/changelog.d/inbound-source-envelope.added.md b/changelog.d/inbound-source-envelope.added.md index 9afd3c00c..ff472c601 100644 --- a/changelog.d/inbound-source-envelope.added.md +++ b/changelog.d/inbound-source-envelope.added.md @@ -8,12 +8,15 @@ ordinary and the RFC-006 coalesced paths; a coalesced occurrence freezes its envelope at acceptance, so deferral, fan-out and replay deliver it unchanged and a later rename never rewrites it. Adapter-supplied metadata cannot set - the field (an adapter's own `origin.source` string is untouched). - `readInboundSource`, `conversationKey` and `INBOUND_SOURCE_KEY` are exported. + the field, and one a module's messages carry is dropped (an adapter's own + `origin.source` string is untouched). `readInboundSource`, + `conversationKey` (a JSON-encoded key, so no server-chosen part can run + into another) and `INBOUND_SOURCE_KEY` are exported. - **Inbound source envelope: admission facts.** An ordinary `channels/incoming` message or `push/event` now has its envelope frozen at admission, before it is queued or acknowledged, so a rename or rebind while it waits cannot change it. Every channel and unscoped envelope names the admission `lane` - (`channels/incoming` or `push/event`) and whether it came through RFC-006 - coalesced admission (`coalesced: true`): facts a consumer needs to know what - an `eventId` is worth. `readInboundSource` validates every field it returns. + (`channels/incoming` or `push/event`), and one that came through RFC-006 + coalesced admission has `coalesced: true` (others have no `coalesced` + field): facts a consumer needs to know what an `eventId` is worth. + `readInboundSource` validates every field it returns. diff --git a/changelog.d/prose-drafts.added.md b/changelog.d/prose-drafts.added.md index 85c6607ad..f55b6d5a2 100644 --- a/changelog.d/prose-drafts.added.md +++ b/changelog.d/prose-drafts.added.md @@ -22,7 +22,12 @@ re-bounce rule (a bare `{{unsent}}` leaves the bounce as the one draft). - Drafts live in typed Chronicle records (`RecordJournal`) that do not follow branch switches: undo, rollback and checkout never change them, and they - survive restarts. Nothing is ever sent from them on its own. + survive restarts. Nothing is ever sent from them on its own. Each + resident's 100 newest drafts are kept, and an older one for 7 days after + anything last happened to it (held, named, attempted, answered, dismissed); + the journal checkpoints what it keeps every 500 entries, so opening it + replays only what follows. A request naming a draft that was let go is + refused, and nothing is sent. - A private notice names held drafts at the next tool boundary when the live stream presents mid-turn messages (a draft holding words copied from one that may already have been posted is named with that risk and the @@ -32,29 +37,42 @@ (listed where it landed), unconfirmed, or dismissed. A dismissed draft whose own attempt may already have been posted is named with that risk, and counts like an unconfirmed one: a receipt that names one, with nothing - confirmed delivered, reads "nothing confirmed", not "nothing". A draft no - notice reached (a crash, an aborted turn) is named at the next turn's - start. + confirmed delivered, reads "nothing confirmed", not "nothing". A draft + counts as named while a message naming it is on the current branch, so one + no notice reached (a crash, an aborted turn), or whose notices a rollback + or branch move took away, is named at the next turn's start: every such + draft by id, the newest eight with a preview. - A draft is never published except by its resident's explicit resend (or `{{unsent}}`), and only that resident can list or act on it. Its notices go into the resident's own history, which residents sharing one message slot already share, as they share the assistant turn the words came from. - New `drafts` tool (whenever MCPL channels are configured): `list`, `read`, - `resend` and `dismiss`, scoped to the calling resident. `resend` publishes + `resend` and `dismiss`, scoped to the calling resident. Every such + resident's tool list gains it, so each one's first request after upgrading + misses the prompt cache once. `resend` publishes drafts verbatim, in order, to one destination through the normal publish path, journaling each attempt durably before the request leaves; it counts as an explicit send. A delivered draft returns its historical receipt instead of sending again; a draft with any attempt whose outcome is unknown (and none confirmed) needs `confirmDuplicate: true`, and a resend - owns its drafts until it finishes, re-checking each before it is sent. - Dismissing a draft (with the tool or `>>skip_reply {{unsent}}`) is - refused while that draft's attempt is out, since a dismissal can't call - back words already on their way; the resident is told to wait for that - result. A draft only queued in a resend can still be dismissed, and is - then not sent. Unused fields may be passed as null. + owns its drafts until it finishes, re-checking each before it is sent, and + a draft it doesn't attempt is told why the batch stopped. Dismissing a + draft (with the tool or `>>skip_reply {{unsent}}`) is refused while that + draft's attempt is out, since a dismissal can't call back words already on + their way; the resident is told to wait for that result. A draft only + queued in a resend can still be dismissed, and is then not sent. A + dismissal of a draft that may already have been posted names that risk. + Unused fields may be passed as null. +- `{{unsent}}` whose draft is being sent, queued in a resend, or unconfirmed + sends nothing, and the envelope's own words are held as a bounce draft + with `{{unsent}}` written out and the risk carried, so nothing the + resident wrote has to be typed again. - Prose segments also break at XML tool mode's refused attempts and their notices (`tool_attempt`, `tool_notice`): the words before an all-refused call and after its notice are two messages, routed or held separately. - `ChannelRegistry.deliverSpeech` (routeSpeech with its `PublishOutcome`) and - `ChannelRegistry.resolveDestination` are new; routeSpeech is unchanged. + `ChannelRegistry.resolveDestination` are new. `routeSpeech` now delegates + to `deliverSpeech` and returns null unless delivery was confirmed, so a + missing receipt and a channel id two servers share now return null (see + the publish-outcomes entry). - The bounce notice now goes to the bouncing resident rather than the primary. diff --git a/changelog.d/publish-outcomes.fixed.md b/changelog.d/publish-outcomes.fixed.md index 659f0742b..6dd88191e 100644 --- a/changelog.d/publish-outcomes.fixed.md +++ b/changelog.d/publish-outcomes.fixed.md @@ -16,4 +16,8 @@ `mcpl:speech-route-failed` trace carry `outcome`. A channel id registered by more than one MCPL server is refused rather than routed through whichever server registered it first, and a supplied but empty or - non-string server selector is refused rather than read as omitted. + non-string server selector is refused rather than read as omitted. A send + refused for its server or grant opens no closed channel. A + `channels/publish` request always has a deadline: with `requestTimeoutMs` + 0 it keeps the 60 s default, since its outcome is told to the resident and + recorded for its draft; its expiry reads as unknown. diff --git a/docs/explicit-prose-routing.md b/docs/explicit-prose-routing.md index b719b400c..8096ac7fc 100644 --- a/docs/explicit-prose-routing.md +++ b/docs/explicit-prose-routing.md @@ -45,12 +45,25 @@ breaking any loop). `{{unsent}}` means the latest bounce while that draft is open (latest-wins, as the clipboard it replaced was), and `>>skip_reply {{unsent}}` dismisses it. -Earlier bounces stay in the agent's drafts, by id, until resent or dismissed; -drafts survive restarts and branch moves, and a delivery through `{{unsent}}` -is recorded on the draft like any resend. When the latest bounce's last -delivery attempt may already have been posted (an unknown outcome), -`{{unsent}}` is not substituted: the agent is told, and can resend it with the -`drafts` tool's `confirmDuplicate`. +Earlier bounces stay in the agent's drafts, by id, until resent, dismissed or +let go (an agent's 100 newest drafts are always kept, and an older one for 7 +days after anything last happened to it); drafts survive restarts and branch +moves, and a delivery through `{{unsent}}` is recorded on the draft like any +resend. + +When the latest bounce may already have been posted (an attempt's outcome is +unknown), or is being sent or queued in a resend right now, what happens +depends on the route: +- **Delivery** (`>>#channel … {{unsent}} …`): nothing is sent. The agent is + told, and can resend the bounce with the `drafts` tool's `confirmDuplicate` + after checking the channel. The envelope's own words are held as a new + bounce draft, with `{{unsent}}` written out and the risk carried, so that + draft too can be resent only with `confirmDuplicate`. A bare `{{unsent}}` + holds nothing new: its words already are the bounce. +- **Hold** (an envelope that bounces, or one an explicit send held back): the + bounce's words are copied into the new draft with `{{unsent}}` written out, + and the draft carries the same risk (`inheritedRisk`), so it reads + unconfirmed until a confirmed delivery. ## What explicit mode retires (for that agent) diff --git a/src/framework.ts b/src/framework.ts index 783d02b31..6656a87be 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -1056,8 +1056,9 @@ export class AgentFramework { private processLoggingPersist: boolean; private processLoggingBroadcast: boolean; private activeStreams: Map> = new Map(); - /** Told once per inbound acceptance (mcpl/inbound-source.ts); unset unless - * a receipt consumer installs one. */ + /** Told once per inbound acceptance (mcpl/inbound-source.ts). An internal + * seam, unset here: the receipt work (shelf-354) assigns it inside the + * framework, and tests set it by cast. */ private inboundAcceptanceObserver?: InboundAcceptanceObserver; /** Per-agent output locus FROZEN for the CURRENT logical turn. Resolved @@ -7129,10 +7130,17 @@ export class AgentFramework { /** The event's surface envelope, when it is console/API input. */ surface?: InboundSource, ): Promise { - // Add messages + // Add messages. Only the framework writes the inbound envelope: one a + // module's metadata carries (echoed from a server's event, say) is + // dropped, and a surface event's messages carry the framework's own. if (response.addMessages) { for (const msg of response.addMessages) { - const metadata = surface ? { ...msg.metadata, [INBOUND_SOURCE_KEY]: surface } : msg.metadata; + let metadata = msg.metadata; + if (metadata && INBOUND_SOURCE_KEY in metadata) { + const { [INBOUND_SOURCE_KEY]: _dropped, ...rest } = metadata; + metadata = rest; + } + if (surface) metadata = { ...metadata, [INBOUND_SOURCE_KEY]: surface }; const id = this.addMessage(msg.participant, msg.content, metadata); this.emitTrace({ type: 'message:added', messageId: id, source: event.type }); } diff --git a/src/mcpl/inbound-source.ts b/src/mcpl/inbound-source.ts index 29c2e65e0..766eb07cf 100644 --- a/src/mcpl/inbound-source.ts +++ b/src/mcpl/inbound-source.ts @@ -2,18 +2,18 @@ * Where an inbound item came from, decided once at Agent Framework's * ingestion boundary and stored on the message as `metadata.inboundSource`. * - * Every later consumer reads this one value instead of re-deriving identity - * from the lane it happened to arrive on: speech routing compares - * conversations with it, the visible provenance header renders from it, and - * delivery bookkeeping keys on it. It is frozen when the host ACCEPTS the - * item. A coalesced occurrence (RFC-006) carries the value stamped at its - * acceptance through deferral, fan-out and replay, so a later rename, - * reconnect or re-render never rewrites where an old item says it came from. + * A consumer of where an item came from reads this one value + * (readInboundSource) rather than re-deriving identity from the lane the + * item happened to arrive on, and compares conversations with + * conversationKey. It is frozen when the host ACCEPTS the item. A coalesced + * occurrence (RFC-006) carries the value stamped at its acceptance through + * deferral, fan-out and replay, so a later rename, reconnect or re-render + * never rewrites where an old item says it came from. * * The field is framework-owned. It is written after any adapter-supplied - * metadata is spread, so an adapter cannot supply or spoof it (adapter - * `origin` objects already use a free-form `source` string, which is why this - * is not called `source`). + * metadata is spread, so an adapter cannot supply or spoof it, and one a + * module's messages carry is dropped (adapter `origin` objects already use a + * free-form `source` string, which is why this is not called `source`). * * Facts only, no policy: whether an item is a body, an edit or a deletion * stays in its tags (`chat:edited`, `chat:deleted`); `deferred` and @@ -143,12 +143,14 @@ export function readInboundSource(metadata: unknown): InboundSource | undefined /** * The conversation an item belongs to, as a comparable key: server + channel * + thread for a channel item, the surface for console input. Undefined for an - * unscoped push, which belongs to no conversation. + * unscoped push, which belongs to no conversation. The parts are server-chosen + * strings, so they are encoded as a JSON array, never joined with a separator + * one of them could contain: two keys are equal only for the same parts. */ export function conversationKey(source: InboundSource): string | undefined { if (source.kind === 'channel') { - return `channel\u0000${source.serverId}\u0000${source.channelId}\u0000${source.threadId ?? ''}`; + return JSON.stringify(['channel', source.serverId, source.channelId, source.threadId ?? null]); } - if (source.kind === 'surface') return `surface\u0000${source.surface}`; + if (source.kind === 'surface') return JSON.stringify(['surface', source.surface]); return undefined; } diff --git a/src/prose-segments.ts b/src/prose-segments.ts index 6e2b96f7e..d10ce357d 100644 --- a/src/prose-segments.ts +++ b/src/prose-segments.ts @@ -22,7 +22,7 @@ import type { ContentBlock } from '@animalabs/membrane'; * The runs exactly as written: contiguous text blocks joined with a newline, * nothing trimmed. Whitespace-only runs are dropped. Held drafts keep these, * so a draft's words (indentation included) are the resident's own, byte for - * byte. + * byte; speech routes each run trimmed of surrounding whitespace. */ export function splitProseRuns(content: readonly ContentBlock[]): string[] { const runs: string[] = []; @@ -54,8 +54,3 @@ export function splitProseRuns(content: readonly ContentBlock[]): string[] { return runs; } - -/** The runs as speech is routed: each trimmed of surrounding whitespace. */ -export function splitProseSegments(content: readonly ContentBlock[]): string[] { - return splitProseRuns(content).map((run) => run.trim()); -} diff --git a/test/inbound-source.test.ts b/test/inbound-source.test.ts index 5516268dc..22814a316 100644 --- a/test/inbound-source.test.ts +++ b/test/inbound-source.test.ts @@ -50,9 +50,15 @@ class ConsoleModule implements Module { getTools(): ToolDefinition[] { return []; } async handleToolCall(_call: ToolCall): Promise { return { success: true }; } async onProcess(event: ProcessEvent, _state: ProcessState): Promise { + // A module that relays an event's metadata as it came, envelope and all. + if ((event as { type: string }).type === 'relay') { + const relayed = event as unknown as { content: string; metadata: Record }; + return { addMessages: [{ participant: 'Relay', content: [{ type: 'text', text: relayed.content }], metadata: relayed.metadata }] }; + } if (event.type !== 'external-message') return {}; const text = String((event as { content?: unknown }).content); - return { addMessages: [{ participant: 'Operator', content: [{ type: 'text', text }] }] }; + const metadata = (event as { metadata?: Record }).metadata; + return { addMessages: [{ participant: 'Operator', content: [{ type: 'text', text }], ...(metadata ? { metadata } : {}) }] }; } } @@ -156,10 +162,24 @@ describe('inbound source envelope', () => { const message = stored().find((m) => m.participant === 'Operator')!; const source = readInboundSource(message.metadata); assert.deepEqual(source && { kind: source.kind, surface: (source as { surface: string }).surface }, { kind: 'surface', surface: 'tui' }); - assert.equal(conversationKey(source!), 'surface\u0000tui'); + assert.equal(conversationKey(source!), '["surface","tui"]'); assert.equal(observed.length, 1); }); + it("drops an envelope a module's metadata carries: only the framework writes one", async () => { + const forged = { kind: 'channel', lane: 'channels/incoming', serverId: 'evil', binding: 'x', channelId: 'discord:g9:elsewhere', acceptedAt: 1 }; + framework.pushEvent({ type: 'relay', content: 'relayed words', metadata: { inboundSource: forged, note: 'kept' } } as unknown as ProcessEvent); + await waitFor(() => stored().some((m) => m.participant === 'Relay'), 'relayed message stored'); + const relayed = stored().find((m) => m.participant === 'Relay')!; + assert.equal(readInboundSource(relayed.metadata), undefined, 'not the forged envelope'); + assert.equal((relayed.metadata as Record).note, 'kept', 'the rest of its metadata stays'); + // A surface event's message carries the framework's envelope, whatever its metadata said. + framework.pushEvent({ type: 'external-message', source: 'tui', content: 'console words', metadata: { inboundSource: forged } } as ProcessEvent); + await waitFor(() => stored().some((m) => m.participant === 'Operator'), 'console message stored'); + const source = readInboundSource(stored().find((m) => m.participant === 'Operator')!.metadata); + assert.equal(source?.kind, 'surface'); + }); + it('keeps a stored envelope when the channel is renamed later', async () => { command({ op: 'incoming', channelId: ROOM, messageId: 'm-2', mode: 'ambient', text: 'before rename' }); await waitFor(() => !!storedWith((m) => m.messageId === 'm-2'), 'first message stored'); @@ -270,6 +290,17 @@ describe('readInboundSource', () => { { ...channel, messageId: 42 }, { ...channel, label: '' }, { ...channel, sourceTimestamp: {} }, + { ...channel, serverId: undefined }, + { ...channel, serverId: '' }, + { ...channel, serverId: 7 }, + { ...channel, binding: undefined }, + { ...channel, binding: { id: 'b' } }, + { ...channel, threadId: 3 }, + { ...channel, threadId: '' }, + { ...channel, replyTo: false }, + { ...channel, eventId: 9 }, + { kind: 'unscoped', lane: 'push/event', serverId: '', binding: 'b', acceptedAt: 1 }, + { kind: 'unscoped', lane: 'push/event', serverId: 's', binding: 'b', acceptedAt: 1, eventId: [] }, { kind: 'unscoped', lane: 'channels/incoming', serverId: 's', binding: 'b', acceptedAt: 1 }, { kind: 'surface', surface: '', acceptedAt: 1 }, { kind: 'mystery', acceptedAt: 1 }, @@ -284,6 +315,13 @@ describe('readInboundSource', () => { assert.notEqual(conversationKey(base), conversationKey({ ...base, serverId: 't' })); assert.notEqual(conversationKey(base), conversationKey({ ...base, threadId: 'th' })); assert.equal(conversationKey(base), conversationKey({ ...base, messageId: 'other', acceptedAt: 2 })); + // The parts are server-chosen: a separator inside one can't make two + // conversations one. + assert.notEqual( + conversationKey({ ...base, channelId: 'c\u0000x', threadId: 'y' }), + conversationKey({ ...base, channelId: 'c', threadId: 'x\u0000y' }), + ); + assert.notEqual(conversationKey({ ...base, threadId: '' }), conversationKey(base), 'an empty thread is not no thread'); }); }); diff --git a/test/prose-drafts.test.ts b/test/prose-drafts.test.ts index 9635ad821..1068ff601 100644 --- a/test/prose-drafts.test.ts +++ b/test/prose-drafts.test.ts @@ -333,14 +333,17 @@ async function harness(opts: { : [], /** One addressed message in #room starts a turn (or, with `request`, a * direct inference request for just that resident); resolves when it settles. */ - turn: async (responses: Array>, agentName = 'scout', via: 'incoming' | 'request' = 'incoming') => { + turn: async (responses: Array>, agentName = 'scout', via: 'incoming' | 'request' | 'silent' = 'incoming') => { for (const r of responses) membrane.pushResponse(r); const before = membrane.calls.length; if (via === 'incoming') { h.command({ op: 'incoming', channelId: ROOM, messageId: `in-${++incoming}`, mode: 'addressed', text: `message ${incoming}` }); } else { + // 'silent': a control-plane wake whose plain prose stays private, as + // a silent heartbeat's does. (framework as unknown as { pendingRequests: unknown[] }).pendingRequests.push({ agentName, reason: 'test', source: 'test', timestamp: Date.now(), + ...(via === 'silent' ? { suppressProse: true } : {}), }); } await until(() => membrane.calls.length > before, 'turn start'); @@ -594,6 +597,21 @@ describe('held prose drafts, end to end', () => { } }); + it("a silent turn's prose stays private beside a send: counted, never held or named", async () => { + const h = await harness(); + try { + await h.turn([ + createMockResponse([text('private narration'), explicitSend('s1')], 'tool_use'), + createMockResponse([text('more private words')]), + ], 'scout', 'silent'); + assert.deepEqual(h.drafts(), [], 'nothing was held as a draft'); + assert.ok(!h.texts().some((t) => t.includes('held as draft')), 'and no notice names one'); + assert.deepEqual(h.publishes().map((p) => p.text), ['an explicit note elsewhere'], 'only the explicit send went out'); + } finally { + await h.close(); + } + }); + it('an unknown outcome is not cleared by a later failure: the draft stays unconfirmed', async () => { const h = await harness(); try { diff --git a/test/prose-segments.test.ts b/test/prose-segments.test.ts index 98ffda842..4fd2ff4f7 100644 --- a/test/prose-segments.test.ts +++ b/test/prose-segments.test.ts @@ -1,7 +1,11 @@ import { test } from 'node:test'; import assert from 'node:assert/strict'; import type { ContentBlock } from '@animalabs/membrane'; -import { splitProseRuns, splitProseSegments } from '../src/prose-segments.js'; +import { splitProseRuns } from '../src/prose-segments.js'; + +/** The runs as speech routes them: each trimmed, as the framework does. */ +const splitProseSegments = (content: Parameters[0]): string[] => + splitProseRuns(content).map((run) => run.trim()); const textBlock = (text: string): ContentBlock => ({ type: 'text', text } as ContentBlock); const toolUse = (id: string): ContentBlock => From 7cc9c216cf39e4620f8707159abf8bbb4ad5fd0f Mon Sep 17 00:00:00 2001 From: Petra-1012 Date: Sun, 11 Oct 2026 15:02:15 -0400 Subject: [PATCH 0109/1112] test(publish): the refused-before-open case uses a grant that could open the channel but not publish into it So the test tells the new order (server and channels.publish checked before any open) from the old one, which opened first: with an empty grant, the old order failed at channels.lifecycle and opened nothing either, so only the refusal's wording told them apart. --- test/publish-outcome.test.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/test/publish-outcome.test.ts b/test/publish-outcome.test.ts index 13f298add..846a9835f 100644 --- a/test/publish-outcome.test.ts +++ b/test/publish-outcome.test.ts @@ -25,7 +25,7 @@ import { MockMembrane, createMockResponse } from './helpers/mock-membrane.js'; type Publish = (params: { channelId: string }) => Promise; function registryWith( - servers: Record Promise }>, + servers: Record Promise }>, failures: unknown[] = [], ) { const published: Array<{ serverId: string; channelId: string }> = []; @@ -34,7 +34,9 @@ function registryWith( for (const [id, s] of Object.entries(servers)) { mocks.set(id, { id, - grant: new CapabilityGrant(new Set(s.grant === false ? [] : ALL_CAPABILITY_PATHS), []), + grant: new CapabilityGrant(new Set( + s.grant === false ? [] : ALL_CAPABILITY_PATHS.filter((p) => !(typeof s.grant === 'object' && s.grant.without.includes(p))), + ), []), sendChannelsPublish: async (params: { channelId: string }) => { published.push({ serverId: id, channelId: params.channelId }); return s.publish(params); @@ -90,7 +92,8 @@ describe('ChannelRegistry.publish outcomes', () => { }); it('refuses a send the grant forbids before opening anything', async () => { - const { registry, seed, opened, published } = registryWith({ discord: { publish: ok, grant: false } }); + // The grant could open the channel (channels.lifecycle), but not publish into it. + const { registry, seed, opened, published } = registryWith({ discord: { publish: ok, grant: { without: ['channels.publish'] } } }); seed('discord', 'c', '#c', false); const outcome = await registry.publish('agent', 'x', { channelId: 'c' }); assert.equal(outcome.status, 'failed'); From 97d860dea8efa30f5c73965d6ca3db2e4e811447 Mon Sep 17 00:00:00 2001 From: Petra-1012 Date: Sun, 11 Oct 2026 15:13:16 -0400 Subject: [PATCH 0110/1112] test(digests): every storage site and store normalization is pinned, and the texts say where a stored copy can be checked sw-af252 (the house's haiku review of #252), its fix round. Text: - I2, I5, I9, I11, I17, I18 (changelog, sourceBodyDigest's, storedBlock's, canonicalJson's and withStoredDigest's docs): - the digests are on every stored MCPL item; console and API input and the host's notices carry none; - canonicalJson mirrors the store's append: an undefined object entry is dropped, an undefined array element is null, a lone surrogate is U+FFFD, and an ill-formed key is dropped; editMessage writes JSON instead; - the copy check is of an unsharded copy: a body the store shards carries the whole body's digests on every shard (bodyGroupId and shardIndex mark them), and no shard hashes to them; - a blob_ref is digested as given, which its copy read back (resolved) doesn't match; the MCPL lanes never store one; - sourceBodyDigest no longer claims a live-turn injection this change neither performs nor tests. Tests (inbound-source): - I1: the conversation router's site and a fork correction's site (a coalesced replacement following its prior into an unbound fork), each with forged digests in the adapter's metadata, store the host's. - I10: a channels/incoming item's forged digests are replaced. - I2: with a strategy whose store shards a two-block push, both shards carry the whole body's digests and neither hashes to them; a one-block body is not sharded and matches its own. - I7: the materialized RFC-006 copy's digests are the host's, over the rendered body, not the deferred placeholder. - I13: a channel message deferred mid-turn and stored at the boundary still hashes to its stored digest. - I3, I4, I12 (the real-store table): a video with a duration; an image without a media type, and with mediaType undefined (both take their bytes' type); a text block with an undefined field; a URL image with a stray data string, kept as given. A document without a media type, and an image without one whose bytes have no signature, are refused at write. A blob_ref is resolved on read and doesn't match its digest. A URL image and a text block are digested as given (plain canonical hash). Red checks: dropping withStoredDigest at the router site, at the fork correction site or on the channels/incoming path fails its test; so do blobbing a URL image and keeping undefined object entries. Runs (on-vps, Node 24.21.0): typecheck clean; inbound-source 28/28. --- changelog.d/ingestion-body-digests.added.md | 34 ++-- src/framework.ts | 6 +- src/mcpl/inbound-source.ts | 41 +++-- test/inbound-source.test.ts | 165 ++++++++++++++++++++ 4 files changed, 214 insertions(+), 32 deletions(-) diff --git a/changelog.d/ingestion-body-digests.added.md b/changelog.d/ingestion-body-digests.added.md index 6e181ed8c..558bf4832 100644 --- a/changelog.d/ingestion-body-digests.added.md +++ b/changelog.d/ingestion-body-digests.added.md @@ -1,15 +1,18 @@ -- **Delivered-body digests on every stored inbound item.** Both MCPL lanes - stamp two fields, outside the frozen source envelope: +- **Delivered-body digests on every stored MCPL inbound item.** Both MCPL + lanes stamp two fields, outside the frozen source envelope (console and API + input, and the host's own notices, carry neither): - `metadata.sourceBodyDigest` is the version identity of the body as delivered, taken at ingestion, after MCPL conversion and before any host decoration or storage sharding. It is the SHA-256 (hex) of `[blocks]` - as canonical JSON (keys sorted at every level, `undefined` dropped), with - the blocks as the store keeps them: inline media as the store hands it - back (base64 re-encoded from its bytes, an image's media type taken from - its bytes' signature, only `type` and `source` kept) and strings - well-formed (a lone surrogate as U+FFFD). So the body hashes alike as - delivered, as handed to storage and as read back, and an undecorated, - unsharded stored copy hashes to it. + as canonical JSON (keys sorted at every level, `undefined` object entries + dropped and an `undefined` array element written as `null`, as JSON + writes them), with the blocks as the store keeps them: inline media as + the store hands it back (base64 re-encoded from its bytes, an image's + media type taken from its bytes' signature, only `type` and `source` + kept), strings well-formed (a lone surrogate as U+FFFD) and an entry + whose key isn't well-formed dropped, as the store's append drops it. So + the body hashes alike as delivered, as handed to storage and as read + back, and an undecorated, unsharded stored copy hashes to it. - `metadata.storedBodyDigest` uses the same function over exactly the blocks handed to storage, decorations included, taken where each path stores. @@ -17,7 +20,12 @@ Both are always the host's own: values an adapter supplies are replaced. A stored copy, read back with its blobs resolved, that no longer hashes to its `storedBodyDigest` was changed after it arrived (`editMessage` keeps - metadata). `sourceBodyDigest(blocks)` is exported from the package root as - the one implementation consumers share; it carries the framing and the - store's normalization, so check a copy with it rather than hashing your own - serialization. + metadata). That check is of an unsharded copy: a body the store shards + (context-manager's adaptive ingress chunking) carries the whole body's two + digests on every shard (`bodyGroupId` and `shardIndex` mark them), and no + shard hashes to them. A `blob_ref` block is digested as given, while the + store resolves it on read, so a copy holding one doesn't match; the MCPL + lanes never store one. `sourceBodyDigest(blocks)` is exported from the + package root as the one implementation consumers share; it carries the + framing and the store's normalization, so check a copy with it rather than + hashing your own serialization. diff --git a/src/framework.ts b/src/framework.ts index 5e44cc63c..63a72154f 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -7838,8 +7838,10 @@ export class AgentFramework { * (`storedBodyDigest`): the delivered-body digest's function over exactly * the blocks handed to storage here — every decoration included, before * storage shards them. That function hashes blocks as the store keeps them - * (inline media re-encoded and relabeled from its bytes), so the copy read - * back hashes to this too. Taken at each storage site, never earlier, because + * (inline media re-encoded and relabeled from its bytes), so an unsharded + * copy read back hashes to this too. A body the store shards (adaptive + * ingress chunking) carries this on every shard, with its metadata, and no + * shard hashes to it. Taken at each storage site, never earlier, because * a path can still decorate the body after ingestion stamped it (the * closed-channel invitation; room-220 #48282). An edit through * editMessage keeps metadata but not this hash. diff --git a/src/mcpl/inbound-source.ts b/src/mcpl/inbound-source.ts index b7c39c107..3d53dc307 100644 --- a/src/mcpl/inbound-source.ts +++ b/src/mcpl/inbound-source.ts @@ -156,13 +156,16 @@ export function conversationKey(source: InboundSource): string | undefined { } /** - * JSON with object keys sorted at every level, written as the store keeps a - * value: `undefined` values are dropped, a string is well-formed (the store - * writes UTF-8, so each lone surrogate comes back as U+FFFD), and an entry - * whose key isn't well-formed is dropped (the store doesn't keep it). So a - * value hashes alike however its keys were ordered (content read back from - * the store has them in a different order than it was written), and alike - * before and after the store. + * JSON with object keys sorted at every level, written as the store's append + * keeps a value (its native UTF-8 write, which the real-store digest table + * checks): an `undefined` object entry is dropped and an `undefined` array + * element is written as `null`, as JSON writes them; a string is + * well-formed (each lone surrogate comes back as U+FFFD); and an entry whose + * key isn't well-formed is dropped (the store doesn't keep it). So a value + * hashes alike however its keys were ordered (content read back from the + * store has them in a different order than it was written), and alike + * before and after the store. editMessage writes JSON instead, so an edited + * copy can keep what an append wouldn't; it has changed anyway. * * It expects JSON-shaped values, as the MCPL lanes deliver them (parsed from * JSON). Any other object is written by its own enumerable entries, unlike @@ -191,7 +194,10 @@ export function canonicalJson(value: unknown): string { * document, audio or video block, comes back as `{ type, source: { type: * 'base64', data, mediaType } }` and nothing else, its data re-encoded from * the decoded bytes, and an image's media type taken from its bytes' - * signature where they have one. Any other block comes back as it was given. + * signature where they have one. Any other block comes back as it was given, + * except a `blob_ref`, which the store resolves to its inline media on read: + * it is digested as given here, so a copy holding one doesn't match its + * digest (the MCPL lanes never store one). */ function storedBlock(block: unknown): unknown { const media = block as { type?: unknown; source?: { type?: unknown; data?: unknown; mediaType?: unknown } } | null; @@ -252,21 +258,22 @@ function sniffRasterImageMediaType(bytes: Uint8Array): string | null { * ContentBlock[] the framework stores for the body — after MCPL conversion, * before any host decoration is added and before storage shards it — with * each block as the store keeps it (storedBlock, canonicalJson). So a body - * hashes alike as delivered, as handed to storage, as injected into a live - * turn, and as read back from the store. The one-element array keeps the - * framing an undecorated, unsharded stored copy has always hashed to, so a - * stamped copy and an older copy of the same body share a version. + * hashes alike as delivered, as handed to storage, and as read back from + * the store. The one-element array keeps the framing an undecorated, + * unsharded stored copy has always hashed to, so a stamped copy and an + * older copy of the same body share a version. * * Ingestion stamps it as `metadata.sourceBodyDigest`, and the same function * over exactly the blocks handed to storage (decorations included) as * `metadata.storedBodyDigest`, on both MCPL lanes. Both live outside the * frozen admission envelope: they describe the body actually delivered (a * materialization, a correction), not the admission. Neither proves later - * presence or completeness; a stored copy that no longer hashes to its - * storedBodyDigest was changed after delivery (editMessage keeps metadata). - * Check a copy read back with its blobs resolved (getAllMessages, getMessage, - * or getMessageWindow without `resolveBlobs: false`): a blob reference never - * matches. + * presence or completeness; an unsharded stored copy that no longer hashes + * to its storedBodyDigest was changed after delivery (editMessage keeps + * metadata). A body the store shards carries the whole body's two digests + * on every shard, and no shard hashes to them. Check a copy read back with + * its blobs resolved (getAllMessages, getMessage, or getMessageWindow + * without `resolveBlobs: false`): a blob reference never matches. * * Exported from the package root, so a consumer checks a copy against either * field with this same function rather than a copy of it. diff --git a/test/inbound-source.test.ts b/test/inbound-source.test.ts index b50b14a70..d632dc1d8 100644 --- a/test/inbound-source.test.ts +++ b/test/inbound-source.test.ts @@ -30,6 +30,7 @@ import { type ToolResult, } from '../src/index.js'; import { ContextManager } from '@animalabs/context-manager'; +import type { ContentBlock } from '@animalabs/membrane'; import { MockMembrane } from './helpers/mock-membrane.js'; import { canonicalJson } from '../src/mcpl/inbound-source.js'; import { createHash } from 'node:crypto'; @@ -261,6 +262,18 @@ describe('inbound source envelope', () => { assert.notEqual(meta.storedBodyDigest, meta.sourceBodyDigest, 'the invitation decorates the stored copy'); }); + it('a channels/incoming item carries the host\'s digests, never the adapter\'s', async () => { + command({ + op: 'incoming', channelId: ROOM, messageId: 'm-forged', mode: 'ambient', text: 'forged digests', + metadata: { sourceBodyDigest: 'adapter-value', storedBodyDigest: 'adapter-value' }, + }); + await waitFor(() => !!storedWith((m) => m.messageId === 'm-forged'), 'message stored'); + const message = storedWith((m) => m.messageId === 'm-forged')!; + const meta = message.metadata as Record; + assert.equal(meta.sourceBodyDigest, sourceBodyDigest([{ type: 'text', text: 'forged digests' }]), 'the delivered body, not the adapter\'s value'); + assert.equal(meta.storedBodyDigest, sourceBodyDigest(message.content), 'exactly what was stored, not the adapter\'s value'); + }); + it('an image the store relabels or re-encodes still matches both its digests once read back', async () => { // slimepriestess's review of #252: the store takes an image's media type // from its bytes and re-encodes its base64, so a digest of the blocks as @@ -411,6 +424,61 @@ describe('readInboundSource', () => { }); }); +describe('stored digests when the store shards a body', () => { + /** A strategy whose store splits a body of more than one block into one + * shard per block, as adaptive ingress chunking does. */ + class ShardingStrategy extends PassthroughStrategy { + chunkIngressMessage(_participant: string, content: ContentBlock[]) { + if (content.length < 2) return null; + return { bodyGroupId: `group-${content.length}-${Date.now()}`, shards: content.map((block, shardIndex) => ({ content: [block], shardIndex })) }; + } + } + + it("every shard carries the whole body's digests, and no shard hashes to them; an unsharded copy matches its own", async () => { + const tempDir = mkdtempSync(join(tmpdir(), 'inbound-shards-')); + const commandPath = join(tempDir, 'commands.jsonl'); + writeFileSync(commandPath, ''); + const framework = await AgentFramework.create({ + storePath: join(tempDir, 'test.chronicle'), + membrane: new MockMembrane().asMembrane(), + agents: [{ name: 'scout', model: 'test-model', systemPrompt: 'You are scout.', strategy: new ShardingStrategy() }], + mcplServers: [{ + id: 'discord', + command: process.execPath, + args: [FIXTURE], + env: { STATUS_PATH: join(tempDir, 'status.jsonl'), COMMAND_PATH: commandPath }, + }], + modules: [], + }); + try { + await framework.start(); + const command = (c: Record): void => appendFileSync(commandPath, JSON.stringify(c) + '\n'); + const blocks = [{ type: 'text', text: 'first part' }, { type: 'text', text: 'second part' }]; + command({ op: 'push', eventId: 'sharded', content: blocks }); + command({ op: 'push', eventId: 'whole', text: 'one block' }); + const stored = () => framework.getAgent('scout')!.getContextManager().getAllMessages(); + await waitFor(() => stored().some((m) => (m.metadata as { eventId?: string } | undefined)?.eventId === 'whole'), 'both pushes stored'); + const shards = stored().filter((m) => (m.metadata as { eventId?: string } | undefined)?.eventId === 'sharded'); + assert.equal(shards.length, 2, 'stored as two shards'); + assert.ok(shards[0]!.bodyGroupId && shards.every((m) => m.bodyGroupId === shards[0]!.bodyGroupId)); + assert.deepEqual(shards.map((m) => m.shardIndex), [0, 1]); + const whole = sourceBodyDigest(blocks); + for (const shard of shards) { + const meta = shard.metadata as Record; + assert.equal(meta.sourceBodyDigest, whole, 'the whole delivered body'); + assert.equal(meta.storedBodyDigest, whole, 'the whole body handed to storage'); + assert.notEqual(sourceBodyDigest(shard.content), whole, 'a shard alone is not the body'); + } + const one = stored().find((m) => (m.metadata as { eventId?: string } | undefined)?.eventId === 'whole')!; + assert.equal(one.bodyGroupId, undefined, 'a one-block body is not sharded'); + assert.equal(sourceBodyDigest(one.content), (one.metadata as Record).storedBodyDigest, 'an unsharded copy matches'); + } finally { + await framework.stop(); + rmSync(tempDir, { recursive: true, force: true }); + } + }); +}); + describe('inbound source envelope under RFC-006 coalescing', () => { const install = (framework: AgentFramework): InboundSource[] => { const observed: InboundSource[] = []; @@ -452,6 +520,14 @@ describe('inbound source envelope under RFC-006 coalescing', () => { assert.equal(observed.length, 1, 'materialization is not a second acceptance'); const stored = sourceOf(f, 'document_diff'); assert.deepEqual(stored, { ...accepted, materialized: true }); + // The materialized copy is its own storage site: both digests are the + // host's, over the rendered body, not the deferred placeholder. + const copy = f.framework.getAgent('agent')!.getContextManager().getAllMessages() + .find((m) => JSON.stringify(m.content).includes('document_diff'))!; + const meta = copy.metadata as Record; + assert.equal(meta.storedBodyDigest, sourceBodyDigest(copy.content), 'exactly what was stored'); + assert.equal(meta.sourceBodyDigest, sourceBodyDigest(copy.content), 'the rendered body, undecorated'); + assert.notEqual(meta.sourceBodyDigest, sourceBodyDigest([{ type: 'text', text: 'fallback_unavailable' }]), 'not the placeholder'); }); it('a channel-scoped coalesced message belongs to its channel, frozen at acceptance', async (t) => { @@ -470,6 +546,46 @@ describe('inbound source envelope under RFC-006 coalescing', () => { assert.equal(stored.sourceTimestamp, TS); assert.deepEqual(observed, [stored]); }); + + it('a channel message deferred mid-turn and stored at the boundary still hashes to its stored digest', async (t) => { + const f = await fixture(); t.after(f.close); + await f.register('chat'); + const internals = f.framework as unknown as { activeTurnTokens: Map; flushDeferredWrites(label: string): Promise }; + internals.activeTurnTokens.set('agent', 100); + await f.send('channels/incoming', { messages: [f.channel('e1', 'deferred_words', { initial: true })] }); + assert.ok(!f.context().includes('deferred_words'), 'held until the boundary'); + internals.activeTurnTokens.delete('agent'); + await internals.flushDeferredWrites('test'); + const copy = f.framework.getAgent('agent')!.getContextManager().getAllMessages() + .find((m) => JSON.stringify(m.content).includes('deferred_words'))!; + const meta = copy.metadata as Record; + assert.equal(meta.storedBodyDigest, sourceBodyDigest(copy.content), 'the copy stored at the boundary matches'); + assert.equal(meta.sourceBodyDigest, sourceBodyDigest([{ type: 'text', text: 'deferred_words' }]), 'the delivered body, unchanged'); + }); + + it("the conversation router's and a fork correction's storage sites carry the host's digests, never the adapter's", async (t) => { + const f = await fixture({ framework: { conversations: { templateAgent: 'agent', bind: { channel: 'always' }, trigger: { channel: 'always' } } } as never }); t.after(f.close); + await f.register(); + const forged = { sourceBodyDigest: 'adapter-value', storedBodyDigest: 'adapter-value' }; + const router = (f.framework as unknown as { conversationRouter: { getBinding(id: string): { agentName: string } | undefined; unbind(id: string): void } }).conversationRouter; + const check = (text: string, fork: string) => { + const copy = f.framework.getAgent(fork)!.getContextManager().getAllMessages().find((m) => JSON.stringify(m.content).includes(text)); + assert.ok(copy, `${text} stored in ${fork}`); + const meta = copy.metadata as Record; + assert.equal(meta.sourceBodyDigest, sourceBodyDigest([{ type: 'text', text }]), `${text}: the delivered body, not the adapter's value`); + assert.equal(meta.storedBodyDigest, sourceBodyDigest(copy.content), `${text}: exactly what was stored, not the adapter's value`); + }; + // Routed into a fresh fork by the conversation router. + await f.send('channels/incoming', { messages: [{ ...f.channel('c1', 'routed_original', { initial: true }), metadata: forged }] }); + const fork = router.getBinding('chat')!.agentName; + check('routed_original', fork); + // A correction follows the prior delivery into that fork, though the + // channel is no longer bound to it. + router.unbind('chat'); + await f.send('channels/incoming', { messages: [{ ...f.channel('c2', 'fork_correction'), metadata: forged }] }); + assert.equal(router.getBinding('chat'), undefined, 'no fresh fork was spawned for the correction'); + check('fork_correction', fork); + }); }); describe('sourceBodyDigest hashes a body as the store keeps it', () => { @@ -510,6 +626,15 @@ describe('sourceBodyDigest hashes a body as the store keeps it', () => { 'text with lone surrogates': { blocks: [{ type: 'text', text: 'cut \ud83d here, \ude00 there, \ud83d\ud83d twice' }], changed: true }, 'a key that is not well-formed': { blocks: [{ type: 'text', text: 'x', rawItem: { 'k\ud800': 1, kept: 2 } }], changed: true }, 'a relabeled image with a text decoration': { blocks: [image(png, 'image/jpeg'), { type: 'text', text: '[invitation]' }], changed: true }, + 'a video with a duration': { blocks: [media('video', 'AAAAGGZ0eXBtcDQy', 'video/mp4', { duration: 2 })], changed: true }, + // No media type: an image takes its bytes' own (an image without a + // signature, or other media, is refused at write: see below). + 'an image without a media type': { blocks: [{ type: 'image', source: { type: 'base64', data: png } }], changed: true }, + 'an image whose media type is undefined': { blocks: [{ type: 'image', source: { type: 'base64', data: png, mediaType: undefined } }], changed: true }, + 'a text block with an undefined field': { blocks: [{ type: 'text', text: 'x', note: undefined }], changed: true }, + // Only a non-URL image is blobbed: a URL image keeps even a stray data + // string as given, and storedBlock must not re-encode it. + 'an image by URL with a stray data string': { blocks: [{ type: 'image', source: { type: 'url', url: 'https://cdn.example/a.png', data: 'not base64 at all' } }], changed: false }, }; it('every body hashes alike as handed to the store and as read back, live and after reopening', async () => { @@ -540,6 +665,46 @@ describe('sourceBodyDigest hashes a body as the store keeps it', () => { } }); + it('a block the store refuses is refused at write, and a blob_ref is digested as given, which its copy read back does not match', async () => { + const dir = mkdtempSync(join(tmpdir(), 'stored-digest-refused-')); + try { + const cm = await ContextManager.open({ path: join(dir, 'store.chronicle'), strategy: new PassthroughStrategy() }); + try { + const svg = Buffer.from('').toString('base64'); + for (const [what, block] of [ + ['a document without a media type', { type: 'document', source: { type: 'base64', data: 'JVBERi0xLjQ=' } }], + ['an image without a media type whose bytes have no signature', { type: 'image', source: { type: 'base64', data: svg } }], + ] as const) { + assert.throws(() => cm.addMessage('user', [block] as never), Error, `${what}: refused at write`); + } + // A blob_ref names media the store already holds; it is resolved to + // inline media on read, so the copy read back hashes differently. + const held = cm.addMessage('user', [image(png)] as never); + const ref = cm.getMessageWindow(0, cm.getMessageCount(), { resolveBlobs: false }).messages + .find((m) => m.id === held)!.content[0] as { type: string }; + assert.equal(ref.type, 'blob_ref'); + const given = [ref]; + const id = cm.addMessage('user', given as never, { storedBodyDigest: sourceBodyDigest(given) }); + const back = cm.getMessage(id)!; + assert.deepEqual(back.content, [image(png)], 'resolved on read'); + assert.notEqual(sourceBodyDigest(back.content), (back.metadata as Record).storedBodyDigest, 'the documented mismatch'); + } finally { + cm.close(); + } + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it('a block the store keeps as given is digested as given', () => { + const plain = (blocks: unknown[]) => createHash('sha256').update(canonicalJson([blocks])).digest('hex'); + // Only a non-URL image is blobbed, so a URL image's stray data string is + // neither decoded nor re-encoded. + const urlImage = { type: 'image', source: { type: 'url', url: 'https://cdn.example/a.png', data: 'not base64 at all' } }; + assert.equal(sourceBodyDigest([urlImage]), plain([urlImage])); + assert.equal(sourceBodyDigest([{ type: 'text', text: 'x' }]), plain([{ type: 'text', text: 'x' }])); + }); + it('what the store keeps still changes the digest', () => { const digest = (blocks: unknown[]) => sourceBodyDigest(blocks); assert.notEqual(digest([image(png)]), digest([image(Buffer.from([...PNG, 0]).toString('base64'))]), 'other bytes'); From 7ce228ed9f2b789adfb0e23de98587e7bbb3a80b Mon Sep 17 00:00:00 2001 From: Petra-1012 Date: Sun, 11 Oct 2026 15:32:12 -0400 Subject: [PATCH 0111/1112] test(digests): the store refuses an undefined array element, and canonicalJson writes one as null Oscar-1808's take-or-leave in his review of 97d860d (room-203 #72199): no test pinned the new text's claim that canonicalJson writes an undefined array element as null, so a canonicalJson that skipped the element passed. He suggested a real-store row, expecting the store's append to write the element as null. The row showed the store doesn't: context-manager 0.13.0's append throws ("undefined cannot be represented as a serde_json::Value"), so no stored copy holds one. So instead: - the refused-at-write test gains a body with ['a', undefined]; - the digested-as-given test asserts the element digests as null, as JSON writes it, which fails for a canonicalJson that skips it (red-checked); - canonicalJson's doc and the changelog say an undefined array element is written as null as JSON writes it, and that the store refuses a body holding one. The doc no longer implies the store writes null. Runs (on-vps, Node 24.21.0): typecheck clean; inbound-source 29/29. --- changelog.d/ingestion-body-digests.added.md | 5 +++-- src/mcpl/inbound-source.ts | 9 +++++---- test/inbound-source.test.ts | 8 ++++++++ 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/changelog.d/ingestion-body-digests.added.md b/changelog.d/ingestion-body-digests.added.md index 558bf4832..0242f0bf8 100644 --- a/changelog.d/ingestion-body-digests.added.md +++ b/changelog.d/ingestion-body-digests.added.md @@ -5,8 +5,9 @@ delivered, taken at ingestion, after MCPL conversion and before any host decoration or storage sharding. It is the SHA-256 (hex) of `[blocks]` as canonical JSON (keys sorted at every level, `undefined` object entries - dropped and an `undefined` array element written as `null`, as JSON - writes them), with the blocks as the store keeps them: inline media as + dropped, and an `undefined` array element written as `null` as JSON + writes it, though the store refuses a body holding one), with the blocks + as the store keeps them: inline media as the store hands it back (base64 re-encoded from its bytes, an image's media type taken from its bytes' signature, only `type` and `source` kept), strings well-formed (a lone surrogate as U+FFFD) and an entry diff --git a/src/mcpl/inbound-source.ts b/src/mcpl/inbound-source.ts index d11a8f8de..e2bc75645 100644 --- a/src/mcpl/inbound-source.ts +++ b/src/mcpl/inbound-source.ts @@ -160,10 +160,11 @@ export function conversationKey(source: InboundSource): string | undefined { /** * JSON with object keys sorted at every level, written as the store's append * keeps a value (its native UTF-8 write, which the real-store digest table - * checks): an `undefined` object entry is dropped and an `undefined` array - * element is written as `null`, as JSON writes them; a string is - * well-formed (each lone surrogate comes back as U+FFFD); and an entry whose - * key isn't well-formed is dropped (the store doesn't keep it). So a value + * checks): an `undefined` object entry is dropped; a string is well-formed + * (each lone surrogate comes back as U+FFFD); and an entry whose key isn't + * well-formed is dropped (the store doesn't keep it). An `undefined` array + * element is written as `null`, as JSON writes it: the store's append + * refuses a body holding one, so no stored copy does. So a value * hashes alike however its keys were ordered (content read back from the * store has them in a different order than it was written), and alike * before and after the store. editMessage writes JSON instead, so an edited diff --git a/test/inbound-source.test.ts b/test/inbound-source.test.ts index 12de580de..b68e0e495 100644 --- a/test/inbound-source.test.ts +++ b/test/inbound-source.test.ts @@ -712,6 +712,8 @@ describe('sourceBodyDigest hashes a body as the store keeps it', () => { for (const [what, block] of [ ['a document without a media type', { type: 'document', source: { type: 'base64', data: 'JVBERi0xLjQ=' } }], ['an image without a media type whose bytes have no signature', { type: 'image', source: { type: 'base64', data: svg } }], + // The store's append can't represent an undefined array element. + ['an array with an undefined element', { type: 'text', text: 'x', parts: ['a', undefined] }], ] as const) { assert.throws(() => cm.addMessage('user', [block] as never), Error, `${what}: refused at write`); } @@ -741,6 +743,12 @@ describe('sourceBodyDigest hashes a body as the store keeps it', () => { const urlImage = { type: 'image', source: { type: 'url', url: 'https://cdn.example/a.png', data: 'not base64 at all' } }; assert.equal(sourceBodyDigest([urlImage]), plain([urlImage])); assert.equal(sourceBodyDigest([{ type: 'text', text: 'x' }]), plain([{ type: 'text', text: 'x' }])); + // An undefined array element is written as null, as JSON writes it (the + // store refuses such a body, above, so no stored copy holds one). + assert.equal( + sourceBodyDigest([{ type: 'text', text: 'x', parts: ['a', undefined] }]), + sourceBodyDigest([{ type: 'text', text: 'x', parts: ['a', null] }]), + ); }); it('what the store keeps still changes the digest', () => { From 0c5c0b0d0d02bbf52cf408d823d2f63b701223ce Mon Sep 17 00:00:00 2001 From: Olive-1011 Date: Sun, 11 Oct 2026 16:08:00 -0400 Subject: [PATCH 0112/1112] fix(deferral): one rule for every write; stop() admits no turn or stream; a failed store never drops a drained entry; the watermark only advances A review of this pull request's first two commits found that the deferral rules still disagreed with each other, that stop()'s drain could meet a live turn, that a failed store could drop a drained entry, and that a compile could roll the consumed watermark back. This commit answers each. One rule for every write: - addMessage's default path checked every agent's tool cycle (pendingAssistantBlocks.size > 0, from efc7e7af, when the agents shared one store), while a targeted write checked only its target's. Now every write waits while its target's own turn, stream or tool cycle is alive, or the host is quiesced. So the primary's writes no longer wait behind a fork's, subagent's or the subconscious's tools, which protected nothing in the primary's window, and one drained queue is judged by one rule: the turn-end flush stores the drained prefix in order, or it goes back whole when a successor turn has taken the agent. Before, a later targeted entry could be stored ahead of an earlier primary-bound one. - Every flush gates on its own agent's cycle: the post-surgery flush (per reserved agent), the withdrawn-turn flush and the puppet's finally, as the turn-end flush already did. The puppet's comment now says what is true. The post-surgery flush also acks after its re-adds, as the other flushes do; before, its entries stayed un-acked until some later ack. - Residents that share the un-namespaced slot with overlapping turns put each other's rounds and writes inside the other's live turn whatever this check does. That limit is stated in the pull request, not fixed here. stop()'s premise made true: once stop() begins, no turn starts, for any agent (startAgentStream checks providerAdmissionClosed first, so a fork's or subagent's retry frame and a wake the loop already dequeued are refused, not only gate-owning residents), and no stream registers (beginAgentTurn refuses beside the identity check, cancelling the stream; the caller's finally releases the token). Before, a turn mid-compile at the cancel pass registered afterwards and outlived stop(), and the drain re-deferred its agent's entries into memory, where they were lost at exit. An ephemeral run refused either way, or one admitted while stopping (runEphemeralToCompletion), ends at once with "the framework is stopping" rather than at its startup watchdog. So "every turn is over" holds at stop()'s drain, which stores every registered target's entries; a quiesced host still re-defers into the persisted queue. A failed store never drops a drained entry: writeDrained, through which every site writes a drained entry (stop's drain, the post-surgery, withdrawn-turn, turn-end and puppet flushes and resume/boot recovery via readdDrained; the turn-start flush and the tool boundary, which store straight to the context manager), logs a write that throws, moves the entry from the un-acked set back into the queue under its id and seq (persisting a persisted queue), and lets the rest continue. At a tool boundary such an entry isn't injected. Before, it stayed un-acked and the next ack dropped it from the persisted queue; at a turn start or from a memory-only queue it was lost at once. A write that always throws is retried, and logged, at each later flush of its target; it is never dropped. drainDeferredFor now returns the queue's own entries, which carry seq. The watermark only advances: compileContext and compileWithInjections write their watermark back through applyCompileWatermark, which counts marks: if one was made while the compile ran (a tool boundary's, a branch switch's), the compile's older watermark applies only where it is later on that mark's branch. Before, a preview compiling during a live turn rolled a boundary's mark back. The puppet's defensive branch persists the queue after queueing its pair, as addMessage's deferral does, so a crash before the next persist can't lose the pair from a persisted queue. The boundary's mark is the store's head, a scalar, as a compile's is, so it also covers what was stored since the compile without being carried: another slot's messages, and a write that bypassed deferral into this slot. The comment there now names that over-cover, says that under anima-research/context-manager#160 such a write already gives the replies stored after it stamps their requests didn't carry, whatever the mark says, and points at deferring those writers as the fix. Changelog: stop() stores what is deferred for a registered target, a quiesced host keeps its persisted queue, and an entry for an unregistered target is dropped as before; a re-deferral's cause is the target's own turn, stream or tool cycle, or a quiesced host; the cost to an addressed message queued behind a held one is named; context-manager references are fully qualified; the one rule, the stop premise, a failed store and the watermark are described. Tests (test/deferred-window-only.test.ts, 16, was 9): - the interjecting tool returns once its interjections are queued (bounded) instead of after 30 ms, and the hold tests record what the boundary's drain found queued, so the cut is exercised rather than reached by a late message; - the held-for-turn-end test records, with a spy at the flush itself, that the other agent's cycle was pending there; - a write waits on its target's own turn and cycle, not another agent's, primary-bound or targeted; - the puppet's flush lands what was deferred behind it while another agent's cycle is pending; - stop() while a turn is mid-compile under the real event loop, with an entry deferred for that agent: no stream is left, the refused turn produced nothing, and after reopening the entry is stored; - a drained write whose store throws at stop() stays in the persisted queue and is stored at the next boot; - at a tool boundary, a write whose store throws stays queued and isn't injected, the next is stored and injected, and the first lands at the turn's end; - a message stored straight into the context during a round, with nothing injected at the boundary, stays unread; - a compile's watermark doesn't roll back a mark made while it ran, and a later compile still advances it; - a held message waits out a successor turn's tool cycle (the stream generation, token and cycle the successor's when the first turn's teardown flushes) and lands once it is over; - the re-deferral test also re-defers plain text coming back from the queue. - test/host-quiesce.test.ts's poison-write test pinned the old drop ("the queue is drained even though one write failed"); resume's flush writes drained entries too, so it now pins that the write that failed stays queued (deferredWrites 1), while the others land and the reopen completes. Each of these fixes, undone alone, fails test/deferred-window-only.test.ts: the default path waiting on every agent's cycle; the puppet's flush waiting on every agent's cycle; a stream registering while stopping; a failed store only logged; the boundary's store unguarded; a compile's watermark always written back; every continuing boundary marking the watermark. --- .../mid-turn-window-only-held.fixed.md | 54 +++- src/agent.ts | 25 +- src/framework.ts | 229 ++++++++------ test/deferred-window-only.test.ts | 298 ++++++++++++++++-- test/host-quiesce.test.ts | 6 +- 5 files changed, 472 insertions(+), 140 deletions(-) diff --git a/changelog.d/mid-turn-window-only-held.fixed.md b/changelog.d/mid-turn-window-only-held.fixed.md index 36641ad8e..0b741f96b 100644 --- a/changelog.d/mid-turn-window-only-held.fixed.md +++ b/changelog.d/mid-turn-window-only-held.fixed.md @@ -3,24 +3,46 @@ waits for the turn's end, after its last round, instead of being stored at the boundary between rounds the stream never showed it. Messages queued behind it wait with it, so the queue keeps its order; messages queued - before it are still heard at the boundary. Stored between rounds, it sat - before the turn's later replies in every later compile although their - requests never held it: a prompt-cache divergence, and on a provider that - binds signed thinking to the prefix it was minted under (context-manager - #155), a reply whose thinking is refused. + before it are still heard at the boundary. The cost: a message addressed to + the agent but queued behind a held one isn't heard mid-turn, and waits for + the turn's end. Stored between rounds, a held message sat before the turn's + later replies in every later compile although their requests never held + it: a prompt-cache divergence, and on a provider that binds signed thinking + to the prefix it was minted under (anima-research/context-manager#155, + whose binding anima-research/context-manager#160 walks), a reply whose + thinking is refused. - A message injected into the live stream at a tool boundary now counts as read for RFC-006 coalescing: the consumed watermark moves when messages are injected, not only at a compile. Before, an injected message sat above the watermark of the turn's compile, so a coalesced replacement for its - subject could remove a message the agent had heard and answered. -- A deferred write that goes back into the queue because a turn is alive now - does so whatever its content. A `tool_result` used to land regardless, + subject could remove a message the agent had heard and answered. Within a + branch the watermark only advances: a compile that started before such a + mark doesn't roll it back when it finishes. +- A deferred write that goes back into the queue does so whatever its + content. That happens while its target's own turn, stream or tool cycle is + alive, or the host is quiesced. A `tool_result` used to land regardless, apart from the `tool_use` it answers, which went back into the queue. -- A turn's end now delivers the agent's deferred messages even while another - agent's tool cycle is pending; only the agent's own cycle holds them. A - write still deferred then (a primary-bound write waits out any agent's - cycle) lands at the agent's next turn start, and `stop()` now stores - whatever is still deferred before the store closes. Before, a message - deferred for an agent whose turn ended during another agent's tool cycle - stayed queued until that agent's next turn, and a memory-only queue lost it - at `stop()`. +- Every write now waits on its own target's state: the target's turn, stream + and tool cycle, or a quiesced host. Before, a write for the primary agent + waited out every agent's tool cycle, while one addressed to an agent waited + only on that agent's. So a turn's end could store a later addressed message + ahead of an earlier one for the primary, and a fork's, subagent's or the + subconscious's tools held the primary's writes, which protected nothing in + the primary's window. Each flush, at a turn's end, after surgery, after a + withdrawn turn and after a puppet call, now delivers the agent's deferred + messages while another agent's tool cycle is pending; only the agent's own + cycle holds them. +- `stop()` now stores what is still deferred for each registered agent + before the store closes; before, a memory-only queue lost it. Once `stop()` + begins, no turn starts and no stream registers, for any agent, so nothing + is mid-turn when it does: a turn that was mid-compile when the streams + were cancelled is refused rather than left running, and an ephemeral run + refused this way ends at once rather than at its startup watchdog. A + quiesced host keeps what is deferred in its persisted queue for boot + recovery. An entry for a target no longer registered (a disposed fork) is + dropped from a memory-only queue at exit, as before. +- A deferred write whose store throws goes back into the queue under its + id, and the others continue. Before, it stayed in the un-acked set and the + next acknowledgement dropped it from the persisted queue (at a turn start + or a flush it was lost even from memory). At a tool boundary, such a write + isn't injected either. diff --git a/src/agent.ts b/src/agent.ts index 4b3ad9f2c..57a4d1a24 100644 --- a/src/agent.ts +++ b/src/agent.ts @@ -560,6 +560,9 @@ export class Agent { * request and may still be replaced or withdrawn in place by its sender. */ private consumedWatermark: { branch: string; sequence: number } | null = null; + /** Counts marks: a compile writes its watermark back only if no mark was + * made while it ran (a tool boundary's, a branch switch's). */ + private watermarkMarks = 0; getConsumedWatermark(): { branch: string; sequence: number } | null { return this.consumedWatermark; @@ -571,6 +574,22 @@ export class Agent { // Tolerates partial context-manager doubles (tests build Agents over // stubs): without a watermark nothing is ever treated as unread. this.consumedWatermark = this.pendingWatermark(); + this.watermarkMarks++; + } + + /** + * A finished compile's watermark, taken before it awaited. Within a branch + * the watermark only advances: if a mark was made while the compile ran, + * the compile's (older) one is applied only where it is later on that + * mark's branch, so a boundary's mark is never rolled back by a compile + * that started before it (a preview during a live turn). + */ + private applyCompileWatermark(watermark: { branch: string; sequence: number }, marksBefore: number): void { + const current = this.consumedWatermark; + if (this.watermarkMarks !== marksBefore && current + && (current.branch !== watermark.branch || current.sequence >= watermark.sequence)) return; + this.consumedWatermark = watermark; + this.watermarkMarks++; } /** The watermark a successful compile will establish (taken before, applied after). */ @@ -597,8 +616,9 @@ export class Agent { // RFC-006 §3.3: consumption is determined at assembly — a compile that // fails assembled nothing, so the watermark moves only on success. const watermark = this.pendingWatermark(); + const marksBefore = this.watermarkMarks; const result = await this.contextManager.compile(this.compileBudget(budget)); - if (watermark) this.consumedWatermark = watermark; + if (watermark) this.applyCompileWatermark(watermark, marksBefore); if (!budget) this.settleRuntimeSettingsTransition(); return result; } @@ -614,10 +634,11 @@ export class Agent { opts?: { kvUnifiedImmutablePrefixHash?: string }, ): Promise { const watermark = this.pendingWatermark(); + const marksBefore = this.watermarkMarks; const result = await this.contextManager.compile( this.compileBudget(budget), injections, opts as never, ); - if (watermark) this.consumedWatermark = watermark; + if (watermark) this.applyCompileWatermark(watermark, marksBefore); if (!budget) this.settleRuntimeSettingsTransition(); return result; } diff --git a/src/framework.ts b/src/framework.ts index 9f6a45760..14176841e 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -2043,25 +2043,21 @@ export class AgentFramework { } } - // Every turn is over, so nothing else will deliver what is still - // deferred: a message held for a turn's end whose flush found the write - // still deferred (a primary-bound write waits out any agent's tool - // cycle), or one that arrived for an idle agent during another's cycle. - // Land each at its target now, while the store is open; a memory-only - // queue would otherwise lose it. One addMessage still defers (quiesced) - // stays in the persisted queue for boot recovery. + // Every turn is over: from the start of stop() no turn starts and no + // stream registers (providerAdmissionClosed, checked for every agent), + // and every stream registered before was cancelled and awaited above. So + // nothing else will deliver what is still deferred: a message held for + // a turn's end whose flush found the target busy again, or one that + // arrived for an idle agent and was deferred behind its own state. Land + // each at its registered target now, while the store is open; a + // memory-only queue would otherwise lose it. A quiesced host defers it + // again, into the persisted queue, for boot recovery; a write that throws + // stays queued (writeDrained). An entry for a target no longer registered + // (a disposed fork) is left: dropped with a memory-only queue at exit, and + // logged and dropped from a persisted one at boot. if (this.deferredMessages.length > 0) { for (const name of this.agents.keys()) { - for (const msg of this.drainDeferredFor(name)) { - try { - this.addMessage(msg.participant, msg.content, msg.metadata, { - deferredWriteId: msg.id, - ...(msg.forAgent ? { forAgent: msg.forAgent } : {}), - }); - } catch (error) { - console.error(`[stop] deferred write for ${name} could not be stored: ${error instanceof Error ? error.message : String(error)}`); - } - } + for (const msg of this.drainDeferredFor(name)) this.readdDrained(msg, 'stop'); } this.ackDeferredWrites(); } @@ -3378,19 +3374,7 @@ export class AgentFramework { this.handOffDeferredWrites(flush); let stored = 0; for (const msg of flush) { - try { - this.addMessage(msg.participant, msg.content, msg.metadata, { - deferredWriteId: msg.id, - ...(msg.forAgent ? { forAgent: msg.forAgent } : {}), - }); - stored++; - } catch (err) { - console.error( - `[host-mode] ${label}: failed to store a deferred context write ` + - `(participant=${msg.participant}, forAgent=${msg.forAgent ?? 'primary'}):`, - err, - ); - } + if (this.readdDrained(msg, `host-mode ${label}`)) stored++; } // Ack = sync the chronicle FIRST, then rewrite the durable queue without // the batch. Never the other way round. @@ -4198,6 +4182,15 @@ export class AgentFramework { } } + /** A turn refused because stop() has begun: an ephemeral run waiting on it + * ends now, rather than when its startup watchdog fires. */ + private endEphemeralRunRefused(agent: Agent): void { + const run = this.ephemeralRuns.get(agent.name); + if (run && this.agents.get(agent.name) === agent) { + run.settle.reject(new Error(`Ephemeral agent "${agent.name}" refused: the framework is stopping`)); + } + } + private createDeferred(): Deferred { let resolve!: (value: T) => void; let reject!: (reason?: unknown) => void; @@ -4284,6 +4277,11 @@ export class AgentFramework { `refusing new ephemeral run for ${agent.name} — resume() first`, ); } + // Once stop() has begun no turn starts (startAgentStream), so a run + // admitted now could only wait out its startup watchdog. + if (this.providerAdmissionClosed) { + throw new Error(`framework is stopping: refusing new ephemeral run for ${agent.name}`); + } // Only a fresh object returned by createEphemeralAgent may enter this path. // Never overwrite a resident/conversation owner or a concurrent run: their // cleanup is name-keyed and could cancel/deregister the legitimate owner. @@ -5951,20 +5949,12 @@ export class AgentFramework { // Writers that deferred behind the reservation land now, on the branch // the operator chose (or the restored source) — the same end-of-turn // flush driveStream/puppet perform, under the same guard. - if (this.deferredMessages.length > 0 && this.pendingAssistantBlocks.size === 0) { + if (this.deferredMessages.length > 0) { for (const name of reserved.keys()) { - if (this.activeTurnTokens.has(name)) continue; - for (const msg of this.drainDeferredFor(name)) { - try { - this.addMessage(msg.participant, msg.content, msg.metadata, { - deferredWriteId: msg.id, - ...(msg.forAgent ? { forAgent: msg.forAgent } : {}), - }); - } catch (error) { - console.error(`[operator] post-surgery deferred flush failed for ${name}: ${error instanceof Error ? error.message : String(error)}`); - } - } + if (this.activeTurnTokens.has(name) || this.pendingAssistantBlocks.has(name)) continue; + for (const msg of this.drainDeferredFor(name)) this.readdDrained(msg, 'post-surgery flush'); } + this.ackDeferredWrites(); } }; } @@ -6744,7 +6734,12 @@ export class AgentFramework { { const deferred = this.drainDeferredFor(agent.name, (msg) => !isInjectableMidTurn(msg, agent.name)); for (const msg of deferred) { - agent.getContextManager().addMessage(msg.participant, msg.content, withDeferredWriteId(msg.metadata, msg.id)); + // One that can't be stored stays queued and isn't injected; the + // rest are, so the boundary still hands the round's results on. + const stored = this.writeDrained(msg, `tool boundary for ${agent.name}`, () => { + agent.getContextManager().addMessage(msg.participant, msg.content, withDeferredWriteId(msg.metadata, msg.id)); + }); + if (!stored) continue; midTurnInjections.push({ participant: msg.participant, content: msg.content, @@ -6958,6 +6953,14 @@ export class AgentFramework { // the turn's compile and read as unread, so a coalesced // replacement could remove a message the agent heard and // answered (and that its later replies were minted after). + // The mark is the store's head, a scalar, as a compile's is: it + // also covers what was stored since the compile without being + // carried, such as another slot's messages, and a write that + // bypassed deferral into this slot (a fresh tool_result, a + // routed channel message). That over-cover can't be left out. + // Under context-manager#160 such a write already gives the + // replies stored after it stamps their requests didn't carry, + // whatever the mark says; deferring those writers is the fix. if (midTurnInjections.length > 0) agent.markContextConsumed(); agent.setStreaming(currentState.stream); this.emitTrace({ @@ -9022,6 +9025,15 @@ export class AgentFramework { attempt = 0, providerGateAlreadyHeld = false, ): Promise { + // Once stop() has begun, no turn starts, for any agent: a fork's or a + // subagent's retry after its delay, or a wake the loop dequeued before it + // ended, would otherwise begin a turn stop() never cancels, and the + // deferred drain at stop's end would meet it live. + if (this.providerAdmissionClosed) { + if (providerGateAlreadyHeld) this.releasePrimaryProviderGate(agent.name); + this.endEphemeralRunRefused(agent); + return; + } const ownsProviderGate = !this.ephemeralRuns.has(agent.name) && !this.conversationAgentHomes.has(agent.name); if (ownsProviderGate && !providerGateAlreadyHeld) { @@ -9147,21 +9159,15 @@ export class AgentFramework { { const deferred = this.drainDeferredFor(agent.name); for (const msg of deferred) { - // Per-message try/catch (mirrors announceLocusIfChanged): one - // poison message — or a transient store-write failure — must not - // abort the turn or drop the messages behind it in the queue. - try { + // One poison message, or a transient store-write failure, must not + // abort the turn or drop the messages behind it in the queue; the + // one that failed stays queued (writeDrained). + this.writeDrained(msg, `turn-start flush for ${agent.name}`, () => { const id = agent.getContextManager().addMessage( msg.participant, msg.content, withDeferredWriteId(msg.metadata, msg.id), ); this.emitTrace({ type: 'message:added', messageId: id, source: 'deferred-flush:turn-start' }); - } catch (err) { - console.error( - `[deferred-flush] turn-start flush failed to store a message for ${agent.name} ` + - `(participant=${msg.participant}):`, - err, - ); - } + }); } this.ackDeferredWrites(); } @@ -9201,13 +9207,8 @@ export class AgentFramework { this.activeTurnTriggers.delete(agent.name); } if (!this.activeTurnTokens.has(agent.name) - && this.deferredMessages.length > 0 && this.pendingAssistantBlocks.size === 0) { - for (const msg of this.drainDeferredFor(agent.name)) { - this.addMessage(msg.participant, msg.content, msg.metadata, { - deferredWriteId: msg.id, - ...(msg.forAgent ? { forAgent: msg.forAgent } : {}), - }); - } + && this.deferredMessages.length > 0 && !this.pendingAssistantBlocks.has(agent.name)) { + for (const msg of this.drainDeferredFor(agent.name)) this.readdDrained(msg, 'withdrawn-turn flush'); this.ackDeferredWrites(); } return false; @@ -9413,11 +9414,16 @@ export class AgentFramework { } = await agent.startStreamWithInjections(tools, injections, undefined, compressionTools, { silentHeartbeat: trigger?.silentHeartbeat, }); - if (this.agents.get(agent.name) !== agent) { + // A turn that was mid-compile when stop() cancelled every stream would + // register after that pass and outlive stop(): refuse it here, as a + // replaced agent is. The caller's finally releases the turn token. + const stopping = this.providerAdmissionClosed; + if (this.agents.get(agent.name) !== agent || stopping) { stream.cancel(); agent.cancelStream(); this.channelRegistry?.stopTyping(); this.eventGate?.onInferenceEnded(agent.name); + if (stopping) this.endEphemeralRunRefused(agent); return false; } @@ -10979,17 +10985,11 @@ export class AgentFramework { // target has meanwhile started a turn). Gated on this agent's own tool // cycle, not every agent's: messages held for this turn's end depend on // this flush, and another agent's pending tools say nothing about this - // window. A write addMessage still defers (a primary-bound write waits - // out any agent's cycle) goes back into the queue under its id, for the - // target's next turn start or stop(). + // window. addMessage judges each entry by the same rule, so the drained + // prefix is stored in order, or goes back into the queue whole under + // its ids when a successor turn has already taken the agent. if (frameReachedTerminal && this.deferredMessages.length > 0 && !this.pendingAssistantBlocks.has(agent.name)) { - const deferred = this.drainDeferredFor(agent.name); - for (const msg of deferred) { - this.addMessage(msg.participant, msg.content, msg.metadata, { - deferredWriteId: msg.id, - ...(msg.forAgent ? { forAgent: msg.forAgent } : {}), - }); - } + for (const msg of this.drainDeferredFor(agent.name)) this.readdDrained(msg, 'turn-end flush'); this.ackDeferredWrites(); } } @@ -11144,6 +11144,9 @@ export class AgentFramework { { id: randomUUID(), seq: ++this.deferredSeq, participant: agentName, content: toolUse, forAgent: agentName }, { id: randomUUID(), seq: ++this.deferredSeq, participant: 'user', content: blocks, forAgent: agentName }, ); + // As addMessage's own deferral does: a crash before the next persist + // must not lose the pair from a persisted queue. + if (this.quiesced || this.deferredWritesPersisted) this.persistDeferredWrites(); } this.emitTrace({ @@ -11164,19 +11167,14 @@ export class AgentFramework { if (this.activeTurnTokens.get(agentName) === turnToken) this.activeTurnTokens.delete(agentName); // Messages deferred while we held the token land now, after the pair // — the same end-of-turn flush driveStream performs, under the same - // tool-cycle guard. (No wake is requested: the agent sees the pair, - // and anything that arrived meanwhile, on its next turn.) Keyed on - // "no turn alive", not on token ownership: if a turn did replace us - // and has already ended, its flush ran before our pair was queued — - // nobody else will flush it, so we do. + // guard: this agent's own tool cycle. (No wake is requested: the agent + // sees the pair, and anything that arrived meanwhile, on its next + // turn.) Keyed on "no turn alive", not on token ownership: if a turn + // did replace us and has already ended, its flush ran before our pair + // was queued — nobody else will flush it, so we do. if (!this.activeTurnTokens.has(agentName) - && this.deferredMessages.length > 0 && this.pendingAssistantBlocks.size === 0) { - for (const msg of this.drainDeferredFor(agentName)) { - this.addMessage(msg.participant, msg.content, msg.metadata, { - deferredWriteId: msg.id, - ...(msg.forAgent ? { forAgent: msg.forAgent } : {}), - }); - } + && this.deferredMessages.length > 0 && !this.pendingAssistantBlocks.has(agentName)) { + for (const msg of this.drainDeferredFor(agentName)) this.readdDrained(msg, 'puppet flush'); this.ackDeferredWrites(); } } @@ -12602,13 +12600,17 @@ export class AgentFramework { // without it would split the pair. (A turn-end flush can run after a // successor turn has taken the agent: the 2026-07-31 interleave.) const freshToolResult = opts?.deferredWriteId === undefined && content.some(b => b.type === 'tool_result'); - // Explicitly-targeted deliveries scope the tool-cycle check to the - // target (pendingAssistantBlocks is keyed by agent); the default path - // keeps the historical GLOBAL check byte-for-byte (conservative: any - // agent's pending cycle defers primary-bound messages). - const midToolCycle = opts?.forAgent - ? this.pendingAssistantBlocks.has(agent.name) - : this.pendingAssistantBlocks.size > 0; + // One rule for every write, primary-bound or targeted: it waits while ITS + // TARGET's own tool cycle is pending (pendingAssistantBlocks is keyed by + // agent). Another agent's cycle says nothing about this window. The + // default path used to check every agent's cycle (efc7e7af, from when + // all agents shared one message store), which held the primary's writes + // behind a fork's, subagent's or the subconscious's tools, and judged a + // drained queue by two rules, so a later targeted entry could land ahead + // of an earlier primary-bound one. Residents that share the un-namespaced + // slot with overlapping turns put each other's writes inside the other's + // live turn whatever this check does: the limit #286 states. + const midToolCycle = this.pendingAssistantBlocks.has(agent.name); if ( !freshToolResult && opts?.bypassDeferralFor !== agent.name && @@ -12649,15 +12651,7 @@ export class AgentFramework { * message and every later one stay queued, so what is drained is always a * prefix of the agent's queue. */ - private drainDeferredFor(agentName: string, stopAt?: (msg: DeferredWrite) => boolean): Array<{ - /** Durable identity — every write of this entry must carry it - * (`withDeferredWriteId` / `opts.deferredWriteId`). */ - id: string; - participant: string; - content: ContentBlock[]; - metadata?: MessageMetadata; - forAgent?: string; - }> { + private drainDeferredFor(agentName: string, stopAt?: (msg: DeferredWrite) => boolean): DeferredWrite[] { if (this.deferredMessages.length === 0) return []; const mine: typeof this.deferredMessages = []; const rest: typeof this.deferredMessages = []; @@ -12677,6 +12671,41 @@ export class AgentFramework { return drained; } + /** + * Write one drained entry. If the write throws, the entry goes back into + * the queue under its id and seq, and the caller goes on with the rest: + * left in the un-acked set, the next ack would drop it from the durable + * queue, and a memory-only queue would lose it at once. Every site that + * writes a drained entry writes through this. Returns whether it was + * written (stored, or deferred again by addMessage). + */ + private writeDrained(msg: DeferredWrite, where: string, write: () => void): boolean { + try { + write(); + return true; + } catch (error) { + console.error( + `[deferral] ${where}: a deferred write for ${msg.forAgent ?? this.primaryAgentName ?? 'the primary'} ` + + `(participant=${msg.participant}) could not be stored; it stays queued: ` + + `${error instanceof Error ? error.message : String(error)}`, + ); + this.unackedDeferredWrites = this.unackedDeferredWrites.filter((m) => m.id !== msg.id); + if (!this.deferredMessages.some((m) => m.id === msg.id)) this.deferredMessages.push(msg); + if (this.quiesced || this.deferredWritesPersisted) this.persistDeferredWrites(); + return false; + } + } + + /** Re-add one drained entry through addMessage, which stores it or defers it again. */ + private readdDrained(msg: DeferredWrite, where: string): boolean { + return this.writeDrained(msg, where, () => { + this.addMessage(msg.participant, msg.content, msg.metadata, { + deferredWriteId: msg.id, + ...(msg.forAgent ? { forAgent: msg.forAgent } : {}), + }); + }); + } + private editMessage(id: MessageId, content: ContentBlock[]): void { const agent = this.primaryAgentName ? this.agents.get(this.primaryAgentName) diff --git a/test/deferred-window-only.test.ts b/test/deferred-window-only.test.ts index 246293027..e6d93a526 100644 --- a/test/deferred-window-only.test.ts +++ b/test/deferred-window-only.test.ts @@ -65,9 +65,17 @@ class InterjectingModule implements Module { for (const interjection of pending) { this.framework!.pushEvent({ type: 'interjection', source: 'test', interjection } as unknown as ProcessEvent); } - // Let the run loop handle them while this round's tools are pending, so - // they land in the deferred queue before the tool-result boundary. - if (pending.length > 0) await new Promise((r) => setTimeout(r, 30)); + // Let the run loop handle them while this round's tools are pending, and + // return only once each is in the deferred queue, so the tool-result + // boundary meets them there (bounded: a run that never queues them fails). + if (pending.length > 0) { + const queue = () => (this.framework as unknown as { deferredMessages: unknown[] }).deferredMessages.length; + const end = Date.now() + 5000; + while (queue() < pending.length) { + if (Date.now() > end) throw new Error(`only ${queue()} of ${pending.length} interjection(s) were queued`); + await new Promise((r) => setTimeout(r, 5)); + } + } const during = this.duringTool; this.duringTool = null; during?.(); @@ -129,6 +137,7 @@ describe('mid-turn deliveries keep the window what the stream carried', () => { } /** A turn of two rounds: `tool` with the interjections, then a reply. */ + let drains: Array<{ atBoundary: boolean; queued: string[] }> = []; async function runTurn(tool: 'move' | 'halt', interjections: Interjection[]): Promise { membrane.pushResponse(createMockResponse([ { type: 'text', text: 'Moving.' }, @@ -136,12 +145,37 @@ describe('mid-turn deliveries keep the window what the stream carried', () => { ] as ContentBlock[], 'tool_use')); membrane.pushResponse(createMockResponse(text('Done.'))); const framework = await createFramework(); + drains = recordDrains(framework); module.interjections = interjections; framework.pushEvent({ type: 'external-message', source: 'test', content: 'go' } as unknown as ProcessEvent); await framework.runUntilIdle(); return framework; } + /** What the tool boundary's drain found queued. */ + const atBoundary = () => drains.find((d) => d.atBoundary)?.queued; + + /** What each drain for the agent found queued, in order, and whether it + * stopped at a held message (the tool boundary's drain does). */ + function recordDrains(framework: AgentFramework): Array<{ atBoundary: boolean; queued: string[] }> { + const drains: Array<{ atBoundary: boolean; queued: string[] }> = []; + const internals = framework as unknown as { + deferredMessages: Array<{ participant: string; content: ContentBlock[] }>; + drainDeferredFor: (name: string, stopAt?: unknown) => unknown[]; + }; + const drain = internals.drainDeferredFor.bind(framework); + internals.drainDeferredFor = (name: string, stopAt?: unknown) => { + if (name === 'assistant' && internals.deferredMessages.length > 0) { + drains.push({ + atBoundary: stopAt !== undefined, + queued: internals.deferredMessages.map((m) => `${m.participant}: ${m.content.map((b) => (b.type === 'text' ? (b as { text: string }).text : `<${b.type}>`)).join(' ')}`), + }); + } + return drain(name, stopAt); + }; + return drains; + } + function injected(): string[] { const options = membrane.lastStream!.receivedToolResultOptions[0]; return (options?.injectedMessages ?? []).map((m) => @@ -150,6 +184,7 @@ describe('mid-turn deliveries keep the window what the stream carried', () => { it('a message named as the agent waits for the turn to end, after its last round', async () => { const framework = await runTurn('move', [{ participant: 'assistant', content: text('my own words') }]); + assert.deepEqual(atBoundary(), ['assistant: my own words'], 'it was queued when the boundary drained'); assert.deepEqual(injected(), []); assert.deepEqual(windowOf(framework), [ 'Antra: go', @@ -176,6 +211,7 @@ describe('mid-turn deliveries keep the window what the stream carried', () => { { participant: 'assistant', content: text('my own words') }, { participant: 'Antra', content: text('look left') }, ]); + assert.deepEqual(atBoundary(), ['assistant: my own words', 'Antra: look left'], 'both were queued when the boundary drained'); assert.deepEqual(injected(), []); assert.deepEqual(windowOf(framework).slice(2), [ 'user: ', @@ -191,6 +227,7 @@ describe('mid-turn deliveries keep the window what the stream carried', () => { { participant: 'Antra', content: text('look left') }, { participant: 'assistant', content: text('my own words') }, ]); + assert.deepEqual(atBoundary(), ['Antra: look left', 'assistant: my own words'], 'both were queued when the boundary drained'); assert.deepEqual(injected(), ['Antra: look left']); assert.deepEqual(windowOf(framework).slice(2), [ 'user: ', @@ -226,14 +263,19 @@ describe('mid-turn deliveries keep the window what the stream carried', () => { await framework.stop(); }); - /** The framework's own queue and its record of pending tool cycles. */ + /** The framework's own queue and its record of turns and tool cycles. */ function internalsOf(framework: AgentFramework) { return framework as unknown as { pendingAssistantBlocks: Map; - deferredMessages: Array<{ id: string }>; - addMessage(participant: string, content: ContentBlock[], metadata?: undefined, opts?: { forAgent?: string }): string; + activeTurnTokens: Map; + activeStreams: Map; + deferredMessages: Array<{ id: string; participant: string }>; + deferredWritesPersisted: boolean; + drainDeferredFor: (name: string, stopAt?: unknown) => unknown[]; + addMessage(participant: string, content: ContentBlock[], metadata?: undefined, opts?: { forAgent?: string; deferredWriteId?: string }): string; }; } + const otherCycle = (): ContentBlock[] => [{ type: 'tool_use', id: 'b1', name: 'robot--move', input: {} } as ContentBlock]; it('a message held for the turn\'s end lands then, while another agent\'s tool cycle is pending', async () => { membrane.pushResponse(createMockResponse([ @@ -243,43 +285,256 @@ describe('mid-turn deliveries keep the window what the stream carried', () => { membrane.pushResponse(createMockResponse(text('Done.'))); const framework = await createFramework(); const internals = internalsOf(framework); + // Whether the other cycle was pending at the turn-end flush itself. + const pendingAtFlush: boolean[] = []; + const drain = internals.drainDeferredFor.bind(framework); + internals.drainDeferredFor = (name: string, stopAt?: unknown) => { + if (name === 'assistant' && stopAt === undefined && internals.deferredMessages.length > 0) { + pendingAtFlush.push(internals.pendingAssistantBlocks.has('builder')); + } + return drain(name, stopAt); + }; module.duringTool = () => { // Another agent is mid tool cycle when this turn ends, and a write // named as this agent (held: it can't join the live stream) waits. - internals.pendingAssistantBlocks.set('builder', [{ type: 'tool_use', id: 'b1', name: 'robot--move', input: {} } as ContentBlock]); + internals.pendingAssistantBlocks.set('builder', otherCycle()); assert.equal(internals.addMessage('assistant', text('my own words'), undefined, { forAgent: 'assistant' }), '', 'deferred'); }; framework.pushEvent({ type: 'external-message', source: 'test', content: 'go' } as unknown as ProcessEvent); await framework.runUntilIdle(); - assert.ok(internals.pendingAssistantBlocks.has('builder'), 'the other cycle was still pending at the flush'); + assert.deepEqual(pendingAtFlush, [true], 'the other cycle was pending at the turn-end flush'); assert.deepEqual(windowOf(framework).slice(-2), ['assistant: Done.', 'assistant: my own words']); internals.pendingAssistantBlocks.delete('builder'); await framework.stop(); }); - it('a message still deferred when the framework stops lands before the store closes', async () => { + it("a write waits on its target's own turn and tool cycle, not another agent's, primary-bound or targeted", async () => { const framework = await createFramework(); const internals = internalsOf(framework); - // A primary-bound write waits out any agent's tool cycle, even with this - // agent idle; that cycle's stream then ends without delivering it. - internals.pendingAssistantBlocks.set('builder', [{ type: 'tool_use', id: 'b1', name: 'robot--move', input: {} } as ContentBlock]); - assert.equal(internals.addMessage('Antra', text('late word')), '', 'deferred'); - internals.pendingAssistantBlocks.delete('builder'); - assert.equal(internals.deferredMessages.length, 1); + internals.pendingAssistantBlocks.set('builder', otherCycle()); + try { + assert.notEqual(internals.addMessage('Antra', text('primary-bound')), '', 'stored: the primary is idle'); + assert.notEqual(internals.addMessage('Antra', text('targeted'), undefined, { forAgent: 'assistant' }), '', 'stored too'); + internals.pendingAssistantBlocks.set('assistant', otherCycle()); + assert.equal(internals.addMessage('Antra', text('in its own cycle')), '', "deferred: the primary's own cycle is pending"); + } finally { + internals.pendingAssistantBlocks.delete('builder'); + internals.pendingAssistantBlocks.delete('assistant'); + } await framework.stop(); + }); + + it("the puppet's flush lands what was deferred behind it while another agent's tool cycle is pending", async () => { + const framework = await createFramework(); + const internals = internalsOf(framework); + module.duringTool = () => { + internals.pendingAssistantBlocks.set('builder', otherCycle()); + assert.equal(internals.addMessage('Antra', text('while puppeted')), '', 'deferred behind the puppet'); + }; + try { + await framework.puppetToolCall('assistant', 'robot--move', {}); + assert.equal(internals.deferredMessages.length, 0); + assert.deepEqual(windowOf(framework).slice(-3), ['assistant: ', 'user: ', 'Antra: while puppeted']); + } finally { + internals.pendingAssistantBlocks.delete('builder'); + } + await framework.stop(); + }); + + it('a turn mid-compile when the framework stops never starts a stream, and what was deferred for it is stored', async () => { + membrane.pushResponse(createMockResponse(text('too late'))); + const framework = await createFramework(); + const internals = internalsOf(framework); + const agent = framework.getAgent('assistant')!; + let compiling!: () => void; + const inCompile = new Promise((resolve) => { compiling = resolve; }); + let release!: () => void; + const released = new Promise((resolve) => { release = resolve; }); + const start = agent.startStreamWithInjections.bind(agent); + (agent as unknown as { startStreamWithInjections: typeof start }).startStreamWithInjections = async (...args) => { + compiling(); + await released; + return start(...args); + }; + // The event loop starts the turn, as in a host (stop() awaits the loop). + framework.start(); + framework.pushEvent({ type: 'external-message', source: 'test', content: 'go' } as unknown as ProcessEvent); + await inCompile; + assert.equal(internals.addMessage('Antra', text('late word')), '', 'deferred: the turn is alive'); + const stopped = framework.stop(); + release(); + await stopped; + assert.equal(internals.activeStreams.size, 0, 'no stream outlived stop()'); const reopened = await createFramework(); assert.ok(windowOf(reopened).includes('Antra: late word'), 'stored before the store closed'); + assert.ok(!windowOf(reopened).includes('assistant: too late'), 'the refused turn produced nothing'); await reopened.stop(); }); - it('a write coming back from the deferred queue re-defers while a turn is alive, tool_result or not', async () => { + it('a drained write whose store throws stays in the persisted queue, and is stored on the next boot', async () => { const framework = await createFramework(); - const internals = framework as unknown as { - activeTurnTokens: Map; - deferredMessages: Array<{ id: string }>; - addMessage: (p: string, c: ContentBlock[], m?: unknown, o?: { deferredWriteId?: string }) => string; + const internals = internalsOf(framework); + internals.deferredWritesPersisted = true; + internals.activeTurnTokens.set('assistant', 1); + assert.equal(internals.addMessage('Antra', text('kept')), '', 'deferred'); + internals.activeTurnTokens.delete('assistant'); + const cm = framework.getAgent('assistant')!.getContextManager(); + const add = cm.addMessage.bind(cm); + (cm as unknown as { addMessage: typeof add }).addMessage = (...args) => { + if ((args[1] as Array<{ text?: string }>).some((b) => b.text === 'kept')) throw new Error('disk full'); + return add(...args); }; + const errors: string[] = []; + const error = console.error; + console.error = (...args: unknown[]) => { errors.push(args.map(String).join(' ')); }; + try { await framework.stop(); } finally { console.error = error; } + assert.ok(errors.some((e) => e.includes('could not be stored; it stays queued') && e.includes('disk full')), errors.join('\n')); + assert.equal(internals.deferredMessages.length, 1, 'still queued'); + + const reopened = await createFramework(); + assert.ok(windowOf(reopened).includes('Antra: kept'), 'boot recovery stored it'); + await reopened.stop(); + }); + + it('at a tool boundary, a write whose store throws stays queued and is not injected; the rest are', async () => { + membrane.pushResponse(createMockResponse([ + { type: 'text', text: 'Moving.' }, + { type: 'tool_use', id: 'c1', name: 'robot--move', input: {} }, + ] as ContentBlock[], 'tool_use')); + membrane.pushResponse(createMockResponse(text('Done.'))); + const framework = await createFramework(); + const cm = framework.getAgent('assistant')!.getContextManager(); + const add = cm.addMessage.bind(cm); + let failOnce = true; + (cm as unknown as { addMessage: typeof add }).addMessage = (...args) => { + if (failOnce && (args[1] as Array<{ text?: string }>).some((b) => b.text === 'look left')) { + failOnce = false; + throw new Error('transient'); + } + return add(...args); + }; + module.interjections = [ + { participant: 'Antra', content: text('look left') }, + { participant: 'Antra', content: text('look right') }, + ]; + const error = console.error; + console.error = () => {}; + try { + framework.pushEvent({ type: 'external-message', source: 'test', content: 'go' } as unknown as ProcessEvent); + await framework.runUntilIdle(); + } finally { console.error = error; } + assert.deepEqual(injected(), ['Antra: look right']); + assert.deepEqual(windowOf(framework).slice(2), [ + 'user: ', + 'Antra: look right', + 'assistant: Done.', + 'Antra: look left', + ], 'the one that failed was kept, and stored at the turn end'); + await framework.stop(); + }); + + it('a message stored straight into the context during a round, with nothing injected at the boundary, stays unread', async () => { + membrane.pushResponse(createMockResponse([ + { type: 'text', text: 'Moving.' }, + { type: 'tool_use', id: 'c1', name: 'robot--move', input: {} }, + ] as ContentBlock[], 'tool_use')); + membrane.pushResponse(createMockResponse(text('Done.'))); + const framework = await createFramework(); + const agent = framework.getAgent('assistant')!; + let bypassed = ''; + module.duringTool = () => { + // A writer that bypasses deferral (an older route): the stream never carried it. + bypassed = agent.getContextManager().addMessage('Antra', text('bypassed')); + }; + framework.pushEvent({ type: 'external-message', source: 'test', content: 'go' } as unknown as ProcessEvent); + await framework.runUntilIdle(); + assert.deepEqual(injected(), []); + const isUnread = (framework as unknown as { + isUnreadStoredMessage: (a: typeof agent, id: string) => boolean; + }).isUnreadStoredMessage.bind(framework); + assert.equal(isUnread(agent, bypassed), true, 'a boundary that injected nothing marks nothing read'); + await framework.stop(); + }); + + it("a compile's watermark never rolls back a mark made while it ran", async () => { + const framework = await createFramework(); + const agent = framework.getAgent('assistant')!; + const cm = agent.getContextManager(); + cm.addMessage('Antra', text('before')); + const compile = cm.compile.bind(cm); + let release!: () => void; + const released = new Promise((resolve) => { release = resolve; }); + (cm as unknown as { compile: typeof compile }).compile = async (...args) => { + await released; + return compile(...args); + }; + const compiling = agent.compileContext(); + cm.addMessage('Antra', text('heard at a boundary')); + agent.markContextConsumed(); + const marked = agent.getConsumedWatermark(); + release(); + await compiling; + assert.deepEqual(agent.getConsumedWatermark(), marked, "the boundary's mark stands"); + (cm as unknown as { compile: typeof compile }).compile = compile; + await agent.compileContext(); + assert.ok(agent.getConsumedWatermark()!.sequence >= marked!.sequence, 'a later compile still advances it'); + await framework.stop(); + }); + + it("a held message waits out a successor turn's tool cycle, and lands once it is over", async () => { + membrane.pushResponse(createMockResponse([ + { type: 'text', text: 'Moving.' }, + { type: 'tool_use', id: 'c1', name: 'robot--move', input: {} }, + ] as ContentBlock[], 'tool_use')); + membrane.pushResponse(createMockResponse(text('Done.'))); + const framework = await createFramework(); + const internals = internalsOf(framework) as ReturnType & { releasePrimaryProviderGate: (name: string) => void }; + const agent = framework.getAgent('assistant')!; + module.duringTool = () => { + assert.equal(internals.addMessage('assistant', text('my own words'), undefined, { forAgent: 'assistant' }), '', 'held'); + }; + // As the turn's stream completes, a successor turn takes the agent and is + // mid tool cycle (the 2026-07-31 interleave): the stream generation, the + // turn token and the cycle are the successor's when the first turn's + // teardown flushes. + let completed = false; + const off = framework.onTrace((event) => { + if ((event as { type: string }).type !== 'inference:completed') return; + off(); + completed = true; + (agent as unknown as { _streamId: number })._streamId++; + internals.activeTurnTokens.set('assistant', 1_000_000); + internals.pendingAssistantBlocks.set('assistant', otherCycle()); + }); + // The first turn's teardown releases its provider gate, then flushes, + // synchronously: once the release is seen, the flush has run. + let tornDown!: () => void; + const teardown = new Promise((resolve) => { tornDown = resolve; }); + const release = internals.releasePrimaryProviderGate.bind(framework); + internals.releasePrimaryProviderGate = (name: string) => { + release(name); + if (completed && name === 'assistant') tornDown(); + }; + framework.start(); + framework.pushEvent({ type: 'external-message', source: 'test', content: 'go' } as unknown as ProcessEvent); + await teardown; + await new Promise((resolve) => setImmediate(resolve)); + assert.ok(!windowOf(framework).includes('assistant: my own words'), "not stored inside the successor's cycle"); + assert.ok(internals.deferredMessages.some((m) => m.participant === 'assistant'), 'still queued'); + // The successor's cycle and turn end; what is still queued lands (here at stop's drain). + internals.pendingAssistantBlocks.delete('assistant'); + internals.activeTurnTokens.delete('assistant'); + internals.activeStreams.delete('assistant'); + await framework.stop(); + const reopened = await createFramework(); + assert.equal(windowOf(reopened).at(-1), 'assistant: my own words'); + await reopened.stop(); + }); + + it('a write coming back from the deferred queue re-defers while a turn is alive, tool_result or not', async () => { + const framework = await createFramework(); + const internals = internalsOf(framework); internals.activeTurnTokens.set('assistant', 1); try { // A queued pair's tool_result must follow its tool_use back into the @@ -290,6 +545,9 @@ describe('mid-turn deliveries keep the window what the stream carried', () => { ], undefined, { deferredWriteId: 'w1' }); assert.equal(requeued, ''); assert.ok(internals.deferredMessages.some((m) => m.id === 'w1')); + // So does plain text. + assert.equal(internals.addMessage('Antra', text('plain'), undefined, { deferredWriteId: 'w2' }), ''); + assert.ok(internals.deferredMessages.some((m) => m.id === 'w2')); // A fresh tool_result write keeps landing as before. const fresh = internals.addMessage('user', [ { type: 'tool_result', toolUseId: 'p2', content: 'ok' } as ContentBlock, diff --git a/test/host-quiesce.test.ts b/test/host-quiesce.test.ts index cae60190e..c0a08f611 100644 --- a/test/host-quiesce.test.ts +++ b/test/host-quiesce.test.ts @@ -683,7 +683,7 @@ test('a batch carrying a budget restart re-parks its sibling wakes instead of co }); }); -test('resume flushes deferred writes per message: one poison write neither drops the rest nor skips the reopen', async () => { +test('resume flushes deferred writes per message: one poison write neither drops the rest nor skips the reopen, and stays queued', async () => { const membrane = new MockMembrane(); await withFramework(membrane, async (framework) => { const traces: TraceEvent[] = []; @@ -709,7 +709,9 @@ test('resume flushes deferred writes per message: one poison write neither drops const status = await framework.resume(); assert.equal(status.quiesced, false); - assert.equal(status.deferredWrites, 0, 'the queue is drained even though one write failed'); + // A drained write whose store throws is never dropped: it goes back into + // the queue under its id, for the next flush. + assert.equal(status.deferredWrites, 1, 'only the write that failed is still queued'); assert.deepEqual( cm.getAllMessages().slice(before).map((m) => (m.content[0] as { text: string }).text), ['first', 'third'], From 1dd57c1c4df283e722650d79046e3ffd753f044a Mon Sep 17 00:00:00 2001 From: Basil-1007 Date: Sun, 11 Oct 2026 16:26:35 -0400 Subject: [PATCH 0113/1112] fix(requests): a reply follows [Continue] only when what it is sent ends on it or on its tool loop The prompt a reply follows was set at each stream start and carried on the turn's trigger, kept for a restart only if the restart's request rendered it. Two shapes slipped through: - A round inside one stream. At a tool boundary, hear-while-acting stores the messages that arrived while the tool ran after the tool results, and hands them to the live stream, so the next round follows them. Its rows still took the stream-start prompt, and once the tool exchange folded, later requests rendered [Continue] between those messages and a reply that followed them. - A stream resumed after a provider cooldown, whose compile ends on a message that arrived meanwhile, kept the carried prompt whenever the turn's earlier rows were still in the window, with the same result. And the carried prompt lived only in memory, so a stream after a process restart lost it while the stored rows still carried it. The prompt is now read from what the reply is sent, and never carried: - Agent: a new prompt when the request ends on [Continue]; else, when it ends on tool results, the prompt stamped on the reply that called those tools (toolLoopPrompt, through the index the render already builds, so the store is the only source); else none. - Framework: each stream start sets the turn's holder from that, and a tool boundary whose deferred flush stores anything after the results clears it, so the next round's rows carry no prompt. The holder is private turn state keyed by the trigger (turnPrompts), not a field on InferenceRequest: nothing passes a prompt in any more, and a value a host set would only be overwritten. ActivationRequestOptions loses continuePrompt, and RenderedRequestOnlyRows loses promptedTurns, which only the old rule read. A tick's stamp stays turn-level: its instruction is true of every round of the tick. Also: - A tick is keyed by its server and event id together (tickKey), in the index, the separator dedupe and the opening check, since two servers' events can share an id. - indexRequestOnlyRows reads the stamps before computing identity keys, so a history walk costs a metadata read per unstamped row. - The stamps have their own type, RequestOnlyStamps, a type alias that fits MessageMetadata without a cast, so a misspelled key fails to compile. - The non-streaming tool_results_ready continuation carries the turn's suppressProse and silentHeartbeat, as the budget restart does. The framework's own streams never reach it; an agent the direct API left waiting on tools does. - Docs and changelog say which rows are stamped (the reply's own: assistant responses, tool results and the contained wrapper), that a stamp on a user-role row marks it as the turn's rather than anchoring a render, and that the direct runInference paths render no request-only turns. The unreleased tick fragment now describes the recorded separator text and the server-and-event key. Tests (silent-heartbeat-rows): - a later round after a message injected at its tool boundary follows the message: its rows carry no prompt, and once the round folds, the message precedes it, not [Continue]; - a reply resumed after a real provider cooldown (a 429 holds the restart, and a message arrives during it) carries no prompt, and no [Continue] renders before it once the round folds; - a reply after the process restarts (a new framework over the same store) reads the prompt from the stored rows; - a reply the error policy retries past its first round follows the prompt; - a request that ends on the agent's own message mints a new [Continue], even with an older one rendered earlier in it; - a reply whose only identity is a redacted_thinking block gets its [Continue]; - two servers' ticks that share an event id each get their separator; - the budget-restart test now has the restart read the prompt from its own request. At the parent's source, five of these fail: the injected round, the cooldown, the process restart and the shared event id on behaviour, and the recorded-text test on the tick key's new form. --- changelog.d/request-only-rows-stay.changed.md | 32 +- .../silent-heartbeat-turn-rows.fixed.md | 24 +- src/agent.ts | 36 ++- src/framework.ts | 88 +++-- src/silent-heartbeat.ts | 94 ++++-- src/types/agent.ts | 18 +- test/silent-heartbeat-rows.test.ts | 303 +++++++++++++++--- 7 files changed, 450 insertions(+), 145 deletions(-) diff --git a/changelog.d/request-only-rows-stay.changed.md b/changelog.d/request-only-rows-stay.changed.md index 5365f9f38..a57eaceb2 100644 --- a/changelog.d/request-only-rows-stay.changed.md +++ b/changelog.d/request-only-rows-stay.changed.md @@ -19,16 +19,22 @@ `[Continue]`, and later compiles put the separator before a reply whose request never held it. - When a compile ends on the agent's own message and the request gets a - trailing `[Continue]` turn, every row the reply's turn stores now records - that prompt (`metadata.promptedBy`, a `{ turn, text }` pair), and every - later request renders `[Continue]` again just before the first of them - that survives, the way a silent tick's separator is rendered. A restart or - retry of the turn keeps the prompt only while its own request renders it. - Without it, later requests merged the reply into the previous assistant - message (one message carrying signed thinking from two responses, a 400 - when it is the latest), and dropped a turn the reply's request held, which - a provider that binds thinking refuses. A reply with neither a tool call - nor signed thinking can't be found in a compiled window and merges as - before. `StartStreamResult.continuePrompt` and - `InferenceRequest.continuePrompt` carry it (new, optional); the agent and - the framework set them. + trailing `[Continue]` turn, the reply's rows (its assistant responses and + tool results) now record that prompt (`metadata.promptedBy`, a + `{ turn, text }` pair), and every later request renders `[Continue]` + again just before the first of its assistant rows that survives, the way a + silent tick's separator is rendered. A later reply follows the same prompt + when what it is sent ends on the tool results of a reply that did: a next + round of the stream, a budget restart, an error retry, or a new process + reading the stored rows. A reply sent anything else after those results, + such as a message that arrived at the tool boundary or during a provider + cooldown, follows that instead and records no prompt. Without the turn, + later requests merged the reply into the previous assistant message (one + message carrying signed thinking from two responses, a 400 when it is the + latest), and dropped a turn the reply's request held, which a provider + that binds thinking refuses. A reply with neither a tool call nor signed or + redacted thinking can't be found in a compiled window and merges as + before. `StartStreamResult.continuePrompt` carries the prompt a stream's + request ends on (new, optional); the agent sets it. The direct + `runInference` paths compile without these request-only turns, as they + did without the tick separator, so their replies render none. diff --git a/changelog.d/silent-heartbeat-turn-rows.fixed.md b/changelog.d/silent-heartbeat-turn-rows.fixed.md index f56239fa8..7c294acae 100644 --- a/changelog.d/silent-heartbeat-turn-rows.fixed.md +++ b/changelog.d/silent-heartbeat-turn-rows.fixed.md @@ -1,14 +1,18 @@ - Silent heartbeat ticks no longer merge into the previous assistant message - on the wire. Every row a tick's turn stores (assistant responses, - tool_result rows) now carries - `metadata.silentHeartbeat: { eventId, serverId, agentName }`, and request - builds render a request-only `[heartbeat tick]` user turn immediately - before the first of the tick's rows that survives compilation (the agent's - own opening request for a tick ends on that same turn instead of - `[Continue]`). Nothing new is stored, and the separator's position follows - from the stamped rows alone, so it is identical on every compile and keeps - the cached prefix stable. Rows are recognised by tool_use ids and thinking - signatures only: an unsigned, tool-free tick reply still merges as before + on the wire. The rows a tick's reply stores (assistant responses, + tool_result rows) now carry + `metadata.silentHeartbeat: { eventId, serverId, agentName, separator }`, + and request builds render a request-only separator user turn, carrying + the tick's instruction, immediately before the first of the tick's rows + that survives compilation (the agent's own opening request for a tick + ends on that same turn instead of `[Continue]`). The separator's text is + the one the tick's request rendered, recorded in the stamp; rows stamped + without it render the bare `[heartbeat tick]` their requests carried. + Nothing new is stored, and the separator's position follows from the + stamped rows alone, so it is identical on every compile and keeps the + cached prefix stable. A tick is told apart by its server and event id + together. Rows are recognised by tool_use ids and signed or redacted + thinking only: an unsigned, tool-free tick reply still merges as before (no signed thinking, so no provider rejection). Previously back-to-back ticks, or a tick after a reply that left no delivery receipt, were sent as one assistant message carrying signed thinking from two responses. diff --git a/src/agent.ts b/src/agent.ts index 24200ba12..2348fb520 100644 --- a/src/agent.ts +++ b/src/agent.ts @@ -9,6 +9,8 @@ import { indexRequestOnlyRows, renderRequestOnlyRows, silentHeartbeatSeparatorTurn, + tickKey, + toolLoopPrompt, type RequestOnlyPrompt, type RequestOnlyRowIndex, type SilentHeartbeatTick, @@ -22,9 +24,6 @@ import { } from './tool-result-history.js'; export interface ActivationRequestOptions { - /** The `[Continue]` prompt this turn's earlier streams replied to, if any: - * kept for this request's reply only while this request renders it. */ - continuePrompt?: RequestOnlyPrompt; /** The silent heartbeat tick this request answers, if any. */ silentHeartbeat?: SilentHeartbeatTick; } @@ -33,10 +32,11 @@ export interface StartStreamResult { stream: YieldingStream; request: NormalizedRequest; /** The request-only `[Continue]` prompt this request's reply follows, if - * any: a new one when the request ends on `[Continue]`, else the turn's - * prompt (`ActivationRequestOptions.continuePrompt`) only if this request - * renders it. The framework stamps the reply's rows with it - * (`metadata.promptedBy`), for later compiles to render it before them. */ + * any, read from the request: a new one when it ends on `[Continue]`; + * else, when it ends on tool results, the prompt of the reply that called + * those tools (`toolLoopPrompt`); else none. The framework stamps the + * reply's rows with it (`metadata.promptedBy`), for later compiles to + * render it before them. */ continuePrompt?: RequestOnlyPrompt; takeKvSubmission?: () => { submissionId: string; wireReceipt: CacheWireReceipt } | undefined; drainKvSubmissionIds?: () => string[]; @@ -876,19 +876,22 @@ export class Agent { // that same turn, so the prefix it caches is the one every later compile // renders. "First" is per agent: a broadcast tick shares its eventId with // every resident. - const rendered = renderRequestOnlyRows(messages, this.indexRequestOnlyRows(), this.name); + const index = this.indexRequestOnlyRows(); + const rendered = renderRequestOnlyRows(messages, index, this.name); messages = rendered.messages; // A tick opens wherever its separator isn't already rendered: on its first // request, and on a restart whose compile left none of the tick's rows, // so the restart's reply follows the separator, where every later compile // renders it. const openingTick = options.silentHeartbeat !== undefined - && !rendered.separatedTicks.has(options.silentHeartbeat.eventId); - - // The [Continue] prompt this request's reply follows: a new one when it - // ends on [Continue]; else the turn's prompt only where this request - // renders it (a stream resumed after a cooldown, say, whose compile now - // ends on a message that arrived meanwhile, renders none); else none. + && !rendered.separatedTicks.has(tickKey(options.silentHeartbeat)); + + // The [Continue] prompt this request's reply follows, read from the + // request itself: a new one when it ends on [Continue]; else, when it + // ends on tool results, the prompt of the reply that called those tools, + // whose tool loop this reply continues; else none. A request that ends + // on anything else (a message that arrived during a cooldown, say) + // prompts a reply that follows that. let continuePrompt: RequestOnlyPrompt | undefined; if (openingTick) { messages = [...messages, silentHeartbeatSeparatorTurn()]; @@ -898,9 +901,8 @@ export class Agent { // and after context compression a stale assistant turn can end up last. messages = [...messages, continueTurn()]; continuePrompt = { turn: randomUUID(), text: CONTINUE_ROW }; - } - if (!continuePrompt && options.continuePrompt && rendered.promptedTurns.has(options.continuePrompt.turn)) { - continuePrompt = options.continuePrompt; + } else { + continuePrompt = toolLoopPrompt(messages.at(-1), index, this.name); } const request: NormalizedRequest = { diff --git a/src/framework.ts b/src/framework.ts index 8c9a0d891..43a60c9c2 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -8,7 +8,7 @@ import type { Membrane, ContentBlock, NormalizedRequest, YieldingStream, ToolRes import { MembraneError } from '@animalabs/membrane'; import { ContextManager, PassthroughStrategy, WindowedPassthroughStrategy, OverBudgetError, UncoveredDropError } from '@animalabs/context-manager'; import type { CacheWireReceipt } from './kv-unified-wire.js'; -import { SILENT_HEARTBEAT_SEPARATOR, type SilentHeartbeatStamp } from './silent-heartbeat.js'; +import { SILENT_HEARTBEAT_SEPARATOR, type RequestOnlyPrompt, type RequestOnlyStamps } from './silent-heartbeat.js'; import { SUBCONSCIOUS_TOOLS, SUBCONSCIOUS_TOOL_NAMES, type SubconsciousConfig } from './tune-out/tools.js'; import { TuneOutCoordinator, TUNE_OUT_DEFAULTS } from './tune-out/coordinator.js'; import type { @@ -1241,6 +1241,16 @@ export class AgentFramework { */ private activeTurnTriggers: Map = new Map(); + /** + * The `[Continue]` prompt the current round of each turn follows, keyed by + * the turn's trigger. Each stream start sets it from what the Agent read in + * its request (`StartStreamResult.continuePrompt`), and a tool boundary + * that stores anything after the results clears it, since the next round + * follows those messages. `silentTurnRowMetadata` stamps the round's rows + * with it. + */ + private readonly turnPrompts = new WeakMap(); + /** The trigger of `agentName`'s turn in progress, if a turn is running. */ getActiveTurnTrigger(agentName: string): InferenceRequest | undefined { return this.activeTurnTriggers.get(agentName); @@ -6698,6 +6708,10 @@ export class AgentFramework { { const deferred = this.drainDeferredFor(agent.name); if (deferred.length > 0) { + // The next round follows these messages, not the [Continue] + // the turn's earlier rounds followed: its rows carry no prompt. + const turnTrigger = this.activeTurnTriggers.get(agent.name); + if (turnTrigger) this.turnPrompts.delete(turnTrigger); for (const msg of deferred) { agent.getContextManager().addMessage(msg.participant, msg.content, withDeferredWriteId(msg.metadata, msg.id)); // Injection guards: tool blocks would corrupt the tool-cycle @@ -6905,7 +6919,6 @@ export class AgentFramework { reason: 'context_budget_restart', source: 'framework', timestamp: Date.now(), - continuePrompt: this.activeTurnTriggers.get(agent.name)?.continuePrompt, suppressProse: this.activeTurnTriggers.get(agent.name)?.suppressProse, silentHeartbeat: this.activeTurnTriggers.get(agent.name)?.silentHeartbeat, }); @@ -6925,12 +6938,16 @@ export class AgentFramework { ...(midTurnInjections.length > 0 ? { injectedMessages: midTurnInjections.length } : {}), }); } else { - // Non-streaming fallback: schedule re-inference + // Non-streaming fallback: schedule re-inference. It continues the + // same turn, so it carries the turn's properties, as the budget + // restart does. this.pendingRequests.push({ agentName: agent.name, reason: 'tool_results_ready', source: 'framework', timestamp: Date.now(), + suppressProse: this.activeTurnTriggers.get(agent.name)?.suppressProse, + silentHeartbeat: this.activeTurnTriggers.get(agent.name)?.silentHeartbeat, }); } } @@ -8515,32 +8532,39 @@ export class AgentFramework { } /** - * Metadata for rows the agent's current turn stores (assistant responses, - * tool_result rows), for the request-only turns that came before them: - * - a silent heartbeat tick stamps each row, with the separator text its - * request rendered, so they stay identifiable after the turn — request - * builds render the separator before the first that survives, and a later - * retention policy can find or collapse them. The stamp names the agent: - * residents share one message slot and a broadcast tick reaches each of - * them with the same eventId; - * - a turn whose stream opened on `[Continue]` stamps each row from then on - * with that prompt (`promptedBy`), so request builds render it before the - * first that survives. + * Stamps for the rows the agent's reply stores in its current turn: its + * assistant responses, its tool_result rows (through `storeResults`), and + * the contained-wrapper row that stands in for a response. The turn's other + * stores (a delivery receipt, a routing notice, a deferred flush) aren't the + * reply's, and carry none. + * - A silent heartbeat tick stamps each of them for the whole turn, with the + * separator text its request rendered. Request builds render the + * separator before the first of the tick's assistant rows that survives, + * and the stamps keep every row of the tick identifiable after the turn, + * for a later retention policy to find or collapse. The stamp names the + * agent: residents share one message slot, and a broadcast tick reaches + * each of them with the same eventId. + * - A `[Continue]` prompt stamps the rows of each round that follows it + * (`turnPrompts`). Request builds render it before the first of its + * assistant rows that survives. On the round's tool_result rows it marks + * them as the reply's, as the tick stamp does. * Undefined for ordinary turns, whose rows are stored exactly as before. * (Named for its first use, the silent tick.) */ - private silentTurnRowMetadata(agentName: string): MessageMetadata | undefined { + private silentTurnRowMetadata(agentName: string): RequestOnlyStamps | undefined { const trigger = this.activeTurnTriggers.get(agentName); - const tick = trigger?.silentHeartbeat; - const stamp: SilentHeartbeatStamp | undefined = tick - ? { eventId: tick.eventId, serverId: tick.serverId, agentName, separator: SILENT_HEARTBEAT_SEPARATOR } - : undefined; - const prompt = trigger?.continuePrompt; - if (!stamp && !prompt) return undefined; - return { - ...(stamp ? { silentHeartbeat: stamp } : {}), - ...(prompt ? { promptedBy: prompt } : {}), - } as MessageMetadata; + if (!trigger) return undefined; + const tick = trigger.silentHeartbeat; + const prompt = this.turnPrompts.get(trigger); + if (!tick && !prompt) return undefined; + const stamps: RequestOnlyStamps = {}; + if (tick) { + stamps.silentHeartbeat = { + eventId: tick.eventId, serverId: tick.serverId, agentName, separator: SILENT_HEARTBEAT_SEPARATOR, + }; + } + if (prompt) stamps.promptedBy = prompt; + return stamps; } /** Record prose segments suppressed by explicit-send silencing. */ @@ -9376,13 +9400,15 @@ export class AgentFramework { continuePrompt, } = await agent.startStreamWithInjections(tools, injections, undefined, compressionTools, { silentHeartbeat: trigger?.silentHeartbeat, - continuePrompt: trigger?.continuePrompt, }); - // The [Continue] prompt this stream's reply follows, if any: every row - // the turn stores from here on carries it (silentTurnRowMetadata), and the - // turn's restarts and retries carry it in the trigger to the next - // stream start, which keeps it only if its request renders it. - if (trigger) trigger.continuePrompt = continuePrompt; + // The [Continue] prompt this stream's first round follows, if any, as + // the Agent read it from the request: the round's rows carry it + // (silentTurnRowMetadata), and so do later rounds' until a tool + // boundary stores something after the results. + if (trigger) { + if (continuePrompt) this.turnPrompts.set(trigger, continuePrompt); + else this.turnPrompts.delete(trigger); + } if (this.agents.get(agent.name) !== agent) { stream.cancel(); agent.cancelStream(); diff --git a/src/silent-heartbeat.ts b/src/silent-heartbeat.ts index 1a2211c31..d819adb08 100644 --- a/src/silent-heartbeat.ts +++ b/src/silent-heartbeat.ts @@ -12,9 +12,14 @@ * first of the tick's rows that survives compilation. * - A compile that ends on the agent's own message gets a trailing * `[Continue]` turn (some models reject a trailing assistant message). The - * reply it prompts is stored straight after that message, so every row the - * reply's turn stores is stamped `metadata.promptedBy` (`RequestOnlyPrompt`), - * and the turn is rendered before the first of them that survives. + * reply it prompts is stored straight after that message, so the reply's + * rows are stamped `metadata.promptedBy` (`RequestOnlyPrompt`), and the turn + * is rendered before the first of them that survives. A reply follows the + * prompt when what it was sent ends on that `[Continue]`, or on the tool + * results of a reply that followed it (`toolLoopPrompt`): a stream start + * and each round of a stream alike. A reply sent anything else after the + * results (a message that arrived meanwhile) follows that, and carries no + * prompt. * * Without the row, the wire formatter merges consecutive same-role messages: * the reply would read as an unprompted continuation of the earlier response, @@ -81,6 +86,12 @@ export interface SilentHeartbeatStamp extends SilentHeartbeatTick { separator?: string; } +/** One tick's key in an index or a rendered window: the server and its event + * id together, since two servers' events can share an id. */ +export function tickKey(tick: SilentHeartbeatTick): string { + return JSON.stringify([tick.serverId, tick.eventId]); +} + /** The tick stamp on a stored row's metadata, if any. */ export function silentHeartbeatOf(metadata: unknown): SilentHeartbeatStamp | undefined { const stamp = (metadata as { silentHeartbeat?: unknown } | undefined)?.silentHeartbeat; @@ -92,13 +103,22 @@ export function silentHeartbeatOf(metadata: unknown): SilentHeartbeatStamp | und } /** A request-only turn a request ended on before the agent's reply, as - * `metadata.promptedBy` on every row the reply's turn stores: `turn` names - * the one prompt, and `text` is what the request rendered. */ + * `metadata.promptedBy` on the rows of each reply that follows it: `turn` + * names the one prompt, and `text` is what the request rendered. */ export interface RequestOnlyPrompt { turn: string; text: string; } +/** The stamps on a row a turn stores after a request-only turn + * (`silentHeartbeatOf`, `promptedByOf` read them back). A type alias, so a + * value fits `MessageMetadata`'s index signature without a cast, and a + * misspelled key fails to compile. */ +export type RequestOnlyStamps = { + silentHeartbeat?: SilentHeartbeatStamp; + promptedBy?: RequestOnlyPrompt; +}; + /** The request-only prompt a stored row replies to, if any. */ export function promptedByOf(metadata: unknown): RequestOnlyPrompt | undefined { const prompt = (metadata as { promptedBy?: unknown } | undefined)?.promptedBy; @@ -129,7 +149,7 @@ function requestOnlyTurn(text: string): NormalizedMessage { function identityKeys(content: ContentBlock[]): string[] { const keys: string[] = []; for (const block of content) { - if (block.type === 'tool_use' && typeof block.id === 'string' && block.id) keys.push(`tool_use:${block.id}`); + if (block.type === 'tool_use' && typeof block.id === 'string' && block.id) keys.push(toolUseKey(block.id)); else if (block.type === 'thinking' && typeof block.signature === 'string' && block.signature) { keys.push(`thinking:${block.signature}`); } else if (block.type === 'redacted_thinking' && typeof block.data === 'string' && block.data) { @@ -139,6 +159,10 @@ function identityKeys(content: ContentBlock[]): string[] { return keys; } +function toolUseKey(id: string): string { + return `tool_use:${id}`; +} + export interface StoredRowLike { participant: string; content: ContentBlock[]; @@ -147,9 +171,9 @@ export interface StoredRowLike { export interface RequestOnlyRowIndex { /** Identity key of every assistant row this agent stored during a silent - * tick → the tick, and the separator text to render before its first - * surviving row. */ - ticks: Map; + * tick → the tick (`tickKey`), and the separator text to render before + * its first surviving row. */ + ticks: Map; /** Identity key of every assistant row this agent stored in reply to a * request-only prompt → that prompt. */ prompts: Map; @@ -157,33 +181,53 @@ export interface RequestOnlyRowIndex { /** Index the agent's own rows that follow request-only turns: the rows of its * silent ticks, and the rows of each reply to a `[Continue]`. The two are - * independent, so a row can be both. */ + * independent, so a row can be both. Only stamped rows are keyed, so an + * agent's history walk costs a metadata read per row. */ export function indexRequestOnlyRows(stored: readonly StoredRowLike[], agentName: string): RequestOnlyRowIndex { - const ticks = new Map(); + const ticks = new Map(); const prompts = new Map(); for (const row of stored) { if (row.participant !== agentName) continue; - const keys = identityKeys(row.content); - if (keys.length === 0) continue; const stamp = silentHeartbeatOf(row.metadata); - if (stamp && stamp.agentName === agentName) { - const tick = { eventId: stamp.eventId, text: stamp.separator ?? LEGACY_SILENT_HEARTBEAT_SEPARATOR }; + const ownTick = stamp !== undefined && stamp.agentName === agentName; + const prompt = promptedByOf(row.metadata); + if (!ownTick && !prompt) continue; + const keys = identityKeys(row.content); + if (ownTick) { + const tick = { tick: tickKey(stamp), text: stamp.separator ?? LEGACY_SILENT_HEARTBEAT_SEPARATOR }; for (const key of keys) ticks.set(key, tick); } - const prompt = promptedByOf(row.metadata); if (prompt) for (const key of keys) prompts.set(key, prompt); } return { ticks, prompts }; } +/** + * The prompt a reply continues when what it is sent ends on `last`: if + * `last` holds tool results, the prompt of the reply that called those tools + * (this reply continues that reply's tool loop); otherwise none. Read from + * the stored stamps, so it doesn't depend on anything held in memory. A + * request that ends on its own `[Continue]` mints a new prompt instead. + */ +export function toolLoopPrompt( + last: NormalizedMessage | undefined, + index: RequestOnlyRowIndex, + agentName: string, +): RequestOnlyPrompt | undefined { + if (!last || last.participant === agentName) return undefined; + for (const block of last.content) { + if (block.type !== 'tool_result') continue; + const prompt = index.prompts.get(toolUseKey(block.toolUseId)); + if (prompt) return prompt; + } + return undefined; +} + export interface RenderedRequestOnlyRows { messages: NormalizedMessage[]; - /** The ticks whose separator this window renders. A tick none of whose - * rows survive compilation renders none. */ + /** The ticks (`tickKey`) whose separator this window renders. A tick none + * of whose rows survive compilation renders none. */ separatedTicks: Set; - /** The prompts (`RequestOnlyPrompt.turn`) this window renders, the same - * way. */ - promptedTurns: Set; } /** @@ -201,14 +245,14 @@ export function renderRequestOnlyRows( ): RenderedRequestOnlyRows { const separatedTicks = new Set(); const promptedTurns = new Set(); - if (index.ticks.size === 0 && index.prompts.size === 0) return { messages, separatedTicks, promptedTurns }; + if (index.ticks.size === 0 && index.prompts.size === 0) return { messages, separatedTicks }; const out: NormalizedMessage[] = []; for (const message of messages) { if (message.participant === agentName) { const keys = identityKeys(message.content); const tick = keys.map((key) => index.ticks.get(key)).find((t) => t !== undefined); - if (tick !== undefined && !separatedTicks.has(tick.eventId)) { - separatedTicks.add(tick.eventId); + if (tick !== undefined && !separatedTicks.has(tick.tick)) { + separatedTicks.add(tick.tick); out.push(requestOnlyTurn(tick.text)); } const prompt = keys.map((key) => index.prompts.get(key)).find((p) => p !== undefined); @@ -219,5 +263,5 @@ export function renderRequestOnlyRows( } out.push(message); } - return { messages: out, separatedTicks, promptedTurns }; + return { messages: out, separatedTicks }; } diff --git a/src/types/agent.ts b/src/types/agent.ts index a4fc4b9e1..267e41ecc 100644 --- a/src/types/agent.ts +++ b/src/types/agent.ts @@ -1,7 +1,7 @@ import type { ContentBlock, YieldingStream } from '@animalabs/membrane'; import type { ContextStrategy } from '@animalabs/context-manager'; import type { ToolCallId, ToolResult, ToolCall } from './events.js'; -import type { RequestOnlyPrompt, SilentHeartbeatTick } from '../silent-heartbeat.js'; +import type { SilentHeartbeatTick } from '../silent-heartbeat.js'; export type SameRoundThinkTextPolicy = 'public' | 'private'; export type SameRoundThinkTextPolicySource = @@ -360,16 +360,10 @@ export interface InferenceRequest { /** Suppress every automatic plain-prose delivery for this logical turn. * Explicit tool calls remain available. Used by authenticated silent wakes. */ suppressProse?: boolean; - /** Set by the framework at each stream start of this turn: the request-only - * `[Continue]` prompt the stream's reply follows (`StartStreamResult. - * continuePrompt`), or none. Every row the turn stores is stamped - * `metadata.promptedBy` with it, so request builds render the turn before - * the first that survives. Carried into the turn's restarts and retries, - * whose stream start keeps it only if its own request renders it. */ - continuePrompt?: RequestOnlyPrompt; - /** The authenticated silent heartbeat tick this turn answers. Every row the - * turn stores is stamped `metadata.silentHeartbeat` with it, and request - * builds render a request-only separator carrying the tick's instruction - * before the tick's first row (`SILENT_HEARTBEAT_SEPARATOR`). */ + /** The authenticated silent heartbeat tick this turn answers. The rows the + * turn's reply stores (its assistant responses and tool results) are + * stamped `metadata.silentHeartbeat` with it, and request builds render a + * request-only separator carrying the tick's instruction before the tick's + * first surviving row (`SILENT_HEARTBEAT_SEPARATOR`). */ silentHeartbeat?: SilentHeartbeatTick; } diff --git a/test/silent-heartbeat-rows.test.ts b/test/silent-heartbeat-rows.test.ts index a0fd91fae..ecfaccff1 100644 --- a/test/silent-heartbeat-rows.test.ts +++ b/test/silent-heartbeat-rows.test.ts @@ -16,24 +16,38 @@ import { mkdtempSync, rmSync } from 'node:fs'; import { join } from 'node:path'; import { tmpdir } from 'node:os'; import { AgentFramework, SILENT_HEARTBEAT_SEPARATOR } from '../src/index.js'; -import { indexRequestOnlyRows, renderRequestOnlyRows } from '../src/silent-heartbeat.js'; +import { indexRequestOnlyRows, renderRequestOnlyRows, tickKey } from '../src/silent-heartbeat.js'; import type { AgentConfig, Module, ModuleContext, ToolCall, ToolDefinition, ToolResult } from '../src/index.js'; import { TOOL_RESULT_GUARD_NOTICE } from '../src/tool-result-guard.js'; import { PassthroughStrategy } from '@animalabs/context-manager'; import { MockMembrane, MockYieldingStream, createMockResponse } from './helpers/mock-membrane.js'; -import { NativeFormatter } from '@animalabs/membrane'; -import type { ContentBlock, NormalizedMessage, NormalizedRequest, NormalizedResponse, YieldingStream } from '@animalabs/membrane'; +import { MembraneError, NativeFormatter } from '@animalabs/membrane'; +import type { ContentBlock, NormalizedMessage, NormalizedRequest, NormalizedResponse, StreamEvent, YieldingStream } from '@animalabs/membrane'; class ToolModule implements Module { readonly name = 'tools'; calls: ToolCall[] = []; + /** A message someone sends while the next call runs: the agent's turn is + * alive, so it is deferred to the tool boundary. */ + arrive: string | null = null; + /** The next call's result ends the turn. */ + endTurn = false; private ctx: ModuleContext | null = null; async start(ctx: ModuleContext) { this.ctx = ctx; ctx.registerSpeechHandler('*'); } async stop() { this.ctx?.unregisterSpeechHandler(); this.ctx = null; } async onProcess() { return {}; } async onAgentSpeech() {} getTools(): ToolDefinition[] { return [{ name: 'ping', description: 'explicit action', inputSchema: { type: 'object', properties: {} } }]; } - async handleToolCall(call: ToolCall): Promise { this.calls.push(call); return { success: true, data: { ok: true } }; } + async handleToolCall(call: ToolCall): Promise { + this.calls.push(call); + if (this.arrive !== null) { + this.ctx!.addMessage('nissa', [{ type: 'text', text: this.arrive }] as ContentBlock[]); + this.arrive = null; + } + const endTurn = this.endTurn; + this.endTurn = false; + return { success: true, data: { ok: true }, ...(endTurn ? { endTurn } : {}) }; + } } let seq = 0; @@ -65,17 +79,55 @@ function signed(signature: string, text: string, extra: ContentBlock[] = []): Co ]; } -/** One explicit response queue per stream, so a restart opens a fresh one. */ +/** A stream the provider fails at once. */ +class ErrorStream implements YieldingStream { + isWaitingForTools = false; + pendingToolCallIds: string[] = []; + toolDepth = 0; + constructor(private readonly error: Error) {} + provideToolResults(): void { throw new Error('not waiting'); } + cancel(): void {} + async *[Symbol.asyncIterator](): AsyncIterator { + yield { type: 'error', error: this.error } as StreamEvent; + } +} + +/** One explicit queue per stream, so a restart opens a fresh one: the + * responses of its tool rounds, or an error the stream fails with. */ class QueuedStreamsMembrane extends MockMembrane { - queues: NormalizedResponse[][] = []; + queues: Array = []; override streamYielding(request: NormalizedRequest, _options?: unknown): YieldingStream { this.calls.push(request); - const stream = new MockYieldingStream(this.queues.shift() ?? []); + const next = this.queues.shift() ?? []; + if (next instanceof Error) return new ErrorStream(next); + const stream = new MockYieldingStream(next); this.lastStream = stream; return stream; } } +/** A 429 that holds the turn for a provider cooldown. */ +const accelerationLimit = () => new MembraneError({ + type: 'rate_limit', retryable: true, httpStatus: 429, + message: "This request would exceed your organization's maximum usage increase rate for input tokens per minute", + rawError: { status: 429 }, +}); + +/** A failure the default error policy retries at once. */ +const transientFailure = () => new MembraneError({ + type: 'server', retryable: true, httpStatus: 500, retryAfterMs: 1, + message: 'internal server error', rawError: { status: 500 }, +}); + +/** A tool round whose usage puts the next round past the physical window, so + * the stream restarts at its boundary. */ +function nearCapToolRound(signature: string, text: string, id: string): NormalizedResponse { + const response = createMockResponse( + signed(signature, text, [{ type: 'tool_use', id, name: 'tools--ping', input: {} } as ContentBlock]), 'tool_use'); + (response as { usage: unknown }).usage = { inputTokens: 5_000, outputTokens: 5, cacheReadTokens: 195_000 }; + return response; +} + /** A strategy that leaves every tool exchange out of the compiled window * (from the moment `omit` is set, for a window that folds them later). */ class OmitToolExchange extends PassthroughStrategy { @@ -94,8 +146,12 @@ const refused = () => ({ async function make(opts: { deliver?: boolean; membrane?: MockMembrane; physicalWindowTokens?: number; agents?: string[]; agent?: Partial; + /** An earlier framework's directory, to open its store again. */ + dir?: string; + /** Where the agent's plain speech lands; null for no channel. */ + locus?: string | null; } = {}) { - const dir = mkdtempSync(join(tmpdir(), 'silent-hb-rows-')); + const dir = opts.dir ?? mkdtempSync(join(tmpdir(), 'silent-hb-rows-')); const membrane = opts.membrane ?? new MockMembrane(); const mod = new ToolModule(); const framework = await AgentFramework.create({ @@ -108,7 +164,8 @@ async function make(opts: { modules: [mod], }); (framework as any).channelRegistry = new Proxy({ - resolveLocus: () => 'world:commons', getDefaultPublishChannel: () => 'world:commons', + resolveLocus: () => (opts.locus === undefined ? 'world:commons' : opts.locus), + getDefaultPublishChannel: () => (opts.locus === undefined ? 'world:commons' : opts.locus), routeSpeech: async () => ({ delivered: opts.deliver ?? true, channelId: 'world:commons' }), startTyping: () => {}, stopTyping: () => {}, getChannelTools: () => [], getDescriptor: () => undefined, @@ -152,6 +209,14 @@ function stored(framework: AgentFramework, agent = 'assistant') { }>; } +function rowWith(framework: AgentFramework, text: string) { + const row = stored(framework).find((m) => m.content.some((b) => (b as { text?: string }).text === text)); + assert.ok(row, `a stored row reads ${JSON.stringify(text)}`); + return row; +} + +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + describe('silent heartbeat stored rows', () => { it('keeps a tick reply out of the previous assistant message on the wire', async () => { const x = await make(); @@ -567,7 +632,26 @@ describe('replies to a [Continue] turn', () => { assert.deepEqual(rendered.messages.map(textOf), [ '[heartbeat tick]', 'old tick reply', 'recorded separator', 'new tick reply', 'recorded prompt', 'continued reply', ]); - assert.deepEqual([...rendered.separatedTicks].sort(), ['hb-new', 'hb-old']); + assert.deepEqual([...rendered.separatedTicks].sort(), [ + tickKey({ serverId: 'heartbeat', eventId: 'hb-new' }), tickKey({ serverId: 'heartbeat', eventId: 'hb-old' }), + ].sort()); + }); + + it('keys a tick by its server and its event id together', () => { + // Two servers' events can share an id: each tick still gets its own + // separator, before its own first row. + const a = { eventId: 'hb-1', serverId: 'heartbeat-a', agentName: 'assistant', separator: 'separator a' }; + const b = { eventId: 'hb-1', serverId: 'heartbeat-b', agentName: 'assistant', separator: 'separator b' }; + const stored = [ + { participant: 'assistant', content: signed('sig-a', 'tick a reply'), metadata: { silentHeartbeat: a } }, + { participant: 'nissa', content: [{ type: 'text', text: 'between the ticks' }] as ContentBlock[] }, + { participant: 'assistant', content: signed('sig-b', 'tick b reply'), metadata: { silentHeartbeat: b } }, + ]; + const index = indexRequestOnlyRows(stored, 'assistant'); + const rendered = renderRequestOnlyRows(stored.map(({ participant, content }) => ({ participant, content })), index, 'assistant'); + assert.deepEqual(rendered.messages.map(textOf), + ['separator a', 'tick a reply', 'between the ticks', 'separator b', 'tick b reply']); + assert.deepEqual([...rendered.separatedTicks].sort(), [tickKey(a), tickKey(b)].sort()); }); it('marks every row the reply\'s turn stores, and renders one [Continue] before the first', async () => { @@ -594,7 +678,7 @@ describe('replies to a [Continue] turn', () => { } finally { await x.framework.stop(); rmSync(x.dir, { recursive: true, force: true }); } }); - it('carries the prompt into the reply\'s budget restart, for a window that later folds the first round', async () => { + it('a budget restart that ends on the round\'s tool results reads its prompt, for a window that later folds the round', async () => { const membrane = new QueuedStreamsMembrane(); const strategy = new OmitToolExchange(); strategy.omit = false; @@ -602,13 +686,10 @@ describe('replies to a [Continue] turn', () => { try { membrane.queues.push([createMockResponse(signed('sig-1', 'first reply'))]); await wake(x, ordinaryEvent('hello')); - // The reply to [Continue] restarts at its tool boundary; the restart's - // compile ends on the tool_result, so it opens on no prompt of its own. - const nearCap = createMockResponse( - signed('sig-c', 'checking', [{ type: 'tool_use', id: 'c-t1', name: 'tools--ping', input: {} } as ContentBlock]), - 'tool_use'); - (nearCap as { usage: unknown }).usage = { inputTokens: 5_000, outputTokens: 5, cacheReadTokens: 195_000 }; - membrane.queues.push([nearCap], [createMockResponse(signed('sig-c2', 'all done'))]); + // The reply to [Continue] restarts at its tool boundary. The restart's + // compile ends on the round's tool_result, so its reply continues that + // round's tool loop and follows the same prompt. + membrane.queues.push([nearCapToolRound('sig-c', 'checking', 'c-t1')], [createMockResponse(signed('sig-c2', 'all done'))]); (x.framework as any).pendingRequests.push({ agentName: 'assistant', reason: 'timer', source: 'framework', timestamp: Date.now() }); await x.framework.runUntilIdle(); assert.equal(membrane.calls.length, 3, 'first, the continued reply, its restart'); @@ -630,31 +711,179 @@ describe('replies to a [Continue] turn', () => { } finally { await x.framework.stop(); rmSync(x.dir, { recursive: true, force: true }); } }); - it('drops a carried prompt its request doesn\'t render (a stream resumed after a cooldown)', async () => { - // The shape a provider cooldown leaves (Basil-1007's probe): the held - // trigger carries the prompt its first stream start set, and a message - // lands meanwhile, so the resumed compile ends on that message and - // renders no [Continue]. - const x = await make({ deliver: false }); + it('a reply resumed after a provider cooldown follows the message that arrived during it, not [Continue]', async () => { + // A reply to [Continue] stores a tool round and restarts at its boundary, + // and the restart meets a 429 that holds the turn. A message arrives + // during the cooldown, so the resumed compile ends on it, while the + // round, still in the window, renders [Continue] earlier. + const membrane = new QueuedStreamsMembrane(); + const strategy = new OmitToolExchange(); + strategy.omit = false; + const x = await make({ membrane, physicalWindowTokens: 200_000, deliver: false, agent: { strategy } }); + const internal = x.framework as unknown as { + providerAccelerationDefaultCooldownMs: number; providerAccelerationJitterMs: number; + providerAccelerationCooldowns: Map; + }; + internal.providerAccelerationDefaultCooldownMs = 1_000; // the shortest hold there is + internal.providerAccelerationJitterMs = 0; + try { + membrane.queues.push([createMockResponse(signed('sig-1', 'first reply'))]); + await wake(x, ordinaryEvent('hello')); + membrane.queues.push([nearCapToolRound('sig-c', 'checking', 'c-t1')], accelerationLimit(), + [createMockResponse(signed('sig-r', 'resumed reply'))]); + (x.framework as any).pendingRequests.push({ agentName: 'assistant', reason: 'timer', source: 'framework', timestamp: Date.now() }); + await x.framework.runUntilIdle(); + assert.equal(internal.providerAccelerationCooldowns.size, 1, 'the restart is held'); + await wake(x, ordinaryEvent('arrived during the cooldown')); + await sleep(1_100); + await x.framework.runUntilIdle(); + assert.equal(membrane.calls.length, 4, 'first, the continued reply, its held restart, the resumed attempt'); + + const resumed = membrane.calls[3]!.messages; + assert.equal(textOf(resumed.at(-1)!), 'arrived during the cooldown'); + assert.ok(resumed.some((m) => textOf(m) === '[Continue]'), 'the round renders its [Continue]'); + assert.equal(rowWith(x.framework, 'resumed reply').metadata?.promptedBy, undefined); + + // Later, the window folds the round: no [Continue] is left to render. + strategy.omit = true; + membrane.queues.push([createMockResponse(signed('sig-later', 'later reply'))]); + await wake(x, ordinaryEvent('later')); + const later = membrane.calls[4]!.messages; + const replyAt = later.findIndex((m) => textOf(m) === 'resumed reply'); + assert.equal(textOf(later[replyAt - 1]!), 'arrived during the cooldown'); + assert.ok(!later.some((m) => textOf(m) === '[Continue]')); + } finally { await x.framework.stop(); rmSync(x.dir, { recursive: true, force: true }); } + }); + + it('a later round that follows a message injected at its tool boundary follows the message, not [Continue]', async () => { + // Hear-while-acting: a message that arrives while the round's tool runs + // is stored after the tool results and handed to the live stream with + // them, so the stream's next round follows it. + const strategy = new OmitToolExchange(); + strategy.omit = false; + const x = await make({ deliver: false, agent: { strategy } }); try { await wake(x, ordinaryEvent('hello'), signed('sig-1', 'first reply')); - x.framework.getAgent('assistant')!.getContextManager() - .addMessage('nissa', [{ type: 'text', text: 'arrived during the cooldown' }] as ContentBlock[]); - x.membrane.pushResponse(createMockResponse(signed('sig-r', 'resumed reply'), 'end_turn')); - (x.framework as any).pendingRequests.push({ - agentName: 'assistant', reason: 'timer', source: 'framework', timestamp: Date.now(), - continuePrompt: { turn: 'stale-p1', text: '[Continue]' }, - }); + x.mod.arrive = 'arrived mid-turn'; + await timerWake(x, + signed('sig-c', 'checking', [{ type: 'tool_use', id: 'c-t1', name: 'tools--ping', input: {} } as ContentBlock]), + signed('sig-c2', 'all done')); + const injected = x.membrane.lastStream!.receivedToolResultOptions[0]?.injectedMessages ?? []; + assert.deepEqual(injected.map((m) => textOf(m as NormalizedMessage)), ['arrived mid-turn']); + + const marked = stored(x.framework).filter((m) => m.metadata?.promptedBy !== undefined); + assert.deepEqual(marked.map((m) => m.participant), ['assistant', 'user'], + 'the round that followed [Continue], and its tool_result row'); + assert.equal(rowWith(x.framework, 'all done').metadata?.promptedBy, undefined); + + // Later, the window folds the round: the reply follows the message. + strategy.omit = true; + await wake(x, ordinaryEvent('later'), signed('sig-later', 'later reply')); + const later = x.membrane.calls[2]!.messages; + const doneAt = later.findIndex((m) => textOf(m) === 'all done'); + assert.equal(textOf(later[doneAt - 1]!), 'arrived mid-turn'); + assert.ok(!later.some((m) => textOf(m) === '[Continue]')); + } finally { await x.framework.stop(); rmSync(x.dir, { recursive: true, force: true }); } + }); + + it('a reply retried after a failure past its first round keeps following the prompt', async () => { + const membrane = new QueuedStreamsMembrane(); + const strategy = new OmitToolExchange(); + strategy.omit = false; + const x = await make({ membrane, physicalWindowTokens: 200_000, deliver: false, agent: { strategy } }); + try { + membrane.queues.push([createMockResponse(signed('sig-1', 'first reply'))]); + await wake(x, ordinaryEvent('hello')); + // The round is stored, its restart fails, and the error policy retries: + // the retry's compile ends on the round's tool results. + membrane.queues.push([nearCapToolRound('sig-c', 'checking', 'c-t1')], transientFailure(), + [createMockResponse(signed('sig-c2', 'all done'))]); + (x.framework as any).pendingRequests.push({ agentName: 'assistant', reason: 'timer', source: 'framework', timestamp: Date.now() }); + await x.framework.runUntilIdle(); + await sleep(50); await x.framework.runUntilIdle(); + assert.equal(membrane.calls.length, 4, 'first, the continued reply, its failed restart, the retry'); + assert.ok(membrane.calls[3]!.messages.at(-1)!.content.some((b) => b.type === 'tool_result')); + + const marked = stored(x.framework).filter((m) => m.metadata?.promptedBy !== undefined); + assert.equal(marked.length, 3, 'the round, its result and the retry\'s reply'); + assert.equal(new Set(marked.map((m) => m.metadata!.promptedBy.turn)).size, 1); + + strategy.omit = true; + membrane.queues.push([createMockResponse(signed('sig-later', 'later reply'))]); + await wake(x, ordinaryEvent('later')); + const later = membrane.calls[4]!.messages; + assert.equal(textOf(later[later.findIndex((m) => textOf(m) === 'all done') - 1]!), '[Continue]'); + } finally { await x.framework.stop(); rmSync(x.dir, { recursive: true, force: true }); } + }); + + it('a request that ends on the agent\'s own message mints a new [Continue], with an older one rendered before it', async () => { + const x = await make({ deliver: false }); + try { + await wake(x, ordinaryEvent('hello'), signed('sig-1', 'first reply')); + await timerWake(x, signed('sig-c1', 'continued once')); + await timerWake(x, signed('sig-c2', 'continued twice')); + await wake(x, ordinaryEvent('later'), signed('sig-later', 'later reply')); + + const second = x.membrane.calls[2]!.messages; + assert.equal(textOf(second.at(-1)!), '[Continue]'); + assert.equal(second.filter((m) => textOf(m) === '[Continue]').length, 2, 'the first prompt renders before its reply'); + const once = rowWith(x.framework, 'continued once').metadata?.promptedBy; + const twice = rowWith(x.framework, 'continued twice').metadata?.promptedBy; + assert.ok(once && twice); + assert.notEqual(twice.turn, once.turn); + + const later = x.membrane.calls[3]!.messages; + for (const text of ['continued once', 'continued twice']) { + assert.equal(textOf(later[later.findIndex((m) => textOf(m) === text) - 1]!), '[Continue]', text); + } + } finally { await x.framework.stop(); rmSync(x.dir, { recursive: true, force: true }); } + }); + + it('finds a reply whose only identity is a redacted_thinking block', async () => { + const x = await make({ deliver: false }); + try { + await wake(x, ordinaryEvent('hello'), signed('sig-1', 'first reply')); + await timerWake(x, [ + { type: 'redacted_thinking', data: 'opaque-reasoning' } as ContentBlock, + { type: 'text', text: 'redacted reply' } as ContentBlock, + ]); await wake(x, ordinaryEvent('later'), signed('sig-later', 'later reply')); - const minting = x.membrane.calls[1]!.messages; - assert.equal(textOf(minting.at(-1)!), 'arrived during the cooldown'); - const reply = stored(x.framework).find((m) => m.content.some((b) => (b as { text?: string }).text === 'resumed reply'))!; - assert.equal(reply.metadata?.promptedBy, undefined); + const continued = x.membrane.calls[1]!.messages; const later = x.membrane.calls[2]!.messages; - const replyAt = later.findIndex((m) => textOf(m) === 'resumed reply'); - assert.deepEqual(later.slice(0, replyAt), minting, 'what precedes the reply is what its request held'); + assert.equal(textOf(later[later.findIndex((m) => textOf(m) === 'redacted reply') - 1]!), '[Continue]'); + assert.deepEqual(later.slice(0, continued.length), continued); + } finally { await x.framework.stop(); rmSync(x.dir, { recursive: true, force: true }); } + }); + + it('a reply after the process restarts reads its prompt from the stored rows', async () => { + // The first process ends the turn at the round's tool boundary, so its + // store ends on the tool results. A new framework over the same store + // holds nothing in memory from that turn. The agent's plain speech has no + // channel, so no turn announces one (a locus announced at the first turn + // after boot would come last, and the reply would follow it instead). + const first = await make({ deliver: false, locus: null }); + await wake(first, ordinaryEvent('hello'), signed('sig-1', 'first reply')); + first.mod.endTurn = true; + await timerWake(first, + signed('sig-c', 'checking', [{ type: 'tool_use', id: 'c-t1', name: 'tools--ping', input: {} } as ContentBlock])); + const prompt = rowWith(first.framework, 'checking').metadata?.promptedBy; + assert.ok(prompt); + await first.framework.stop(); + + const strategy = new OmitToolExchange(); + strategy.omit = false; + const x = await make({ deliver: false, locus: null, dir: first.dir, agent: { strategy } }); + try { + await timerWake(x, signed('sig-c2', 'picked up')); + assert.ok(x.membrane.calls[0]!.messages.at(-1)!.content.some((b) => b.type === 'tool_result')); + assert.deepEqual(rowWith(x.framework, 'picked up').metadata?.promptedBy, prompt); + + strategy.omit = true; + await wake(x, ordinaryEvent('later'), signed('sig-later', 'later reply')); + const later = x.membrane.calls[1]!.messages; + assert.equal(textOf(later[later.findIndex((m) => textOf(m) === 'picked up') - 1]!), '[Continue]'); } finally { await x.framework.stop(); rmSync(x.dir, { recursive: true, force: true }); } }); From 68f529f071dcbffc9f4310822db4c52976559129 Mon Sep 17 00:00:00 2001 From: Basil-1007 Date: Sun, 11 Oct 2026 16:30:09 -0400 Subject: [PATCH 0114/1112] fix(requests): end a reply's tool loop where its next round is handed messages, not where they are drained The previous commit cleared the turn's [Continue] holder whenever the tool boundary drained any deferred message. What the next round follows is what the live stream is handed after the results: the injected messages, and the routing notice that rides only an injection. A drained message that stays window-only (tool blocks, or named as the agent) isn't in the next round's request, and a drain that stores an entry without injecting it leaves the round following the results. So the holder is now cleared only when the boundary has injections, once they are collected. Placement, so other open changes still merge as text: - the clear sits after the conversational-input signal, outside the drain block (which another change restructures) and outside the streaming branch's hand-off (where others add receipts and logging); - the stream start sets the holder after the replaced-agent check rather than before it, so a stream cancelled there sets nothing, and the check itself stays as it is on main; - the test harness's channel registry line stays as on main, which other changes edit: the process-restart test gives the agent no locus in its own body instead of through a make() option. --- src/framework.ts | 35 +++++++++++++++++------------- test/silent-heartbeat-rows.test.ts | 12 +++++----- 2 files changed, 26 insertions(+), 21 deletions(-) diff --git a/src/framework.ts b/src/framework.ts index 43a60c9c2..a62081dac 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -1245,9 +1245,9 @@ export class AgentFramework { * The `[Continue]` prompt the current round of each turn follows, keyed by * the turn's trigger. Each stream start sets it from what the Agent read in * its request (`StartStreamResult.continuePrompt`), and a tool boundary - * that stores anything after the results clears it, since the next round - * follows those messages. `silentTurnRowMetadata` stamps the round's rows - * with it. + * that hands the live stream messages after the results clears it, since + * the next round follows those messages. `silentTurnRowMetadata` stamps the + * round's rows with it. */ private readonly turnPrompts = new WeakMap(); @@ -6708,10 +6708,6 @@ export class AgentFramework { { const deferred = this.drainDeferredFor(agent.name); if (deferred.length > 0) { - // The next round follows these messages, not the [Continue] - // the turn's earlier rounds followed: its rows carry no prompt. - const turnTrigger = this.activeTurnTriggers.get(agent.name); - if (turnTrigger) this.turnPrompts.delete(turnTrigger); for (const msg of deferred) { agent.getContextManager().addMessage(msg.participant, msg.content, withDeferredWriteId(msg.metadata, msg.id)); // Injection guards: tool blocks would corrupt the tool-cycle @@ -6750,6 +6746,15 @@ export class AgentFramework { this.midTurnInputSignals.add(agent.name); } + // The next round is handed these messages after the results, so it + // follows them, not the tool loop of a reply to [Continue]: its rows + // carry no prompt. (The routing notice below rides only an + // injection already here.) + if (midTurnInjections.length > 0) { + const turnTrigger = this.activeTurnTriggers.get(agent.name); + if (turnTrigger) this.turnPrompts.delete(turnTrigger); + } + // Check if any tool result requested endTurn const shouldEndTurn = currentState.toolResults.some(tc => tc.result.endTurn); @@ -9401,14 +9406,6 @@ export class AgentFramework { } = await agent.startStreamWithInjections(tools, injections, undefined, compressionTools, { silentHeartbeat: trigger?.silentHeartbeat, }); - // The [Continue] prompt this stream's first round follows, if any, as - // the Agent read it from the request: the round's rows carry it - // (silentTurnRowMetadata), and so do later rounds' until a tool - // boundary stores something after the results. - if (trigger) { - if (continuePrompt) this.turnPrompts.set(trigger, continuePrompt); - else this.turnPrompts.delete(trigger); - } if (this.agents.get(agent.name) !== agent) { stream.cancel(); agent.cancelStream(); @@ -9416,6 +9413,14 @@ export class AgentFramework { this.eventGate?.onInferenceEnded(agent.name); return false; } + // The [Continue] prompt this stream's first round follows, if any, as + // the Agent read it from the request: the round's rows carry it + // (silentTurnRowMetadata), and so do later rounds' until a tool + // boundary hands the stream messages after the results. + if (trigger) { + if (continuePrompt) this.turnPrompts.set(trigger, continuePrompt); + else this.turnPrompts.delete(trigger); + } const handle = this.driveStream( agent, diff --git a/test/silent-heartbeat-rows.test.ts b/test/silent-heartbeat-rows.test.ts index ecfaccff1..b29a78904 100644 --- a/test/silent-heartbeat-rows.test.ts +++ b/test/silent-heartbeat-rows.test.ts @@ -148,8 +148,6 @@ async function make(opts: { agents?: string[]; agent?: Partial; /** An earlier framework's directory, to open its store again. */ dir?: string; - /** Where the agent's plain speech lands; null for no channel. */ - locus?: string | null; } = {}) { const dir = opts.dir ?? mkdtempSync(join(tmpdir(), 'silent-hb-rows-')); const membrane = opts.membrane ?? new MockMembrane(); @@ -164,8 +162,7 @@ async function make(opts: { modules: [mod], }); (framework as any).channelRegistry = new Proxy({ - resolveLocus: () => (opts.locus === undefined ? 'world:commons' : opts.locus), - getDefaultPublishChannel: () => (opts.locus === undefined ? 'world:commons' : opts.locus), + resolveLocus: () => 'world:commons', getDefaultPublishChannel: () => 'world:commons', routeSpeech: async () => ({ delivered: opts.deliver ?? true, channelId: 'world:commons' }), startTyping: () => {}, stopTyping: () => {}, getChannelTools: () => [], getDescriptor: () => undefined, @@ -863,7 +860,9 @@ describe('replies to a [Continue] turn', () => { // holds nothing in memory from that turn. The agent's plain speech has no // channel, so no turn announces one (a locus announced at the first turn // after boot would come last, and the reply would follow it instead). - const first = await make({ deliver: false, locus: null }); + const noLocus = (x: { framework: AgentFramework }) => { (x.framework as any).channelRegistry.resolveLocus = () => null; }; + const first = await make({ deliver: false }); + noLocus(first); await wake(first, ordinaryEvent('hello'), signed('sig-1', 'first reply')); first.mod.endTurn = true; await timerWake(first, @@ -874,7 +873,8 @@ describe('replies to a [Continue] turn', () => { const strategy = new OmitToolExchange(); strategy.omit = false; - const x = await make({ deliver: false, locus: null, dir: first.dir, agent: { strategy } }); + const x = await make({ deliver: false, dir: first.dir, agent: { strategy } }); + noLocus(x); try { await timerWake(x, signed('sig-c2', 'picked up')); assert.ok(x.membrane.calls[0]!.messages.at(-1)!.content.some((b) => b.type === 'tool_result')); From d096f6dbea2348ca34cb0718bddbca4c7faac6c3 Mon Sep 17 00:00:00 2001 From: Olive-1011 Date: Sun, 11 Oct 2026 16:42:13 -0400 Subject: [PATCH 0115/1112] fix(usage): a stream's usage is logged once, however it ends; the guard keeps its response-usage fallback A review of this pull request found that the count and the log disagreed for some ends: driveStream's finally counts every stream's usage once, but a turn abandoned by an operator quiesce returned before any log write, a superseded stream (its natural completion, its abort, the generation check at the top of the loop, a throw) returned or broke without one, and an entry written after another entry of the same stream copied the stream's latest usage sample again. So session totals and log-backed health totals disagreed, and the text contradicted itself. Counted once and logged once: - Every entry of a stream goes through logStream, which now carries the stream's usage on the first entry that has any (the value the entry names, else the latest cumulative sample) and on no later entry of that stream. Before, a failure entry written after a completion's carried the rounds again, and log-backed totals counted them twice. - countUsage records the value it counted. In finally, after counting, a stream whose counted usage no entry carried writes one entry, success false, naming what ended it: "Turn abandoned by operator quiesce" (set where the quiesce-abandoned abort returns), "Stream superseded: a newer stream of the agent took over" when a newer stream of the same agent took over, or "Stream abandoned: its agent was disposed or replaced" when the agent itself is gone (a disposed ephemeral, a replaced conversation). The entry carries exactly what was counted. A stream with no usage writes nothing there, as before. - InferenceLogEntry.tokenUsage's comment says that exactly one entry of a stream carries its usage. #159's guard counts and notes one value again with main's fallback: response.details?.usage ?? latestUsage ?? response.usage. Before this commit the fallback stopped at the running sample, so a guard refusal whose stream emitted no usage event counted and noted nothing, where main noted the response's own usage. Comments that list the ends taken at a tool boundary (StreamBoundaryEnd, streamBoundaryEnds, BOUNDARY_END_REQUEST, InferenceLogEntry.stopReason) now name the physical-window restart beside the context-budget one, as the code handles both. Changelog: consumers that end a turn's display on inference:completed are a host's, such as connectome-host's; quiesce-abandoned and superseded streams, and those of a disposed agent, now write a failure entry carrying their rounds, and exactly one entry of each stream carries its usage; the new boundary entries still grow the log, which keeps no retention, by one small entry per tool-ended turn and restart. Tests: - end-turn-usage: a turn abandoned by an operator quiesce after a finished round, and a stream superseded by a newer one after a finished round, each count that round once and write one failure entry carrying it, naming what ended it; a failure after a completion (on a resident, whose stream's catch logs it) is logged without carrying the usage again. - end-turn-usage: the stream error, abort and thrown-stream tests wait for the run to settle, bounded, and check it failed with the stream and that no retry had been made by then, instead of swallowing the rejection and sleeping 50 ms; the provider-cooldown test waits until the retried call happens, with time only as the outer bound, instead of sleeping 1.1 s; the forced /unstick tests read the posted outcome ("the model responded"), not only that the session was cleared. - tool-result-guard: the abandoned-round test scripts membrane's cumulative samples (10, then 20 input tokens) in the test, so a count taken from the first sample fails it; a new test strips the usage events, so the guard counts and logs the refused response's own usage. Each of these fixes, undone alone, fails end-turn-usage or tool-result-guard: the finally entry; the quiesce-abandoned name; the once-only entry; the guard's response-usage fallback; keeping the latest sample; the forced /unstick's success report. --- .../fix-67-end-turn-accounting.fixed.md | 14 +- src/framework.ts | 59 +++++-- src/types/framework.ts | 8 +- test/end-turn-usage.test.ts | 148 +++++++++++++++++- test/tool-result-guard.test.ts | 38 ++++- 5 files changed, 236 insertions(+), 31 deletions(-) diff --git a/changelog.d/fix-67-end-turn-accounting.fixed.md b/changelog.d/fix-67-end-turn-accounting.fixed.md index 41c6895fd..8eaffbd16 100644 --- a/changelog.d/fix-67-end-turn-accounting.fixed.md +++ b/changelog.d/fix-67-end-turn-accounting.fixed.md @@ -4,14 +4,18 @@ and adds its usage once to the session totals (`usage:updated`, `getSessionUsage()`). Like a completion, it resets the consecutive-failure streak and the poison-history rewind budget, ends a refusal-rewind episode (a forced `/unstick` reports that the model responded), and records a - provider-cooldown recovery. Consumers that end a turn's display on `inference:completed`, such as - typing indicators, voice relays and alerts that close on success, now end it on these turns too. + provider-cooldown recovery. In hosts such as connectome-host, consumers that end a turn's display + on `inference:completed` (a typing indicator, notices that close on success, a voice relay) now + end it on these turns too. - **Every stream's billed rounds reach the session totals once, however the stream ends.** A context-budget or physical-window restart is counted and logged where it is decided (`stopReason: 'context_budget'` or `'physical_window'`), without `inference:completed`. A stream that errors, aborts or throws after finishing rounds counts them, and its failure entry in the - inference log carries them. `inferenceCount` counts these streams too, and log-backed health - totals include them. + inference log carries them. A turn abandoned by an operator quiesce, a stream superseded by a + newer one, or one whose agent was disposed, which wrote no entry before, now writes a failure + entry naming what ended it and carrying its rounds. `inferenceCount` counts these streams too. Exactly one entry of each stream + carries its usage, so log-backed health totals agree with the session totals. - An inference-log entry for a stream the framework ended at a tool boundary (a tool-ended turn or a restart) records a note in place of the compiled request, so these frequent entries don't each - add a whole-context blob to the store. + add a whole-context blob to the store. They are still new entries: the inference log keeps no + retention, so it grows by one small entry for every tool-ended turn and restart. diff --git a/src/framework.ts b/src/framework.ts index 50e26ca6d..2ea0f4b2f 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -491,13 +491,15 @@ const TURN_CHECKPOINTS_ID = 'framework/turn-checkpoints'; // legacy single-map l const TURN_CHECKPOINTS_TREE_ID = 'framework/turn-checkpoints/tree'; /** Why the framework ended a stream at a tool boundary: an endTurn tool - * result, or a context-budget restart (the `inference:stream_restarted` - * reasons). It is the stream's inference-log stop reason. */ + * result, or a context-budget or physical-window restart (the + * `inference:stream_restarted` reasons). It is the stream's inference-log + * stop reason. */ type StreamBoundaryEnd = 'turn_ended' | 'context_budget' | 'physical_window'; /** The request a boundary-ended stream's inference-log entry records in * place of its body. A body is the whole compiled context, one blob per - * entry; these ends happen every tool-ended turn and budget restart. */ + * entry; these ends happen every tool-ended turn and every context-budget + * or physical-window restart. */ const BOUNDARY_END_REQUEST = { note: 'request body not kept for a stream ended at a tool boundary' }; /** Maximum number of turn checkpoints to keep per agent. */ @@ -1026,12 +1028,12 @@ export class AgentFramework { private processLoggingBroadcast: boolean; private activeStreams: Map> = new Map(); // The framework ends a stream at a tool boundary from handleProcessEvent: - // an endTurn result, or a context-budget restart. Each live stream - // registers here how to record that end (its usage, its log entry and, for - // endTurn, the completed turn), so the record is made where the decision - // is taken, before the cancellation's asynchronous teardown or an - // ephemeral's disposal. Keyed by the physical stream, so a successor's - // record is never touched. + // an endTurn result, or a context-budget or physical-window restart. Each + // live stream registers here how to record that end (its usage, its log + // entry and, for endTurn, the completed turn), so the record is made where + // the decision is taken, before the cancellation's asynchronous teardown + // or an ephemeral's disposal. Keyed by the physical stream, so a + // successor's record is never touched. private streamBoundaryEnds = new WeakMap void>(); /** Per-agent output locus FROZEN for the CURRENT logical turn. Resolved @@ -9542,26 +9544,37 @@ export class AgentFramework { // response without usage, the boundary ends below, or `finally` for // every other end). Each caller passes the one value it also reports. let usageCounted = false; + let countedUsage: DetailedUsage | undefined; const countUsage = (usage: DetailedUsage | undefined): void => { if (usageCounted) return; usageCounted = true; + countedUsage = usage; if (!usage) return; this.usageTracker.onInferenceCompleted(agent.name, usage, usage.estimatedCost ? { total: usage.estimatedCost.total, currency: usage.estimatedCost.currency } : undefined); this.persistUsageState(); }; - // Every inference-log entry of this stream carries the usage of the - // rounds it finished, unless the entry names its own. + // ...and is carried by exactly one of its inference-log entries, so + // log-backed totals agree with the session's. The first entry carries + // it (the value the entry names, else the rounds finished so far); no + // later entry of this stream carries it again. An end that writes no + // entry of its own is logged in `finally`. + let usageLogged = false; const logStream = (entry: Omit): void => { + const tokenUsage = usageLogged ? undefined : (entry.tokenUsage ?? tokenUsageOf(latestUsage)); + if (tokenUsage) usageLogged = true; this.logInference({ timestamp: startTime, agentName: agent.name, requestId, ...entry, - tokenUsage: entry.tokenUsage ?? tokenUsageOf(latestUsage), + tokenUsage, }); }; + // What ended a stream whose end writes no log entry of its own, for the + // entry `finally` writes for it. + let unloggedEnd: string | undefined; // A turn that ended successfully: with its final response (natural // completion), or without one, when a tool result with endTurn ended it. const completeTurn = (usage: DetailedUsage | undefined, durationMs: number, response?: NormalizedResponse): void => { @@ -10052,7 +10065,7 @@ export class AgentFramework { // cumulative across this physical tool loop): count it before // the retry opens a fresh stream with its own usage. One value // for the count, the log entry and the refusal note. - const abandoned = response.details?.usage ?? latestUsage; + const abandoned: DetailedUsage | undefined = response.details?.usage ?? latestUsage ?? response.usage; const tokenUsage = tokenUsageOf(abandoned); this.noteRefusal(agent.name, category, tokenUsage); countUsage(abandoned); @@ -10675,6 +10688,7 @@ export class AgentFramework { this.eventGate?.onInferenceEnded(agent.name); } if (cancelKind === 'quiesce_abandoned') { + unloggedEnd = 'Turn abandoned by operator quiesce'; // Operator-initiated cancel during a quiesce drain. Settle // the turn honestly — deliberately NO inference:exhausted: // that trace centrally drives the consecutive-failure @@ -10892,6 +10906,25 @@ export class AgentFramework { // Every end the cases above didn't count (a stream error or abort, a // throw, a superseded generation) still billed the rounds it finished. countUsage(latestUsage); + // An end that wrote no entry carrying them (a quiesce-abandoned turn, + // a stream superseded by a newer one, a disposed agent's stream) writes + // one now, as a failure naming what ended it, so the log carries what + // the totals count. + if (countedUsage && !usageLogged) { + const replaced = this.agents.get(agent.name) !== agent; + const superseded = generationLost || agent.streamId !== myStreamId; + logStream({ + success: false, + error: unloggedEnd ?? (replaced + ? 'Stream abandoned: its agent was disposed or replaced' + : superseded + ? 'Stream superseded: a newer stream of the agent took over' + : 'Stream ended without a final response'), + request: compiledRequest ?? { note: 'streaming request' }, + durationMs: Date.now() - startTime, + tokenUsage: tokenUsageOf(countedUsage), + }); + } const unsettledKvSubmissions = drainKvSubmissionIds?.() ?? []; if (unsettledKvSubmissions.length > 0) { try { diff --git a/src/types/framework.ts b/src/types/framework.ts index 87fb67491..8daf6e6b2 100644 --- a/src/types/framework.ts +++ b/src/types/framework.ts @@ -411,7 +411,10 @@ export interface InferenceLogEntry { response?: unknown | { blobId: string }; durationMs: number; /** The stream's usage: its final response's, or, for a stream that ended - * without one, the provider rounds it finished before it ended. */ + * without one, the provider rounds it finished before it ended. Exactly + * one entry of a stream carries it, the one its end wrote; a stream whose + * end writes none (a quiesce-abandoned turn, a superseded stream, a + * disposed agent's stream) gets a failure entry that carries it. */ tokenUsage?: { input: number; output: number; @@ -420,7 +423,8 @@ export interface InferenceLogEntry { }; /** Stop reason from the model, or why the framework ended the stream at a * tool boundary: 'turn_ended' (a tool result with endTurn), or - * 'context_budget' / 'physical_window' (a context-budget restart). */ + * 'context_budget' / 'physical_window' (a context-budget or + * physical-window restart). */ stopReason?: string; } diff --git a/test/end-turn-usage.test.ts b/test/end-turn-usage.test.ts index 4636625e7..2e026490f 100644 --- a/test/end-turn-usage.test.ts +++ b/test/end-turn-usage.test.ts @@ -442,11 +442,17 @@ describe('a stream counts its usage once, however it ends', () => { name: 'worker', model: 'test-model', systemPrompt: 'Do the task.', allowedTools: 'all', proseRouting: 'disabled', }); contextManager.addMessage('user', [{ type: 'text', text: 'Run once.' }]); - await framework.runEphemeralToCompletion(agent, contextManager).catch(() => undefined); + // The run settles once the failure path has decided: it rejects with + // the stream's failure, or, had a retry been scheduled, it would wait + // for it and resolve on the default response. + const outcome = await Promise.race([ + framework.runEphemeralToCompletion(agent, contextManager).then(() => 'resolved', (error: unknown) => error), + new Promise((resolve) => setTimeout(() => resolve('timed out'), 10_000)), + ]); + assert.ok(outcome instanceof Error, `the run failed with the stream: ${String(outcome)}`); + assert.equal(membrane.calls.length, 1, 'no retry was scheduled by the time the run failed'); assert.ok(await pollUntil(() => traces.some((e) => e.type === 'usage:updated')), 'the finished round was counted'); - await new Promise((r) => setTimeout(r, 50)); - assert.equal(membrane.calls.length, 1, 'no retry'); assert.equal(traces.filter((e) => e.type === 'inference:completed').length, 0); assert.equal(traces.filter((e) => e.type === 'usage:updated').length, 1); assert.deepEqual(framework.getSessionUsage().totals, expectedTotals(firstUsage)); @@ -537,9 +543,15 @@ describe('a tool-ended turn ends what a successful turn ends', () => { internal[key].set('worker', 2); } internal.rewindEpisode.set('worker', { markerId: 'marker', count: 1, category: 'cyber' }); - internal.forcedRewind.set('worker', { remaining: 1, removed: [], serverId: 'none', channelId: '' }); + internal.forcedRewind.set('worker', { remaining: 1, removed: [], serverId: 'ops', channelId: 'ops-channel' }); + const posted: Array<{ channelId: string; content: string }> = []; + (f.framework as unknown as { postToChannel: (s: string, c: string, content: string) => Promise }) + .postToChannel = async (_server, channelId, content) => { posted.push({ channelId, content }); }; f.membrane.pushResponse(response()); await run(f.framework, 'worker'); + assert.equal(posted.length, 1, 'the /unstick outcome was posted'); + assert.equal(posted[0].channelId, 'ops-channel'); + assert.match(posted[0].content, /the model responded/); assert.equal(internal.consecutiveInferenceFailures.get('worker'), 0, 'the hard-down streak'); assert.equal(internal.exhaustionRewinds.get('worker'), 0, 'the poison-history rewind budget'); assert.equal(internal.refusalRewinds.get('worker'), 0); @@ -563,8 +575,7 @@ describe('a tool-ended turn ends what a successful turn ends', () => { r.framework.pushEvent({ type: 'external-message', source: 'test', content: 'first', metadata: {} } as unknown as ProcessEvent); await r.framework.runUntilIdle(); assert.equal(r.membrane.calls.length, 1, 'held for the cooldown'); - await new Promise((resolve) => setTimeout(resolve, 1_100)); - await r.framework.runUntilIdle(); + assert.ok(await pollUntil(() => r.membrane.calls.length === 2, 10_000), 'the cooldown ended and the call was retried'); assert.ok(await pollUntil(() => r.traces.some((e) => e.type === 'inference:turn_ended')), 'the retried turn ended by tool'); assert.equal(r.membrane.calls.length, 2); assert.equal(r.internal.providerAccelerationLastRecovery.get('resident')?.stopReason, 'turn_ended'); @@ -593,3 +604,128 @@ describe('a tool-ended turn ends what a successful turn ends', () => { } }); }); + +/** A completion the stream would yield next; these tests end it before it's handled. */ +const unseenCompletion = (): StreamEvent => + ({ type: 'complete', response: createMockResponse([{ type: 'text', text: 'unseen' }]) }) as StreamEvent; + +/** A tool that returns only once released, so the stream waits on it. */ +class HeldTools implements Module { + readonly name = 'test'; + release: () => void = () => {}; + private readonly held = new Promise((resolve) => { this.release = resolve; }); + calledWith: string[] = []; + async start() {} + async stop() {} + async onProcess() { return {}; } + getTools() { + return [{ name: 'echo', description: 'echo', inputSchema: { type: 'object' as const, properties: {} } }]; + } + async handleToolCall(call: ToolCall) { + this.calledWith.push(call.id); + await this.held; + return { success: true, data: {} }; + } +} + +describe('a stream that ends without writing its own entry is logged once, in finally', () => { + async function residentWith(stream: YieldingStream, tools: Module) { + const dir = mkdtempSync(join(tmpdir(), 'af-end-turn-usage-')); + const membrane = new MockMembrane(); + let made = false; + membrane.streamYielding = (request: NormalizedRequest): YieldingStream => { + membrane.calls.push(request); + if (made) return new MockYieldingStream([createMockResponse([{ type: 'text', text: 'done' }])]); + made = true; + return stream; + }; + const framework = await AgentFramework.create({ + storePath: join(dir, 'store.chronicle'), membrane: membrane.asMembrane(), + agents: [{ name: 'resident', model: 'test-model', systemPrompt: 'system', allowedTools: 'all' }], + modules: [new InputModule(), tools], syncIntervalMs: 0, maintenanceIntervalMs: 0, + }); + const traces: TraceEvent[] = []; + framework.onTrace((event) => traces.push(event)); + framework.start(); + framework.pushEvent({ type: 'external-message', source: 'test', content: 'go', metadata: {} } as unknown as ProcessEvent); + return { dir, membrane, framework, traces }; + } + + function loggedOnce(framework: AgentFramework, traces: TraceEvent[], error: RegExp) { + assert.equal(traces.filter((e) => e.type === 'usage:updated').length, 1, 'counted once'); + assert.deepEqual(framework.getSessionUsage().totals, expectedTotals(firstUsage)); + const logs = framework.queryInferenceLogs({ agentName: 'resident' }).entries.map((e) => e.entry); + assert.equal(logs.length, 1, 'one entry'); + assert.equal(logs[0].success, false); + assert.match(String(logs[0].error), error); + assert.deepEqual(logs[0].tokenUsage, expectedTokens(firstUsage), 'the entry carries what was counted'); + } + + it('a turn abandoned by an operator quiesce after a finished round', async () => { + const tools = new HeldTools(); + const r = await residentWith(new RoundThenEnd(firstUsage, unseenCompletion()), tools); + try { + assert.ok(await pollUntil(() => tools.calledWith.length === 1), 'the round finished and its tool is running'); + await r.framework.quiesce({ timeoutMs: 50, abandon: true }); + assert.ok(await pollUntil(() => r.framework.queryInferenceLogs({ agentName: 'resident' }).total > 0), 'logged'); + loggedOnce(r.framework, r.traces, /^Turn abandoned by operator quiesce$/); + } finally { + // Let the held tool finish while the framework's queue is still open. + tools.release(); + await pollUntil(() => r.traces.some((e) => + (e.type === 'tool:completed' || e.type === 'tool:failed') && (e as { callId?: string }).callId === 'echo-1')); + await r.framework.stop(); + rmSync(r.dir, { recursive: true, force: true }); + } + }); + + it('a failure after a completion is logged without carrying the usage again', async () => { + const r = await resident([() => new MockYieldingStream([finalResponse('Done.', totalUsage)])]); + // Something after the completion's entry throws: the stream's catch + // writes a failure entry too. (A resident: an ephemeral is deregistered + // once its completion settles it, so its catch treats the throw as a + // superseded stream's.) + const internal = r.framework as unknown as { endRefusalEpisode: (name: string) => void }; + const end = internal.endRefusalEpisode.bind(r.framework); + internal.endRefusalEpisode = () => { + internal.endRefusalEpisode = end; + throw new Error('after the completion'); + }; + r.framework.start(); + try { + r.framework.pushEvent({ type: 'external-message', source: 'test', content: 'go', metadata: {} } as unknown as ProcessEvent); + assert.ok(await pollUntil(() => r.framework.queryInferenceLogs({ agentName: 'resident' }).total === 2), 'both entries'); + // Both entries carry the stream's start time, so find each by outcome. + const logs = r.framework.queryInferenceLogs({ agentName: 'resident' }).entries.map((e) => e.entry); + const completed = logs.find((e) => e.success); + const failed = logs.find((e) => !e.success); + assert.deepEqual(completed?.tokenUsage, expectedTokens(totalUsage), 'the completion carries the usage'); + assert.equal(failed?.error, 'Stream threw: after the completion'); + assert.equal(failed?.tokenUsage, undefined, 'the later failure does not carry it again'); + assert.deepEqual(r.framework.getSessionUsage().totals, expectedTotals(totalUsage)); + } finally { + await r.framework.stop(); + rmSync(r.dir, { recursive: true, force: true }); + } + }); + + it('a stream superseded by a newer one after a finished round', async () => { + const stream = new RoundThenEnd(firstUsage, unseenCompletion()); + let framework: AgentFramework | undefined; + const provide = stream.provideToolResults.bind(stream); + // A newer stream of the agent takes over while this one waits on its tool. + stream.provideToolResults = () => { + (framework!.getAgent('resident') as unknown as { _streamId: number })._streamId++; + provide(); + }; + const r = await residentWith(stream, new TurnTools()); + framework = r.framework; + try { + assert.ok(await pollUntil(() => r.framework.queryInferenceLogs({ agentName: 'resident' }).total > 0), 'logged'); + loggedOnce(r.framework, r.traces, /^Stream superseded/); + } finally { + await r.framework.stop(); + rmSync(r.dir, { recursive: true, force: true }); + } + }); +}); diff --git a/test/tool-result-guard.test.ts b/test/tool-result-guard.test.ts index 510ca08b9..9fc1f5001 100644 --- a/test/tool-result-guard.test.ts +++ b/test/tool-result-guard.test.ts @@ -528,17 +528,45 @@ test('usage of an abandoned guarded round is counted in session totals', async ( }); test('an abandoned guarded round whose response carries no usage still counts its rounds', async () => { - // Each scripted response emits a usage sample of 10 input tokens; neither - // final response carries details.usage, so both streams count their samples. - const h = await harness([[calls('one'), refused()], [answer()]], { toolResultGuard: true }); + // Membrane's usage samples are cumulative across a stream's rounds: the + // withheld stream's two rounds emit 10, then 20 input tokens, and the + // retry's one round 10. No final response carries details.usage, so each + // stream counts its latest sample. + const sampled = (response: NormalizedResponse, inputTokens: number, outputTokens: number) => + ({ ...response, usage: { inputTokens, outputTokens } }) as NormalizedResponse; + const h = await harness([[sampled(calls('one'), 10, 5), sampled(refused(), 20, 10)], [sampled(answer(), 10, 5)]], { toolResultGuard: true }); try { await h.run(); const totals = h.framework.getSessionUsage().totals as unknown as Record; - assert.equal(totals.inputTokens, 20, `each stream is counted once, from its samples: ${JSON.stringify(totals)}`); + assert.equal(totals.inputTokens, 30, `each stream is counted once, from its latest sample: ${JSON.stringify(totals)}`); // The log says what the totals count: the withheld stream, then the retry. const logs = h.framework.queryInferenceLogs({ agentName: 'assistant' }).entries.map((e) => e.entry).reverse(); assert.deepEqual(logs.map((e) => [e.success, e.tokenUsage]), - [[false, { input: 10, output: 5 }], [true, { input: 10, output: 5 }]]); + [[false, { input: 20, output: 10 }], [true, { input: 10, output: 5 }]]); + } finally { await h.framework.stop(); } +}); + +test("an abandoned guarded round with no usage sample counts and notes its response's own usage", async () => { + const h = await harness([[calls('one'), refused()], [answer()]], { toolResultGuard: true }); + // A stream that emits no usage events: only the refused response's own + // usage (10 input tokens, from the shared mock) says what it was billed. + const make = h.membrane.streamYielding.bind(h.membrane); + h.membrane.streamYielding = (request, options) => { + const stream = make(request, options); + const events = stream[Symbol.asyncIterator].bind(stream); + stream[Symbol.asyncIterator] = async function* () { + const it = events(); + for (let r = await it.next(); !r.done; r = await it.next()) if (r.value.type !== 'usage') yield r.value; + }; + return stream; + }; + try { + await h.run(); + const totals = h.framework.getSessionUsage().totals as unknown as Record; + assert.equal(totals.inputTokens, 10, `the refused round is counted from its response: ${JSON.stringify(totals)}`); + const withheld = h.framework.queryInferenceLogs({ agentName: 'assistant' }).entries.map((e) => e.entry) + .find((e) => e.stopReason === 'refusal'); + assert.deepEqual(withheld?.tokenUsage, { input: 10, output: 5 }, "the withheld stream's entry carries the response's usage"); } finally { await h.framework.stop(); } }); From c4f1f1a9afd04b41979930ac7df5bf6ca2453544 Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Sun, 11 Oct 2026 16:47:01 -0400 Subject: [PATCH 0116/1112] fix(mcpl): report a handler's own failure, never let a report end the host, and answer what nothing handles A review of this change found the containment it adds still had ways to lose a failure or end the host, and a few inbound paths outside it. The handler wrapper (now AgentFramework.containedInboundHandler, so it can be tested directly): - A failure report names the handler's own error when it threw or rejected. A reply write that failed before it is reported beside it (trace field responseFailure), instead of replacing it: a handler that swallowed an encoding failure and then failed with 'db down' used to report only the encoding failure. - A reply the handler tries after one was already attempted is refused without throwing, and reported (refusedResponse: { kind: 'result' }, or { kind: 'error', code, message } with the message bounded). So channels/register's early acknowledgement followed by a failed store append, whose catch answers again and rethrows, reports the append's error, not 'MCPL response already attempted'. A handler that answers twice and resolves is reported too. - responseAttempted in the trace is what the handler's own writes did, read when the failure is reported, not a snapshot from the first failed write. - Nothing in the report can throw: errors are described by describeCause, which handles a null-prototype reason, a throwing toPrimitive or a throwing message getter, and the report's body is guarded. On the async path it runs as the rejection handler, where an exception was an unhandled rejection, which ends the host. The trace is emitted before the stderr line: it's built from strings alone, while printing hands the raw value to util.inspect, which a value with a throwing inspect method makes throw. - The -32603 reply carries the standard message, "Internal error", not the handler's own text, which can name internal paths and errors and goes to a server that may be a third party's. Eight of the nine handlers answered nothing before this change. The detail stays in the trace and stderr. - The stderr line also prints a failed internal-error reply, so a headless host sees it. - mcpl/manifestChanged and notifications/tools/list_changed go through the same containment, so their failures emit mcpl:request-handler-error too. The Discord awareness accounting a tools-list refresh waits on stays on its own path: its failure is global, and is reported for every connection as each is recycled. The connection: - A request (it has an id) whose method isn't a string, or isn't one this host handles, is answered -32601 "Method not found", with a stderr line, as the function's own comment says a request that will never be answered must be. So is a request whose event nothing on the host listens for (checked in emit, after the capability gate, so a gated event still gets the gate's -32002 first). A notification nobody handles is ignored, as JSON-RPC asks. - A dispatch failure's error, which the framework traces as mcpl:server-error, names the event or method, the request id and the cause, bounded and with control characters replaced. - A server's live stderr lines go through the same containment as buffered ones, so a stderr listener that throws is reported, not thrown into the transport. - StdioTransport listens for 'error' on the child's stdin. A write to a child that has just exited fails there with EPIPE, and unlistened that would end the host. It's reported as the transport's error, or logged when nothing listens; the exit closes the transport. - McplResponseSerializationError's message says why encoding failed (a BigInt, a circular structure, a throwing toJSON). The changelog fragment is rewritten to match: which handlers are contained, what each trace field holds, when -32603 and -32601 are sent and with what message, and that a reply to a connection that has already closed is dropped without a trace, as before. Tests (mcpl-request-handler-errors, 43 to 57): - a swallowed encoding failure followed by the handler's own error; - an answer tried after the reply, refused, with the handler's error reported; and a handler that answers twice and resolves; - a rejection with a null-prototype reason, and one that neither String() nor util.inspect can print: each reported in the trace, answered, with no unhandled rejection; - -32601 for an unknown method, for non-string methods, and for an event nothing listens for, and nothing for an unknown notification; - a dispatch failure's trace naming the event, the request and the cause; a throwing 'error' listener caught and logged with the next message still delivered; a throwing live stderr listener contained; - a failed manifest refresh reported as a handler failure; - a failed internal-error reply on stderr; - a stdin EPIPE with and without a transport listener; - a synchronous handler that swallows an encoding failure, reported and answered once; - existing tests now assert the reply's code and message, the acknowledgement written before reconciliation's first await settles, and why encoding failed; the two negative routing tests gain positive controls in the same setup; the stdio probe waits up to 120 s for its child (request timeout 60 s) instead of 5 s and 1 s. Red checks on dist, each alone (15): reporting the write failure over the handler's error, a throwing refusal, a resolved double answer left unreported, describeCause without its fallback, no -32601, no failure context, the handler's text on the wire, an uncontained manifest listener, no reply failure on stderr, no stdin listener, sync success unchecked, an encoding error that says nothing, live stderr emitted directly, printing before the trace, and no answer for an unheard request: each fails the test aimed at it. Local: tsc clean; mcpl-request-handler-errors 57/57 on Node 24.19.0. --- .../208-inbound-handler-containment.fixed.md | 10 +- src/framework.ts | 214 ++++++++----- src/mcpl/errors.ts | 26 +- src/mcpl/server-connection.ts | 76 +++-- src/mcpl/transport.ts | 7 + src/types/trace.ts | 18 +- test/fixtures/mcpl-inbound-host-probe.mjs | 2 +- test/mcpl-request-handler-errors.test.ts | 285 +++++++++++++++++- 8 files changed, 531 insertions(+), 107 deletions(-) diff --git a/changelog.d/208-inbound-handler-containment.fixed.md b/changelog.d/208-inbound-handler-containment.fixed.md index c37e96a8c..c6ba2feb7 100644 --- a/changelog.d/208-inbound-handler-containment.fixed.md +++ b/changelog.d/208-inbound-handler-containment.fixed.md @@ -1 +1,9 @@ -- MCPL request and notification handlers contain synchronous throws and async rejections (#186), including channel reconciliation failures after an acknowledgement. Failures emit `mcpl:request-handler-error`; requests receive JSON-RPC `-32603` only when no response write has been attempted. Pre-write JSON encoding failures can receive an internal-error reply; possibly partial transport writes remain single-attempt. Failed error-response writes are included in the trace, failures also reach stderr for headless hosts, and notifications receive no response. Non-object JSON is ignored before handshake/dispatch property access; synchronous inbound dispatch failures are reported without interrupting later messages, including buffered plane flushes. +- MCPL inbound handlers, for requests and for notifications (`mcpl/manifestChanged` and `notifications/tools/list_changed` included), contain synchronous throws and async rejections (#186), including channel reconciliation failures after an acknowledgement. Nothing a server sends, and nothing a handler throws or rejects with, ends the host: a rejection reason that can't be printed is reported as one that can. + - **Each failure emits `mcpl:request-handler-error` and a stderr line.** Its `error` is the handler's own error. A reply write that failed before it is `responseFailure`, and a reply the handler tried to send after one was already attempted is `refusedResponse` (a result, or an error with its code and message): refused, never written. `responseError` is the internal-error reply's own failure, which the stderr line prints too. A failure of the Discord awareness accounting that a tools-list refresh waits on is global: it's reported as `mcpl:server-error` for every connection, as each is recycled. + - **A request gets JSON-RPC `-32603` "Internal error"** when its handler fails before any reply bytes may have gone out. That's when it attempted no reply, or its reply failed while encoding. A transport write that threw may have sent part of a reply, so it isn't followed by another. The reply carries the standard message, not the handler's own error text, which stays in the trace and stderr. A notification gets no reply. + - **A request whose method isn't a string,** or isn't one the host handles, or whose event nothing on the host listens for, is answered `-32601` "Method not found", with a stderr line. A notification nobody handles is ignored. + - **Non-object JSON** is ignored before property access, in the handshake reader and in routing, as a non-JSON line already is. + - **A synchronous failure while dispatching an inbound message**, buffered plane flushes and a server's stderr lines included, is reported without interrupting later messages. Its `mcpl:server-error` names the event or method, the request id, and the cause. + - **A failed write to a stdio server's stdin** (a child that has just exited) is reported instead of ending the host. + - **A response that can't be encoded** says why in its error (a BigInt, a circular structure, a throwing `toJSON`). + - **A reply to a connection that has already closed** is dropped without a trace, on either transport, as before. diff --git a/src/framework.ts b/src/framework.ts index 0d28e5ab7..163edac0d 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -387,7 +387,7 @@ import { EventGate, formatShadowWarning } from './gate/event-gate.js'; import { UsageTracker, type PersistedUsageState } from './usage/usage-tracker.js'; import type { SessionUsageSnapshot, UsageUpdatedEvent } from './usage/types.js'; import type { McplServerConnection } from './mcpl/server-connection.js'; -import { McplResponseSerializationError } from './mcpl/errors.js'; +import { McplResponseSerializationError, boundedForLog, describeCause } from './mcpl/errors.js'; import type { McplServerConfig, McplHostCapabilities, @@ -470,6 +470,13 @@ interface DeferredWrite { } const bySeq = (a: { seq: number }, b: { seq: number }): number => a.seq - b.seq; +/** The reply helpers an MCPL connection hands an inbound request's handler. */ +type InboundResponder = { + id: string | number; + respond: (result: unknown) => void; + respondError: (code: number, message: string, data?: unknown) => void; +}; + /** Stamp a deferred write's durable id into the metadata it is stored with * (boot recovery dedups replays by it). Idempotent for an already-stamped * message. */ @@ -13690,88 +13697,144 @@ export class AgentFramework { } /** - * Wire event listeners on an MCPL server connection. - * Push events and inference requests are deferred to Steps 6/7. + * An inbound MCPL handler, contained: EventEmitter doesn't consume the + * promise a listener returns, so this keeps each handler's throw or + * rejection at this boundary, including failures after channels/register or + * channels/changed has acknowledged the request. It's invoked at once, so + * control-plane replies keep their acknowledge-before-reconcile order. */ - private wireMcplEvents(connection: McplServerConnection): void { - type Responder = { - id: string | number; - respond: (result: unknown) => void; - respondError: (code: number, message: string, data?: unknown) => void; - }; - - // EventEmitter does not consume promises returned by listeners. Keep - // each RPC handler's rejection at this boundary, including failures after - // channels/register or channels/changed has acknowledged the request. - const onRequest = ( - event: string, - handler: (params: T, responder?: Responder) => void | Promise, - ): void => { - connection.on(event, (params: T, responder?: Responder) => { - let responseAttempted = false; - let responseFailure: { error: unknown; responseAttempted: boolean } | undefined; - const writeResponse = (write: () => void): void => { - if (responseAttempted) { - // Inner handler catches can try to answer a failed result with - // an error. Preserve the first failure and refuse the second write. - throw responseFailure ? responseFailure.error : new Error('MCPL response already attempted'); - } - // A throwing transport write may have partially sent the reply. - responseAttempted = true; - try { - write(); - } catch (error) { - // Only the encoder can establish that no write was attempted. - // Transport exceptions stay outcome-unknown and cannot be retried. - if (error instanceof McplResponseSerializationError) responseAttempted = false; - responseFailure = { error, responseAttempted }; - throw error; + private containedInboundHandler( + serverId: string, + event: string, + handler: (params: T, responder?: InboundResponder) => void | Promise, + ): (params: T, responder?: InboundResponder) => void { + return (params: T, responder?: InboundResponder) => { + // Whether a reply's bytes may have reached the transport. Only an + // encoding failure proves none did; a transport exception may have + // written part of one, so it can't be retried. + let responseAttempted = false; + // The first reply write that threw, if any, which a handler may have + // swallowed. + let writeFailure: unknown; + let writeFailed = false; + // A write the handler tried after a reply was attempted. It's refused + // without throwing, so a handler that answers again in its own catch + // and then rethrows reports its own error; what it tried to send is + // kept for the report. + let refused: { kind: 'result' } | { kind: 'error'; code: number; message: string } | undefined; + const writeResponse = ( + write: () => void, + refusedAs: { kind: 'result' } | { kind: 'error'; code: number; message: string }, + ): void => { + if (responseAttempted) { + refused ??= refusedAs; + return; + } + responseAttempted = true; + try { + write(); + } catch (error) { + if (error instanceof McplResponseSerializationError) responseAttempted = false; + if (!writeFailed) { + writeFailed = true; + writeFailure = error; } - }; - const tracked = responder && { - ...responder, - respond: (result: unknown) => writeResponse(() => responder.respond(result)), - respondError: (code: number, message: string, data?: unknown) => - writeResponse(() => responder.respondError(code, message, data)), - }; - const failed = (error: unknown): void => { - if (responseFailure) error = responseFailure.error; - const message = error instanceof Error ? error.message : String(error); - const attemptedBeforeFailure = responseFailure?.responseAttempted ?? responseAttempted; + throw error; + } + }; + const tracked = responder && { + ...responder, + respond: (result: unknown) => writeResponse(() => responder.respond(result), { kind: 'result' }), + respondError: (code: number, message: string, data?: unknown) => + writeResponse(() => responder.respondError(code, message, data), { + kind: 'error', code, message: boundedForLog(typeof message === 'string' ? message : describeCause(message), 500), + }), + }; + // Report a failure: the handler's own error when it threw or + // rejected, else the reply write it swallowed, else the reply it + // tried to send twice. Nothing in here may throw: on the async path + // it runs as a rejection handler, and an exception there would be an + // unhandled rejection, which ends the host. + const report = (handlerError: { error: unknown } | undefined): void => { + try { + const primary = handlerError + ? handlerError.error + : writeFailed ? writeFailure : new Error('the handler tried to answer a second time after its reply'); + const message = describeCause(primary); + // What the handler's own writes did, before any internal-error + // reply sent here. + const handlerResponseAttempted = responseAttempted; let responseError: string | undefined; if (tracked && !responseAttempted) { + // The server gets the standard message, not the handler's own + // text, which may name internal paths or errors; the trace and + // stderr carry the detail. try { - tracked.respondError(-32603, message); + tracked.respondError(-32603, 'Internal error'); } catch (sendError) { - responseError = sendError instanceof Error ? sendError.message : String(sendError); + responseError = describeCause(sendError); } } - // Keep headless hosts diagnosable even without a trace subscriber. - console.error(`[mcpl] ${connection.id} ${event} handler failed:`, error); + const responseFailure = writeFailed && handlerError && handlerError.error !== writeFailure + ? describeCause(writeFailure) : undefined; + // The trace first: it's built from strings alone, while printing + // hands the raw thrown value to util.inspect, which an exotic value + // (a Proxy whose traps throw) can make throw. this.emitTrace({ type: 'mcpl:request-handler-error', - serverId: connection.id, + serverId, event, requestId: responder?.id, - responseAttempted: attemptedBeforeFailure, + responseAttempted: handlerResponseAttempted, error: message, + ...(responseFailure === undefined ? {} : { responseFailure }), + ...(refused === undefined ? {} : { refusedResponse: refused }), ...(responseError === undefined ? {} : { responseError }), }); - }; - try { - // Invoke immediately: control-plane responses must retain their - // ACK-before-reconcile ordering and data-plane barrier semantics. - const succeeded = (): void => { - // A handler may swallow an encoding/transport failure without rethrowing. - if (responseFailure) failed(responseFailure.error); - }; - const result = handler(params, tracked); - if (result) void result.then(succeeded, failed); - else succeeded(); - } catch (error) { - failed(error); + // Keep headless hosts diagnosable even without a trace subscriber. + console.error( + `[mcpl] ${serverId} ${event} handler failed:`, primary, + ...(responseFailure === undefined ? [] : ['| its reply write had failed:', writeFailure]), + ...(refused === undefined ? [] : ['| it then tried to answer again:', refused]), + ...(responseError === undefined ? [] : ['| the internal-error reply failed too:', responseError]), + ); + } catch (reportError) { + try { console.error(`[mcpl] ${serverId} ${event} handler failed, and reporting it failed too:`, reportError); } + catch { /* stderr itself is gone */ } } - }); + }; + const failed = (error: unknown): void => report({ error }); + // A handler may swallow a write failure, or answer twice, and still + // resolve. + const succeeded = (): void => { + if (writeFailed || refused) report(undefined); + }; + try { + // Invoke immediately: control-plane responses must retain their + // ACK-before-reconcile ordering and data-plane barrier semantics. + const result = handler(params, tracked); + if (result) void Promise.resolve(result).then(succeeded, failed); + else succeeded(); + } catch (error) { + failed(error); + } + }; + } + + /** + * Wire event listeners on an MCPL server connection. + * Push events and inference requests are deferred to Steps 6/7. + */ + private wireMcplEvents(connection: McplServerConnection): void { + type Responder = InboundResponder; + + // Every inbound handler, request or notification, is contained + // (containedInboundHandler). + const onRequest = ( + event: string, + handler: (params: T, responder?: Responder) => void | Promise, + ): void => { + connection.on(event, this.containedInboundHandler(connection.id, event, handler)); }; // Forward subprocess stderr lines as trace events so consumers (conhost, @@ -13845,11 +13908,10 @@ export class AgentFramework { // §17.3: a manifest-change announcement. Rate-limited re-fetch → diff // → §6.7 consequences → one receipt. The announcement itself carries no // payload and no authority; everything below acts on FETCHED content. - connection.on('manifest-changed', (params: { revision?: string; domains?: string[] }) => { - void this.handleManifestChanged(connection, params).catch((err) => { - console.error(`[mcpl] ${connection.id} manifest refresh failed:`, err instanceof Error ? err.message : err); - }); - }); + // A notification, contained and reported like every inbound handler: a + // refresh that fails emits mcpl:request-handler-error. + onRequest('manifest-changed', (params: { revision?: string; domains?: string[] }) => + this.handleManifestChanged(connection, params)); onRequest('channels-list', ( _params: unknown, @@ -13915,8 +13977,12 @@ export class AgentFramework { responder.respond(result); }); - // Handle dynamic tool list changes (notifications/tools/list_changed) - connection.on('tools-list-changed', () => { + // Handle dynamic tool list changes (notifications/tools/list_changed). + // Contained like every inbound handler: a synchronous failure here emits + // mcpl:request-handler-error. A failure of the awareness accounting its + // barrier waits on is global, and failMcplDataPlaneGate reports it for + // every connection (mcpl:server-error) as it recycles them. + onRequest('tools-list-changed', () => { // Pause before starting any async refresh. This listener runs // synchronously in the transport line callback, so a following inbound // data event cannot pass before the new barrier exists. diff --git a/src/mcpl/errors.ts b/src/mcpl/errors.ts index b6b01bad8..bd6afd08a 100644 --- a/src/mcpl/errors.ts +++ b/src/mcpl/errors.ts @@ -40,10 +40,32 @@ export const CHANNEL_OPEN_FAILED = -32024; // Error Factories // ============================================================================ -/** Response encoding failed before the transport was given any bytes. */ +/** The text of anything thrown or rejected. Never throws, whatever it is: a + * null-prototype object, or a message getter or toString that throws. */ +export function describeCause(cause: unknown): string { + try { + return cause instanceof Error ? String(cause.message) : String(cause); + } catch { + try { + return Object.prototype.toString.call(cause); + } catch { + return 'an error that could not be printed'; + } + } +} + +/** Text a server or a handler chose, made fit for one log or trace line: + * control characters replaced, and cut to `max` code points. */ +export function boundedForLog(text: string, max = 200): string { + const points = [...text.replace(/[\u0000-\u001f\u007f-\u009f]/g, '?')]; + return points.length > max ? `${points.slice(0, max).join('')}…` : points.join(''); +} + +/** Response encoding failed before the transport was given any bytes. Its + * message says why (a BigInt, a circular structure, a throwing toJSON). */ export class McplResponseSerializationError extends Error { constructor(cause: unknown) { - super('MCPL response could not be serialized', { cause }); + super(`MCPL response could not be serialized: ${boundedForLog(describeCause(cause))}`, { cause }); this.name = 'McplResponseSerializationError'; } } diff --git a/src/mcpl/server-connection.ts b/src/mcpl/server-connection.ts index 2d81de9dc..69a0f5761 100644 --- a/src/mcpl/server-connection.ts +++ b/src/mcpl/server-connection.ts @@ -14,7 +14,7 @@ import { EventEmitter } from 'node:events'; import { openTransport, type McplTransport, type TransportCloseInfo } from './transport.js'; import { maskNegotiatedCapabilities } from './capability-mask.js'; import { CapabilityGrant, expandAdvertisementShorthand } from './capability-grant.js'; -import { CAPABILITY_DISABLED, McplResponseSerializationError } from './errors.js'; +import { CAPABILITY_DISABLED, McplResponseSerializationError, boundedForLog, describeCause } from './errors.js'; import type { McplServerConfig, @@ -80,6 +80,13 @@ interface RequestOptions { surfaceOrphanedResponse?: boolean; } +/** A request's method for a log line, whatever the server sent there. */ +function describeMethod(method: unknown): string { + return typeof method === 'string' + ? `method "${boundedForLog(method)}"` + : `a method of type ${method === null ? 'null' : typeof method}`; +} + /** * McplServerConnection manages a single JSON-RPC 2.0 connection to an * MCPL server over stdio. Use the static `connect()` factory to create @@ -379,6 +386,14 @@ export class McplServerConnection extends EventEmitter { } return true; } + // §6.6: a request that will never be answered MUST get an error. A + // request whose event nothing on this host listens for would never be. + const request = args[1] as { respondError?: (code: number, message: string, data?: unknown) => void } | undefined; + if (typeof request?.respondError === 'function' && this.listenerCount(event) === 0) { + request.respondError(-32601, 'Method not found'); + console.error(`[mcpl] ${this.id}: answered -32601 to ${name}, which nothing on this host handles`); + return true; + } return super.emit(event, ...args); } // Buffer cap (issue #122): a long data-plane pause (quiesce window) on a @@ -1098,7 +1113,9 @@ export class McplServerConnection extends EventEmitter { this.setupLifecycle(transport); // Surface transport diagnostics (stdio child stderr) as 'stderr' events, // preserving the `{ line }` payload existing consumers expect. - transport.on('stderr', (line: string) => this.emit('stderr', { line })); + // Through the same containment boundary as every inbound event, so a + // stderr listener that throws is reported, not thrown into the transport. + transport.on('stderr', (line: string) => this.dispatchInboundEvent('stderr', { line })); } /** @@ -1120,7 +1137,11 @@ export class McplServerConnection extends EventEmitter { try { this.dispatchMessage(msg); } catch (error) { - this.reportInboundFailure(error); + const request = msg as { method?: unknown; id?: unknown }; + this.reportInboundFailure(error, { + ...(typeof request.method === 'string' ? { event: request.method } : {}), + ...(typeof request.id === 'string' || typeof request.id === 'number' ? { requestId: request.id } : {}), + }); } }); } @@ -1167,20 +1188,30 @@ export class McplServerConnection extends EventEmitter { // and staged-buffer flushes share. See the comment there for why // rejecting here, at line receipt, was wrong for staged connections. - if (eventName) { - // Emit the typed event with params and (for requests) a respond callback + if (!eventName) { + // §6.6: a request that will never be answered MUST get an error, so a + // request (it has an id) whose method isn't a string, or isn't one this + // host handles, is answered -32601. A notification nobody handles is + // ignored, as JSON-RPC asks. if (request.id != null) { - // Server expects a response — provide a respond helper - this.dispatchInboundEvent(eventName, request.params, { - id: request.id, - respond: (result: unknown) => this.sendResponse(request.id!, result), - respondError: (code: number, message: string, data?: unknown) => - this.sendErrorResponse(request.id!, code, message, data), - }); - } else { - // Notification — no response expected - this.dispatchInboundEvent(eventName, request.params); + this.sendErrorResponse(request.id, -32601, 'Method not found'); + console.error(`[mcpl] ${this.id}: answered -32601 to a request for ${describeMethod(request.method)}, which this host doesn't handle`); } + return; + } + + // Emit the typed event with params and (for requests) a respond callback + if (request.id != null) { + // Server expects a response — provide a respond helper + this.dispatchInboundEvent(eventName, request.params, { + id: request.id, + respond: (result: unknown) => this.sendResponse(request.id!, result), + respondError: (code: number, message: string, data?: unknown) => + this.sendErrorResponse(request.id!, code, message, data), + }); + } else { + // Notification — no response expected + this.dispatchInboundEvent(eventName, request.params); } } @@ -1189,12 +1220,21 @@ export class McplServerConnection extends EventEmitter { try { this.emit(event, ...args); } catch (error) { - this.reportInboundFailure(error); + const requestId = (args[1] as { id?: unknown } | undefined)?.id; + this.reportInboundFailure(error, { + event, + ...(typeof requestId === 'string' || typeof requestId === 'number' ? { requestId } : {}), + }); } } - private reportInboundFailure(cause: unknown): void { - const error = new Error(`MCPL server "${this.id}" inbound dispatch failed`, { cause }); + /** Report a failure the containment boundary caught, naming the event (or + * method) and request it was dispatching, and what failed. The message is + * what the framework's 'error' listener traces. */ + private reportInboundFailure(cause: unknown, context: { event?: string; requestId?: string | number } = {}): void { + const where = context.event === undefined ? '' : ` for ${boundedForLog(context.event)}` + + (context.requestId === undefined ? '' : ` (request ${boundedForLog(String(context.requestId))})`); + const error = new Error(`MCPL server "${this.id}" inbound dispatch failed${where}: ${boundedForLog(describeCause(cause))}`, { cause }); console.error(error.message, cause); // EventEmitter's unhandled 'error' behavior would defeat containment before // framework wiring, so stderr remains the fallback when no listener exists. diff --git a/src/mcpl/transport.ts b/src/mcpl/transport.ts index 2fc8a523e..40275d118 100644 --- a/src/mcpl/transport.ts +++ b/src/mcpl/transport.ts @@ -156,6 +156,13 @@ export class StdioTransport extends McplTransport { }); this.child.on('error', (err) => this.emit('error', err)); + // A write to a child that has just exited fails asynchronously on its + // stdin (EPIPE). Unlistened, that 'error' would end the host; the exit + // itself closes the transport, so the write's failure is only reported. + this.child.stdin?.on('error', (err) => { + if (this.listenerCount('error') > 0) this.emit('error', err); + else console.error(`[mcpl] a write to the server's stdin failed: ${err.message}`); + }); this.child.on('exit', (code, signal) => this.markClosed({ code, signal, reason: 'child process exited' }), ); diff --git a/src/types/trace.ts b/src/types/trace.ts index 47baeada2..2bef7a57b 100644 --- a/src/types/trace.ts +++ b/src/types/trace.ts @@ -438,10 +438,24 @@ export type TraceEvent = event: string; /** Absent for notifications. */ requestId?: string | number; - /** True if a response write was attempted before the handler failed. */ + /** Whether the handler's own reply may have reached the server: true + * once it attempted one, unless the attempt failed while encoding, + * before any bytes were written. When it's false and a responder was + * given, the host answered -32603 "Internal error" itself, unless + * `responseError` says that reply failed. */ responseAttempted: boolean; + /** The handler's own error when it threw or rejected; otherwise the + * reply write that failed and that it swallowed, or that it tried to + * answer a second time. */ error: string; - /** Failure while trying to send an internal-error response. */ + /** A reply write that failed, when the handler then failed with an + * error of its own (which is `error`). */ + responseFailure?: string; + /** What the handler tried to send after a reply was already attempted: + * refused, never written. A result, or an error reply with its code + * and (bounded) message. */ + refusedResponse?: { kind: 'result' } | { kind: 'error'; code: number; message: string }; + /** Failure while trying to send the internal-error response. */ responseError?: string; }) | (TraceEventBase & { diff --git a/test/fixtures/mcpl-inbound-host-probe.mjs b/test/fixtures/mcpl-inbound-host-probe.mjs index 888e39248..9906d7615 100644 --- a/test/fixtures/mcpl-inbound-host-probe.mjs +++ b/test/fixtures/mcpl-inbound-host-probe.mjs @@ -30,7 +30,7 @@ process.stdin.on('data', (chunk) => { }); `; const connection = await McplServerConnection.connect({ - id: 'probe', command: process.execPath, args: ['-e', server, mode], requestTimeoutMs: 1000, + id: 'probe', command: process.execPath, args: ['-e', server, mode], requestTimeoutMs: 60_000, }, { version: '0.5' }); try { const received = []; diff --git a/test/mcpl-request-handler-errors.test.ts b/test/mcpl-request-handler-errors.test.ts index 18b8858c6..004dd0c3d 100644 --- a/test/mcpl-request-handler-errors.test.ts +++ b/test/mcpl-request-handler-errors.test.ts @@ -3,13 +3,15 @@ import assert from 'node:assert/strict'; import { EventEmitter } from 'node:events'; import { spawnSync } from 'node:child_process'; import { fileURLToPath } from 'node:url'; +import { inspect } from 'node:util'; import { AgentFramework } from '../src/framework.js'; import { ChannelRegistry } from '../src/mcpl/channel-registry.js'; import { InferenceRouter } from '../src/mcpl/inference-router.js'; import { McplServerConnection } from '../src/mcpl/server-connection.js'; -import { McplTransport } from '../src/mcpl/transport.js'; +import { McplTransport, StdioTransport } from '../src/mcpl/transport.js'; import { CapabilityGrant, ALL_CAPABILITY_PATHS } from '../src/mcpl/capability-grant.js'; +import { McplResponseSerializationError } from '../src/mcpl/errors.js'; function harness(connection = Object.assign(new EventEmitter(), { id: 'srv' })) { const traces: any[] = []; @@ -59,9 +61,11 @@ for (const [event, field] of [['channels-register', 'channels'], ['channels-chan framework.channelRegistry = registry; const reply = responder(); connection.emit(event, { [field]: [{ id: 'room', type: 'test', direction: 'bidirectional' }] }, reply); + // Synchronously, before reconciliation's first await settles. + assert.equal(reply.results.length, 1, 'ACK precedes reconciliation'); await settle(); - assert.equal(reply.results.length, 1, 'ACK still precedes reconciliation'); + assert.equal(reply.results.length, 1, 'and nothing else is answered'); assert.equal(reply.errors.length, 0, 'an accepted registration must not get a second response'); const failures = failureTraces(traces); assert.equal(failures.length, 1); @@ -93,11 +97,13 @@ for (const [event, component, method] of [ await settle(); assert.equal(reply.results.length, 0); - assert.deepEqual(reply.errors, [{ code: -32603, message: 'handler failed before response', data: undefined }]); + // The server gets the standard message; the handler's own text stays home. + assert.deepEqual(reply.errors, [{ code: -32603, message: 'Internal error', data: undefined }]); const failures = failureTraces(traces); assert.equal(failures.length, 1); assert.equal(failures[0].event, event); assert.equal(failures[0].responseAttempted, false); + assert.match(failures[0].error, /handler failed before response/); }); } @@ -118,8 +124,9 @@ test('a synchronous handler throw is contained and answered', async () => { const reply = responder(); assert.doesNotThrow(() => connection.emit('channels-list', {}, reply)); await settle(); - assert.equal(reply.errors.length, 1); + assert.deepEqual(reply.errors, [{ code: -32603, message: 'Internal error', data: undefined }]); assert.equal(failureTraces(traces).length, 1); + assert.match(failureTraces(traces)[0].error, /list failed/); }); test('a throwing success response is not retried as an error response', async () => { @@ -285,9 +292,14 @@ for (const kind of ['bigint', 'circular', 'throwing toJSON'] as const) { assert.equal(reply.id, 17); assert.equal(reply.error.code, -32603); assert.equal(reply.result, undefined); + assert.equal(reply.error.message, 'Internal error'); const failures = failureTraces(traces); assert.equal(failures.length, 1); assert.equal(failures[0].responseAttempted, false); + // The encoding failure says why. + assert.match(failures[0].error, kind === 'bigint' ? /could not be serialized: .*BigInt/i + : kind === 'circular' ? /could not be serialized: .*circular/i + : /could not be serialized: snapshot serialization failed/); if (kind === 'throwing toJSON') assert.equal(calls, 1, 'results are encoded only once'); }); } @@ -356,7 +368,7 @@ for (const mode of ['handshake', 'connected', 'throwing-listener']) { test('separate stdio host survives inbound ' + mode + ' faults and processes later messages', () => { const probe = spawnSync(process.execPath, [ '--import', 'tsx', fileURLToPath(new URL('./fixtures/mcpl-inbound-host-probe.mjs', import.meta.url)), mode, - ], { encoding: 'utf8', timeout: 5000 }); + ], { encoding: 'utf8', timeout: 120_000 }); assert.equal(probe.error, undefined); assert.equal(probe.status, 0, probe.stderr); assert.match(probe.stdout, /continued/); @@ -434,6 +446,9 @@ test('a method that is not a string is not dispatched, even if it stringifies to connection.on('tools-list-changed', () => { received = true; }); transport.emit('line', JSON.stringify({ jsonrpc: '2.0', method: ['notifications/tools/list_changed'] })); assert.equal(received, false); + // The control: the same method as a string is dispatched in this setup. + transport.emit('line', JSON.stringify({ jsonrpc: '2.0', method: 'notifications/tools/list_changed' })); + assert.equal(received, true); }); test('tools-observe: a synchronous failure is answered with one internal error', async () => { @@ -442,21 +457,26 @@ test('tools-observe: a synchronous failure is answered with one internal error', const reply = responder(); assert.doesNotThrow(() => connection.emit('tools-observe', { rules: [] }, reply)); await settle(); - assert.deepEqual(reply.errors, [{ code: -32603, message: 'filter store failed', data: undefined }]); + assert.deepEqual(reply.errors, [{ code: -32603, message: 'Internal error', data: undefined }]); assert.equal(failureTraces(traces).length, 1); + assert.match(failureTraces(traces)[0].error, /filter store failed/); }); test('model-info: a throwing error write is traced without escaping or a second write', async () => { const { connection, traces } = harness(); const reply = responder(); - let writes = 0; - reply.respondError = () => { writes++; throw new Error('response transport failed'); }; + const writes: Array<{ code: number; message: string }> = []; + reply.respondError = (code: number, message: string) => { writes.push({ code, message }); throw new Error('response transport failed'); }; assert.doesNotThrow(() => connection.emit('model-info', {}, reply)); await settle(); - assert.equal(writes, 1); + assert.equal(writes.length, 1); + assert.equal(writes[0]!.code, -32601); + assert.match(writes[0]!.message, /model\/info unavailable/); const failures = failureTraces(traces); assert.equal(failures.length, 1); assert.equal(failures[0].responseAttempted, true); + assert.match(failures[0].error, /response transport failed/); + assert.equal(failures[0].responseFailure, undefined, 'the write failure is the error itself, not repeated'); }); test('a method named after an Object.prototype member is not buffered while the planes are paused', () => { @@ -468,4 +488,251 @@ test('a method named after an Object.prototype member is not buffered while the transport.emit('line', JSON.stringify({ jsonrpc: '2.0', method })); } assert.equal((connection as any).bufferedEvents.length, 0); + // The control: a known notification does wait in the buffer here. + transport.emit('line', JSON.stringify({ jsonrpc: '2.0', method: 'notifications/tools/list_changed' })); + assert.equal((connection as any).bufferedEvents.length, 1); +}); + +test('a handler that swallows an encoding failure and then fails reports its own error, and the failed write beside it', async () => { + const { framework, traces, transport, request } = wireHarness(); + framework.channelRegistry = { + async handleRegister(_server: string, _params: unknown, reply: any) { + try { reply.respond({ count: 1n }); } catch { /* swallowed */ } + throw new Error('db down'); + }, + }; + request('channels/register'); + await settle(); + assert.equal(transport.lines.length, 1, 'no bytes of the result went out, so one internal-error reply does'); + assert.deepEqual(JSON.parse(transport.lines[0]!).error, { code: -32603, message: 'Internal error' }); + const failures = failureTraces(traces); + assert.equal(failures.length, 1); + assert.match(failures[0].error, /db down/); + assert.match(failures[0].responseFailure, /could not be serialized: .*BigInt/i); + assert.equal(failures[0].responseAttempted, false); +}); + +test('an answer tried after the reply is refused without throwing, so the handler reports its own error', async () => { + const { framework, connection, traces } = harness(); + framework.channelRegistry = { + async handleRegister(_server: string, _params: unknown, reply: any) { + reply.respond({ registered: [] }); + try { + await Promise.resolve(); + throw new Error('store append failed'); + } catch (error) { + reply.respondError(-32603, 'could not record the registration'); + throw error; + } + }, + }; + const reply = responder(); + connection.emit('channels-register', {}, reply); + await settle(); + assert.equal(reply.results.length, 1); + assert.equal(reply.errors.length, 0, 'the second answer is never written'); + const failures = failureTraces(traces); + assert.equal(failures.length, 1); + assert.match(failures[0].error, /store append failed/); + assert.deepEqual(failures[0].refusedResponse, { kind: 'error', code: -32603, message: 'could not record the registration' }); + assert.equal(failures[0].responseAttempted, true); +}); + +test('a handler that answers twice and resolves is still reported', async () => { + const { framework, connection, traces } = harness(); + framework.channelRegistry = { + async handleRegister(_server: string, _params: unknown, reply: any) { + reply.respond({ registered: [] }); + reply.respondError(-32602, 'second thoughts'); + }, + }; + const reply = responder(); + connection.emit('channels-register', {}, reply); + await settle(); + assert.equal(reply.results.length, 1); + assert.equal(reply.errors.length, 0); + const failures = failureTraces(traces); + assert.equal(failures.length, 1); + assert.match(failures[0].error, /second time/); + assert.deepEqual(failures[0].refusedResponse, { kind: 'error', code: -32602, message: 'second thoughts' }); +}); + +test('a rejection whose reason cannot be printed is reported and answered, with no unhandled rejection', async () => { + const { framework, connection, traces } = harness(); + const unhandled: unknown[] = []; + const onUnhandled = (reason: unknown) => { unhandled.push(reason); }; + process.on('unhandledRejection', onUnhandled); + try { + framework.channelRegistry = { + async handleRegister() { throw Object.create(null); }, + }; + const reply = responder(); + connection.emit('channels-register', {}, reply); + await settle(); + await settle(); + assert.deepEqual(unhandled, []); + assert.deepEqual(reply.errors, [{ code: -32603, message: 'Internal error', data: undefined }]); + const failures = failureTraces(traces); + assert.equal(failures.length, 1); + assert.equal(typeof failures[0].error, 'string'); + } finally { + process.off('unhandledRejection', onUnhandled); + } +}); + +test('a request for a method nothing handles is answered -32601; such a notification is ignored', (t) => { + const { transport } = wireHarness(); + t.mock.method(console, 'error', () => {}); + transport.emit('line', JSON.stringify({ jsonrpc: '2.0', id: 31, method: 'no/such-method' })); + transport.emit('line', JSON.stringify({ jsonrpc: '2.0', id: 32, method: ['host/command'] })); + transport.emit('line', JSON.stringify({ jsonrpc: '2.0', id: 33, method: 7 })); + transport.emit('line', JSON.stringify({ jsonrpc: '2.0', method: 'no/such-notification' })); + const replies = transport.lines.map((line) => JSON.parse(line)); + assert.deepEqual(replies.map((r) => [r.id, r.error?.code, r.error?.message]), [ + [31, -32601, 'Method not found'], + [32, -32601, 'Method not found'], + [33, -32601, 'Method not found'], + ]); +}); + +test('a dispatch failure names the event, the request and what failed', (t) => { + const { connection, transport, traces } = wireHarness(); + t.mock.method(console, 'error', () => {}); + connection.prependListener('channels-list', () => { throw new Error('listener exploded'); }); + transport.emit('line', JSON.stringify({ jsonrpc: '2.0', id: 41, method: 'channels/list' })); + const errors = traces.filter((e) => e.type === 'mcpl:server-error'); + assert.equal(errors.length, 1); + assert.match(errors[0].error, /inbound dispatch failed for channels-list \(request 41\): listener exploded/); +}); + +test('a manifest refresh that fails is reported as a handler failure', async (t) => { + const { framework, connection, traces } = harness(); + t.mock.method(console, 'error', () => {}); + framework.handleManifestChanged = async () => { throw new Error('manifest fetch failed'); }; + connection.emit('manifest-changed', { revision: 'r2' }); + await settle(); + const failures = failureTraces(traces); + assert.equal(failures.length, 1); + assert.equal(failures[0].event, 'manifest-changed'); + assert.equal(failures[0].requestId, undefined); + assert.match(failures[0].error, /manifest fetch failed/); +}); + +test('a failed internal-error reply reaches stderr too', async (t) => { + const { framework, connection } = harness(); + framework.traceListeners = []; + framework.channelRegistry = { async handleRegister() { throw new Error('register failed'); } }; + const lines: string[] = []; + t.mock.method(console, 'error', (...args: unknown[]) => { lines.push(args.map(String).join(' ')); }); + const reply = responder(); + reply.respondError = () => { throw new Error('internal-error reply could not be written'); }; + connection.emit('channels-register', {}, reply); + await settle(); + assert.equal(lines.length, 1); + assert.match(lines[0]!, /channels-register handler failed: .*register failed/); + assert.match(lines[0]!, /the internal-error reply failed too: internal-error reply could not be written/); +}); + +test("an 'error' listener that throws is caught and logged, and the next message still arrives", (t) => { + const { connection, transport } = wireHarness(); + const logged: string[] = []; + t.mock.method(console, 'error', (...args: unknown[]) => { logged.push(String(args[0])); }); + connection.removeAllListeners('error'); + connection.on('error', () => { throw new Error('error listener exploded'); }); + connection.prependListener('channels-list', () => { throw new Error('listener exploded'); }); + let received = false; + connection.removeAllListeners('tools-list-changed'); + connection.on('tools-list-changed', () => { received = true; }); + assert.doesNotThrow(() => transport.emit('line', JSON.stringify({ jsonrpc: '2.0', id: 51, method: 'channels/list' }))); + assert.ok(logged.some((line) => /MCPL error listener failed/.test(line)), logged.join('\n')); + transport.emit('line', JSON.stringify({ jsonrpc: '2.0', method: 'notifications/tools/list_changed' })); + assert.equal(received, true); +}); + +test("a failed write to a stdio server's stdin is reported, never thrown", (t) => { + const child = Object.assign(new EventEmitter(), { stdin: new EventEmitter(), stderr: new EventEmitter(), killed: false, exitCode: null }); + const rl = new EventEmitter(); + const transport = new (StdioTransport as any)(child, rl) as StdioTransport; + const logged: string[] = []; + t.mock.method(console, 'error', (...args: unknown[]) => { logged.push(String(args[0])); }); + const epipe = Object.assign(new Error('write EPIPE'), { code: 'EPIPE' }); + // With nobody listening on the transport, it's logged. + assert.doesNotThrow(() => child.stdin.emit('error', epipe)); + assert.match(logged[0] ?? '', /stdin failed: write EPIPE/); + // With a listener, it's the transport's error. + const reported: unknown[] = []; + transport.on('error', (error) => reported.push(error)); + child.stdin.emit('error', epipe); + assert.deepEqual(reported, [epipe]); +}); + +test('a synchronous handler that swallows an encoding failure is reported and answered once', async () => { + const { framework, traces } = harness(); + const listener = framework.containedInboundHandler('srv', 'test-sync', (_params: unknown, reply: any) => { + try { reply.respond({ count: 1n }); } catch { /* swallowed, synchronously */ } + }); + const writes: unknown[] = []; + listener({}, { + id: 61, + // Encode as the connection does: an encoding failure is typed, and + // proves no bytes went out. + respond(result: unknown) { + try { JSON.stringify(result); } catch (cause) { throw new McplResponseSerializationError(cause); } + writes.push(result); + }, + respondError(code: number, message: string) { writes.push({ code, message }); }, + }); + await settle(); + assert.deepEqual(writes, [{ code: -32603, message: 'Internal error' }]); + const failures = failureTraces(traces); + assert.equal(failures.length, 1); + assert.equal(failures[0].event, 'test-sync'); + assert.match(failures[0].error, /BigInt/); +}); + +test("a stderr line whose listener throws is contained like any inbound event", (t) => { + const { connection, transport, traces } = wireHarness(); + t.mock.method(console, 'error', () => {}); + connection.prependListener('stderr', () => { throw new Error('stderr listener exploded'); }); + assert.doesNotThrow(() => transport.emit('stderr', 'a line from the server')); + const errors = traces.filter((e) => e.type === 'mcpl:server-error'); + assert.equal(errors.length, 1); + assert.match(errors[0].error, /inbound dispatch failed for stderr: stderr listener exploded/); +}); + +test('a rejection no printer can inspect still reaches the trace first', async (t) => { + const { framework, connection, traces } = harness(); + // Print as console.error does: every argument goes through util.inspect. + t.mock.method(console, 'error', (...args: unknown[]) => { for (const arg of args) inspect(arg); }); + const unhandled: unknown[] = []; + const onUnhandled = (reason: unknown) => { unhandled.push(reason); }; + process.on('unhandledRejection', onUnhandled); + try { + // String() throws on it, and so does printing it. + const hostile = { + [inspect.custom]() { throw new Error('inspect refused'); }, + [Symbol.toPrimitive]() { throw new Error('toPrimitive refused'); }, + }; + framework.channelRegistry = { async handleRegister() { throw hostile; } }; + const reply = responder(); + connection.emit('channels-register', {}, reply); + await settle(); + await settle(); + assert.deepEqual(unhandled, []); + assert.deepEqual(reply.errors, [{ code: -32603, message: 'Internal error', data: undefined }]); + const failures = failureTraces(traces); + assert.equal(failures.length, 1); + assert.equal(typeof failures[0].error, 'string'); + } finally { + process.off('unhandledRejection', onUnhandled); + } +}); + +test('a request for an event nothing on the host listens for is answered -32601', (t) => { + const { connection, transport } = wireHarness(); + t.mock.method(console, 'error', () => {}); + connection.removeAllListeners('channels-list'); + transport.emit('line', JSON.stringify({ jsonrpc: '2.0', id: 71, method: 'channels/list' })); + const replies = transport.lines.map((line) => JSON.parse(line)); + assert.deepEqual(replies.map((r) => [r.id, r.error?.code, r.error?.message]), [[71, -32601, 'Method not found']]); }); From d65cb181f3448c29203ba165e9e542ae314d786e Mon Sep 17 00:00:00 2001 From: Oscar-1808 Date: Sun, 11 Oct 2026 16:58:39 -0400 Subject: [PATCH 0117/1112] fix(routing): every notice goes to the agent it's about, and a released agent's unread failures reach the primary Review of this branch found that a failed reply already written into a fork's or a run's window was never passed on, that a release while quiesced lost its writes, and that some notices still went to the wrong place or left a misleading trace: - At release, every failed-reply notice the agent never read reaches the primary by name, once: each one still queued for it, and each one stored in its window above its consumed watermark (flushed there at its turn's end, or stored while it was idle). A notice injected into its live stream at a tool boundary was read, though no compile ran after it, so it is recorded at injection and not passed on. - A release asked for while the host is quiesced waits for resume(), which runs it before its own flush; the agent stays registered meanwhile and can't start a turn. Before, its queued writes were dropped and acked, lost from the window and the recovery queue. - The fallback reaper treats a held turn token as busy, so a closing fork mid-compile, before its stream registers, isn't disposed. - noticeTargetFor addresses every registered agent's notice to that agent, residents included, so a resident's notice waits on its own turn, stream and tool cycle rather than the primary's. Residents share one slot, so one may land there during another resident's turn; agent-framework #291 covers that. - An agent no longer registered gets nothing: noticeTargetFor logs the drop once and returns null, and its caller writes nothing, traces nothing and requests no wake (a bounce's wake and a background script's wake included). - A channel opened by an agent's own send is announced to that agent; a policy open, with no agent behind it, still goes to the primary. - A restored deferred write whose target is gone (boot recovery, or a resume's flush) passes a failed-reply notice to the primary by name; other writes are dropped, logged, as before. - The by-name notice says only what it has: an entry queued by an earlier build, without textLen or reason, no longer reads "undefined chars". - A background script's deferred wake is traced as deferred, not with an empty id. Tests (test/route-failure-targeting.test.ts, 33, up from 20): - a resident's notice addressed to the speaker and waiting on its own turn and tool cycle, and the primary's own wording pinned; - a fork's stored, unread notice passed on at its release, and one its later turn compiled kept its own; - the reaper and a held turn token; - a queued ordinary write landing in a released fork's window with its durable id, and acked; - a fork's notice waiting on its own tool cycle and its own stream, and landing while another agent's cycle is pending; - the auto-open notice to its opener, to the primary for a policy open, and dropped for an agent that is gone; - a background-script wake for an agent that is gone: no write, trace or wake; - restored notices for gone agents passed on by name, an earlier build's without "undefined"; - the TTL closure while quiesced, through resume() itself; - with a module tool in an ephemeral run: a notice heard at a tool boundary isn't passed on; a bounced prose notice is injected at the next tool boundary and follows its prose in the window; a run that ends while quiesced is released at resume, its queued write landed. Red checks: against the previous commit's framework.ts, 13 of the tests fail, the four residents' tests and nine aimed at the fixes. Each of these single changes to the new framework.ts fails the test aimed at it: no stored notice counted unread; a live-injected notice counted unread; the reaper ignoring turn tokens; releases run at once while quiesced; a fork's dispose not waiting; resume running no postponed release; a restored notice for a gone agent not passed on; a missing textLen printed; an unregistered agent's notice still addressed; an auto-open notice ignoring who caused it; a release not acking what it landed; a fork's notice waiting on every agent's tool cycle; a notice not waiting on its agent's stream. Changelog: the header names its exceptions; a subagent's notices moved from the primary, not from a parent; residents' notices are addressed to them, with #291 named; every failed-reply notice carries textLen; with no primary, no one is told, and it's logged. Co-Authored-By: Claude Opus 5.5 --- .../fix-188-route-failure-owner.fixed.md | 14 +- src/framework.ts | 278 +++++++++---- test/route-failure-targeting.test.ts | 376 ++++++++++++++++-- 3 files changed, 561 insertions(+), 107 deletions(-) diff --git a/changelog.d/fix-188-route-failure-owner.fixed.md b/changelog.d/fix-188-route-failure-owner.fixed.md index e1d289d61..86c49735a 100644 --- a/changelog.d/fix-188-route-failure-owner.fixed.md +++ b/changelog.d/fix-188-route-failure-owner.fixed.md @@ -1,6 +1,8 @@ -- Notices about an agent's own action now reach the window that agent reads (#188): - - A conversation fork's or an ephemeral run's failed-reply notice (`[discord-send-failed]`), bounced-prose notice and background-script wake land in its own window, deferred against its own turn, instead of the primary's. In connectome-host this moves a subagent's notices from its parent to the subagent. - - Residents share one message slot, so a resident's notices stay on the default path, unchanged. - - When a fork or an ephemeral run has ended without reading that its reply to a channel failed, the primary is told by name instead: "A reply by `` (N chars) could not be delivered to … (…). `` has ended, and the human did not receive it." That notice's metadata carries `speaker`, and every failed-reply notice's metadata now carries `textLen`. - - Writes still queued for a fork or an ephemeral run when it is released land in its own window first, which Chronicle keeps. While the host is quiesced they are dropped instead, logged. - - A background-script wake for an agent that is no longer registered is dropped, logged, rather than given to the primary. +- Notices about an agent's own action now reach the window that agent reads, deferred against that agent's own turn, stream and tool cycle (#188). Two exceptions: a failed reply its speaker will never read goes to the primary by name, and a notice for an agent no longer registered goes nowhere, logged. + - The notices: a failed reply (`[discord-send-failed]`), bounced prose (beside its `{{unsent}}` clipboard), a background script's wake, and a channel opened by the agent's own send. A channel opened by subscription policy, with no agent behind it, is still announced to the primary. + - A conversation fork's or an ephemeral run's notices land in its own window instead of the primary's. In connectome-host this moves a subagent's notices from the primary, where the default path put them, to the subagent. + - A resident's notices are addressed to it too, in the slot the residents share, and wait on its own turn rather than the primary's. Whether it may land there while another resident's turn is alive is the question of #291. + - When a fork or an ephemeral run is released, each failed-reply notice it never read is passed to the primary, once, by name: "A reply by `` (N chars) could not be delivered to … (…). `` has ended, and the human did not receive it." Never read means still queued for it, or stored in its window since its last compile and not heard mid-turn at a tool boundary. The same goes for a delivery that fails after its speaker is released, and for such a notice restored at boot whose agent is gone. That notice's metadata carries `speaker`, and every failed-reply notice's metadata now carries `textLen`. With no primary registered, no one is told, and it's logged. + - Writes still queued for a fork or an ephemeral run when it is released land in its own window first, which Chronicle keeps. A release asked for while the host is quiesced waits for resume, and lands them then. + - The fallback reaper no longer disposes a closing fork whose turn is still alive before its stream has registered. + - A notice or a background-script wake for an agent that is no longer registered is dropped, logged once, with no trace and no wake. diff --git a/src/framework.ts b/src/framework.ts index 664578c3b..84a5b3834 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -1277,6 +1277,15 @@ export class AgentFramework { * ephemeral agents have isolated slots and are their own only readers. */ private readonly sharedSlotAgents = new Set(); + /** + * A fork's or an ephemeral run's failed-reply notices that reached its live + * stream at a tool boundary, by deferred-write id. Read, though stored above + * its consumed watermark, since no compile ran after them; a release + * (unreadFailedReplies) doesn't pass them on. Forgotten at release. + */ + private readonly injectedFailureNotices = new Map>(); + /** Releases asked for while the host was quiesced, by agent name; resume() runs them. */ + private readonly releasesAfterResume = new Map void>(); /** Group commit: one fsync per event-loop turn, shared by every waiter. */ private coalescingCommit: Promise | null = null; private inferenceRouter: InferenceRouter | null = null; @@ -3060,10 +3069,14 @@ export class AgentFramework { this.eventGate?.setQuiesced(false); try { - // Flush context writes deferred by the quiesce window (module events, - // api message.send) — with no turn alive there is no other flush - // point. Per-message try/catch and per-message durable ack (see - // flushDeferredWrites); nothing here may reach the plane reopen below. + // Releases asked for during the window run first, each landing its + // agent's queued writes and passing on its unread failed replies, as at + // any release. Then flush context writes deferred by the quiesce window + // (module events, api message.send) — with no turn alive there is no + // other flush point. Per-message try/catch and per-message durable ack + // (see flushDeferredWrites); nothing here may reach the plane reopen + // below. + this.runReleasesAfterResume(); await this.flushDeferredWrites('resume'); this.persistHostMode(null); } finally { @@ -3132,7 +3145,20 @@ export class AgentFramework { this.deferredMessages = keep; this.handOffDeferredWrites(flush); let stored = 0; + let passed = 0; for (const msg of flush) { + // A target no longer registered (a fork or a run gone before a crash + // that left this write to boot recovery) has no window to land in. A + // failed reply it never read is passed to the primary, by name; any + // other write is dropped, logged, by addMessage. + if ( + msg.forAgent && !this.agents.has(msg.forAgent) && + (msg.metadata as { kind?: unknown } | undefined)?.kind === 'discord-send-failed' + ) { + this.passFailedReplyToPrimary(msg.forAgent, msg.metadata); + passed++; + continue; + } try { this.addMessage(msg.participant, msg.content, msg.metadata, { deferredWriteId: msg.id, @@ -3152,6 +3178,7 @@ export class AgentFramework { this.ackDeferredWrites(); console.error( `[host-mode] ${label}: flushed ${stored}/${flush.length} deferred context write(s)` + + (passed > 0 ? `, ${passed} failed repl${passed === 1 ? 'y' : 'ies'} of a released agent passed to the primary` : '') + (keep.length > 0 ? `, ${keep.length} kept for a still-alive turn` : ''), ); } @@ -4137,8 +4164,11 @@ export class AgentFramework { // The late stream finalizer intentionally refuses name-keyed cleanup once // deregistered, so disposal must release EventGate liveness itself. this.eventGate?.onInferenceEnded(agent.name); - this.landQueuedWritesBeforeRelease(agent); - this.agents.delete(agent.name); + this.releaseNowOrAfterResume(agent.name, () => { + if (this.agents.get(agent.name) !== agent) return; + this.releaseWindow(agent); + this.agents.delete(agent.name); + }); } this.toolImageLedgers.delete(agent.name); // Spawn-and-dispose bookkeeping (main, d453165/fee96a7): without this, @@ -6486,6 +6516,7 @@ export class AgentFramework { content: msg.content, ...(msg.metadata ? { metadata: msg.metadata } : {}), }); + this.noteInjectedFailureNotice(agent, msg.id, msg.metadata); } } } @@ -7170,10 +7201,16 @@ export class AgentFramework { * forever. The fork's context stays in Chronicle for investigation. */ private disposeConversationAgent(agentName: string): void { + if (this.quiesced) { + // Released by resume() (releaseNowOrAfterResume). It stays closing, so + // a sweep meanwhile reaches here again and waits too. + this.releasesAfterResume.set(agentName, () => this.disposeConversationAgent(agentName)); + return; + } this.closingConversationAgents.delete(agentName); const channelId = this.conversationAgentHomes.get(agentName); const agent = this.agents.get(agentName); - if (agent) this.landQueuedWritesBeforeRelease(agent); + if (agent) this.releaseWindow(agent); this.agents.delete(agentName); this.toolImageLedgers.delete(agentName); this.agentConfigs.delete(agentName); @@ -7259,9 +7296,12 @@ export class AgentFramework { // Fallback reaper: a closing fork whose closure inference never ran // (request dropped as stale, inference policy veto) would otherwise stay - // registered forever. No active stream + no pending request = done. + // registered forever. No turn, no active stream and no pending request = + // done. A turn holds its token from dequeue, through its hooks and + // compile, before its stream registers, so a fork mid-compile is busy. for (const agentName of [...this.closingConversationAgents]) { if ( + !this.activeTurnTokens.has(agentName) && !this.activeStreams.has(agentName) && !this.pendingRequests.some((r) => r.agentName === agentName) ) { @@ -8335,13 +8375,21 @@ export class AgentFramework { (plural ? 'these channels' : 'this channel') + ` will reach you. If you don't want ${plural ? 'one of them' : 'it'}, use channel_close — ` + 'your choice sticks; neither policy nor a later send will silently reopen it.'; + // An open caused by an agent's own send is that agent's to hear about, in + // the window it reads; a policy admission with no agent behind it goes to + // the primary. + let target: { forAgent: string } | undefined; + if (agentName) { + const addressed = this.noticeTargetFor(agentName, 'channel auto-open'); + if (!addressed) return; + target = addressed; + } try { // Through framework.addMessage for mid-turn safety (deferral + // hear-while-acting injection) — a delivery-forced open happens inside // a live turn, and a direct context-manager append there would store - // the notice out of order. Targets the primary agent; connectome-host - // runs trunk-only, so this is the delivering agent in practice. - this.addMessage('user', [{ type: 'text', text }], { system: true, kind: 'channel-notice' }); + // the notice out of order. + this.addMessage('user', [{ type: 'text', text }], { system: true, kind: 'channel-notice' }, target); console.error(`[channel] auto-open notice (${cause}) -> ${agentName ?? 'primary'}: ${shown}`); } catch (err) { console.error('recordChannelAutoOpenNotice failed:', err); @@ -8546,31 +8594,34 @@ export class AgentFramework { `destination plus the token {{unsent}}, e.g. "${prefix}#channel {{unsent}}" or "${prefix}@person {{unsent}}". ` + `"${prefix}skip_reply {{unsent}}" keeps it in context only.` + (prefix === '>>' ? ' The prose_help tool shows the full syntax.' : ''); - try { - // Through framework.addMessage, NOT the context manager directly: while - // the turn is still streaming this defers the notice to the next tool - // boundary, where it is BOTH stored in order and injected into the live - // stream (hear-while-acting) — the model sees the error and can correct - // the prefix within the same turn. A direct append here would land the - // notice before the turn's assistant blocks (cache bust, invisible - // until the next compile). - const id = this.addMessage( - 'user', - [{ type: 'text', text: notice }], - { system: true, kind: 'prose-bounce' }, - this.noticeTargetFor(name), - ); - if (id) this.emitTrace({ type: 'message:added', messageId: id, source: 'prose-bounce' }); - else this.emitTrace({ type: 'message:added', messageId: 'deferred', source: 'prose-bounce' }); - } catch (err) { - console.error('bounceProse: failed to record bounce notice:', err); + const target = this.noticeTargetFor(name, 'prose-bounce'); + if (target) { + try { + // Through framework.addMessage, NOT the context manager directly: while + // the turn is still streaming this defers the notice to the next tool + // boundary, where it is BOTH stored in order and injected into the live + // stream (hear-while-acting) — the model sees the error and can correct + // the prefix within the same turn. A direct append here would land the + // notice before the turn's assistant blocks (cache bust, invisible + // until the next compile). + const id = this.addMessage( + 'user', + [{ type: 'text', text: notice }], + { system: true, kind: 'prose-bounce' }, + target, + ); + if (id) this.emitTrace({ type: 'message:added', messageId: id, source: 'prose-bounce' }); + else this.emitTrace({ type: 'message:added', messageId: 'deferred', source: 'prose-bounce' }); + } catch (err) { + console.error('bounceProse: failed to record bounce notice:', err); + } } const capped = streak > AgentFramework.PROSE_BOUNCE_WAKE_CAP; console.error( `[prose] ${name}: BOUNCED ${text.length} chars — ${reason}` + (capped ? ' (bounce-wake cap reached; notice appended without wake)' : ''), ); - if (!capped) { + if (!capped && target) { // Wake so the resend can happen immediately — an unsent DM reply must // not sit invisible until the next unrelated event. this.pendingRequests.push({ @@ -11031,12 +11082,14 @@ export class AgentFramework { * later if that ever needs revisiting. */ private injectScriptWake(record: BackgroundScriptRecord, envelope: string): void { + const target = this.noticeTargetFor(record.agentName, `background-script ${record.id} wake`); + if (!target) return; try { const messageId = this.addMessage('user', [{ type: 'text', text: envelope }], { source: 'background-script', scriptId: record.id, tags: ['script:wake'], - }, this.noticeTargetFor(record.agentName)); + }, target); this.pendingRequests.push({ agentName: record.agentName, reason: 'script:wake', @@ -11045,7 +11098,7 @@ export class AgentFramework { }); this.emitTrace({ type: 'message:added', - messageId, + messageId: messageId || 'deferred', source: `background-script:${record.id}`, }); } catch (err) { @@ -11956,60 +12009,131 @@ export class AgentFramework { /** * Where a notice about an agent's own action belongs: its failed reply, its - * bounced prose, its background script's wake. A conversation fork or an - * ephemeral run reads only its own isolated slot, so the notice is addressed - * to it. Residents share one message slot, which the default path already - * writes, under the guard that waits for any agent's tool cycle and the - * primary's turn; addressing a resident would narrow that guard to its own - * turn and cycle. A name no longer registered (a fork or run already - * released) is addressed too, so addMessage drops the notice, logged, rather - * than give it to the primary as if it were the primary's; a failed reply is - * passed on by name instead (passFailedReplyToPrimary). - */ - private noticeTargetFor(agentName: string): { forAgent: string } | undefined { - const agent = this.agents.get(agentName); - return agent && this.sharedSlotAgents.has(agent) ? undefined : { forAgent: agentName }; + * bounced prose (beside its `{{unsent}}` clipboard), its background script's + * wake, a channel its own send opened. Every registered agent's notice is + * addressed to that agent, so it lands in the window that agent reads and + * waits on that agent's own turn, stream and tool cycle: a conversation + * fork or an ephemeral run reads only its isolated slot, and a resident's + * notice would otherwise wait on the primary's turn rather than its own. + * Residents share one message slot, so a resident's notice can land there + * while another resident's turn is alive; agent-framework #291 covers that. + * + * null when the agent is no longer registered (a fork or a run already + * released): there is no window to write, so the caller writes nothing, + * traces nothing and requests no wake. The drop is logged here, once. A + * failed reply is passed on by name instead (passFailedReplyToPrimary). + */ + private noticeTargetFor(agentName: string, notice: string): { forAgent: string } | null { + if (this.agents.has(agentName)) return { forAgent: agentName }; + console.error(`[notice] ${agentName} is no longer registered, so its ${notice} notice is dropped`); + return null; } /** - * A conversation fork or an ephemeral run is about to be deregistered. - * Writes still queued for its window would wait for a boundary it never - * reaches, and the flush that finds it gone would drop them. Land them in - * its own window now, which Chronicle keeps for investigation; its stream - * is over or cancelled, so nothing it compiles can diverge from them. A - * failed reply among them is also passed to the primary, because the - * speaker will never read it. While quiesced, a write could land - * mid-surgery, so the writes are dropped instead, logged. - */ - private landQueuedWritesBeforeRelease(agent: Agent): void { + * A conversation fork or an ephemeral run is about to be deregistered. A + * release never runs while the host is quiesced: it waits for resume() + * (releaseNowOrAfterResume), since a write could land mid-surgery. + * + * The agent never reads its window again, so every failed-reply notice it + * never read is passed to the primary, by name, once: each one stored in + * its window above its consumed watermark (flushed there at its turn's end, + * or stored while it was idle) and not heard at a tool boundary, and each + * one still queued for it. Writes still + * queued would wait for a boundary it never reaches, and the flush that + * finds it gone would drop them, so they land in its own window now, which + * Chronicle keeps for investigation; its stream is over or cancelled, so + * nothing it compiles can diverge from them. + */ + private releaseWindow(agent: Agent): void { + for (const metadata of this.unreadFailedReplies(agent)) this.passFailedReplyToPrimary(agent.name, metadata); const queued = this.drainDeferredFor(agent.name); if (queued.length === 0) return; const cm = agent.getContextManager(); for (const msg of queued) { - if (this.quiesced) { + try { + cm.addMessage(msg.participant, msg.content, withDeferredWriteId(msg.metadata, msg.id)); + } catch (err) { console.error( - `[deferred-flush] ${agent.name}: released while quiesced, so a queued write for its window ` + - `is dropped (participant=${msg.participant})`, + `[deferred-flush] ${agent.name}: failed to store a queued write before release ` + + `(participant=${msg.participant}):`, + err, ); - } else { - try { - cm.addMessage(msg.participant, msg.content, withDeferredWriteId(msg.metadata, msg.id)); - } catch (err) { - console.error( - `[deferred-flush] ${agent.name}: failed to store a queued write before release ` + - `(participant=${msg.participant}):`, - err, - ); - } } this.passFailedReplyToPrimary(agent.name, msg.metadata); } this.ackDeferredWrites(); } + /** + * The failed-reply notices stored in `agent`'s window that it never read: + * above its consumed watermark (or any, when it never compiled on this + * branch), and not injected into its live stream at a tool boundary. + */ + private unreadFailedReplies(agent: Agent): MessageMetadata[] { + const injected = this.injectedFailureNotices.get(agent.name); + this.injectedFailureNotices.delete(agent.name); + try { + const cm = agent.getContextManager(); + const watermark = agent.getConsumedWatermark(); + const floor = watermark && watermark.branch === cm.currentBranch().name + ? watermark.sequence + : Number.NEGATIVE_INFINITY; + return cm.getAllMessages() + .filter((m) => { + const metadata = m.metadata as { kind?: unknown; deferredWriteId?: unknown } | undefined; + return m.sequence > floor && metadata?.kind === 'discord-send-failed' && + !(typeof metadata.deferredWriteId === 'string' && injected?.has(metadata.deferredWriteId)); + }) + .map((m) => m.metadata as MessageMetadata); + } catch (err) { + // A release must complete whatever its window's read meets. + console.error(`[route-failure] ${agent.name}: could not read its window for unread failed replies:`, err); + return []; + } + } + + /** + * Record a failed-reply notice injected into `agent`'s live stream at a + * tool boundary (injectedFailureNotices). Residents are never released, so + * only an isolated agent's are kept. + */ + private noteInjectedFailureNotice(agent: Agent, id: string, metadata: MessageMetadata | undefined): void { + if (this.sharedSlotAgents.has(agent)) return; + if ((metadata as { kind?: unknown } | undefined)?.kind !== 'discord-send-failed') return; + let ids = this.injectedFailureNotices.get(agent.name); + if (!ids) this.injectedFailureNotices.set(agent.name, (ids = new Set())); + ids.add(id); + } + + /** + * Run `release` now, or, while the host is quiesced, when it resumes. A + * release lands the agent's queued writes in its window, and while quiesced + * a write could land mid-surgery; dropping them instead would lose them + * from both the window and the recovery queue. The agent stays registered + * meanwhile, and can't start a turn while quiesced. + */ + private releaseNowOrAfterResume(agentName: string, release: () => void): void { + if (this.quiesced) this.releasesAfterResume.set(agentName, release); + else release(); + } + + /** resume(): the releases asked for during the quiesce window, each on its own. */ + private runReleasesAfterResume(): void { + const releases = [...this.releasesAfterResume.values()]; + this.releasesAfterResume.clear(); + for (const release of releases) { + try { + release(); + } catch (err) { + console.error('[host-mode] resume: a release postponed by the quiesce window failed:', err); + } + } + } + /** * A fork or an ephemeral run ended before reading that its reply never - * reached the human: it was released with the notice still queued, or its + * reached the human: it was released with the notice unread or still + * queued, its queued notice was restored at boot after it was gone, or its * delivery failed after release. Tell the primary instead, naming the * speaker, so an agent who can still act learns of it: to resend, or to * tell the human. Only for a delivery to a channel; a reply that had no @@ -12029,15 +12153,22 @@ export class AgentFramework { console.error(`[route-failure] ${speaker} has ended, and there is no primary to tell that its reply to ${failure.channelId} failed`); return; } + // A notice queued by an earlier build may carry neither; say only what it has. + const textLen = typeof failure.textLen === 'number' ? failure.textLen : undefined; + const reason = typeof failure.reason === 'string' && failure.reason !== '' ? failure.reason : undefined; try { this.addMessage( 'user', [{ type: 'text', - text: `[discord-send-failed] A reply by ${speaker} (${String(failure.textLen)} chars) could not be delivered to ` + - `${this.channelForNotice(failure.channelId)} (${String(failure.reason)}). ${speaker} has ended, and the human did not receive it.`, + text: `[discord-send-failed] A reply by ${speaker}${textLen !== undefined ? ` (${textLen} chars)` : ''} could not be delivered to ` + + `${this.channelForNotice(failure.channelId)}${reason !== undefined ? ` (${reason})` : ''}. ${speaker} has ended, and the human did not receive it.`, }], - { system: true, kind: 'discord-send-failed', channelId: failure.channelId, reason: failure.reason, textLen: failure.textLen, speaker } as MessageMetadata, + { + system: true, kind: 'discord-send-failed', channelId: failure.channelId, speaker, + ...(reason !== undefined ? { reason } : {}), + ...(textLen !== undefined ? { textLen } : {}), + } as MessageMetadata, ); } catch (err) { console.error(`[route-failure] failed to tell the primary that ${speaker}'s reply to ${failure.channelId} failed:`, err); @@ -12657,6 +12788,7 @@ export class AgentFramework { this.passFailedReplyToPrimary(conversationId, metadata); return; } + // A registered speaker's notice is addressed to it (noticeTargetFor). this.addMessage( 'user', [{ @@ -12664,7 +12796,7 @@ export class AgentFramework { text: `[discord-send-failed] Your previous reply (${textLen} chars) could not be delivered to ${this.channelForNotice(channelId)} (${reason}). It was saved to your archive but the human did not receive it.`, }], metadata, - this.noticeTargetFor(conversationId), + { forAgent: conversationId }, ); } catch (err) { console.error('onRouteFailure: failed to record send-failure marker:', err); diff --git a/test/route-failure-targeting.test.ts b/test/route-failure-targeting.test.ts index a28c3db10..41ebe7242 100644 --- a/test/route-failure-targeting.test.ts +++ b/test/route-failure-targeting.test.ts @@ -7,7 +7,8 @@ import { AgentFramework } from '../src/index.js'; import { ContextManager, PassthroughStrategy } from '@animalabs/context-manager'; import { CapabilityGrant } from '../src/mcpl/capability-grant.js'; import type { McplServerRegistry } from '../src/mcpl/server-registry.js'; -import type { ProcessEvent } from '../src/index.js'; +import type { ProcessEvent, Module, ModuleContext, ToolDefinition, ToolCall, ToolResult, EventResponse } from '../src/index.js'; +import type { ContentBlock } from '@animalabs/membrane'; import type { ChannelRegistry } from '../src/mcpl/channel-registry.js'; import { MockMembrane, createMockResponse } from './helpers/mock-membrane.js'; @@ -27,9 +28,39 @@ function internals(framework: AgentFramework) { unackedDeferredWrites: unknown[]; flushDeferredWrites(label: string): Promise; injectScriptWake(record: { id: string; agentName: string }, envelope: string): void; + closingConversationAgents: Set; + activeStreams: Map>; + recordChannelAutoOpenNotice(agentName: string | undefined, channels: Array<{ channelId: string; label?: string }>, cause: 'subscription-policy' | 'opened-by-delivery' | 'opened-by-reply'): void; + addMessage(participant: string, content: ContentBlock[], metadata?: Record, opts?: { forAgent?: string }): string; }; } +/** A tool whose call makes `speaker`'s reply fail to reach a channel while its + * turn is alive, and a tool that does nothing; either can quiesce the host + * from inside the turn. */ +class SenderModule implements Module { + readonly name = 'sender'; + framework: AgentFramework | null = null; + speaker = 'worker'; + quiesceOnCall = false; + async start(_ctx: ModuleContext): Promise {} + async stop(): Promise {} + getTools(): ToolDefinition[] { + return [ + { name: 'post', description: 'Post a reply', inputSchema: { type: 'object', properties: {} } }, + { name: 'noop', description: 'Do nothing', inputSchema: { type: 'object', properties: {} } }, + ]; + } + async handleToolCall(call: ToolCall): Promise { + if (call.name.endsWith('post')) { + await internals(this.framework!).channelRegistry.routeSpeech(this.speaker, 'mid-turn reply', 'missing-channel'); + } + if (this.quiesceOnCall) internals(this.framework!).quiesced = true; + return { success: true, data: { ok: true } }; + } + async onProcess(): Promise { return {}; } +} + const SEND_FAILED = '[discord-send-failed]'; const BOUNCED = 'was not delivered —'; const count = (messages: Array<{ content: unknown }>, needle: string) => @@ -103,24 +134,25 @@ describe('notices about an agent\'s own action', () => { } describe('residents share one message slot', () => { - it('keeps a secondary resident\'s notice on the default path, where it reads it', async (t) => { + it('addresses a secondary resident\'s notice to it, in the slot every resident reads', async (t) => { const primary = t.mock.method(framework.getAgent('primary')!.getContextManager(), 'addMessage'); const secondary = t.mock.method(framework.getAgent('secondary')!.getContextManager(), 'addMessage'); const result = await internals(framework).channelRegistry.routeSpeech('secondary', 'hello', 'missing-channel'); assert.equal(result, null); - assert.equal(primary.mock.callCount(), 1, 'the default path writes the shared slot'); - assert.equal(secondary.mock.callCount(), 0); - const [participant, content, metadata] = primary.mock.calls[0]!.arguments; + assert.equal(primary.mock.callCount(), 0); + assert.equal(secondary.mock.callCount(), 1, 'written through the speaker\'s own window'); + const [participant, content, metadata] = secondary.mock.calls[0]!.arguments; assert.equal(participant, 'user'); - assert.match(JSON.stringify(content), /\[discord-send-failed\].*5 chars.*missing-channel/); + assert.match(JSON.stringify(content), /\[discord-send-failed\] Your previous reply \(5 chars\) could not be delivered to missing-channel/); assert.equal(metadata?.system, true); assert.equal(metadata?.kind, 'discord-send-failed'); assert.equal(metadata?.channelId, 'missing-channel'); assert.match(String(metadata?.reason), /no registered channel/); + assert.equal(metadata?.textLen, 5); t.mock.restoreAll(); const { messages } = await framework.getAgent('secondary')!.getContextManager().compile(); - assert.equal(count(messages, SEND_FAILED), 1, 'the speaking resident reads the shared slot'); + assert.equal(count(messages, SEND_FAILED), 1, 'the speaking resident reads it'); assert.equal(internals(framework).pendingRequests.length, 0, 'a failure notice must not request inference'); assert.equal(membrane.calls.length, 0); }); @@ -134,30 +166,32 @@ describe('notices about an agent\'s own action', () => { assert.equal(primary.mock.callCount(), 1); assert.equal(secondary.mock.callCount(), 0); assert.equal(primary.mock.calls[0]!.arguments[2]?.kind, 'discord-send-failed'); + assert.match(JSON.stringify(primary.mock.calls[0]!.arguments[1]), /Your previous reply \(5 chars\) could not be delivered to the channel/); assert.equal(internals(framework).pendingRequests.length, 0); }); - it('waits for the primary\'s live turn before writing a secondary resident\'s notice', async () => { + it('waits for the speaking resident\'s own live turn', async () => { const state = internals(framework); - state.activeTurnTokens.set('primary', 1); + state.activeTurnTokens.set('secondary', 1); await state.channelRegistry.routeSpeech('secondary', 'hello', null); - assert.equal(state.deferredMessages.length, 1, 'the shared slot is mid-turn for the primary'); - assert.equal(state.deferredMessages[0]!.forAgent, undefined); - const { messages } = await framework.getAgent('primary')!.getContextManager().compile(); + assert.equal(state.deferredMessages.length, 1, 'the speaker is mid-turn'); + assert.equal(state.deferredMessages[0]!.forAgent, 'secondary'); + const { messages } = await framework.getAgent('secondary')!.getContextManager().compile(); assert.equal(count(messages, SEND_FAILED), 0); }); - for (const [speaker, cycling] of [['secondary', 'primary'], ['primary', 'secondary']] as const) { - it(`waits for ${cycling}'s tool cycle before writing ${speaker}'s notice`, async () => { + for (const speaker of ['secondary', 'primary'] as const) { + it(`waits for ${speaker}'s own tool cycle before writing its notice`, async () => { const state = internals(framework); - state.pendingAssistantBlocks.set(cycling, []); + state.pendingAssistantBlocks.set(speaker, []); await state.channelRegistry.routeSpeech(speaker, 'hello', null); - assert.equal(state.deferredMessages.length, 1, 'a write now would split a tool_use from its result'); - const { messages } = await framework.getAgent('primary')!.getContextManager().compile(); + assert.equal(state.deferredMessages.length, 1, 'a write now would split its tool_use from its result'); + assert.equal(state.deferredMessages[0]!.forAgent, speaker); + const { messages } = await framework.getAgent(speaker)!.getContextManager().compile(); assert.equal(count(messages, SEND_FAILED), 0); }); } @@ -322,15 +356,20 @@ describe('notices about an agent\'s own action', () => { assert.equal(internals(framework).deferredMessages.length, 0); }); - it('drops, logged, a background-script wake for an agent no longer registered', async (t) => { + it('drops, logged once, a background-script wake for an agent no longer registered: no write, trace or wake', async (t) => { const primary = t.mock.method(framework.getAgent('primary')!.getContextManager(), 'addMessage'); const errors: string[] = []; t.mock.method(console, 'error', (...args: unknown[]) => { errors.push(args.map(String).join(' ')); }); + const traces: string[] = []; + framework.onTrace((event) => { if (event.type === 'message:added') traces.push(JSON.stringify(event)); }); internals(framework).injectScriptWake({ id: 's-2', agentName: 'released-run' }, '[background script s-2] Your background script DIED.'); assert.equal(primary.mock.callCount(), 0, 'the primary started no such script'); - assert.ok(errors.some((e) => /"released-run" is not a registered agent — message dropped/.test(e))); + assert.deepEqual(errors, ['[notice] released-run is no longer registered, so its background-script s-2 wake notice is dropped']); + assert.deepEqual(traces, [], 'nothing was queued or stored'); + assert.equal(internals(framework).deferredMessages.length, 0); + assert.equal(internals(framework).pendingRequests.length, 0, 'no wake for an agent that is gone'); }); it('wakes a fork\'s background script into the fork\'s own window', async () => { @@ -345,6 +384,187 @@ describe('notices about an agent\'s own action', () => { assert.equal(count(primaryMessages, envelope), 0); }); + /** Run the fallback reaper over a closing fork, as a minute's sweep would. */ + function reap(forkName: string) { + const state = internals(framework); + state.closingConversationAgents.add(forkName); + state.lastConversationSweep = 0; + state.sweepExpiredConversations(); + } + + it('tells the primary, once, of a failure notice a fork stored but never read, when the fork is released', async () => { + const reply = 'Reply from the fork.'; + const fork = await spawnFork(reply); + // The notice landed in the fork's window after its turn: no compile has read it. + + reap(fork.name); + + assert.equal(framework.getAgent(fork.name), null, 'released with no closure turn'); + const { messages } = await framework.getAgent('primary')!.getContextManager().compile(); + const told = messages.filter((m) => JSON.stringify(m.content).includes(SEND_FAILED)); + assert.equal(told.length, 1); + assert.match(JSON.stringify(told[0]!.content), + new RegExp(`A reply by ${fork.name} \\(${reply.length} chars\\) could not be delivered to missing-channel \\(no registered channel[^)]*\\)\\. ${fork.name} has ended, and the human did not receive it\\.`)); + }); + + it('passes on only what a fork never read: a notice its later turn compiled stays its own', async () => { + const fork = await spawnFork('Reply from the fork.'); + const second = 'A second reply, which fails as well.'; + membrane.pushResponse(createMockResponse([{ type: 'text', text: second }])); + framework.pushEvent({ + type: 'mcpl:channel-incoming', + serverId: 'srv', + channelId: 'missing-channel', + messageId: 'incoming-2', + author: { id: 'user-1', name: 'User' }, + content: [{ type: 'text', text: 'Hello again.' }], + timestamp: new Date().toISOString(), + metadata: { channel_type: 'im' }, + triggerInference: true, + } as unknown as ProcessEvent); + await framework.runUntilIdle(); + assert.equal(membrane.calls.length, 2, 'the fork\'s second turn compiled its first notice'); + + reap(fork.name); + + const { messages } = await framework.getAgent('primary')!.getContextManager().compile(); + const told = messages.filter((m) => JSON.stringify(m.content).includes(SEND_FAILED)); + assert.equal(told.length, 1); + assert.match(JSON.stringify(told[0]!.content), new RegExp(`A reply by ${fork.name} \\(${second.length} chars\\)`)); + }); + + it('does not reap a closing fork whose turn holds its token before its stream registers', async () => { + const fork = await spawnFork(); + const state = internals(framework); + state.activeTurnTokens.set(fork.name, 7); + + reap(fork.name); + assert.equal(framework.getAgent(fork.name), fork, 'a turn mid-compile is busy'); + + state.activeTurnTokens.delete(fork.name); + reap(fork.name); + assert.equal(framework.getAgent(fork.name), null); + }); + + it('lands a queued ordinary write in a released fork\'s window, with its durable id, and acks it', async () => { + const fork = await spawnFork(); + const state = internals(framework); + state.deferredWritesPersisted = true; + state.activeTurnTokens.set(fork.name, 3); + state.addMessage('user', [{ type: 'text', text: 'queued for the fork' }], { kind: 'test-note' }, { forAgent: fork.name }); + assert.equal(state.deferredMessages.length, 1); + state.activeTurnTokens.delete(fork.name); + + reap(fork.name); + + assert.equal(framework.getAgent(fork.name), null); + assert.equal(state.deferredMessages.length, 0); + assert.equal(state.unackedDeferredWrites.length, 0, 'acked out of the recovery queue'); + const archive = await ContextManager.open({ + store: framework.getStore(), namespace: 'conversations/' + fork.name, isolate: true, strategy: new PassthroughStrategy(), + }); + const stored = archive.getAllMessages().find((m) => JSON.stringify(m.content).includes('queued for the fork')); + assert.ok(stored, 'it landed in the fork\'s own window'); + assert.equal(typeof (stored.metadata as { deferredWriteId?: unknown } | undefined)?.deferredWriteId, 'string'); + }); + + for (const [what, hold] of [ + ['its own tool cycle', (state: ReturnType, fork: string) => { state.pendingAssistantBlocks.set(fork, []); }], + ['its own stream', (state: ReturnType, fork: string) => { state.activeStreams.set(fork, new Promise(() => {})); }], + ] as const) { + it(`defers a fork's notice while only ${what} is alive`, async (t) => { + const fork = await spawnFork(); + const state = internals(framework); + const forkCm = t.mock.method(fork.getContextManager(), 'addMessage'); + hold(state, fork.name); + try { + await state.channelRegistry.routeSpeech(fork.name, 'hello', null); + + assert.equal(forkCm.mock.callCount(), 0); + assert.equal(state.deferredMessages.length, 1); + assert.equal(state.deferredMessages[0]!.forAgent, fork.name); + } finally { + // A stream that never settles would hold stop() in afterEach. + state.activeStreams.delete(fork.name); + } + }); + } + + it('lands a fork\'s notice while only another agent\'s tool cycle is pending', async (t) => { + const fork = await spawnFork(); + const state = internals(framework); + const forkCm = t.mock.method(fork.getContextManager(), 'addMessage'); + state.pendingAssistantBlocks.set('primary', []); + + await state.channelRegistry.routeSpeech(fork.name, 'hello', null); + + assert.equal(forkCm.mock.callCount(), 1, 'the fork\'s window has no cycle in flight'); + assert.equal(state.deferredMessages.length, 0); + }); + + it('tells the agent whose send opened a channel, in its own window; a policy open tells the primary', async (t) => { + const fork = await spawnFork(); + const state = internals(framework); + const primary = t.mock.method(framework.getAgent('primary')!.getContextManager(), 'addMessage'); + const forkCm = t.mock.method(fork.getContextManager(), 'addMessage'); + + state.recordChannelAutoOpenNotice(fork.name, [{ channelId: 'c-1', label: '#one' }], 'opened-by-reply'); + assert.equal(forkCm.mock.callCount(), 1); + assert.equal(primary.mock.callCount(), 0); + assert.match(JSON.stringify(forkCm.mock.calls[0]!.arguments[1]), /Now open: #one \(c-1\) — your own message into the closed channel engaged it/); + + state.recordChannelAutoOpenNotice(undefined, [{ channelId: 'c-2' }], 'subscription-policy'); + assert.equal(primary.mock.callCount(), 1, 'no agent caused a policy open'); + assert.equal(forkCm.mock.callCount(), 1); + }); + + it('drops, logged once, an auto-open notice for an agent no longer registered', async (t) => { + const primary = t.mock.method(framework.getAgent('primary')!.getContextManager(), 'addMessage'); + const errors: string[] = []; + t.mock.method(console, 'error', (...args: unknown[]) => { errors.push(args.map(String).join(' ')); }); + + internals(framework).recordChannelAutoOpenNotice('released-conversation', [{ channelId: 'c-1' }], 'opened-by-delivery'); + + assert.equal(primary.mock.callCount(), 0); + assert.equal(internals(framework).deferredMessages.length, 0); + assert.deepEqual(errors, ['[notice] released-conversation is no longer registered, so its channel auto-open notice is dropped']); + }); + + it('passes a restored failure notice whose agent is gone to the primary, by name, and drops its other writes, logged', async (t) => { + const state = internals(framework); + const errors: string[] = []; + t.mock.method(console, 'error', (...args: unknown[]) => { errors.push(args.map(String).join(' ')); }); + (state.deferredMessages as unknown as Array>).push( + { + id: 'w-1', seq: 1, participant: 'user', forAgent: 'conversation-gone', + content: [{ type: 'text', text: '[discord-send-failed] Your previous reply (12 chars) could not be delivered.' }], + metadata: { system: true, kind: 'discord-send-failed', channelId: 'missing-channel', reason: 'no registered channel', textLen: 12 }, + }, + { + // Queued by an earlier build: no textLen, and no reason either. + id: 'w-2', seq: 2, participant: 'user', forAgent: 'conversation-older', + content: [{ type: 'text', text: '[discord-send-failed] Your previous reply could not be delivered.' }], + metadata: { system: true, kind: 'discord-send-failed', channelId: 'missing-channel' }, + }, + { + id: 'w-3', seq: 3, participant: 'user', forAgent: 'conversation-gone', + content: [{ type: 'text', text: 'an ordinary write' }], + }, + ); + + await state.flushDeferredWrites('boot-recovery'); + + const { messages } = await framework.getAgent('primary')!.getContextManager().compile(); + const told = messages.filter((m) => JSON.stringify(m.content).includes('has ended')).map((m) => JSON.stringify(m.content)); + assert.equal(told.length, 2); + assert.match(told[0]!, /A reply by conversation-gone \(12 chars\) could not be delivered to missing-channel \(no registered channel\)\. conversation-gone has ended/); + assert.match(told[1]!, /A reply by conversation-older could not be delivered to missing-channel\. conversation-older has ended/); + assert.ok(!told.some((t) => t.includes('undefined'))); + assert.equal(count(messages, 'an ordinary write'), 0); + assert.ok(errors.some((e) => /"conversation-gone" is not a registered agent — message dropped \(participant=user\)/.test(e))); + assert.equal(state.deferredMessages.length, 0); + }); + for (const condition of ['ordinary', 'other-tool-cycle', 'quiesced', 'write-fails'] as const) { it('handles a failed TTL-closure reply notice at disposal: ' + condition, async (t) => { const state = internals(framework); @@ -398,15 +618,18 @@ describe('notices about an agent\'s own action', () => { await framework.runUntilIdle(); state.pendingAssistantBlocks.clear(); - assert.equal(framework.getAgent(fork.name), null, 'the fork is disposed when its closure turn ends'); if (condition === 'quiesced') { - // A write while quiesced could land mid-surgery: the archive write - // is dropped, logged, and the primary's notice waits for resume. - assert.ok(errors.some((e) => e.includes(`[deferred-flush] ${fork.name}: released while quiesced, so a queued write for its window is dropped`))); + // A write while quiesced could land mid-surgery, so the release + // waits: the fork stays registered with its notice queued, and the + // primary hasn't been told. + assert.equal(framework.getAgent(fork.name), fork, 'released only once the host resumes'); assert.equal(state.deferredMessages.length, 1); - state.quiesced = false; - await state.flushDeferredWrites('closure test resume'); + assert.equal(state.deferredMessages[0]!.forAgent, fork.name); + const waiting = await framework.getAgent('primary')!.getContextManager().compile(); + assert.equal(count(waiting.messages, SEND_FAILED), 0); + await framework.resume({ force: true }); } + assert.equal(framework.getAgent(fork.name), null, 'the fork is disposed when its closure turn ends'); if (condition === 'other-tool-cycle') { // The primary's notice waits for the shared slot's tool cycle. assert.equal(state.deferredMessages.length, 1); @@ -423,9 +646,7 @@ describe('notices about an agent\'s own action', () => { const { messages } = await archive.compile(); const replyIndex = messages.findIndex((m) => m.participant === fork.name && JSON.stringify(m.content).includes(finalReply)); assert.ok(replyIndex >= 0, 'the failed closure reply remains archived'); - if (condition === 'quiesced') { - assert.equal(count(messages, SEND_FAILED), 0); - } else if (condition === 'write-fails') { + if (condition === 'write-fails') { // A write the store refuses is logged; disposal still completes. assert.equal(count(messages, SEND_FAILED), 0); assert.ok(errors.some((e) => e.includes(`[deferred-flush] ${fork.name}: failed to store a queued write before release`))); @@ -449,4 +670,103 @@ describe('notices about an agent\'s own action', () => { }); } }); + + describe('a run\'s notices at its tool boundaries, and its release', () => { + let runDir: string; + let runFramework: AgentFramework; + let runMembrane: MockMembrane; + let sender: SenderModule; + + beforeEach(async () => { + runDir = mkdtempSync(join(tmpdir(), 'route-failure-runs-')); + runMembrane = new MockMembrane(); + sender = new SenderModule(); + runFramework = await AgentFramework.create({ + storePath: join(runDir, 'test.chronicle'), + membrane: runMembrane.asMembrane(), + agents: [{ name: 'primary', model: 'test-model', systemPrompt: 'Primary.', proseRouting: 'locus' }], + modules: [sender], + }); + sender.framework = runFramework; + await internals(runFramework).initializeMcpl([]); + }); + + afterEach(async () => { + internals(runFramework).quiesced = false; + await runFramework.stop(); + rmSync(runDir, { recursive: true, force: true }); + }); + + /** Run 'worker' once through these rounds; every round but the last ends on its tool call. */ + async function run(rounds: ContentBlock[][]) { + const created = await runFramework.createEphemeralAgent({ + name: 'worker', model: 'test-model', systemPrompt: 'Do the task.', allowedTools: 'all', proseRouting: 'explicit', + }); + created.contextManager.addMessage('user', [{ type: 'text', text: 'Run once.' }]); + rounds.forEach((blocks, i) => runMembrane.pushResponse(createMockResponse(blocks, i < rounds.length - 1 ? 'tool_use' : undefined))); + const done = runFramework.runEphemeralToCompletion(created.agent, created.contextManager); + runFramework.start(); + await done; + await runFramework.runUntilIdle(); + } + + async function archived() { + const archive = await ContextManager.open({ + store: runFramework.getStore(), namespace: 'subagent/worker', isolate: true, strategy: new PassthroughStrategy(), + }); + return (await archive.compile()).messages; + } + + const injected = () => runMembrane.lastStream!.receivedToolResultOptions + .flatMap((o) => o?.injectedMessages ?? []) + .map((m) => JSON.stringify(m.content)); + + it('passes on nothing a run read live at a tool boundary', async () => { + await run([ + [{ type: 'tool_use', id: 'p1', name: 'sender--post', input: {} }], + [{ type: 'text', text: '>>skip_reply done' }], + ]); + + assert.ok(injected().some((c) => c.includes(SEND_FAILED)), 'the notice reached the run\'s live stream'); + assert.equal(count(await archived(), SEND_FAILED), 1, 'and its window'); + const { messages } = await runFramework.getAgent('primary')!.getContextManager().compile(); + assert.equal(count(messages, 'A reply by worker'), 0, 'it read it, so no one else is told'); + }); + + it('injects a bounced prose notice at the next tool boundary, after the prose it bounced', async () => { + const prose = 'Worker prose with no destination.'; + await run([ + [{ type: 'text', text: prose }, { type: 'tool_use', id: 'n1', name: 'sender--noop', input: {} }], + [{ type: 'text', text: '>>skip_reply done' }], + ]); + + assert.ok(injected().some((c) => c.includes(BOUNCED)), 'the speaker hears it within the turn'); + const messages = await archived(); + const proseIndex = messages.findIndex((m) => m.participant === 'worker' && JSON.stringify(m.content).includes(prose)); + const noticeIndex = messages.findIndex((m) => JSON.stringify(m.content).includes(BOUNCED)); + assert.ok(proseIndex >= 0, 'the bounced prose is in its window'); + assert.ok(noticeIndex > proseIndex, 'the notice follows it there'); + assert.match(JSON.stringify(messages[noticeIndex]!.content), /\{\{unsent\}\}/, 'naming the clipboard that holds it'); + const { messages: primary } = await runFramework.getAgent('primary')!.getContextManager().compile(); + assert.equal(count(primary, BOUNCED), 0); + }); + + it('releases a run that ends while the host is quiesced only at resume, landing what it left queued', async () => { + sender.quiesceOnCall = true; + await run([ + [{ type: 'tool_use', id: 'n1', name: 'sender--noop', input: {} }], + [{ type: 'text', text: 'Worker prose that bounces at the end.' }], + ]); + const state = internals(runFramework); + assert.equal(state.quiesced, true); + assert.ok(runFramework.getAgent('worker'), 'still registered while quiesced'); + assert.ok(state.deferredMessages.some((m) => m.forAgent === 'worker'), 'its last notice is still queued'); + + await runFramework.resume({ force: true }); + + assert.equal(runFramework.getAgent('worker'), null, 'released at resume'); + assert.equal(state.deferredMessages.length, 0); + assert.equal(count(await archived(), BOUNCED), 1, 'what it left queued landed in its window'); + }); + }); }); From be01bd4b28674eaefb2e783bc654c5c20b5e6089 Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Sun, 11 Oct 2026 17:03:16 -0400 Subject: [PATCH 0118/1112] fix(mcpl): a failed report still names the failure; log text replaces format and separator characters too Two follow-ups from a review of the previous commit: - When printing a failure report fails (the raw thrown value can't be inspected), the fallback stderr line now names the failure and why reporting it failed, as strings from describeCause, instead of printing the raw value again, so a headless host's stderr still says which failure it was. The trace has already been emitted by then. - boundedForLog replaces every \p{Cc} and \p{Cf} character and U+2028 and U+2029, not only C0 and C1 controls: a server-chosen method, request id or cause can carry bidi and other format characters, or a line or paragraph separator, which some log readers reorder or break lines on. Tests (mcpl-request-handler-errors, 57 to 59): a value printing can't handle still leaves a stderr line naming the failure; a dispatch failure's text has control, bidi and separator characters replaced. Red checks on dist, each alone: printing the raw value in the fallback, and replacing controls only, each fail the test aimed at it. Local: tsc clean; mcpl-request-handler-errors 59/59 on Node 24.19.0. --- src/framework.ts | 15 ++++++++----- src/mcpl/errors.ts | 6 ++++-- test/mcpl-request-handler-errors.test.ts | 27 ++++++++++++++++++++++++ 3 files changed, 41 insertions(+), 7 deletions(-) diff --git a/src/framework.ts b/src/framework.ts index 163edac0d..5611571c4 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -13756,10 +13756,10 @@ export class AgentFramework { // it runs as a rejection handler, and an exception there would be an // unhandled rejection, which ends the host. const report = (handlerError: { error: unknown } | undefined): void => { + const primary = handlerError + ? handlerError.error + : writeFailed ? writeFailure : new Error('the handler tried to answer a second time after its reply'); try { - const primary = handlerError - ? handlerError.error - : writeFailed ? writeFailure : new Error('the handler tried to answer a second time after its reply'); const message = describeCause(primary); // What the handler's own writes did, before any internal-error // reply sent here. @@ -13799,8 +13799,13 @@ export class AgentFramework { ...(responseError === undefined ? [] : ['| the internal-error reply failed too:', responseError]), ); } catch (reportError) { - try { console.error(`[mcpl] ${serverId} ${event} handler failed, and reporting it failed too:`, reportError); } - catch { /* stderr itself is gone */ } + // Strings only: the raw values may be what failed to print. + try { + console.error( + `[mcpl] ${serverId} ${event} handler failed: ${describeCause(primary)}` + + ` (and reporting it failed too: ${describeCause(reportError)})`, + ); + } catch { /* stderr itself is gone */ } } }; const failed = (error: unknown): void => report({ error }); diff --git a/src/mcpl/errors.ts b/src/mcpl/errors.ts index bd6afd08a..9fcab5c99 100644 --- a/src/mcpl/errors.ts +++ b/src/mcpl/errors.ts @@ -55,9 +55,11 @@ export function describeCause(cause: unknown): string { } /** Text a server or a handler chose, made fit for one log or trace line: - * control characters replaced, and cut to `max` code points. */ + * control and format characters replaced (C0 and C1 controls, bidi and other + * \p{Cf} characters, and the line and paragraph separators some log readers + * break lines on), and cut to `max` code points. */ export function boundedForLog(text: string, max = 200): string { - const points = [...text.replace(/[\u0000-\u001f\u007f-\u009f]/g, '?')]; + const points = [...text.replace(/[\p{Cc}\p{Cf}\u2028\u2029]/gu, '?')]; return points.length > max ? `${points.slice(0, max).join('')}…` : points.join(''); } diff --git a/test/mcpl-request-handler-errors.test.ts b/test/mcpl-request-handler-errors.test.ts index 004dd0c3d..d21a4fdbc 100644 --- a/test/mcpl-request-handler-errors.test.ts +++ b/test/mcpl-request-handler-errors.test.ts @@ -736,3 +736,30 @@ test('a request for an event nothing on the host listens for is answered -32601' const replies = transport.lines.map((line) => JSON.parse(line)); assert.deepEqual(replies.map((r) => [r.id, r.error?.code, r.error?.message]), [[71, -32601, 'Method not found']]); }); + +test('a value printing cannot handle still leaves a stderr line naming the failure', async (t) => { + const { framework, connection } = harness(); + framework.traceListeners = []; + const lines: string[] = []; + // Print as console.error does, and keep what it printed. + t.mock.method(console, 'error', (...args: unknown[]) => { lines.push(args.map((a) => typeof a === 'string' ? a : inspect(a)).join(' ')); }); + const hostile = { + [inspect.custom]() { throw new Error('inspect refused'); }, + toString() { return 'the hostile failure'; }, + }; + framework.channelRegistry = { async handleRegister() { throw hostile; } }; + connection.emit('channels-register', {}, responder()); + await settle(); + assert.equal(lines.length, 1); + assert.match(lines[0]!, /channels-register handler failed: the hostile failure \(and reporting it failed too: inspect refused\)/); +}); + +test('log text from a server has control, format and separator characters replaced', (t) => { + const { connection, transport, traces } = wireHarness(); + t.mock.method(console, 'error', () => {}); + connection.prependListener('channels-list', () => { throw new Error('a\u2028b\u202ec\u0007d'); }); + transport.emit('line', JSON.stringify({ jsonrpc: '2.0', id: 81, method: 'channels/list' })); + const errors = traces.filter((e) => e.type === 'mcpl:server-error'); + assert.equal(errors.length, 1); + assert.match(errors[0].error, /: a\?b\?c\?d$/); +}); From 9e45ae1d6b415266dba34cbf73cf3821df0cf137 Mon Sep 17 00:00:00 2001 From: Basil-1007 Date: Sun, 11 Oct 2026 17:21:16 -0400 Subject: [PATCH 0119/1112] fix(mcpl): a channel's transport is known only from a receipt for its current target; removal cancels queued lifecycle work A review of the lifecycle ordering found ways the registry still claimed a transport state nothing had confirmed, queued work that outlived its server's registrations, and text that stated more than the code does. One rule, where the host directs a server's lifecycle (its channels.lifecycle grant): a channel's transport state is known only from a receipt for its current target. - Every entry a registration creates (channels/register, channels/changed added) starts unconfirmed, so channel_close's "already closed" shortcut can't answer for a channel nothing confirmed closed. - A failed open or close for the current target leaves the outcome unknown, since an RPC can fail after the server acted: the entry is marked unconfirmed with its open flag unchanged, so a confirmed-open channel stays usable after a failed backscroll (the mark is read only while not open). A failure from an older connection or target also withdraws the open flag, as before. - Incoming traffic confirms nothing unconfirmed: after a retarget it may still come from the old subscription. It marks a channel open only while the entry the message was accepted for is confirmed and decided open or tuned out, so a close receipt stands, whether it landed before the message or during the coalescer's await; an entry a re-registration put in its place meanwhile waits for its own evidence. The mark no longer needs deleting there. - Where the host doesn't direct a server's lifecycle, the server opens and closes its own channels and sends no receipt, so its traffic marks a channel open, as it always has. Lifecycle work: - Removing a server cancels its queued lifecycle operations: each records the server's removal count when queued and, when its turn comes after a removal, rejects with a cancellation error instead of acting on a closing connection or a later re-registration's entries. Reconciliation and the corrective close of a substituted channel treat that, and a channel removed while queued, as cancellation, not a failed reconcile. - disconnectMcplServer removes the channel entries before closing the connection. - A retarget through channels/changed of a channel decided open or tuned out opens the new target. When an operation is already queued or running for the channel, it reads the new target itself before confirming anything, so nothing more is queued; a per-channel count of such operations, dropped synchronously at each one's last look at its target, tells the two apart. - Declining an invitation goes through the lifecycle queue like any closed decision, unless a receipt confirmed the transport closed with nothing pending. A failed close is reported in the result (closeError); the decision stands. Nothing is sent to a server the host doesn't direct. - waitForOpenChannel stops when the tail it awaited is still current after settling, instead of awaiting it again without yielding. - A registration batch reconciles its channels side by side (Promise.allSettled): each channel has its own queue, so one held RPC no longer delays its siblings' opens. The substitution check: - A null or absent id in the answer, or in a history item, names no channel and is accepted as before. Another string is a substitution, as before. Any other type fails the open the same way, unconfirmed and traced, with no corrective close, since it names nothing registered. - Ids compare exactly, as the server's traffic does: every server-facing path keys channels by exact id, and handleIncoming refuses any other, so canonicalising only the open check would make it looser than the same server's traffic. The ChannelsOpenParams.channelId doc and the changelog now say so, instead of "any other channel". The doc also says what a server does without channelId (match by type and address) and that the host closes a channel opened in substitution when it's decided closed. - The mcpl:channel-open-substituted trace records the answered id in the same bounded form as the log line, and that form counts code points, not UTF-16 units. Text: - The delivery and reply notice says what happened ("your own message into the closed channel opened it") and states no rule about later sends; whether speaking opens a closed channel is still open with the maintainers. - The notice builder's doc says a policy admission is announced as the decision, before transport, and names the opened-by-reply cause. - The framework's engagement comment says a channel decided open whose transport is unconfirmed is opened again first, and its engagement recorded when that open returns. The open-on-send failure log says a later registration or delivery retries it, not that it "will reconcile". - The changelog states the one rule and its scope, when channel_open's "stays selected" note appears, the cancellation and side-by-side reconciling, the null and non-string ids, exact comparison, the trace's bounded id, and that either rollout order is safe. Tests (channel-lifecycle-order, channel-open-substitution): - a failed initial close leaves the channel unconfirmed, so channel_close sends a real close, and a re-registration's entry is unconfirmed too, so without the lifecycle grant channel_close is refused for the grant rather than answered "already closed"; - traffic after a retarget confirms nothing, and a failed open for the new target leaves it unconfirmed; after a failed open of the same target, traffic marks nothing open either; - traffic for a channel decided closed leaves its confirmed close standing, and so does a close that lands while the coalescer holds a message; - a retarget opens a channel decided open at its new target and sends nothing for one decided closed, whose close is then real; - removing a server cancels its queued work across a re-registration, a substitute removed while its corrective close is queued is cancelled, not traced as failed, and disconnectMcplServer has removed the channel entries by the time it closes the connection; - declining closes an unconfirmed transport for real, needs no RPC once closed is confirmed, and queues behind pending work; - a settled operation releases its queue key; - a batch's channels reconcile side by side; - a server the host doesn't direct still has its channels marked open by traffic, and a decline there sends nothing; - non-string answered ids, null ids, the trace's bounded id and code-point counting; - the notices' exact wording, including the opt-out; - the tune-out delivery test asserts the one status a reply returns and the RPCs sent, the lost-confirmation delivery tests make their retarget's open fail (a retarget now opens), and the removed-channel batch test checks that its sibling opened. mcpl-gate-roundtrip's minimal registry gains a server registry, which traffic now consults for the lifecycle grant. --- .../210-channel-lifecycle-order.fixed.md | 8 +- src/framework.ts | 37 ++- src/mcpl/channel-registry.ts | 235 +++++++++---- src/mcpl/types.ts | 9 +- test/channel-lifecycle-order.test.ts | 310 +++++++++++++++++- test/channel-open-substitution.test.ts | 31 ++ test/mcpl-gate-roundtrip.test.ts | 4 +- 7 files changed, 541 insertions(+), 93 deletions(-) diff --git a/changelog.d/210-channel-lifecycle-order.fixed.md b/changelog.d/210-channel-lifecycle-order.fixed.md index e5de6a05e..9f3449d9a 100644 --- a/changelog.d/210-channel-lifecycle-order.fixed.md +++ b/changelog.d/210-channel-lifecycle-order.fixed.md @@ -1,9 +1,9 @@ -- Failed backscroll requests preserve the registry's known-open state rather than claiming a close. Automatic delivery joins pending lifecycle work without reopening a channel closed during the wait. A delivery or reply is announced as opening a channel, with the current registration label, only when the channel wasn't already decided open or tuned out; reconfirming the transport of a channel that was isn't announced. Speech engages a channel decided closed as a reply does, even while a failed close has left its transport open, wherever the server's grant lets the host open it. A failed `channel_open` says the channel stays selected for opening, since that decision outlives the failed round-trip and a later delivery or reconnect tries again. +- Failed backscroll requests preserve the registry's known-open state rather than claiming a close. Automatic delivery joins pending lifecycle work without reopening a channel closed during the wait. A delivery or reply is announced as opening a channel, with the current registration label, only when the channel wasn't already decided open or tuned out; reconfirming the transport of a channel that was isn't announced. Speech engages a channel decided closed as a reply does, even while a failed close has left its transport open, wherever the server's grant lets the host open it. When `channel_open` fails for a channel that isn't open and is still decided open, its error says the channel stays selected for opening, since that decision outlives the failed round-trip and a later delivery or reconnect tries again. -- Channel lifecycle operations now run in order per channel and reconcile the latest desired state and registration after an in-flight operation completes. Successful reconciliation confirms the current target and desired state at its receipt boundary; a superseded tool request reports that its requested state no longer applies. Automatic delivery preserves active tune-out attention state while opening or awaiting transport. +- Channel lifecycle operations now run in order per channel and reconcile the latest desired state and registration after an in-flight operation completes. Successful reconciliation confirms the current target and desired state at its receipt boundary; a superseded tool request reports that its requested state no longer applies. Automatic delivery preserves active tune-out attention state while opening or awaiting transport. A registration batch reconciles its channels side by side, so one held RPC doesn't delay its siblings. Removing a server cancels its queued lifecycle operations, and the framework removes its channel entries before closing its connection, so nothing queued runs against a closing connection or a later re-registration. -- Open receipts confirm only the current registration, connection, type, and address. Retargeting invalidates an older open flag; failed corrective opens leave the new target unconfirmed and fail joined delivery, while same-target backscroll failures retain known-open state. Explicit close retries an unconfirmed target instead of reporting it already closed. +- A channel's transport state is known only from a receipt for its current registration, connection, type, and address. A channel a registration creates or replaces, a retarget, a substitution, or a failed open or close leaves it unconfirmed: failed corrective opens fail joined delivery, while a same-target backscroll failure keeps a channel that was open usable. Incoming traffic confirms nothing unconfirmed (after a retarget it may still come from the old subscription), and it marks a channel open only while that channel is decided open or tuned out, so a close receipt stands. Explicit close, and declining an invitation, close an unconfirmed target for real instead of reporting it already closed. A retarget of a channel decided open or tuned out opens the new target. - Lifecycle convergence makes at most five transport attempts per operation. Continued descriptor/intent supersession fails with a diagnostic, releases the queue, and gives joined speech or reply preparation a visible failure instead of hanging. Later stable operations can retry. -- A `channels/open` answered for a different channel than the one requested, by the answer's `channel.id` or a history item's `channelId`, fails the open. The requested channel isn't marked open (its open state becomes unconfirmed), nothing in the answer is used, its history included, and the host logs `[channel-open-substituted]` and emits an `mcpl:channel-open-substituted` trace. When the channel the server opened instead is registered and its desired state is closed, the host closes it again, so its traffic doesn't reach the resident. A substituted answer on a receipt already made stale by retargeting is retried like any stale receipt. An answer that names no channel is accepted as before. `ChannelsOpenParams.channelId` is documented as exact: a server that doesn't know the id should refuse. +- A `channels/open` answered for a different channel than the one requested, by the answer's `channel.id` or a history item's `channelId`, compared exactly as the server's traffic is, fails the open. The requested channel isn't marked open (its open state becomes unconfirmed), nothing in the answer is used, its history included, and the host logs `[channel-open-substituted]` and emits an `mcpl:channel-open-substituted` trace, whose `answeredFor` is the answered id in the bounded form the log line prints. When the channel the server opened instead is registered and its desired state is closed, the host closes it again, so its traffic doesn't reach the resident. A substituted answer on a receipt already made stale by retargeting is retried like any stale receipt. An answer that names no channel, by a null id or none, is accepted as before; one whose id isn't a string fails the open the same way, with nothing to close. Either rollout order is safe: on a server that still falls back, the resident sees a failed open for an id that server no longer knows (a deleted channel), rather than another channel's traffic. `ChannelsOpenParams.channelId` is documented as exact: a server that doesn't know the id should refuse. diff --git a/src/framework.ts b/src/framework.ts index d0e792061..43be0bf5c 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -8359,10 +8359,12 @@ export class AgentFramework { } /** - * Durable window notice for a channel the agent did not ask to open — - * subscription-policy admission, or an open forced by the agent's own - * delivery into a closed channel. The agent must always learn that new - * traffic will start flowing, and that `channel_close` opts out (their + * Durable window notice for a channel the agent did not ask to open: a + * subscription-policy admission, announced as the decision before its + * transport RPC (a failed open keeps the decision, and channel_list shows + * transport state), or an open that the agent's own delivery or reply + * caused (`opened-by-delivery`, `opened-by-reply`), once its transport is + * confirmed. Either way it names `channel_close` as the opt-out (their * decision outranks policy). System message: never triggers inference, * never conversational, never locus-eligible. */ @@ -8380,10 +8382,9 @@ export class AgentFramework { ? `[channels] Selected for opening by subscription policy: ${shown}. ` + 'This records the admission decision; channel_list shows current transport state. ' + 'When open, ' + (plural ? 'their' : 'its') + ' ongoing traffic will reach you. ' - : `[channels] Now open: ${shown} — your own message into the closed channel engaged it. ` + + : `[channels] Now open: ${shown}: your own message into the closed channel opened it. ` + 'Ongoing traffic from ' + (plural ? 'these channels' : 'this channel') + ' will reach you. ') + - `If you don't want ${plural ? 'one of them' : 'it'}, use channel_close; your choice outranks policy. ` + - 'A fresh message into a closed channel engages it again.'; + `If you don't want ${plural ? 'one of them' : 'it'}, use channel_close; your choice outranks policy.`; try { // Through framework.addMessage for mid-turn safety (deferral + // hear-while-acting injection) — a delivery-forced open happens inside @@ -13541,10 +13542,11 @@ export class AgentFramework { } /** - * Disconnect an MCPL server at runtime: close the connection, destroy its - * feature-set and checkpoint state (permanent removal — unlike a transient - * transport close, which preserves checkpoints for the reconnect), remove - * its channels from the registry, drop routing entries, and refresh tools. + * Disconnect an MCPL server at runtime: remove its channels from the + * registry (first, cancelling their queued lifecycle work), close the + * connection, destroy its feature-set and checkpoint state (permanent + * removal — unlike a transient transport close, which preserves checkpoints + * for the reconnect), drop routing entries, and refresh tools. * No-op-ish if the server is not connected (still clears routing state). */ async disconnectMcplServer(id: string): Promise { @@ -13554,8 +13556,10 @@ export class AgentFramework { const config = this.mcplServerConfigs.get(id); const oldToolNames = new Set(this.mcplTools.map(t => t.name)); - await this.mcplServerRegistry.removeServer(id); + // Channel entries go first: removing them cancels the server's queued + // lifecycle work, which must not run against the closing connection. this.channelRegistry?.removeServer(id); + await this.mcplServerRegistry.removeServer(id); // Permanent removal: also destroy feature-set and checkpoint state // explicitly. The 'close' handler usually does this, but a connection that @@ -14243,9 +14247,11 @@ export class AgentFramework { // Recorded here — not eagerly — because injection metadata // carries the CANONICAL channel id (descriptor form), while // send args often carry the raw provider id; this callback - // has the resolved form. The already-open path resolves on + // has the resolved form. A channel confirmed open resolves on // the microtask queue, i.e. before the tool-result event - // that evaluates injections. + // that evaluates injections. One decided open whose transport + // is unconfirmed is opened again first, so its engagement is + // recorded when that open returns. if (status !== 'unknown-channel' && status !== 'ambiguous') { let engaged = this.turnEngagedChannels.get(agentName); if (!engaged) { @@ -14265,7 +14271,8 @@ export class AgentFramework { ); } else if (status === 'open-failed') { console.error( - `[channel] ${agentName}: ${toolName} delivered to ${target} but the open-on-send failed — channel remains half-alive, will reconcile`, + `[channel] ${agentName}: ${toolName} delivered to ${target} but the open-on-send failed; ` + + 'a later registration or delivery tries the open again', ); } }) diff --git a/src/mcpl/channel-registry.ts b/src/mcpl/channel-registry.ts index 9aecdf63d..c3b69e6f0 100644 --- a/src/mcpl/channel-registry.ts +++ b/src/mcpl/channel-registry.ts @@ -52,6 +52,8 @@ const MAX_CHANNEL_LIFECYCLE_ATTEMPTS = 5; class ChannelLifecycleConvergenceError extends Error {} /** A channels/open answered for a channel other than the one it named. */ class ChannelOpenSubstitutionError extends Error {} +/** An operation queued for a server's registrations that were removed. */ +class ChannelLifecycleCancelledError extends Error {} const CHANNEL_LIFECYCLE_LOG_ID = 'mcpl/channel-lifecycle'; /** @@ -62,23 +64,32 @@ const CHANNEL_LIFECYCLE_LOG_ID = 'mcpl/channel-lifecycle'; function plainIdForLog(id: unknown): string { if (typeof id === 'string' && /^[\x21-\x7e]{1,200}$/.test(id)) return id; return typeof id === 'string' - ? `(not a plain id: a ${id.length}-character string)` + ? `(not a plain id: a ${[...id].length}-character string)` : `(not a plain id: ${id === null ? 'null' : typeof id})`; } /** - * The other channel a channels/open answer names, if it names one: its - * `channel.id`, or else the `channelId` of a history item. Each is evidence of - * which channel the server opened. An answer that names no channel at all - * can't be checked, and gives undefined. + * What a channels/open answer says about which channel the server opened: + * its `channel.id`, or else the `channelId` of a history item. Each is + * evidence. A string other than the requested id names another channel (ids + * compare exactly, as the server's traffic is matched). Null or absence names + * no channel, and gives undefined, as does an answer that names none at all. + * Any other value is malformed: it names nothing the host could register. */ -function substitutedChannel(result: ChannelsOpenResult | undefined, requested: string): unknown { - const answered = result?.channel?.id; - if (answered !== undefined && answered !== requested) return answered; +function substitutedChannel( + result: ChannelsOpenResult | undefined, + requested: string, +): { kind: 'other'; id: string } | { kind: 'malformed'; value: unknown } | undefined { + const named = (value: unknown) => { + if (value === undefined || value === null || value === requested) return undefined; + return typeof value === 'string' ? { kind: 'other' as const, id: value } : { kind: 'malformed' as const, value }; + }; + const answered = named(result?.channel?.id); + if (answered) return answered; const history: unknown[] = Array.isArray(result?.history) ? result.history : []; for (const item of history) { - const from = (item as { channelId?: unknown } | null)?.channelId; - if (from !== undefined && from !== requested) return from; + const from = named((item as { channelId?: unknown } | null)?.channelId); + if (from) return from; } return undefined; } @@ -703,6 +714,14 @@ export class ChannelRegistry { // across descriptor replacement. Settled tails release themselves; a // failed operation must not poison later decisions. private channelLifecycleTails = new Map>(); + // Bumped when a server's registrations are removed: an operation queued + // before that is cancelled when it would run, rather than acting on a + // closing connection or on a later re-registration's entries. + private serverRemovals = new Map(); + // Operations queued or running per channel that will still read its + // target before confirming anything. Unlike the tail, which settles a few + // microtasks after its operation's last check, this drops at that check. + private lifecycleOutstanding = new Map(); // Losing an open receipt does not confirm closure. Keep that distinction // private while preserving the public open:boolean availability projection. private unconfirmedChannelTargets = new WeakSet(); @@ -712,6 +731,21 @@ export class ChannelRegistry { this.unconfirmedChannelTargets.add(entry); } + /** Whether the host directs this server's channel lifecycle (§14.1: its + * `channels.lifecycle` grant). Where it doesn't, the server opens and + * closes its own channels, and sends no receipt the host could wait for. */ + private directsLifecycle(serverId: string): boolean { + return CapabilityGrant.of(this.serverRegistry.getServer(serverId)).has('channels.lifecycle'); + } + + /** A registration's entry: nothing about its transport is known until a + * receipt for its target confirms it open or closed. */ + private registerEntry(key: string, serverId: string, descriptor: ChannelDescriptor): void { + const entry: ChannelEntry = { serverId, descriptor, open: false }; + this.channels.set(key, entry); + this.unconfirmedChannelTargets.add(entry); + } + private hasPendingLifecycle(entry: ChannelEntry): boolean { return this.channelLifecycleTails.has(this.lifecycleKey(entry.serverId, entry.descriptor.id)); } @@ -785,11 +819,7 @@ export class ChannelRegistry { continue; } const key = `${serverId}:${channel.id}`; - this.channels.set(key, { - serverId, - descriptor: channel, - open: false, - }); + this.registerEntry(key, serverId, channel); this.appendLabelSighting(channel.id, channel.label, extractDmMeta(channel.metadata)); registeredIds.push(channel.id); results.push({ id: channel.id, accepted: true }); @@ -830,6 +860,7 @@ export class ChannelRegistry { // channels/register — a changed-set cannot smuggle in what register // would have rejected (PR #79 review blocker 5). const addedResults: Array<{ id: string; accepted: boolean; reason?: string }> = []; + const retargeted: ChannelDescriptor[] = []; // Process removed channels — itemized truthfully: removal of a channel // we never had is reported, not silently accepted. if (params.removed) { @@ -860,8 +891,19 @@ export class ChannelRegistry { continue; } // A live subscription to the old target does not confirm the new - // type/address. Label-only updates preserve that confirmation. - if (!sameChannelTarget(existing.descriptor, channel)) this.invalidateTransportConfirmation(existing); + // type/address. Label-only updates preserve that confirmation. A + // channel decided open (or tuned out) is opened at its new target. + if (!sameChannelTarget(existing.descriptor, channel)) { + this.invalidateTransportConfirmation(existing); + // An operation queued or running for this channel reads its target + // again before it confirms anything, so it opens the new one. With + // none, a channel decided open (or tuned out) is reconciled here. + const desired = this.getDesiredState(serverId, channel.id); + if ((desired === 'open' || desired === 'tuned-out') && + !this.lifecycleOutstanding.has(this.lifecycleKey(serverId, channel.id))) { + retargeted.push(channel); + } + } existing.descriptor = channel; this.appendLabelSighting(channel.id, channel.label, extractDmMeta(channel.metadata)); addedResults.push({ id: channel.id, accepted: true }); @@ -879,11 +921,7 @@ export class ChannelRegistry { continue; } const key = `${serverId}:${channel.id}`; - this.channels.set(key, { - serverId, - descriptor: channel, - open: false, - }); + this.registerEntry(key, serverId, channel); this.appendLabelSighting(channel.id, channel.label, extractDmMeta(channel.metadata)); addedResults.push({ id: channel.id, accepted: true }); accepted.push(channel); @@ -898,7 +936,8 @@ export class ChannelRegistry { // settled above, so nothing in the response depends on reconciling. responder?.respond({ results: addedResults }); - if (accepted.length > 0) await this.reconcileChannels(serverId, accepted); + const reconcile = [...accepted, ...retargeted]; + if (reconcile.length > 0) await this.reconcileChannels(serverId, reconcile); this.emitTraceFn({ type: 'mcpl:channels-changed', @@ -996,17 +1035,31 @@ export class ChannelRegistry { // exactly the authority a self-attested channel would be stealing). // A coalesced item is "accepted" only once the coalescer admits it, so // for those this runs after the hook (below). + const incomingEntry = this.channels.get(incomingKey)!; const markAccepted = () => { this.defaultPublishChannel = message.channelId; this.defaultPublishMessageId = message.messageId; this.defaultPublishThreadId = message.threadId; - // A server sending channels/incoming is authoritative evidence that - // the transport is actually open. This repairs transient status only; - // durable desired state still changes exclusively through lifecycle - // operations. - const incomingEntry = this.channels.get(incomingKey)!; + // Where the host directs this server's lifecycle, only a receipt for + // the current target confirms it: traffic shows the transport open + // only for a confirmed target, and only for a channel still decided + // open or tuned out. After a retarget, traffic for this id may still + // come from the old subscription, and a close receipt that landed + // first (or during the coalescer's await) stands. Where the host + // doesn't direct it, the server opens its own channels and its + // traffic is the only evidence the host gets, so it marks the + // channel open, as it always has. The entry is the one the message + // was accepted for: one a re-registration put in its place meanwhile + // waits for its own evidence. Durable desired state still changes + // exclusively through lifecycle operations. + if (this.directsLifecycle(serverId)) { + if (this.unconfirmedChannelTargets.has(incomingEntry)) return; + const desired = this.getDesiredState(serverId, message.channelId); + if (desired !== 'open' && desired !== 'tuned-out') return; + } else { + this.unconfirmedChannelTargets.delete(incomingEntry); + } incomingEntry.open = true; - this.unconfirmedChannelTargets.delete(incomingEntry); }; if (!coalesced) markAccepted(); @@ -1454,6 +1507,10 @@ export class ChannelRegistry { * server's channels don't linger and route speech into the void. */ removeServer(serverId: string): void { + // Every lifecycle operation queued for this server is cancelled when it + // would run (ChannelLifecycleCancelledError), so none acts on a closing + // connection or on a later re-registration's entries. + this.serverRemovals.set(serverId, (this.serverRemovals.get(serverId) ?? 0) + 1); for (const [key, entry] of this.channels) { if (entry.serverId !== serverId) continue; this.channels.delete(key); @@ -2140,11 +2197,42 @@ export class ChannelRegistry { options?: { history?: ChannelHistoryRequest }, ): Promise<{ open: boolean; result?: ChannelsOpenResult }> { const lifecycleKey = this.lifecycleKey(serverId, channelId); - const channelKey = `${serverId}:${channelId}`; + const removals = this.serverRemovals.get(serverId) ?? 0; const previous = this.channelLifecycleTails.get(lifecycleKey) ?? Promise.resolve(); - const operation = previous.catch(() => {}).then(async () => { + this.lifecycleOutstanding.set(lifecycleKey, (this.lifecycleOutstanding.get(lifecycleKey) ?? 0) + 1); + const operation = previous.catch(() => {}).then( + () => this.runLifecycleOperation(serverId, channelId, removals, options), + ); + // Keep the rejection visible to existing joiners, while observing it + // immediately and letting each queued successor start after either result. + const tail = operation.then(() => {}); + this.channelLifecycleTails.set(lifecycleKey, tail); + const cleanup = () => { + if (this.channelLifecycleTails.get(lifecycleKey) === tail) { + this.channelLifecycleTails.delete(lifecycleKey); + } + }; + void tail.then(cleanup, cleanup); + return operation; + } + + /** One queued lifecycle operation, run when its turn in the queue comes. */ + private async runLifecycleOperation( + serverId: string, + channelId: string, + removals: number, + options?: { history?: ChannelHistoryRequest }, + ): Promise<{ open: boolean; result?: ChannelsOpenResult }> { + const channelKey = `${serverId}:${channelId}`; + const lifecycleKey = this.lifecycleKey(serverId, channelId); + try { let attempts = 0; for (;;) { + if ((this.serverRemovals.get(serverId) ?? 0) !== removals) { + throw new ChannelLifecycleCancelledError( + `Channel ${channelId}: its server's registrations were removed while this operation was queued`, + ); + } const entry = this.channels.get(channelKey); if (!entry) throw new Error(`Channel is no longer registered: ${channelId}`); const server = this.serverRegistry.getServer(serverId); @@ -2185,9 +2273,16 @@ export class ChannelRegistry { await server.sendChannelsClose({ channelId }); } } catch (error) { - // A failed same-target backscroll can retain its known-open state. - // A receipt/failure from an older connection or target cannot. - if (this.channels.get(channelKey) === entry && !sameTarget()) this.invalidateTransportConfirmation(entry); + // An RPC can fail after the server acted, so a failure for the + // current target leaves its outcome unknown: it is marked + // unconfirmed with open unchanged, which keeps a confirmed-open + // channel usable after a failed backscroll (the mark is read only + // while not open). A failure from an older connection or target + // also withdraws the open flag. + if (this.channels.get(channelKey) === entry) { + if (sameTarget()) this.unconfirmedChannelTargets.add(entry); + else this.invalidateTransportConfirmation(entry); + } throw error; } // A server can answer for a channel other than the one named: one @@ -2197,11 +2292,14 @@ export class ChannelRegistry { // open it made if the host wants that channel closed. const substitute = open ? substitutedChannel(result, channelId) : undefined; if (substitute !== undefined) { + const answered = plainIdForLog(substitute.kind === 'other' ? substitute.id : substitute.value); console.error( - `[channel-open-substituted] server=${serverId} requested=${plainIdForLog(channelId)} answered=${plainIdForLog(substitute)}`, + `[channel-open-substituted] server=${serverId} requested=${plainIdForLog(channelId)} answered=${answered}`, ); - this.emitTraceFn({ type: 'mcpl:channel-open-substituted', serverId, channelId, answeredFor: substitute }); - if (typeof substitute === 'string') this.closeSubstitutedChannel(serverId, substitute); + this.emitTraceFn({ type: 'mcpl:channel-open-substituted', serverId, channelId, answeredFor: answered }); + // A malformed id names nothing registered here, so there is + // nothing to close. + if (substitute.kind === 'other') this.closeSubstitutedChannel(serverId, substitute.id); } // Never apply an old receipt to the current target, even transiently: // a corrective open might fail before that stale bit is replaced. @@ -2218,8 +2316,9 @@ export class ChannelRegistry { if (substitute !== undefined) { this.invalidateTransportConfirmation(entry); throw new ChannelOpenSubstitutionError( - `Server "${serverId}" answered channels/open for ${channelId} with a different channel; ` + - `${channelId} is not marked open, and nothing in the answer is used`, + `Server "${serverId}" answered channels/open for ${channelId} with ` + + (substitute.kind === 'other' ? 'a different channel' : 'a channel id that is not a string') + + `; ${channelId} is not marked open, and nothing in the answer is used`, ); } entry.open = open; @@ -2228,18 +2327,13 @@ export class ChannelRegistry { if (open !== (now === 'open' || now === 'tuned-out')) continue; return { open, result }; } - }); - // Keep the rejection visible to existing joiners, while observing it - // immediately and letting each queued successor start after either result. - const tail = operation.then(() => {}); - this.channelLifecycleTails.set(lifecycleKey, tail); - const cleanup = () => { - if (this.channelLifecycleTails.get(lifecycleKey) === tail) { - this.channelLifecycleTails.delete(lifecycleKey); - } - }; - void tail.then(cleanup, cleanup); - return operation; + } finally { + // Synchronous with the operation's last look at its target: from here + // on, nothing it does will open a new target. + const left = (this.lifecycleOutstanding.get(lifecycleKey) ?? 1) - 1; + if (left > 0) this.lifecycleOutstanding.set(lifecycleKey, left); + else this.lifecycleOutstanding.delete(lifecycleKey); + } } /** @@ -2255,7 +2349,11 @@ export class ChannelRegistry { if (!this.channels.has(`${serverId}:${channelId}`)) return; if (this.getDesiredState(serverId, channelId) !== 'closed') return; this.applyDesiredChannelState(serverId, channelId).catch((err) => { - if (err instanceof ChannelLifecycleConvergenceError) return; + // Removal while queued is cancellation, as in reconcileChannels, and + // exhausted convergence emitted its own diagnostic. + if (!this.channels.has(`${serverId}:${channelId}`) || + err instanceof ChannelLifecycleCancelledError || + err instanceof ChannelLifecycleConvergenceError) return; this.emitTraceFn({ type: 'mcpl:channel-reconcile-failed', serverId, channelId, desired: 'closed', error: (err as Error).message, @@ -2282,6 +2380,10 @@ export class ChannelRegistry { // the current target, so joined delivery must fail visibly. if (error instanceof ChannelLifecycleConvergenceError) throw error; } + // A settled tail releases its own key. One still current after it + // settled would be awaited again without ever yielding to the event + // loop, so stop waiting rather than spin. + if (this.channelLifecycleTails.get(lifecycleKey) === pending) break; } const current = this.channels.get(`${entry.serverId}:${entry.descriptor.id}`); if (!current) throw new Error('Channel registration was removed while delivery waited'); @@ -2340,8 +2442,10 @@ export class ChannelRegistry { } } - for (const channel of channels) { - if (!this.channels.has(`${serverId}:${channel.id}`)) continue; + // Each channel has its own lifecycle queue, so a held RPC for one (up to + // its timeout, times its attempts) doesn't delay its siblings. + await Promise.allSettled(channels.map(async (channel) => { + if (!this.channels.has(`${serverId}:${channel.id}`)) return; try { await this.applyDesiredChannelState(serverId, channel.id); } catch (err) { @@ -2349,8 +2453,9 @@ export class ChannelRegistry { // Exhausted convergence and a substituted answer already emitted // their own diagnostics. if (!this.channels.has(`${serverId}:${channel.id}`) || + err instanceof ChannelLifecycleCancelledError || err instanceof ChannelLifecycleConvergenceError || - err instanceof ChannelOpenSubstitutionError) continue; + err instanceof ChannelOpenSubstitutionError) return; this.emitTraceFn({ type: 'mcpl:channel-reconcile-failed', serverId, @@ -2359,7 +2464,7 @@ export class ChannelRegistry { error: (err as Error).message, }); } - } + })); } // ========================================================================== @@ -2960,6 +3065,23 @@ export class ChannelRegistry { } } + // Where the host directs the lifecycle, a decline goes through the + // lifecycle queue like any closed decision, unless a receipt confirmed + // the transport closed with nothing pending: an unconfirmed transport may + // be open, and pending work must settle on this decision. A failed close + // is reported; the decision stands. + let closeError: string | undefined; + const current = this.channels.get(`${entry.serverId}:${input.channelId}`); + if (current && this.directsLifecycle(entry.serverId) && + (this.unconfirmedChannelTargets.has(current) || this.hasPendingLifecycle(current))) { + try { + const applied = await this.applyDesiredChannelState(entry.serverId, input.channelId); + if (applied.open) closeError = 'the close was superseded by a newer lifecycle decision'; + } catch (error) { + closeError = (error as Error).message; + } + } + this.appendLifecycleEvent({ kind: 'invitation-declined', serverId: entry.serverId, @@ -2976,6 +3098,7 @@ export class ChannelRegistry { acknowledged, ...(representation ? { representation } : {}), ...(acknowledgmentError ? { acknowledgmentError } : {}), + ...(closeError ? { closeError: `Failed to close the channel's transport: ${closeError}` } : {}), }, }; } diff --git a/src/mcpl/types.ts b/src/mcpl/types.ts index d6c9e21cd..f0eecf44a 100644 --- a/src/mcpl/types.ts +++ b/src/mcpl/types.ts @@ -1049,10 +1049,13 @@ export interface ChannelsListResult { */ export interface ChannelsOpenParams { /** - * The exact registered id to open. A server that doesn't know it should + * The exact registered id to open. Without it, a server matches by `type` + * and `address`, as before. A server that doesn't know the id should * refuse rather than open another channel: the host fails an open whose - * answer names any other channel, by its `channel.id` or a history item's - * `channelId`. + * answer names a different channel id, by its `channel.id` or a history + * item's `channelId`, compared exactly, as the server's traffic is. When + * that other channel is registered and decided closed, the host closes it + * again. */ channelId?: string; type: string; diff --git a/test/channel-lifecycle-order.test.ts b/test/channel-lifecycle-order.test.ts index e916e73d5..a787b74b3 100644 --- a/test/channel-lifecycle-order.test.ts +++ b/test/channel-lifecycle-order.test.ts @@ -10,16 +10,27 @@ import { ChannelRegistry } from '../src/mcpl/channel-registry.js'; import { CapabilityGrant, ALL_CAPABILITY_PATHS } from '../src/mcpl/capability-grant.js'; import type { McplServerRegistry } from '../src/mcpl/server-registry.js'; import type { FeatureSetManager } from '../src/mcpl/feature-set-manager.js'; -import type { ChannelDescriptor, ChannelsOpenResult } from '../src/mcpl/types.js'; +import type { ChannelDescriptor, ChannelIncomingMessage, ChannelsOpenResult } from '../src/mcpl/types.js'; const tick = () => new Promise((resolve) => setImmediate(resolve)); function descriptor(id = 'x', initiallyOpen = false, address = 'original'): ChannelDescriptor { return { id, type: 'test', label: id, direction: 'bidirectional', initiallyOpen, address: { value: address } }; } -type NoticeHandler = NonNullable[4]>['onChannelAutoOpened']>; +type RegistryOptions = NonNullable[4]>; +type NoticeHandler = NonNullable; -function fixture(onNotice?: NoticeHandler, store?: JsStore) { +/** A tool result's data, as the channel tools shape it. */ +const dataOf = (result: { data?: unknown }) => (result.data ?? {}) as { status?: string; closeError?: string }; + +function message(channelId = 'x', messageId = 'm1'): ChannelIncomingMessage { + return { + channelId, messageId, author: { id: 'u1', name: 'someone' }, + timestamp: '2026-10-11T00:00:00Z', content: [{ type: 'text', text: 'hello' }], + }; +} + +function fixture(onNotice?: NoticeHandler, store?: JsStore, coalesce?: RegistryOptions['handleCoalescedIncoming']) { const actual = new Map(); const calls: Array<{ kind: 'open' | 'close'; id: string; address?: unknown }> = []; const pending: Array<{ finish: () => void; fail: () => void }> = []; @@ -59,7 +70,8 @@ function fixture(onNotice?: NoticeHandler, store?: JsStore) { { getServer: () => currentServer } as unknown as McplServerRegistry, {} as FeatureSetManager, () => {}, (event) => traces.push(event), { store, onRouteFailure: (info) => routeFailures.push(info), - onChannelAutoOpened: (info) => { notices.push(info); onNotice?.(info); } }, + onChannelAutoOpened: (info) => { notices.push(info); onNotice?.(info); }, + ...(coalesce ? { handleCoalescedIncoming: coalesce } : {}) }, ); const isOpen = (id = 'x') => registry.getOpenChannels().some((e) => e.descriptor.id === id); const tool = (kind: 'open' | 'close') => registry.handleChannelToolCall('channel_' + kind, { serverId: 'test', channelId: 'x' }); @@ -213,17 +225,22 @@ for (const delivery of ['speech', 'reply'] as const) { // turn a delivery into a decision to cancel attention diversion. const f = fixture(); await f.registry.handleChanged('test', { added: [descriptor('x', true)] }); + f.calls.length = 0; f.held.add('x'); const close = f.tool('close'); await tick(); f.registry.enterTuneOut('test', 'x', { epochId: 'new-epoch', cadenceSeconds: 60, backlogCap: 20, maxWakes: 3, startedAtSequence: 1, }, 'agent-tool'); + // The close in flight converges on the tune-out by opening again, and the + // delivery joins it: the channel was already decided open, so a reply + // reports it already open, and nothing sends a third RPC. const sent = delivery === 'speech' ? f.registry.routeSpeech('resident', 'hello', 'x').then((result) => result?.delivered === true) - : f.registry.openIfClosedForSend('x', 'test').then((result) => result.status === 'opened' || result.status === 'already-open'); + : f.registry.openIfClosedForSend('x', 'test').then((result) => result.status === 'already-open'); await f.drain(close, sent); assert.equal(await sent, true); + assert.deepEqual(f.calls.map((c) => c.kind), ['close', 'open']); assert.equal(f.registry.getTuneOutState('test', 'x')?.params.epochId, 'new-epoch'); assert.equal(f.actual.get('x'), true); if (delivery === 'speech') assert.deepEqual(f.publishedWhileOpen, [true]); @@ -723,8 +740,15 @@ for (const delivery of ['speech', 'reply'] as const) { test(delivery + ' into a channel decided open whose transport lost confirmation is not announced as opening it', async () => { const f = fixture(); await f.registry.handleChanged('test', { added: [descriptor('x', true, 'A')] }); - await f.registry.handleChanged('test', { updated: [descriptor('x', true, 'B')] }); - assert.equal(f.isOpen(), false, 'retargeting leaves the new target unconfirmed'); + // A retarget opens the new target; that open fails, so the channel stays + // decided open with its transport unconfirmed. + f.held.add('x'); + const retarget = f.registry.handleChanged('test', { updated: [descriptor('x', true, 'B')] }); + await tick(); + f.pending.shift()!.fail(); + await retarget; + f.held.delete('x'); + assert.equal(f.isOpen(), false, 'a failed open leaves the new target unconfirmed'); f.calls.length = 0; if (delivery === 'speech') { assert.equal((await f.registry.routeSpeech('resident', 'hello', 'x'))?.delivered, true); @@ -744,7 +768,12 @@ for (const delivery of ['speech', 'reply'] as const) { f.registry.enterTuneOut('test', 'x', { epochId: 'kept-epoch', cadenceSeconds: 60, backlogCap: 20, maxWakes: 3, startedAtSequence: 1, }, 'agent-tool'); - await f.registry.handleChanged('test', { updated: [descriptor('x', true, 'B')] }); + f.held.add('x'); + const retarget = f.registry.handleChanged('test', { updated: [descriptor('x', true, 'B')] }); + await tick(); + f.pending.shift()!.fail(); + await retarget; + f.held.delete('x'); assert.equal(f.isOpen(), false); if (delivery === 'speech') { assert.equal((await f.registry.routeSpeech('resident', 'hello', 'x'))?.delivered, true); @@ -829,24 +858,34 @@ test('a failed reply open reports the label of the registration that replaced it await f.drain(replacement); }); -test('a channel removed earlier in its batch does not hold up the rest of the batch', async () => { +test('a channel removed from its batch holds up none of its siblings, and fails nothing', async () => { const f = fixture(); f.held.add('x'); const first = f.registry.handleChanged('test', { added: [descriptor('x')] }); await tick(); assert.equal(f.pending.length, 1, "x's first operation is in flight"); f.held.add('y'); - let batchSettled = false; - const batch = f.registry.handleChanged('test', { added: [descriptor('y'), descriptor('x')] }) - .then(() => { batchSettled = true; }); + const batch = f.registry.handleChanged('test', { added: [descriptor('y', true), descriptor('x')] }); await tick(); assert.equal(f.pending.length, 2, "y's operation is in flight too"); await f.registry.handleChanged('test', { removed: ['x'] }); f.pending.pop()!.finish(); // y's receipt; x's first operation stays in flight for (let i = 0; i < 5; i++) await tick(); - assert.equal(batchSettled, true, "the batch skips x rather than queueing behind x's operation in flight"); + assert.equal(f.isOpen('y'), true, "y opened while x's operation was still in flight"); await f.drain(first, batch); - assert.equal(f.traces.filter((e) => e.type === 'mcpl:channel-reconcile-failed').length, 0); + assert.equal(f.traces.filter((e) => e.type === 'mcpl:channel-reconcile-failed').length, 0, + 'the removed channel is cancelled, not failed'); +}); + +test("a batch's channels reconcile side by side: one held RPC delays none of its siblings", async () => { + const f = fixture(); + f.held.add('x'); + const batch = f.registry.handleChanged('test', { added: [descriptor('x', true), descriptor('y', true)] }); + for (let i = 0; i < 5; i++) await tick(); + assert.equal(f.isOpen('y'), true, "y's open didn't wait for x's"); + assert.equal(f.isOpen('x'), false); + await f.drain(batch); + assert.equal(f.isOpen('x'), true); }); test('a descriptor the registration rejected is neither admitted nor announced', async () => { @@ -858,3 +897,246 @@ test('a descriptor the registration rejected is neither admitted nor announced', assert.deepEqual(f.notices.map((n) => n.channels.map((c) => c.channelId)), [['y']]); assert.equal(f.registry.getDesiredState('test', ''), undefined); }); + +test('a failed initial close leaves the channel unconfirmed, so channel_close closes it for real', async () => { + const f = fixture(); + f.held.add('x'); + const registration = f.registry.handleChanged('test', { added: [descriptor()] }); + await tick(); + f.pending.shift()!.fail(); + await registration; + f.held.delete('x'); + f.calls.length = 0; + assert.equal(dataOf(await f.tool('close')).status, 'closed'); + assert.deepEqual(f.calls.map((c) => c.kind), ['close'], 'a failed close may have closed nothing'); + assert.equal(dataOf(await f.tool('close')).status, 'already closed', 'its receipt confirms the close'); + assert.equal(f.calls.length, 1); +}); + +test("a re-registration's entry is unconfirmed until a receipt, so channel_close never answers already closed for it", async () => { + const f = fixture(); + await f.registry.handleChanged('test', { added: [descriptor()] }); + assert.equal(dataOf(await f.tool('close')).status, 'already closed', 'its initial close was confirmed'); + // The server registers again, without the lifecycle grant this time: the + // host sends nothing, and nothing confirms the new entry closed. + f.setGrant(ALL_CAPABILITY_PATHS.filter((path) => path !== 'channels.lifecycle')); + f.calls.length = 0; + await f.registry.handleRegister('test', { channels: [descriptor()] }); + assert.equal(f.calls.length, 0); + const result = await f.tool('close'); + assert.equal(result.success, false); + assert.match(String(result.error), /channels\.lifecycle not in/, 'refused for the grant, not answered already closed'); +}); + +test('traffic after a retarget confirms nothing: only a receipt for the new target does', async () => { + const f = fixture(); + await f.registry.handleChanged('test', { added: [descriptor('x', true, 'A')] }); + f.held.add('x'); + const retarget = f.registry.handleChanged('test', { updated: [descriptor('x', true, 'B')] }); + await tick(); + await f.registry.handleIncoming('test', { messages: [message()] }); + assert.equal(f.isOpen(), false, "the old subscription's traffic doesn't confirm the new target"); + f.pending.shift()!.fail(); + await retarget; + assert.equal(f.isOpen(), false, 'and a failed open for the new target leaves it unconfirmed'); +}); + +test('traffic for a channel decided closed leaves its confirmed close standing', async () => { + const f = fixture(); + await f.registry.handleChanged('test', { added: [descriptor('x', true)] }); + assert.equal((await f.tool('close')).success, true); + await f.registry.handleIncoming('test', { messages: [message()] }); + assert.equal(f.isOpen(), false); +}); + +test('a close that lands while the coalescer holds a message stands', async () => { + let admit!: () => void; + const f = fixture(undefined, undefined, async (_serverId, held) => { + await new Promise((resolve) => { admit = resolve; }); + return { messageId: held.messageId, accepted: true }; + }); + await f.registry.handleChanged('test', { added: [descriptor('x', true)] }); + const incoming = f.registry.handleIncoming('test', { messages: [{ ...message(), coalesce: { key: 'subject' } }] }); + await tick(); + assert.equal((await f.tool('close')).success, true); + admit(); + await incoming; + assert.equal(f.isOpen(), false, 'the close receipt stands'); +}); + +test('a retarget of a channel decided open opens the new target; one decided closed sends nothing', async () => { + const f = fixture(); + await f.registry.handleChanged('test', { added: [descriptor('x', true, 'A'), descriptor('y', false, 'A')] }); + f.calls.length = 0; + await f.registry.handleChanged('test', { updated: [descriptor('x', true, 'B'), descriptor('y', false, 'B')] }); + assert.deepEqual(f.calls.map((c) => [c.kind, c.id, c.address]), [['open', 'x', { value: 'B' }]]); + assert.equal(f.isOpen('x'), true); + const closeY = await f.registry.handleChannelToolCall('channel_close', { serverId: 'test', channelId: 'y' }); + assert.equal(dataOf(closeY).status, 'closed', "y's new target is unconfirmed, so its close is sent"); + assert.deepEqual(f.calls.at(-1), { kind: 'close', id: 'y', address: undefined }); +}); + +test("removing a server cancels its queued lifecycle work, even across a re-registration", async () => { + const f = fixture(); + await f.registry.handleChanged('test', { added: [descriptor('x', true)] }); + f.calls.length = 0; + f.held.add('x'); + const closing = f.tool('close'); + await tick(); + const reopening = f.tool('open'); + f.registry.removeServer('test'); + const reregistration = f.registry.handleChanged('test', { added: [descriptor('x', true)] }); + await f.drain(closing, reopening, reregistration); + assert.equal((await reopening).success, false, 'the open queued before the removal never runs'); + assert.match(String((await reopening).error), /removed while this operation was queued/); + assert.deepEqual(f.calls.map((c) => c.kind), ['close', 'open'], "the close in flight, then the re-registration's own open"); + assert.equal(f.isOpen(), true); + assert.equal(f.traces.filter((e) => e.type === 'mcpl:channel-reconcile-failed').length, 0); +}); + +test('a substitute removed while its corrective close is queued is cancelled, not traced as failed', async () => { + const f = fixture(); + f.held.add('y'); + const registration = f.registry.handleChanged('test', { added: [descriptor('x'), descriptor('y')] }); + await tick(); + assert.equal(f.pending.length, 1, "y's initial close is in flight"); + f.answeredFor.set('x', 'y'); + const opening = f.tool('open'); + for (let i = 0; i < 3; i++) await tick(); + await f.registry.handleChanged('test', { removed: ['y'] }); + await f.drain(registration, opening); + assert.equal((await opening).success, false); + assert.equal(f.traces.filter((e) => e.type === 'mcpl:channel-reconcile-failed').length, 0); +}); + +test('declining an invitation closes an unconfirmed transport for real, and waits for pending work', async () => { + const f = fixture(); + const decline = () => f.registry.handleChannelToolCall('channel_decline', { serverId: 'test', channelId: 'x', messageId: 'm1' }); + f.held.add('x'); + const registration = f.registry.handleChanged('test', { added: [descriptor()] }); + await tick(); + f.pending.shift()!.fail(); + await registration; + f.held.delete('x'); + f.calls.length = 0; + const unconfirmed = await decline(); + assert.equal(unconfirmed.success, true); + assert.equal(dataOf(unconfirmed).status, 'remained closed'); + assert.equal(dataOf(unconfirmed).closeError, undefined); + assert.deepEqual(f.calls.map((c) => c.kind), ['close'], 'the unconfirmed transport is closed for real'); + f.calls.length = 0; + assert.equal((await decline()).success, true); + assert.equal(f.calls.length, 0, 'a confirmed close with nothing pending needs no RPC'); + + // A re-registration's close in flight: the decline queues behind it. + f.held.add('x'); + const reregistration = f.registry.handleChanged('test', { removed: ['x'], added: [descriptor()] }); + await tick(); + assert.equal(f.pending.length, 1); + const pending = decline(); + await f.drain(reregistration, pending); + assert.equal((await pending).success, true); + assert.deepEqual(f.calls.map((c) => c.kind), ['close', 'close'], 'the decline went through the queue behind the close'); +}); + +test("a settled operation releases its channel's queue key", async () => { + const f = fixture(); + await f.registry.handleChanged('test', { added: [descriptor('x', true)] }); + const internals = f.registry as unknown as { channelLifecycleTails: Map; lifecycleOutstanding: Map }; + for (let i = 0; i < 3; i++) await tick(); + assert.equal(internals.channelLifecycleTails.size, 0); + assert.equal(internals.lifecycleOutstanding.size, 0); +}); + +test('the notices say what happened, name the opt-out, and state no rule about later sends', async () => { + const dir = mkdtempSync(join(tmpdir(), 'channel-notice-text-')); + const framework = await AgentFramework.create({ + storePath: join(dir, 'store'), membrane: new MockMembrane().asMembrane(), + agents: [{ name: 'resident', model: 'test-model', systemPrompt: 'Test.' }], + modules: [], syncIntervalMs: 0, + }); + try { + const recordNotice = (framework as unknown as { + recordChannelAutoOpenNotice( + agent: undefined, + channels: Array<{ channelId: string; label?: string }>, + cause: 'subscription-policy' | 'opened-by-delivery' | 'opened-by-reply', + ): void; + }).recordChannelAutoOpenNotice.bind(framework); + recordNotice(undefined, [{ channelId: 'discord:g1:c1', label: '#general' }], 'opened-by-reply'); + recordNotice(undefined, [{ channelId: 'c2' }], 'opened-by-delivery'); + recordNotice(undefined, [{ channelId: 'c3' }, { channelId: 'c4', label: 'ops' }], 'subscription-policy'); + const texts = framework.getAgent('resident')!.getContextManager().getAllMessages() + .flatMap((m) => m.content) + .filter((b) => b.type === 'text' && b.text.startsWith('[channels]')) + .map((b) => (b as { text: string }).text); + assert.deepEqual(texts, [ + "[channels] Now open: #general (discord:g1:c1): your own message into the closed channel opened it. " + + "Ongoing traffic from this channel will reach you. If you don't want it, use channel_close; your choice outranks policy.", + "[channels] Now open: c2: your own message into the closed channel opened it. " + + "Ongoing traffic from this channel will reach you. If you don't want it, use channel_close; your choice outranks policy.", + '[channels] Selected for opening by subscription policy: c3, ops (c4). ' + + 'This records the admission decision; channel_list shows current transport state. ' + + "When open, their ongoing traffic will reach you. If you don't want one of them, use channel_close; your choice outranks policy.", + ]); + } finally { + await framework.stop(); + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("a server the host doesn't direct opens its own channels: its traffic marks them open, and a decline sends nothing", async () => { + const f = fixture(); + f.setGrant(ALL_CAPABILITY_PATHS.filter((path) => path !== 'channels.lifecycle')); + await f.registry.handleChanged('test', { added: [descriptor()] }); + await f.registry.handleIncoming('test', { messages: [message()] }); + assert.equal(f.isOpen(), true, 'traffic is the only evidence such a server gives'); + await f.registry.handleChanged('test', { updated: [descriptor('x', false, 'B')] }); + assert.equal(f.isOpen(), false); + await f.registry.handleIncoming('test', { messages: [message('x', 'm2')] }); + assert.equal(f.isOpen(), true); + await f.registry.handleChanged('test', { added: [descriptor('y')] }); + const declined = await f.registry.handleChannelToolCall('channel_decline', { serverId: 'test', channelId: 'y', messageId: 'm3' }); + assert.equal(declined.success, true); + assert.equal(dataOf(declined).closeError, undefined); + assert.equal(f.calls.length, 0, 'the host sends no lifecycle RPC to a server it does not direct'); +}); + +test('a failed open leaves the channel unconfirmed: traffic afterwards marks nothing open', async () => { + const f = fixture(); + await f.registry.handleChanged('test', { added: [descriptor()] }); + f.held.add('x'); + const opening = f.tool('open'); + await tick(); + f.pending.shift()!.fail(); + assert.equal((await opening).success, false); + f.held.delete('x'); + await f.registry.handleIncoming('test', { messages: [message()] }); + assert.equal(f.isOpen(), false, 'an RPC can fail after the server acted, and traffic does not settle which'); +}); + +test('disconnecting a server removes its channel entries before its connection closes', async () => { + const dir = mkdtempSync(join(tmpdir(), 'disconnect-order-')); + const framework = await AgentFramework.create({ + storePath: join(dir, 'store'), membrane: new MockMembrane().asMembrane(), + agents: [{ name: 'resident', model: 'test-model', systemPrompt: 'Test.' }], + modules: [], syncIntervalMs: 0, + }); + try { + const f = fixture(); + await f.registry.handleChanged('test', { added: [descriptor('x', true)] }); + let channelsAtClose: number | undefined; + const internals = framework as unknown as { channelRegistry: ChannelRegistry | null; mcplServerRegistry: unknown }; + internals.channelRegistry = f.registry; + internals.mcplServerRegistry = { + removeServer: async () => { channelsAtClose = f.registry.listChannelsRaw().length; }, + getAllServers: () => [], + closeAll: async () => {}, + }; + await framework.disconnectMcplServer('test'); + assert.equal(channelsAtClose, 0, 'nothing queued for its channels can run against the closing connection'); + } finally { + await framework.stop(); + rmSync(dir, { recursive: true, force: true }); + } +}); diff --git a/test/channel-open-substitution.test.ts b/test/channel-open-substitution.test.ts index bfecb4578..34fa8b12e 100644 --- a/test/channel-open-substitution.test.ts +++ b/test/channel-open-substitution.test.ts @@ -202,6 +202,34 @@ test('odd ids are described in the log line, never printed raw', async () => { assert.deepEqual(logged, [ `[channel-open-substituted] server=test requested=x answered=(not a plain id: a ${forged.length}-character string)`, ]); + assert.equal(f.substituted()[0]?.answeredFor, `(not a plain id: a ${forged.length}-character string)`, + 'the trace records the same bounded form, not the raw value'); +}); + +test('an odd id is measured in characters, not UTF-16 units', async () => { + const f = fixture(() => ({ channel: { ...descriptor('y'), id: 'y\u{1F600}' } })); + await f.registry.handleChanged('test', { added: [descriptor('x')] }); + const { lines } = await capturingErrors(() => f.openX()); + assert.deepEqual(lines.filter((l) => l.includes('channel-open-substituted')), + ['[channel-open-substituted] server=test requested=x answered=(not a plain id: a 2-character string)']); +}); + +test('an answer whose channel id is not a string fails the open, and closes nothing', async () => { + for (const answer of [ + () => ({ channel: { ...descriptor('y'), id: 5 } }) as unknown as ChannelsOpenResult, + () => ({ history: [{ ...historyOf('y')[0]!, channelId: false }] }) as unknown as ChannelsOpenResult, + ]) { + const f = fixture(answer); + await f.registry.handleChanged('test', { added: [descriptor('x'), descriptor('y')] }); + f.calls.length = 0; + const { value: result } = await capturingErrors(() => f.openX()); + await tick(); + assert.equal(result.success, false); + assert.match(String(result.error), /with a channel id that is not a string/); + assert.equal(f.isOpen('x'), false); + assert.match(String(f.substituted()[0]?.answeredFor), /^\(not a plain id: (number|boolean)\)$/); + assert.deepEqual(f.calls, [{ kind: 'open', id: 'x' }], 'nothing registered by that id to close'); + } }); test('an answer for the requested channel, with or without its history, or one naming no channel, opens it as before', async () => { @@ -209,6 +237,9 @@ test('an answer for the requested channel, with or without its history, or one n (requested: string) => ({ channel: descriptor(requested), history: historyOf(requested) }), (requested: string) => ({ channel: descriptor(requested) }), () => ({}) as ChannelsOpenResult, + // A null id names no channel, in the answer or in a history item. + (requested: string) => ({ channel: { ...descriptor(requested), id: null } }) as unknown as ChannelsOpenResult, + (requested: string) => ({ history: [{ ...historyOf(requested)[0]!, channelId: null }] }) as unknown as ChannelsOpenResult, ]) { const f = fixture(answer); await f.registry.handleChanged('test', { added: [descriptor('x'), descriptor('y')] }); diff --git a/test/mcpl-gate-roundtrip.test.ts b/test/mcpl-gate-roundtrip.test.ts index 3ede04df9..1d10485b4 100644 --- a/test/mcpl-gate-roundtrip.test.ts +++ b/test/mcpl-gate-roundtrip.test.ts @@ -35,7 +35,9 @@ function makeGate(config: GateConfig) { function makeRegistry(shouldTriggerInference?: (c: string, m: Record) => boolean) { const pushed: ProcessEvent[] = []; const registry = new ChannelRegistry( - {} as never, + // No server connection: the host directs no channel lifecycle here, so + // traffic is what marks a channel open. + { getServer: () => undefined } as never, {} as never, (ev) => pushed.push(ev), () => {}, From f56d854f2f17a0dee951bcd05fb45702dfe384f7 Mon Sep 17 00:00:00 2001 From: Agnes-1009 Date: Sun, 11 Oct 2026 17:09:08 -0400 Subject: [PATCH 0120/1112] fix(mcp): the tool-result normalizer drops nothing silently, keeps inline only what a provider takes, and a failed save is a lost payload A review of this change found that the standard tool-result reading still lost payloads silently on several paths, inlined what a provider would refuse, and let a failed workspace save escape as a dispatch failure. This commit is that part of the round: src/mcpl/tool-result-normalize.ts and the framework's save closure for it. **Nothing dropped silently** - A failed result is read by the model only as its error text. That covers isError, a payload that couldn't be retained, and malformed content. So there an inline image is a payload like audio: saved to the workspace and named, or marked not retained. A not-retained image counts toward the "N payload(s) could not be retained" total. - An image or audio block with inline data but no mimeType has inline data. It's saved as application/octet-stream and stubbed, instead of being reported as having none. - A malformed entry gets a stub saying it was left out. That covers an entry that isn't an object, and a text block whose text isn't a string. Content that is present but isn't an array makes the result a failure ("malformed result"), not an empty success. Absent content stays no content, as a structured-only result may send it. **What stays inline** - An inline image stays native only when its type is one providers take (png, jpeg, gif, webp, compared without parameters or case, and passed on in canonical spelling), and its decoded size is within 5 MB. That's the same ceiling the framework's read_image guard uses. Any other image is saved and stubbed. A saved stub that wasn't shown for another reason than its type says why ("over the 5.0 MB a model takes per image", "a failed result is read as its text alone"). - A native image is passed on without a wrapped encoding's whitespace, which a provider may refuse, and with it the whole request. - An unknown block is still shown as JSON, but bounded like a stub field. A note says how many characters were left out, so a large or binary payload can't flood the model's view. **Bounds and names** - A URI is the reference itself, so resource links, a media block's uri and a resource's uri are bounded at 2048 code points, not 200. - Stub text is bounded by code points, so a cut never splits a surrogate pair. - A payload's decoded size is computed with its whitespace removed, so a wrapped base64 payload isn't overstated. - The extension table is read through Object.hasOwn, so a subtype such as "constructor" falls through to the sanitized name (or "bin"). It used to make Object's own constructor, stringified, the file's extension. **A failed save** - The framework's save closure catches a rejected writeBinary. Like a refused write, it logs why and resolves null, so the payload is a lost part of a completed call's result, not a dispatch failure. - The normalizer logs a save that rejects, naming the file, the kind, the type and the size, where it used to swallow it with .catch(() => null). - Server-chosen types in those log lines are JSON-quoted. The module doc says the rules above. It also corrects what it said programs get: the direct path returns the server's raw content, and a script receives the server's result whole only when it carries structuredContent. An unstructured result reaches a script as this reading, with its payloads saved and stubbed. The test file's header no longer cites the house's room and shelf. The changelog fragment for standard tool results says the new rules: which images stay native, what is saved, a failed result's images, unknown and malformed blocks, and the logged save failure. The release script folds it cleanly. **Tests** - mcp-tool-result-normalize, 11 to 21: - a large unknown block, bounded with its omitted count; - a long real URI kept whole, and a pathological one cut at 2048; - an isError result's image saved, or marked not retained, and an image beside a lost payload counted; - image and audio without mimeType saved as octet-stream; - another image type saved; an image one byte over 5 MB saved and one exactly at it kept; a parameterized, upper-case type passed canonically; - a rejecting save logged and lost; - malformed entries stubbed, non-array content a malformed result, and absent content still fine; - a cut that would have split a surrogate pair; a CRLF-wrapped payload's size; - "constructor" and "__proto__" subtypes; - a CRLF-wrapped native image passed without its wrapping. The bounded-field test now bounds a name, not a URI. - mcp-modern-framework, 14 to 15: a save whose writeBinary rejects, and one the workspace refuses, both resolve to a lost payload with why logged. The incomplete-result test now counts the inline image beside the lost audio and blob: 3 payload(s), not 2. **The check, from clean worktrees as AF's CI runs it** (npm install, tsc --noEmit, npm run build, npm test; Node 24.19.0, the Node npm's scripts ran on too): - **This commit:** tsc and the build are clean. npm test reported 1483 tests across 125 suites, all passing, with the eleven new ones among them. Five tests at the end of channel-registry-routing.test.ts went unreported, with nothing failing. A file's last reports are cut short under --test-force-exit at times, and its parent's run did report those five. - **Its parent:** 1477 of 1477. - **Red first:** this commit's two test files, against the parent's sources, fail exactly the eleven new tests, plus the incomplete-result test's count of 3. The other 24 pass. - **Removals:** each of fourteen, taken out alone, fails the test aimed at it: - the failed result's image kept native; - data without a type read as no data; - the unlogged rejected save; - the unbounded unknown block; - any image type kept native; - no size ceiling; - the URI cut at 200; - a dropped malformed entry; - a non-list read as an empty success; - a cut by UTF-16 units; - the size counting whitespace; - the prototype extension; - a native image keeping its wrapping; - the save closure rethrowing. Co-Authored-By: Agnes (Claude Opus 5.5) --- .../mcp-standard-tool-results.changed.md | 22 +- src/framework.ts | 13 +- src/mcpl/tool-result-normalize.ts | 203 +++++++++++++----- test/mcp-modern-framework.test.ts | 33 ++- test/mcp-tool-result-normalize.test.ts | 159 +++++++++++++- 5 files changed, 363 insertions(+), 67 deletions(-) diff --git a/changelog.d/mcp-standard-tool-results.changed.md b/changelog.d/mcp-standard-tool-results.changed.md index 27782b0a5..adfe0d362 100644 --- a/changelog.d/mcp-standard-tool-results.changed.md +++ b/changelog.d/mcp-standard-tool-results.changed.md @@ -1,14 +1,22 @@ - What the model sees of an MCP tool result from a server that isn't an MCPL peer (an MCP-only legacy server, or a modern server) now follows its content types: - - text stays text, and inline images stay native; - - audio and binary resources are saved to the workspace (`tool-results/`), - each under a unique name, with a bounded stub saying where. If a payload - can't be saved, the result is reported as incomplete (`isError`), noting - the tool may already have completed, never as a success with part of it - missing; + - text stays text; + - an inline image stays native when a provider can take it: png, jpeg, + gif or webp, within 5 MB decoded; + - audio, binary resources, other images, and media with inline data but + no type are saved to the workspace (`tool-results/`), each under a + unique name, with a bounded stub saying where. If a payload can't be + saved, the result is reported as incomplete (`isError`), noting the tool + may already have completed, never as a success with part of it missing, + and the host's log says why the save failed; + - a failed result is read as its text alone, so there an inline image is + saved and stubbed like any other payload; - a `resource_link` is shown as a reference and never fetched; - - an embedded text resource shows its text. + - an embedded text resource shows its text; + - an unknown block is shown as JSON, bounded, saying how much was left + out. A malformed entry is named and left out, and content that isn't a + list of blocks makes the result a failure. Before, any non-text block turned the whole result into raw JSON. `structuredContent` is kept by presence on the new `ToolResult.structured` diff --git a/src/framework.ts b/src/framework.ts index 194b0fee1..8cd3c75ef 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -15453,11 +15453,20 @@ export class AgentFramework { const label = `${new Date().toISOString().slice(0, 10)}-${callId}`.replace(/[^A-Za-z0-9._-]/g, '_').slice(0, 80) + `-${randomUUID()}`; const workspace = this.getWorkspaceModule(); const mount = workspace ? this.firstWritableMountName(workspace) : null; + // A save that fails resolves null, never rejects: the payload is then a + // lost part of a completed call's result, not a dispatch failure. Why it + // failed goes to the host's log; the stub says only that it did. const save = workspace && mount ? async (fileName: string, bytes: Buffer, mimeType: string) => { const path = `${mount}/tool-results/${fileName}`; - const written = await workspace.writeBinary(path, bytes, mimeType); - return written.success ? path : null; + try { + const written = await workspace.writeBinary(path, bytes, mimeType); + if (written.success) return path; + console.error(`[mcp] saving tool payload ${path} (${JSON.stringify(mimeType)}) failed: ${written.error ?? 'the workspace refused the write'}`); + } catch (error) { + console.error(`[mcp] saving tool payload ${path} (${JSON.stringify(mimeType)}) failed:`, error); + } + return null; } : null; return normalizeStandardToolResult(result, label, save); diff --git a/src/mcpl/tool-result-normalize.ts b/src/mcpl/tool-result-normalize.ts index 2f229c4e5..a0adf48e7 100644 --- a/src/mcpl/tool-result-normalize.ts +++ b/src/mcpl/tool-result-normalize.ts @@ -5,25 +5,27 @@ * uri-form media as references), which is not this module's job. * * An MCP result has two views, and each consumer gets its own: - * - **The content array is the server's model-facing view.** Text stays text - * and inline images stay native. Anything the provider format can't carry - * (audio, binary blobs) is saved to the workspace, and a bounded stub says - * where it went: a stub tells the reader where the payload is; it never - * replaces it silently. If a payload can't be saved, the result is a - * failure to materialize it, not a success. A `resource_link` is shown as a reference and never - * fetched. An embedded resource shows its text, or is saved like a blob. + * - **The content array is the server's model-facing view.** Text stays text, + * and an inline image stays native when a provider can take it: one of the + * types providers accept, within their per-image size. Anything else the + * provider format can't carry (audio, binary blobs, other images) is saved + * to the workspace, and a bounded stub says where it went. A stub tells the + * reader where the payload is; nothing is dropped silently. If a payload + * can't be saved, the result is a failure to materialize it, not a + * success. A `resource_link` is shown as a reference and never fetched. An + * embedded resource shows its text, or is saved like a blob. * - **`structuredContent` is the machine-readable view.** It is kept by * presence on `ToolResult.structured`. The model sees it rendered as JSON * only when the server's content has no text block of its own, because * otherwise the server already chose what the model reads. * - * This is the model's reading. Programs get the server's result itself: - * the direct path returns the raw content, and a script receives a - * structured result whole (framework.ts). + * A failed result is read by the model only as its error text, so there an + * inline image is saved and stubbed too, like any other payload. * - * Before this, a structured-only result reached the model as an empty string, - * and a mixed result became one JSON string with its image as base64 text - * (room-284 probe P1c). + * This is the model's reading. The direct path returns the server's raw + * content instead. A script receives the server's result whole only when it + * carries structuredContent; an unstructured one reaches a script as this + * reading, its payloads saved to the workspace and stubbed (framework.ts). */ import type { ToolResult } from '../types/index.js'; @@ -32,18 +34,61 @@ import type { ToolResult } from '../types/index.js'; export type SaveToolPayload = (fileName: string, bytes: Buffer, mimeType: string) => Promise; /** Stub fields are bounded independently of anything the server sends. */ -const STUB_FIELD_CHARS = 200; +const STUB_FIELD_POINTS = 200; +/** A URI is the reference itself, so it's bounded far more loosely than a + * stub field: a real one survives, and a pathological one can't flood the + * model's view. */ +const URI_POINTS = 2048; +/** The image types providers take inline. Any other image is saved. */ +const NATIVE_IMAGE_TYPES = new Set(['image/png', 'image/jpeg', 'image/gif', 'image/webp']); +/** The decoded size providers take per inline image: the same 5 MB ceiling + * the framework's read_image tool guards. A larger image is saved. */ +const NATIVE_IMAGE_MAX_BYTES = 5 * 1024 * 1024; + +/** Why a payload isn't shown inline, unless said otherwise. */ +const UNSUPPORTED = "the model can't take this type"; type Block = { type: 'text'; text: string } | { type: 'image'; data: string; mimeType: string }; -function bounded(value: unknown): string { - const text = typeof value === 'string' ? value : JSON.stringify(value) ?? ''; - return text.length > STUB_FIELD_CHARS ? `${text.slice(0, STUB_FIELD_CHARS)}…` : text; +/** The first `max` code points of `text`, never splitting a surrogate pair, + * and how many code points were left out. */ +function headPoints(text: string, max: number): { head: string; omitted: number } { + let end = 0; + let points = 0; + while (end < text.length && points < max) { + end += text.codePointAt(end)! > 0xffff ? 2 : 1; + points++; + } + let omitted = 0; + for (let i = end; i < text.length; omitted++) i += text.codePointAt(i)! > 0xffff ? 2 : 1; + return { head: text.slice(0, end), omitted }; +} + +/** Any value as JSON (a string quoted too). Never throws. */ +function asJson(value: unknown): string { + try { + return JSON.stringify(value) ?? 'null'; + } catch { + return Object.prototype.toString.call(value); + } +} + +/** Any value as text: a string as itself, anything else as JSON. Never throws. */ +function asText(value: unknown): string { + return typeof value === 'string' ? value : asJson(value); } +function bounded(value: unknown, max = STUB_FIELD_POINTS): string { + const { head, omitted } = headPoints(asText(value), max); + return omitted > 0 ? `${head}…` : head; +} + +/** A payload's decoded size, its base64 read without the whitespace a + * wrapped encoding carries. */ function decodedSize(base64: string): number { - const padding = base64.endsWith('==') ? 2 : base64.endsWith('=') ? 1 : 0; - return Math.max(0, Math.floor((base64.length * 3) / 4) - padding); + const compact = base64.replace(/\s+/g, ''); + const padding = compact.endsWith('==') ? 2 : compact.endsWith('=') ? 1 : 0; + return Math.max(0, Math.floor((compact.length * 3) / 4) - padding); } function humanSize(bytes: number): string { @@ -52,13 +97,24 @@ function humanSize(bytes: number): string { return `${(bytes / (1024 * 1024)).toFixed(1)} MB`; } +const KNOWN_EXTENSIONS: Record = { 'octet-stream': 'bin', 'mpeg': 'mp3', 'plain': 'txt', 'x-wav': 'wav' }; + function extensionFor(mimeType: string): string { const subtype = mimeType.split('/')[1]?.split(';')[0]?.trim().toLowerCase() ?? ''; - const known: Record = { 'octet-stream': 'bin', 'mpeg': 'mp3', 'plain': 'txt', 'x-wav': 'wav' }; - const ext = known[subtype] ?? subtype.replace(/[^a-z0-9]/g, ''); + // The table's own keys only: a subtype such as 'constructor' must not find + // what Object.prototype holds. + const ext = Object.hasOwn(KNOWN_EXTENSIONS, subtype) ? KNOWN_EXTENSIONS[subtype]! : subtype.replace(/[^a-z0-9]/g, ''); return ext.length > 0 && ext.length <= 10 ? ext : 'bin'; } +/** The image type a provider takes inline, in its canonical spelling, or + * null when it takes none of these. */ +function nativeImageType(mimeType: unknown): string | null { + if (typeof mimeType !== 'string') return null; + const base = mimeType.split(';')[0]!.trim().toLowerCase(); + return NATIVE_IMAGE_TYPES.has(base) ? base : null; +} + export interface StandardToolResult { content?: unknown; isError?: boolean; @@ -70,56 +126,81 @@ export interface StandardToolResult { * `label` names saved payloads (one per call; payloads get an index suffix). * A payload that can't be retained (no `save`, or saving failed) makes the * result a failure to materialize, never a success with part of it lost. + * Content that is present but isn't a list of blocks makes it a malformed + * result, never an empty success. */ export async function normalizeStandardToolResult( result: StandardToolResult, label: string, save: SaveToolPayload | null, ): Promise { - const raw = Array.isArray(result.content) ? result.content : []; + const malformed = result.content !== undefined && !Array.isArray(result.content); + const raw: unknown[] = Array.isArray(result.content) ? result.content : []; const blocks: Block[] = []; let saved = 0; /** Payloads that could be neither shown nor retained. */ let lost = 0; - const payloadStub = async (kind: string, base64: string, mimeType: string, origin?: string): Promise => { + const payloadStub = async ( + kind: string, + base64: string, + mimeType: string, + opts: { origin?: string; why?: string } = {}, + ): Promise => { + const why = opts.why ?? UNSUPPORTED; const size = humanSize(decodedSize(base64)); - const what = `[${kind}${origin ? ` ${bounded(origin)}` : ''}: ${bounded(mimeType)}, ${size}`; + const what = `[${kind}${opts.origin ? ` ${bounded(opts.origin, URI_POINTS)}` : ''}: ${bounded(mimeType)}, ${size}`; if (!save) { lost++; - return `${what}. Not retained: the model can't take this type, and no workspace is mounted to save it]`; + return `${what}. Not retained: ${why}, and no workspace is mounted to save it]`; } - const path = await save(`${label}-${++saved}.${extensionFor(mimeType)}`, Buffer.from(base64, 'base64'), mimeType) - .catch(() => null); - if (path) return `${what}, saved to workspace file ${path}]`; + const fileName = `${label}-${++saved}.${extensionFor(mimeType)}`; + const path = await save(fileName, Buffer.from(base64, 'base64'), mimeType).catch((error: unknown) => { + // The stub says only that saving failed; why goes to the host's log. + console.error(`[mcp] saving tool payload ${fileName} (${kind}, ${JSON.stringify(bounded(mimeType))}, ${size}) failed:`, error); + return null; + }); + if (path) return `${what}, saved to workspace file ${path}${why === UNSUPPORTED ? '' : `; not shown inline: ${why}`}]`; lost++; - return `${what}. Not retained: the model can't take this type, and saving it to the workspace failed]`; + return `${what}. Not retained: ${why}, and saving it to the workspace failed]`; }; for (const entry of raw) { - const block = (entry && typeof entry === 'object' ? entry : {}) as Record; + if (!entry || typeof entry !== 'object' || Array.isArray(entry)) { + blocks.push({ type: 'text', text: `[malformed content entry, left out: ${bounded(entry)}]` }); + continue; + } + const block = entry as Record; switch (block.type) { case 'text': - if (typeof block.text === 'string') blocks.push({ type: 'text', text: block.text }); + blocks.push(typeof block.text === 'string' + ? { type: 'text', text: block.text } + : { type: 'text', text: '[malformed text block, left out: its text is not a string]' }); break; case 'image': - if (typeof block.data === 'string' && typeof block.mimeType === 'string') { - blocks.push({ type: 'image', data: block.data, mimeType: block.mimeType }); - } else { - blocks.push({ type: 'text', text: `[image without inline data, not fetched${typeof block.uri === 'string' ? `: ${bounded(block.uri)}` : ''}]` }); + case 'audio': { + const kind = block.type; + if (typeof block.data !== 'string') { + blocks.push({ type: 'text', text: `[${kind} without inline data, not fetched${typeof block.uri === 'string' ? `: ${bounded(block.uri, URI_POINTS)}` : ''}]` }); + break; } - break; - case 'audio': - if (typeof block.data === 'string' && typeof block.mimeType === 'string') { - blocks.push({ type: 'text', text: await payloadStub('audio', block.data, block.mimeType) }); + // Inline data with no type is still a payload, of no known type. + const mimeType = typeof block.mimeType === 'string' ? block.mimeType : 'application/octet-stream'; + const native = kind === 'image' ? nativeImageType(block.mimeType) : null; + if (native && decodedSize(block.data) <= NATIVE_IMAGE_MAX_BYTES) { + // Passed on without a wrapped encoding's whitespace, which a + // provider may refuse, and with it the whole request. + blocks.push({ type: 'image', data: block.data.replace(/\s+/g, ''), mimeType: native }); } else { - blocks.push({ type: 'text', text: `[audio without inline data, not fetched${typeof block.uri === 'string' ? `: ${bounded(block.uri)}` : ''}]` }); + const why = native ? `over the ${humanSize(NATIVE_IMAGE_MAX_BYTES)} a model takes per image` : UNSUPPORTED; + blocks.push({ type: 'text', text: await payloadStub(kind, block.data, mimeType, { why }) }); } break; + } case 'resource_link': { const name = typeof block.name === 'string' ? block.name : undefined; const mime = typeof block.mimeType === 'string' ? `, ${bounded(block.mimeType)}` : ''; - blocks.push({ type: 'text', text: `[resource link${name ? ` "${bounded(name)}"` : ''}: ${bounded(block.uri)}${mime}; not fetched]` }); + blocks.push({ type: 'text', text: `[resource link${name ? ` "${bounded(name)}"` : ''}: ${bounded(block.uri, URI_POINTS)}${mime}; not fetched]` }); break; } case 'resource': { @@ -127,17 +208,33 @@ export async function normalizeStandardToolResult( const uri = typeof resource.uri === 'string' ? resource.uri : undefined; const mime = typeof resource.mimeType === 'string' ? resource.mimeType : undefined; if (typeof resource.text === 'string') { - blocks.push({ type: 'text', text: `[resource ${bounded(uri ?? '(no uri)')}${mime ? `, ${bounded(mime)}` : ''}]\n${resource.text}` }); + blocks.push({ type: 'text', text: `[resource ${uri === undefined ? '(no uri)' : bounded(uri, URI_POINTS)}${mime ? `, ${bounded(mime)}` : ''}]\n${resource.text}` }); } else if (typeof resource.blob === 'string') { - blocks.push({ type: 'text', text: await payloadStub('resource', resource.blob, mime ?? 'application/octet-stream', uri) }); + blocks.push({ type: 'text', text: await payloadStub('resource', resource.blob, mime ?? 'application/octet-stream', { origin: uri }) }); } else { - blocks.push({ type: 'text', text: `[resource ${bounded(uri ?? '(no uri)')} with no contents]` }); + blocks.push({ type: 'text', text: `[resource ${uri === undefined ? '(no uri)' : bounded(uri, URI_POINTS)} with no contents]` }); } break; } - default: - // Unknown to this client: shown as data rather than dropped. - blocks.push({ type: 'text', text: JSON.stringify(entry) ?? '' }); + default: { + // Unknown to this client: shown as data rather than dropped, bounded + // like a stub field, so a large or binary payload can't flood the + // model's view. + const { head, omitted } = headPoints(asText(entry), STUB_FIELD_POINTS); + blocks.push({ type: 'text', text: omitted > 0 ? `${head}… [${omitted} more characters of this block left out]` : head }); + } + } + } + + const failing = !!result.isError || lost > 0 || malformed; + if (failing) { + // A failed result is read only as its text: an inline image becomes a + // payload like any other, saved and named, or marked not retained. + for (let i = 0; i < blocks.length; i++) { + const block = blocks[i]!; + if (block.type === 'image') { + blocks[i] = { type: 'text', text: await payloadStub('image', block.data, block.mimeType, { why: 'a failed result is read as its text alone' }) }; + } } } @@ -146,10 +243,10 @@ export async function normalizeStandardToolResult( // model. Stubs this module wrote are notices, not the server's text. const serverText = raw.some((entry) => { const block = entry as { type?: unknown; text?: unknown } | null; - return !!block && block.type === 'text' && typeof block.text === 'string' && block.text.trim().length > 0; + return !!block && typeof block === 'object' && block.type === 'text' && typeof block.text === 'string' && block.text.trim().length > 0; }); if (hasStructured && !serverText) { - blocks.push({ type: 'text', text: JSON.stringify(result.structuredContent) ?? 'null' }); + blocks.push({ type: 'text', text: asJson(result.structuredContent) }); } const text = blocks.filter((b): b is Extract => b.type === 'text').map((b) => b.text).join('\n'); @@ -157,6 +254,14 @@ export async function normalizeStandardToolResult( if (result.isError) { return { success: false, error: text || 'Tool call failed', isError: true, ...structured }; } + if (malformed) { + return { + success: false, + error: `[malformed result: its content is not a list of blocks; the tool may already have completed]${text ? `\n${text}` : ''}`, + isError: true, + ...structured, + }; + } if (lost > 0) { // The server answered, but part of its answer is gone: report that as a // failure to materialize the result, never as a success. diff --git a/test/mcp-modern-framework.test.ts b/test/mcp-modern-framework.test.ts index 570993c81..0eac21d63 100644 --- a/test/mcp-modern-framework.test.ts +++ b/test/mcp-modern-framework.test.ts @@ -135,9 +135,12 @@ test('model path: a payload the host cannot retain is reported as an incomplete await waitFor('the tool round', () => (membrane.lastStream?.receivedToolResults.length ?? 0) >= 1); const [delivered] = membrane.lastStream!.receivedToolResults[0] as Array<{ content: unknown; isError: boolean }>; // No workspace here: audio and the blob can be neither shown nor kept. + // That makes the result a failure, read as its text alone, so the inline + // image is a payload too, and can't be kept either. assert.equal(delivered!.isError, true); const text = JSON.stringify(delivered!.content); - assert.match(text, /result incomplete: 2 payload\(s\) could not be retained/); + assert.match(text, /result incomplete: 3 payload\(s\) could not be retained/); + assert.match(text, /\[image: image\/png, 8 B\. Not retained: a failed result is read as its text alone, and no workspace is mounted to save it\]/); assert.match(text, /resource link \\"report\.pdf\\": file:\/\/\/srv\/report\.pdf, application\/pdf; not fetched/); assert.match(text, /embedded note text/); }); @@ -293,6 +296,34 @@ test('saved payloads: call ids that sanitize to one label still get distinct fil }); }); +test("a payload whose workspace write rejects is lost from a completed call's result, with why logged, not a dispatch failure", async (t) => { + await withWorkspace(async ({ framework, workspace }) => { + const read = (framework as unknown as { + mcpToolResult: (result: unknown, callId: string, mcplPeer: boolean) => Promise; + }).mcpToolResult.bind(framework); + const logged: string[] = []; + t.mock.method(console, 'error', (...args: unknown[]) => { + logged.push(args.map((a) => (a instanceof Error ? a.message : String(a))).join(' ')); + }); + const clip = { content: [{ type: 'audio', data: Buffer.from('clip').toString('base64'), mimeType: 'audio/wav' }] }; + + workspace.writeBinary = async () => { throw new Error('disk gone'); }; + const rejected = await read(clip, 'call-1', false); + assert.equal(rejected.success, false); + assert.match(rejected.error!, /saving it to the workspace failed\]/); + + workspace.writeBinary = async () => ({ success: false, error: 'disk full', isError: true }); + const refused = await read(clip, 'call-2', false); + assert.equal(refused.success, false); + assert.match(refused.error!, /saving it to the workspace failed\]/); + + const saves = logged.filter((line) => line.startsWith('[mcp] saving tool payload')); + assert.equal(saves.length, 2, logged.join('\n')); + assert.match(saves[0]!, /^\[mcp\] saving tool payload board\/tool-results\/\S+\.wav \("audio\/wav"\) failed: disk gone$/); + assert.match(saves[1]!, /^\[mcp\] saving tool payload board\/tool-results\/\S+\.wav \("audio\/wav"\) failed: disk full$/); + }); +}); + test('MCPL policy on a modern server is a configuration error at connect', async () => { await withModern(async ({ framework }) => { await assert.rejects( diff --git a/test/mcp-tool-result-normalize.test.ts b/test/mcp-tool-result-normalize.test.ts index 955669120..b864c5cce 100644 --- a/test/mcp-tool-result-normalize.test.ts +++ b/test/mcp-tool-result-normalize.test.ts @@ -1,8 +1,8 @@ /** - * The standard reading of an MCP tool result (room-284, shelf-487): the - * content array is the model's view, and `structuredContent` the - * programmatic one, kept by presence. A payload the model can't take is - * saved and described by a bounded stub; nothing is fetched. + * The standard reading of an MCP tool result: the content array is the + * model's view, and `structuredContent` the programmatic one, kept by + * presence. A payload the model can't take is saved and described by a + * bounded stub; nothing is fetched, and nothing is dropped silently. */ import { test } from 'node:test'; import assert from 'node:assert/strict'; @@ -122,12 +122,155 @@ test('a tool error is a failure with its text; structured survives', async () => }); test('an unknown block is shown as data, and stub fields are bounded', async () => { - const long = `memo://${'x'.repeat(1000)}`; const r = await normalizeStandardToolResult({ - content: [{ type: 'hologram', depth: 3 }, { type: 'resource_link', uri: long }], + content: [{ type: 'hologram', depth: 3 }, { type: 'resource_link', uri: 'memo://r', name: 'n'.repeat(1000) }], }, 'L', null); const [unknown, link] = String(r.data).split('\n'); assert.equal(unknown, '{"type":"hologram","depth":3}'); - assert.ok(link!.length < 260, `bounded: ${link!.length}`); - assert.match(link!, /…; not fetched\]$/); + assert.equal(link, `[resource link "${'n'.repeat(200)}…": memo://r; not fetched]`); +}); + +test('a large unknown block is bounded like a stub field, saying how much was left out', async () => { + const r = await normalizeStandardToolResult({ content: [{ type: 'hologram', blob: 'x'.repeat(10_000) }] }, 'L', null); + const json = JSON.stringify({ type: 'hologram', blob: 'x'.repeat(10_000) }); + assert.equal(r.data, `${json.slice(0, 200)}… [${json.length - 200} more characters of this block left out]`); +}); + +test('a URI is bounded loosely: a real one survives whole, a pathological one is cut', async () => { + const real = `https://example.com/report?${'q'.repeat(1500)}`; + const huge = `memo://${'x'.repeat(5000)}`; + const r = await normalizeStandardToolResult({ + content: [{ type: 'resource_link', uri: real }, { type: 'resource_link', uri: huge }], + }, 'L', null); + const [whole, cut] = String(r.data).split('\n'); + assert.equal(whole, `[resource link: ${real}; not fetched]`); + assert.equal(cut, `[resource link: ${huge.slice(0, 2048)}…; not fetched]`); +}); + +test('in a failed result an inline image is saved and named, or marked not retained, never dropped', async () => { + const { save, saved } = recordingSave(); + const r = await normalizeStandardToolResult({ + content: [{ type: 'text', text: 'render failed' }, { type: 'image', data: b64('partial'), mimeType: 'image/png' }], + isError: true, + }, 'L', save); + assert.equal(r.success, false); + assert.deepEqual(saved, [{ fileName: 'L-1.png', bytes: 'partial', mimeType: 'image/png' }]); + assert.equal(r.error, + 'render failed\n' + + '[image: image/png, 7 B, saved to workspace file home/tool-results/L-1.png; not shown inline: a failed result is read as its text alone]'); + + const unsaved = await normalizeStandardToolResult({ + content: [{ type: 'image', data: b64('png'), mimeType: 'image/png' }], + isError: true, + }, 'L', null); + assert.equal(unsaved.error, + '[image: image/png, 3 B. Not retained: a failed result is read as its text alone, and no workspace is mounted to save it]'); + + // A payload lost beside it makes the result a failure too: the image is + // then a payload like the audio, and both are counted. + const lost = await normalizeStandardToolResult({ + content: [{ type: 'image', data: b64('png'), mimeType: 'image/png' }, { type: 'audio', data: b64('wav'), mimeType: 'audio/wav' }], + }, 'L', null); + assert.equal(lost.success, false); + assert.match(lost.error!, /^\[result incomplete: 2 payload\(s\) could not be retained/); + assert.match(lost.error!, /\[image: image\/png, 3 B\. Not retained: a failed result is read as its text alone/); +}); + +test('image and audio with inline data but no mimeType are payloads of no known type, saved', async () => { + const { save, saved } = recordingSave(); + const r = await normalizeStandardToolResult({ content: [{ type: 'image', data: b64('img') }, { type: 'audio', data: b64('aud') }] }, 'L', save); + assert.deepEqual(saved.map((p) => [p.fileName, p.bytes, p.mimeType]), [ + ['L-1.bin', 'img', 'application/octet-stream'], + ['L-2.bin', 'aud', 'application/octet-stream'], + ]); + assert.equal(r.success, true); + assert.equal(r.data, + '[image: application/octet-stream, 3 B, saved to workspace file home/tool-results/L-1.bin]\n' + + '[audio: application/octet-stream, 3 B, saved to workspace file home/tool-results/L-2.bin]'); +}); + +test("an image a provider can't take inline is saved: another type, or over the per-image size", async () => { + const { save, saved } = recordingSave(); + const atLimit = Buffer.alloc(5 * 1024 * 1024, 1).toString('base64'); + const overLimit = Buffer.alloc(5 * 1024 * 1024 + 1, 1).toString('base64'); + const r = await normalizeStandardToolResult({ + content: [ + { type: 'image', data: b64('tiff'), mimeType: 'image/tiff' }, + { type: 'image', data: overLimit, mimeType: 'image/png' }, + { type: 'image', data: atLimit, mimeType: 'image/png' }, + { type: 'image', data: b64('ok'), mimeType: 'IMAGE/JPEG; q=1' }, + ], + }, 'L', save); + assert.deepEqual(saved.map((p) => [p.fileName, p.mimeType]), [['L-1.tiff', 'image/tiff'], ['L-2.png', 'image/png']]); + const data = r.data as Array<{ type: string; text?: string; data?: string; mimeType?: string }>; + assert.equal(data[0]!.text, '[image: image/tiff, 4 B, saved to workspace file home/tool-results/L-1.tiff]'); + assert.equal(data[1]!.text, + '[image: image/png, 5.0 MB, saved to workspace file home/tool-results/L-2.png; not shown inline: over the 5.0 MB a model takes per image]'); + assert.deepEqual(data[2], { type: 'image', data: atLimit, mimeType: 'image/png' }, 'exactly at the limit stays native'); + assert.deepEqual(data[3], { type: 'image', data: b64('ok'), mimeType: 'image/jpeg' }, 'a type a provider takes, in its canonical spelling'); +}); + +test("a native image is passed without its encoding's line wrapping", async () => { + const flat = b64('a small png, wrapped'); + const wrapped = flat.replace(/(.{8})/g, '$1\r\n'); + assert.notEqual(wrapped, flat); + const r = await normalizeStandardToolResult({ content: [{ type: 'image', data: wrapped, mimeType: 'image/png' }] }, 'L', null); + assert.deepEqual(r.data, [{ type: 'image', data: flat, mimeType: 'image/png' }]); +}); + +test('a save that rejects is a lost payload, and why it failed is logged', async (t) => { + const logged: string[] = []; + t.mock.method(console, 'error', (...args: unknown[]) => { logged.push(args.map((a) => (a instanceof Error ? a.message : String(a))).join(' ')); }); + const r = await normalizeStandardToolResult( + { content: [{ type: 'audio', data: b64('clip'), mimeType: 'audio/wav' }] }, + 'L', + async () => { throw new Error('disk gone'); }, + ); + assert.equal(r.success, false); + assert.match(r.error!, /\[audio: audio\/wav, 4 B\. Not retained: the model can't take this type, and saving it to the workspace failed\]/); + const saves = logged.filter((line) => line.startsWith('[mcp] saving tool payload')); + assert.equal(saves.length, 1, logged.join('\n')); + assert.match(saves[0]!, /saving tool payload L-1\.wav \(audio, "audio\/wav", 4 B\) failed: disk gone/); +}); + +test('malformed content: a bad entry is named and left out; content that is not a list is a malformed result', async () => { + const r = await normalizeStandardToolResult({ content: [{ type: 'text', text: 'ok' }, null, 'stray', { type: 'text', text: 42 }] }, 'L', null); + assert.equal(r.success, true); + assert.equal(r.data, + 'ok\n[malformed content entry, left out: null]\n[malformed content entry, left out: stray]\n' + + '[malformed text block, left out: its text is not a string]'); + + const notList = await normalizeStandardToolResult({ content: { type: 'text', text: 'a block, not a list' }, structuredContent: { n: 1 } }, 'L', null); + assert.deepEqual(notList, { + success: false, + error: '[malformed result: its content is not a list of blocks; the tool may already have completed]\n{"n":1}', + isError: true, + structured: { n: 1 }, + }); + + // Absent content is no content, as a structured-only result may send it. + const absent = await normalizeStandardToolResult({ structuredContent: { n: 1 } }, 'L', null); + assert.equal(absent.success, true); + assert.equal(absent.data, '{"n":1}'); +}); + +test("stub text never splits a surrogate pair, and a wrapped payload's size counts only its bytes", async () => { + // One ASCII character first, so a cut by UTF-16 units would land inside a pair. + const r = await normalizeStandardToolResult({ content: [{ type: 'resource_link', uri: 'memo://x', name: `a${'😀'.repeat(300)}` }] }, 'L', null); + assert.equal(r.data, `[resource link "a${'😀'.repeat(199)}…": memo://x; not fetched]`); + + const wrapped = b64('hello world').replace(/(.{4})/g, '$1\r\n'); + const sized = await normalizeStandardToolResult({ content: [{ type: 'audio', data: wrapped, mimeType: 'audio/wav' }] }, 'L', null); + assert.match(sized.error!, /\[audio: audio\/wav, 11 B\. Not retained/); +}); + +test("a subtype named like one of Object.prototype's members is just a name", async () => { + const { save, saved } = recordingSave(); + await normalizeStandardToolResult({ + content: [ + { type: 'audio', data: b64('a'), mimeType: 'audio/constructor' }, + { type: 'audio', data: b64('b'), mimeType: 'audio/__proto__' }, + ], + }, 'L', save); + assert.deepEqual(saved.map((p) => p.fileName), ['L-1.bin', 'L-2.proto']); }); From 6d0ab4187c2dd9e15ad2dbf2f409ae4c74d22aa0 Mon Sep 17 00:00:00 2001 From: Nell-1783 Date: Sun, 11 Oct 2026 17:22:50 -0400 Subject: [PATCH 0121/1112] fix(mcp): one transport rule for every selector, credentials kept out of logs and status, and refusals for settings that would be ignored Configuration - A configuration with both command and url and no transport is refused before anything is spawned or dialed. Before, it was launched as stdio and the url was silently ignored. With a transport, the transport says which field is used, as before: a host layers a recipe's url and transport: 'websocket' over a file's command (connectome-host's recipe overrides do), and a url that disagrees with the named transport is still refused. resolveServerBinding makes the refusal, so every selector gives the same answer. Listed in the breaking fragment. - The legacy transport selectors (isWebSocketTransport, openTransport) now decide through resolveServerBinding, so they can't disagree with it: an http(s) url is never dialed as a WebSocket, and openTransport refuses a modern binding before anything is spawned or dialed. - A token or accessProvider on a modern stdio server is refused, like the other settings that can't apply there. A legacy stdio server keeps ignoring them, as before. - A legacy requestTimeoutMs above 2^31-1 is refused. Node runs a longer timer after about 1 ms, so every request timed out at once; every value that worked, 0 included, still passes. Credentials - redactUrl shows a url for a person or a log to read: scheme, host and port, and the path with any credential-like segment replaced (16 or more characters with a digit, as a token or uuid has, or 32 or more of anything: some hosted servers put a per-user secret in the path), with the userinfo removed and each query parameter's value replaced. Configuration errors and a failed WebSocket dial's error use it, and so does listMcplServers' url. An invalid url isn't echoed at all, since it can't be redacted. - An accessProvider that throws is logged by its error's class and its message's length, on the WebSocket dial and in the modern engine's bearer auth: the provider's text may quote the credential. - The cleartext warning also covers a secret the url carries itself, to an http:// or ws:// host that isn't this machine: userinfo, a query parameter whose name says credential (token, key, secret, password, auth, signature), or a credential-like path segment, as redactUrl judges it. It names what it found, never the value. - Loopback detection follows the WHATWG parser's forms: 127.0.0.0/8 as an IPv4-mapped IPv6 address ([::ffff:7f00:1]) and a name with its root's trailing dot (localhost.) count as this host. Status - listMcplServers lists a configuration that connecting would refuse with refused (why), and family and transport null when no binding resolves, where it guessed a legacy binding before. policyEstablished is null for a modern server, which has no MCPL policy, rather than false, which read as a pending one. Not changed - The cleartext warning is checked once per connection: neither engine's reconnect attempts run the configuration check again. A probe with a token to a closed port gave one warning over three legacy connect failures, and one over sixteen modern connect and reconnect failures. Text - README: command and url together need a transport; a modern stdio server takes no credential; a -32022 refusal names the revisions the server lists (or its error text), and suggests protocol: 'modern' only when the list includes 2026-07-28. - types.ts: the command, url, transport and token docs say where each applies, and the token doc says where it travels for each transport; the requestTimeoutMs doc gives the upper bound. - Changelog: the cleartext fragment names url-carried secrets and the unspecified addresses (0.0.0.0, [::]) that count as this host; the standard-results fragment covers redaction, the selectors and the status fields; the breaking fragment covers command with url and no transport, and the legacy timeout bound. Tests - mcp-protocol-family: command with url refused without a transport and resolved by an explicit one, a mismatched one still refused; the redacted url, path segments included, in errors, and an invalid one not echoed; the legacy timeout bound; credentials on a modern stdio server; url-carried secrets warned about without quoting them; the IPv4-mapped, trailing-dot and [::] loopback forms; an access provider's failure logged without its text, on the WebSocket dial and over http. - mcpl-list-status: a refused configuration, redacted urls, and a modern server's policyEstablished. - mcpl-ws-transport: the selector follows resolveServerBinding; openTransport refuses modern and refused configurations; an invalid url and a failed dial's error carry no credential. - mcp-modern-connection and mcpl-legacy-version: the refusal tests now use the real fixtures' start logs, so a launch before the refusal would show, with a control that the log records a launch. Co-Authored-By: Claude Opus 5.5 --- README.md | 12 +- .../cleartext-mcp-credential-warning.added.md | 13 +- .../legacy-mcp-protocol-boundary.breaking.md | 10 ++ .../mcp-standard-tool-results.changed.md | 22 +++- src/framework.ts | 44 ++++--- src/mcpl/modern-connection.ts | 7 +- src/mcpl/protocol-family.ts | 119 ++++++++++++++--- src/mcpl/transport.ts | 38 ++++-- src/mcpl/types.ts | 31 +++-- test/mcp-modern-connection.test.ts | 18 ++- test/mcp-protocol-family.test.ts | 124 +++++++++++++++++- test/mcpl-legacy-version.test.ts | 15 ++- test/mcpl-list-status.test.ts | 39 ++++++ test/mcpl-ws-transport.test.ts | 34 ++++- 14 files changed, 447 insertions(+), 79 deletions(-) diff --git a/README.md b/README.md index 6ca53d160..77ef73f74 100644 --- a/README.md +++ b/README.md @@ -288,14 +288,19 @@ configuration chooses the family, and nothing is probed: } ``` +A configuration with both `command` and `url` needs `transport` to say which +one is used; without it, it is refused rather than silently ignoring one. + A modern server's tools get the same prefix, `enabledTools`/`disabledTools`, class overrides and dispatch as an MCPL server's, and `listMcplServers()` shows its `family`, `protocolVersion` and `transport`. Its config differs in -two ways: +three ways: - It has no MCPL surface (grant, push, channels, inference requests, feature sets), so setting MCPL-only fields on one is a configuration error. - `requestTimeoutMs` is one deadline per tool call and must be 1 to 2^31−1; 0 is refused, where MCPL reads it as no watchdog. +- Over stdio it takes no credential: `token` or `accessProvider` on one is + refused. At the deadline, cancellation is requested for a leg still in flight; between the rounds of an `input_required` continuation nothing is in flight @@ -304,7 +309,10 @@ and the call is never retried. A call that ends for any other reason after the server asked for another round, before its final answer, is reported the same way, since the server may already have acted. An MCPL server that refuses `2024-11-05` with `-32022` fails with `McplProtocolVersionError`, -which names the fix, and is not retried. +and is not retried. The error names the revisions the server says it +supports, or its error text when it lists none, and when the list includes +`2026-07-28` it says to configure the server with `protocol: 'modern'` (stdio) +or reach it over http(s). #### Feature sets diff --git a/changelog.d/cleartext-mcp-credential-warning.added.md b/changelog.d/cleartext-mcp-credential-warning.added.md index 4851f6685..dadbe0514 100644 --- a/changelog.d/cleartext-mcp-credential-warning.added.md +++ b/changelog.d/cleartext-mcp-credential-warning.added.md @@ -1,6 +1,11 @@ -- Connecting to an MCP server warns once when its credential (`token` or - `accessProvider`) would cross the network unencrypted: an `http://` or - `ws://` url whose host isn't loopback. It warns rather than refuses, since a - private network or a TLS-terminating proxy can make that deliberate. +- Connecting to an MCP server warns once when a credential would cross the + network unencrypted, to an `http://` or `ws://` url whose host isn't this + machine: its `token` or `accessProvider`, or a secret the url carries + itself (userinfo, a query parameter whose name says credential: `token`, + `key`, `secret`, `password`, `auth` or `signature`, or a path segment + that looks like one, as the redacted url judges it). Loopback hosts are + exempt, and so are the unspecified addresses `0.0.0.0` and `[::]`, which + reach this host too. It warns rather than refuses, since a private network + or a TLS-terminating proxy can make that deliberate. `serverConfigWarnings` is exported beside `serverConfigProblems`, so a host can apply the same check in its own configuration validation. diff --git a/changelog.d/legacy-mcp-protocol-boundary.breaking.md b/changelog.d/legacy-mcp-protocol-boundary.breaking.md index 84cf28ddd..fc815cfa6 100644 --- a/changelog.d/legacy-mcp-protocol-boundary.breaking.md +++ b/changelog.d/legacy-mcp-protocol-boundary.breaking.md @@ -17,3 +17,13 @@ whole inventory reaches the agent; before, every page after the first was silently dropped. A repeated cursor, or more than 100 pages, is an error rather than a partial inventory. +- **Hosts and operators:** a server configuration with both `command` and + `url` and no `transport` is now refused before anything is spawned or + dialed. Before, it was launched as stdio and the url was silently + ignored. Migration: drop the field you don't use, or set `transport` to + say which one is used. Unchanged: a configuration with only one of them, + or with both and a `transport`, resolves as before (a host can still + layer a url and `transport: 'websocket'` over a file's command). A legacy + `requestTimeoutMs` above 2^31−1 is refused too: Node ran it as a timer of + about 1 ms, so every request timed out at once. Every value that worked, + `0` included, passes. diff --git a/changelog.d/mcp-standard-tool-results.changed.md b/changelog.d/mcp-standard-tool-results.changed.md index adfe0d362..4bb318b76 100644 --- a/changelog.d/mcp-standard-tool-results.changed.md +++ b/changelog.d/mcp-standard-tool-results.changed.md @@ -68,7 +68,21 @@ - A server configuration naming no usable transport is now an error before anything is spawned or dialed. That covers a `transport` that doesn't match the url's scheme, `transport: 'http'` without a url, and an - unrecognized scheme. Every valid configuration resolves as before. - `McplServerConnection.connect()` and `connectWithReconnect()` apply the - framework's whole check (so `protocol` on a WebSocket url is refused too), - and a configuration error is never turned into a retry stub. + unrecognized scheme. `McplServerConnection.connect()` and + `connectWithReconnect()` apply the framework's whole check (so `protocol` + on a WebSocket url is refused too), and a configuration error is never + turned into a retry stub. `openTransport()` and `isWebSocketTransport()` + decide by the same rule, so an http(s) url is never dialed as a WebSocket. +- A configuration error, a WebSocket dial failure and `listMcplServers()` + show a server's url redacted, since a credential may ride in it: no + userinfo, each query parameter's value replaced, and so is a path + segment that looks like a credential (16 or more characters with a + digit, as a token or uuid has, or 32 or more of anything), such as the + per-user secret some hosted servers put in the path. A url that doesn't + parse isn't shown. An `accessProvider` that throws is logged by + its error's class and its message's length, not its text, which may quote + the credential. +- `listMcplServers()` lists a configuration that connecting would refuse + with `refused` (why), and `family` and `transport` null, where it guessed + a legacy binding before. Its `policyEstablished` is null for a modern + server, which has no MCPL policy. diff --git a/src/framework.ts b/src/framework.ts index 8cd3c75ef..45f7be15a 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -393,7 +393,7 @@ import type { SessionUsageSnapshot, UsageUpdatedEvent } from './usage/types.js'; import type { McplServerConnection } from './mcpl/server-connection.js'; import { McplProtocolVersionError, McplUnreapedLaunchError } from './mcpl/server-connection.js'; import { ModernMcpConnection } from './mcpl/modern-connection.js'; -import { resolveServerBinding, serverConfigProblems } from './mcpl/protocol-family.js'; +import { redactUrl, resolveServerBinding, serverConfigProblems } from './mcpl/protocol-family.js'; import { normalizeStandardToolResult } from './mcpl/tool-result-normalize.js'; import type { McplServerConfig, @@ -14466,16 +14466,21 @@ export class AgentFramework { connected: boolean; retrying: boolean; /** Which engine serves it: `legacy` (MCP 2024-11-05 + MCPL) or `modern` - * (MCP 2026-07-28), as its configuration selects. */ - family: 'legacy' | 'modern'; + * (MCP 2026-07-28), as its configuration selects; null when the + * configuration names no usable transport (see `refused`). */ + family: 'legacy' | 'modern' | null; /** The MCP revision its last handshake established; null until one has. */ protocolVersion: string | null; - /** How it is reached. */ - transport: 'stdio' | 'websocket' | 'http'; + /** How it is reached; null when the configuration names no usable transport. */ + transport: 'stdio' | 'websocket' | 'http' | null; + /** Why connecting refuses this configuration, before anything is spawned + * or dialed; null when it is usable. */ + refused: string | null; toolPrefix: string; toolCount: number; - /** True only after the §5.3 policy receipt activated the grant. */ - policyEstablished: boolean; + /** True only after the §5.3 policy receipt activated the grant. Null for + * a modern server, which has no MCPL policy. */ + policyEstablished: boolean | null; /** Sole normative authorization allowlist for the connection (§5.4). */ effectiveGrant: string[]; /** Advertised paths removed by host enabled/disabled capability config. */ @@ -14509,13 +14514,16 @@ export class AgentFramework { lastNegotiatedAt: number | null; }; command?: string; + /** The url as {@link redactUrl} shows it, since a credential may ride in + * it: no userinfo, each query parameter's value replaced, and any path + * segment that looks like a credential replaced. */ url?: string; }> { const result: Array<{ id: string; connected: boolean; retrying: boolean; - family: 'legacy' | 'modern'; protocolVersion: string | null; - transport: 'stdio' | 'websocket' | 'http'; - toolPrefix: string; toolCount: number; policyEstablished: boolean; + family: 'legacy' | 'modern' | null; protocolVersion: string | null; + transport: 'stdio' | 'websocket' | 'http' | null; refused: string | null; + toolPrefix: string; toolCount: number; policyEstablished: boolean | null; effectiveGrant: string[]; maskedCapabilities: string[]; deniedCapabilities: string[]; allowHostCommands: boolean; toolObserveFilter: import('./mcpl/tool-lifecycle.js').ToolObserveRule[] | null; @@ -14532,12 +14540,15 @@ export class AgentFramework { const modern = this.modernServers().get(id) ?? null; const connection = modern ? null : this.mcplServerRegistry?.getServer(id) ?? null; const connected = modern?.isConnected ?? connection?.isConnected ?? false; - let binding: { family: 'legacy' | 'modern'; transport: 'stdio' | 'websocket' | 'http' }; + // What connecting would decide, before anything is spawned or dialed: a + // configuration it refuses is listed as refused, not as a guessed binding. + let binding: { family: 'legacy' | 'modern'; transport: 'stdio' | 'websocket' | 'http' } | null; try { binding = resolveServerBinding(config); } catch { - binding = { family: 'legacy', transport: config.url ? 'websocket' : 'stdio' }; + binding = null; } + const problems = serverConfigProblems(config); // Attribute through the dispatch resolver: a bare prefix match would // also count a nested prefix's tools (`foo` vs `foo--bar`) as ours. const ownTools = (this.mcplTools ?? []).flatMap((t) => { @@ -14548,12 +14559,13 @@ export class AgentFramework { id, connected, retrying: !connected && (modern?.willReconnect ?? connection?.willReconnect ?? false), - family: binding.family, + family: binding?.family ?? null, protocolVersion: modern?.protocolVersion ?? connection?.protocolVersion ?? null, - transport: binding.transport, + transport: binding?.transport ?? null, + refused: problems.length > 0 ? problems.join('; ') : null, toolPrefix: prefix, toolCount: ownTools.length, - policyEstablished: connection?.policyEstablished ?? false, + policyEstablished: binding?.family === 'modern' ? null : connection?.policyEstablished ?? false, effectiveGrant: connection?.grant.effectiveList() ?? [], maskedCapabilities: [...(connection?.droppedCapabilities ?? [])].sort(), deniedCapabilities: [...(connection?.grant.deniedPaths ?? [])].sort(), @@ -14567,7 +14579,7 @@ export class AgentFramework { ? { ...connection.manifestState } : { lastValidatedRevision: null, lastFetchedAt: null, lastNegotiatedAt: null }, command: config.command, - url: config.url, + url: config.url === undefined ? undefined : redactUrl(config.url), }); } return result; diff --git a/src/mcpl/modern-connection.ts b/src/mcpl/modern-connection.ts index 2ad44f6d8..7af667292 100644 --- a/src/mcpl/modern-connection.ts +++ b/src/mcpl/modern-connection.ts @@ -57,7 +57,7 @@ import { import { StdioTransport, type McplTransport } from './transport.js'; import { McplRequestError } from './server-connection.js'; -import { MODERN_MCP_PROTOCOL_VERSION, checkServerConfig } from './protocol-family.js'; +import { MODERN_MCP_PROTOCOL_VERSION, checkServerConfig, describeCredentialError } from './protocol-family.js'; import type { McplServerConfig, McpToolDefinition } from './types.js'; const DEFAULT_REQUEST_TIMEOUT_MS = 60_000; @@ -261,9 +261,10 @@ function bearerAuth(config: McplServerConfig): AuthProvider | undefined { if (fresh) return fresh; } catch (error) { // Same fallback as the WebSocket dial: use the static token, and let - // the server refuse a stale credential loudly. + // the server refuse a stale credential loudly. The provider's own + // text may quote the credential, so it isn't logged. console.error( - `[mcp] server "${config.id}": access provider failed (${error instanceof Error ? error.message : error}) — using configured fallback`, + `[mcp] server "${config.id}": access provider failed (${describeCredentialError(error)}) — using configured fallback`, ); } } diff --git a/src/mcpl/protocol-family.ts b/src/mcpl/protocol-family.ts index cea22cedb..54682351c 100644 --- a/src/mcpl/protocol-family.ts +++ b/src/mcpl/protocol-family.ts @@ -60,12 +60,57 @@ export const MCPL_ONLY_POLICY_FIELDS = [ type BindingFields = Pick; +/** + * A path segment that may be a credential. Some hosted servers put a + * per-user secret in the path (`…/mcp/s//mcp`). A random token + * (base64url, hex, a uuid) is long and carries a digit; anything very long + * is treated as one too. A short or plain-word segment can't be told apart + * from a name, so it stays. + */ +function credentialLikeSegment(segment: string): boolean { + return (segment.length >= 16 && /[0-9]/.test(segment)) || segment.length >= 32; +} + +/** + * A url as text a person or a log reads it: its scheme, host and port, and + * its path with any credential-like segment replaced. The userinfo is + * removed and each query parameter's value replaced, since a credential + * rides there too. A url that doesn't parse can't be redacted, so it isn't + * shown at all. + */ +export function redactUrl(url: string | undefined): string { + let parsed: URL; + try { + parsed = new URL(url ?? ''); + } catch { + return '(an invalid url, not shown)'; + } + parsed.username = ''; + parsed.password = ''; + parsed.hash = ''; + parsed.pathname = parsed.pathname.split('/').map((segment) => (credentialLikeSegment(segment) ? 'redacted' : segment)).join('/'); + parsed.search = new URLSearchParams([...parsed.searchParams].map(([name]): [string, string] => [name, 'redacted'])).toString(); + return parsed.toString(); +} + +/** + * A credential provider's failure as a log line may show it: the error's + * class and its message's length, never the text, which may quote the + * credential it failed to refresh. + */ +export function describeCredentialError(error: unknown): string { + if (!(error instanceof Error)) return `a thrown ${typeof error}`; + const length = error.message.length; + return `${error.constructor?.name || error.name || 'Error'}, ${length} character${length === 1 ? '' : 's'}`; +} + function urlScheme(config: BindingFields): string { let parsed: URL; try { parsed = new URL(config.url!); } catch { - throw new Error(`MCP server "${config.id}": invalid url "${config.url}"`); + // Not echoed: a url that doesn't parse can't be redacted. + throw new Error(`MCP server "${config.id}": invalid url (it isn't shown, since it may carry a credential)`); } return parsed.protocol; } @@ -73,9 +118,18 @@ function urlScheme(config: BindingFields): string { /** * Resolve a server's family and transport. Throws on a configuration that * names no usable transport (for example an `ftp://` URL, or - * `transport: 'http'` with a WebSocket URL). + * `transport: 'http'` with a WebSocket URL), or names two without saying + * which: `command` and `url` together need a `transport`, since one of them + * goes unused. (A host may layer a url and its transport over a file's + * command, so both with a `transport` stay valid.) */ export function resolveServerBinding(config: BindingFields): ServerBinding { + if (config.command && config.url && config.transport === undefined) { + throw new Error( + `MCP server "${config.id}": "command" and "url" are both set, and no "transport" says which one to use: ` + + `a server is either launched (command) or dialed (url)`, + ); + } const networkKind = (): McpTransportKind => { const scheme = urlScheme(config); if (scheme === 'ws:' || scheme === 'wss:') return 'websocket'; @@ -95,7 +149,7 @@ export function resolveServerBinding(config: BindingFields): ServerBinding { transport = networkKind(); if (config.transport !== undefined && config.transport !== transport) { throw new Error( - `MCP server "${config.id}": transport "${config.transport}" does not match url "${config.url}"`, + `MCP server "${config.id}": transport "${config.transport}" does not match url ${redactUrl(config.url)}`, ); } } else if (config.transport !== undefined) { @@ -138,8 +192,8 @@ export function serverConfigProblems(config: McplServerConfig): string[] { ); } } + const timeout = config.requestTimeoutMs; if (binding.family === 'modern') { - const timeout = config.requestTimeoutMs; if (timeout !== undefined && !(Number.isInteger(timeout) && timeout >= 1 && timeout <= MAX_TIMER_MS)) { problems.push( `MCP server "${config.id}": requestTimeoutMs must be an integer from 1 to ${MAX_TIMER_MS} for a modern server (got ${String(timeout)}); every modern call has a deadline`, @@ -150,28 +204,50 @@ export function serverConfigProblems(config: McplServerConfig): string[] { problems.push(`MCP server "${config.id}": "${field}" is MCPL policy, which a modern MCP server cannot negotiate`); } } + // A modern stdio server takes no credential: nothing would carry it. + if (binding.transport === 'stdio') { + for (const field of ['token', 'accessProvider'] as const) { + if (config[field]) { + problems.push(`MCP server "${config.id}": "${field}" applies only to a network (url) server; a modern stdio server would ignore it`); + } + } + } + } else if (typeof timeout === 'number' && timeout > MAX_TIMER_MS) { + // Never usable: Node runs a longer timer after about 1 ms, so every + // request timed out at once. Every value that worked (0 included) passes. + problems.push( + `MCP server "${config.id}": requestTimeoutMs must be at most ${MAX_TIMER_MS} (got ${String(timeout)}), or 0 to disable the request timeout; a larger value times every request out at once`, + ); } return problems; } -/** A host a connection to which never leaves this machine. The unspecified - * addresses count: a connection to 0.0.0.0 or [::] reaches this host. */ +/** A host a connection to which never leaves this machine, as the WHATWG URL + * parser gives it. The unspecified addresses count: a connection to 0.0.0.0 + * or [::] reaches this host. So do 127.0.0.0/8 written as an IPv4-mapped + * IPv6 address ([::ffff:127.0.0.1] parses as [::ffff:7f00:1]) and a name + * with its root's trailing dot (localhost.). */ function isLoopbackHost(hostname: string): boolean { - const host = hostname.toLowerCase(); + const host = hostname.toLowerCase().replace(/\.$/, ''); return host === 'localhost' || host.endsWith('.localhost') || host === '[::1]' || /^127(\.\d{1,3}){3}$/.test(host) || - host === '0.0.0.0' || host === '[::]'; + /^\[::ffff:7f[0-9a-f]{2}:[0-9a-f]{1,4}\]$/.test(host) || host === '0.0.0.0' || host === '[::]'; } +/** Query parameter names that say they carry a credential. */ +const CREDENTIAL_PARAM = /token|key|secret|password|auth|signature/i; + /** * What is usable in a server configuration but worth an operator's attention, - * as messages; empty when there is nothing. A credential (`token` or - * `accessProvider`) for an `http://` or `ws://` url whose host isn't loopback - * crosses the network in cleartext, in the `Authorization` header or the - * WebSocket `?token=` query. That can be deliberate, on a private network or - * behind a TLS-terminating proxy, so it is a warning and never a refusal. + * as messages; empty when there is nothing. A credential for an `http://` or + * `ws://` url whose host isn't loopback crosses the network in cleartext: + * `token` or `accessProvider` (in the `Authorization` header, or the + * WebSocket `?token=` query), and a secret the url carries itself (userinfo, + * a query parameter whose name says it's a credential, or a path segment + * that looks like one, as {@link redactUrl} judges it). That can be + * deliberate, on a private network or behind a TLS-terminating proxy, so it + * is a warning and never a refusal. */ export function serverConfigWarnings(config: McplServerConfig): string[] { - if (!config.token && !config.accessProvider) return []; let transport: McpTransportKind; try { transport = resolveServerBinding(config).transport; @@ -181,10 +257,21 @@ export function serverConfigWarnings(config: McplServerConfig): string[] { if (transport === 'stdio') return []; const url = new URL(config.url!); if ((url.protocol !== 'http:' && url.protocol !== 'ws:') || isLoopbackHost(url.hostname)) return []; + const carried: string[] = []; + if (config.token || config.accessProvider) carried.push('its credential'); + if (url.username || url.password) carried.push("its url's userinfo"); + if (url.pathname.split('/').some(credentialLikeSegment)) carried.push("a credential-like segment of its url's path"); + const params = [...new Set([...url.searchParams.keys()].filter((name) => CREDENTIAL_PARAM.test(name)))]; + if (params.length > 0) { + carried.push(`its url's query ${params.length === 1 ? 'parameter' : 'parameters'} ${params.map((name) => JSON.stringify(name)).join(', ')}`); + } + if (carried.length === 0) return []; + const what = carried.length === 1 ? carried[0] : `${carried.slice(0, -1).join(', ')} and ${carried[carried.length - 1]}`; + const one = carried.length === 1 && params.length <= 1; const secure = url.protocol === 'http:' ? 'https' : 'wss'; return [ - `MCP server "${config.id}": its credential goes to ${url.host} unencrypted (${url.protocol}//), where anything on the ` + - `network path can read it; use ${secure}:// unless that network is trusted`, + `MCP server "${config.id}": ${what} ${one ? 'goes' : 'go'} to ${url.host} unencrypted (${url.protocol}//), ` + + `where anything on the network path can read ${one ? 'it' : 'them'}; use ${secure}:// unless that network is trusted`, ]; } diff --git a/src/mcpl/transport.ts b/src/mcpl/transport.ts index 9d2540c03..09ba3de81 100644 --- a/src/mcpl/transport.ts +++ b/src/mcpl/transport.ts @@ -22,6 +22,7 @@ import { EventEmitter } from 'node:events'; import WebSocket from 'ws'; import type { McplServerConfig } from './types.js'; +import { describeCredentialError, redactUrl, resolveServerBinding } from './protocol-family.js'; /** Why/how a transport closed. `code`/`signal` are populated for stdio child * exits; `reason` carries a WebSocket close reason or an error message. */ @@ -317,15 +318,16 @@ export class WebSocketTransport extends McplTransport { } catch (err) { // Fall back to the static token (if any) rather than failing the dial // outright — the server will refuse a stale credential loudly anyway. + // The provider's own text may quote the credential, so it isn't logged. console.error( - `[mcpl] server "${config.id}": access provider failed (${err instanceof Error ? err.message : err}) — dialing with configured fallback`, + `[mcpl] server "${config.id}": access provider failed (${describeCredentialError(err)}) — dialing with configured fallback`, ); } } const url = buildWebSocketUrl(config, token); // Error strings travel: traces, logs, tool results the model reads. They // get the endpoint, never the credential. - const redacted = buildWebSocketUrl({ ...config, token: undefined }, null); + const redacted = redactUrl(config.url); const ws = new WebSocket(url); // A failed dial can emit 'error' MORE THAN ONCE on the raw socket (seen @@ -404,7 +406,8 @@ export function buildWebSocketUrl(config: McplServerConfig, tokenOverride?: stri try { parsed = new URL(config.url); } catch { - throw new Error(`MCPL server "${config.id}": invalid WebSocket url "${config.url}"`); + // Not echoed: a url that doesn't parse can't be redacted. + throw new Error(`MCPL server "${config.id}": invalid WebSocket url (it isn't shown, since it may carry a credential)`); } if (parsed.protocol !== 'ws:' && parsed.protocol !== 'wss:') { throw new Error( @@ -422,22 +425,29 @@ export function buildWebSocketUrl(config: McplServerConfig, tokenOverride?: stri // Factory // --------------------------------------------------------------------------- -/** True when the config selects the WebSocket transport (explicit `transport`, - * or a `url` with no `command`). */ +/** True when the config selects the WebSocket transport, as + * {@link resolveServerBinding} decides it; false for any other transport and + * for a configuration it refuses. */ export function isWebSocketTransport(config: McplServerConfig): boolean { - if (config.transport === 'websocket') return true; - if (config.transport === 'stdio') return false; - return Boolean(config.url) && !config.command; + try { + return resolveServerBinding(config).transport === 'websocket'; + } catch { + return false; + } } /** - * Open the transport selected by `config` (WebSocket when `transport` is - * 'websocket' or a bare `url` is given; stdio otherwise). The returned transport - * is connected at the byte level but has NOT performed the MCPL handshake. + * Open the legacy transport `config` selects, as {@link resolveServerBinding} + * decides it: stdio for a `command`, MCPL's WebSocket binding for a ws(s):// + * url. A configuration it refuses, or one for the modern family (an http(s) + * url, or `protocol: 'modern'`), throws before anything is spawned or dialed. + * The returned transport is connected at the byte level but has NOT + * performed the MCPL handshake. */ export async function openTransport(config: McplServerConfig): Promise { - if (isWebSocketTransport(config)) { - return WebSocketTransport.open(config); + const binding = resolveServerBinding(config); + if (binding.family !== 'legacy') { + throw new Error(`MCP server "${config.id}" is configured for modern MCP; the legacy transports can't open it`); } - return StdioTransport.spawn(config); + return binding.transport === 'websocket' ? WebSocketTransport.open(config) : StdioTransport.spawn(config); } diff --git a/src/mcpl/types.ts b/src/mcpl/types.ts index 3daf4e49e..5ca5f0ff8 100644 --- a/src/mcpl/types.ts +++ b/src/mcpl/types.ts @@ -217,9 +217,10 @@ export interface McplServerConfig { id: string; /** - * Command to spawn the server process (stdio transport). Mutually exclusive - * with `url`. Required for the stdio transport; omit it (and set `url`) for - * the WebSocket transport. + * Command to spawn the server process (stdio transport). Required for the + * stdio transport; omit it (and set `url`) for a network server, over + * WebSocket or HTTP. A configuration with both needs `transport` to say + * which one is used, and is refused without it. */ command?: string; @@ -243,8 +244,9 @@ export interface McplServerConfig { inheritEnv?: boolean; /** - * URL for a network server, used instead of `command`. Its scheme decides - * the protocol family: + * URL for a network server, used instead of `command`. A configuration + * with both needs `transport` to say which one is used, and is refused + * without it. Its scheme decides the protocol family: * - `ws://` or `wss://` is a legacy MCPL server over MCPL's WebSocket * binding. The host dials it and runs the same newline-delimited JSON-RPC * as over stdio. @@ -254,9 +256,11 @@ export interface McplServerConfig { url?: string; /** - * Transport selector. When it is absent, a `command` means stdio and a bare - * `url` means the transport its scheme names. Set it explicitly to - * disambiguate; it must agree with the url's scheme. + * Transport selector. When it is absent, a `command` means stdio and a + * `url` means the transport its scheme names; with both, it is required. + * When it is set, it decides which of the two is used and must agree with + * it: `'stdio'` needs a `command`, `'websocket'` a ws(s):// url and + * `'http'` an http(s):// url. */ transport?: 'stdio' | 'websocket' | 'http'; @@ -270,8 +274,11 @@ export interface McplServerConfig { protocol?: 'legacy' | 'modern'; /** - * Bearer token for WebSocket auth. Appended to `url` as a `token` query - * parameter (`?token=…`) on connect. Ignored by the stdio transport. + * Bearer token for a network server. Where it travels depends on the + * transport: on a ws(s):// url it is appended as a `token` query parameter + * (`?token=…`) on connect; to an http(s):// server it is sent as an + * `Authorization: Bearer` header and never put in the url. A legacy stdio + * server ignores it, and a modern stdio server refuses it. */ token?: string; @@ -387,7 +394,9 @@ export interface McplServerConfig { * accepts a request and never responds would otherwise freeze the awaiting * agent turn forever; with the timeout the pending request rejects with a * descriptive error, which the framework surfaces as a normal isError - * tool_result. Set 0 to disable. Default: 60000 (60 seconds). + * tool_result. Set 0 to disable. Default: 60000 (60 seconds). A value above + * 2^31−1 is refused: Node runs a longer timer after about 1 ms, so every + * request would time out at once. * * For a modern server it is one deadline per tool call, across every leg * of the call, and must be an integer from 1 to 2^31−1. It can't be diff --git a/test/mcp-modern-connection.test.ts b/test/mcp-modern-connection.test.ts index d71e2b6f5..28539665b 100644 --- a/test/mcp-modern-connection.test.ts +++ b/test/mcp-modern-connection.test.ts @@ -182,7 +182,21 @@ test('a first connect that fails keeps retrying with reconnect, and close stops }); test('configuration mistakes throw before anything is launched', async () => { + // x.invalid never resolves, so a dial would fail with a connect error instead. await assert.rejects(ModernMcpConnection.connect({ id: 'x', url: 'https://x.invalid/mcp', protocol: 'modern' }), /applies only to stdio/); - await assert.rejects(ModernMcpConnection.connect({ id: 'x', command: 'srv', protocol: 'modern', requestTimeoutMs: 0 }), /requestTimeoutMs must be an integer/); - await assert.rejects(ModernMcpConnection.connect({ id: 'x', command: 'srv' }), /not configured for modern MCP/); + // Stdio refusals use the real fixture, whose start log records any launch. + const dir = scratch(); + const refused: Array<[Partial, RegExp]> = [ + [{ requestTimeoutMs: 0 }, /requestTimeoutMs must be an integer/], + [{ protocol: undefined }, /not configured for modern MCP/], + [{ token: 't' }, /"token" applies only to a network \(url\) server/], + [{ url: 'https://x.invalid/mcp' }, /"command" and "url" are both set/], + ]; + for (const [extra, pattern] of refused) { + await assert.rejects(ModernMcpConnection.connect(config(dir, extra)), pattern, JSON.stringify(extra)); + } + assert.deepEqual(lines(join(dir, 'starts.log')), [], 'nothing was launched'); + // The same fixture, configured right, does launch: the log above can tell. + await connect(config(dir)); + assert.equal(lines(join(dir, 'starts.log')).length, 1); }); diff --git a/test/mcp-protocol-family.test.ts b/test/mcp-protocol-family.test.ts index 5a0d31d20..761d3a971 100644 --- a/test/mcp-protocol-family.test.ts +++ b/test/mcp-protocol-family.test.ts @@ -12,10 +12,13 @@ import { serverConfigProblems, serverConfigWarnings, checkServerConfig, + redactUrl, + describeCredentialError, MCPL_ONLY_POLICY_FIELDS, MAX_TIMER_MS, } from '../src/mcpl/protocol-family.js'; import { ModernMcpConnection } from '../src/mcpl/modern-connection.js'; +import { WebSocketTransport } from '../src/mcpl/transport.js'; import type { McplServerConfig } from '../src/mcpl/types.js'; const cfg = (c: Partial): McplServerConfig => ({ id: 's', ...c }); @@ -28,12 +31,20 @@ test('family and transport follow the configuration', () => { assert.deepEqual(resolveServerBinding(cfg({ url: 'wss://example.com/mcpl' })), { family: 'legacy', transport: 'websocket' }); assert.deepEqual(resolveServerBinding(cfg({ url: 'http://localhost:8080/mcp' })), { family: 'modern', transport: 'http' }); assert.deepEqual(resolveServerBinding(cfg({ url: 'https://example.com/mcp' })), { family: 'modern', transport: 'http' }); - // Existing semantics: a command wins over a url unless transport says otherwise. - assert.deepEqual(resolveServerBinding(cfg({ command: 'srv', url: 'ws://x/mcpl' })), { family: 'legacy', transport: 'stdio' }); + // With both a command and a url, an explicit transport says which is used + // (a host layers a recipe's url and transport over a file's command). assert.deepEqual( resolveServerBinding(cfg({ command: 'srv', url: 'ws://x/mcpl', transport: 'websocket' })), { family: 'legacy', transport: 'websocket' }, ); + assert.deepEqual( + resolveServerBinding(cfg({ command: 'srv', url: 'ws://x/mcpl', transport: 'stdio' })), + { family: 'legacy', transport: 'stdio' }, + ); + assert.deepEqual( + resolveServerBinding(cfg({ command: 'srv', url: 'https://x/mcp', transport: 'http' })), + { family: 'modern', transport: 'http' }, + ); assert.deepEqual( resolveServerBinding(cfg({ url: 'https://x/mcp', transport: 'http' })), { family: 'modern', transport: 'http' }, @@ -50,6 +61,39 @@ test('a configuration that names no usable transport throws', () => { assert.throws(() => resolveServerBinding(cfg({ command: 'srv', transport: 'http' })), /transport "http" requires "url"/); }); +test('command and url without a transport are refused, rather than one of them silently ignored', () => { + for (const url of ['ws://x/mcpl', 'https://x/mcp']) { + for (const extra of [{}, { protocol: 'legacy' as const }, { protocol: 'modern' as const }]) { + const c = cfg({ command: 'srv', url, ...extra }); + assert.throws(() => resolveServerBinding(c), /"command" and "url" are both set, and no "transport" says which one to use/, JSON.stringify(c)); + assert.match(serverConfigProblems(c).join(), /"command" and "url" are both set/, JSON.stringify(c)); + } + } + // A transport that disagrees with the url it would use is still refused. + assert.throws(() => resolveServerBinding(cfg({ command: 'srv', url: 'https://x/mcp', transport: 'websocket' })), /does not match url/); + assert.throws(() => resolveServerBinding(cfg({ command: 'srv', url: 'ws://x/mcpl', transport: 'http' })), /does not match url/); +}); + +test('errors and the status view show a url redacted: no userinfo, no query values', () => { + assert.equal(redactUrl('https://user:pw@mcp.example:8443/v1/mcp?token=abc®ion=eu&token=def#frag'), + 'https://mcp.example:8443/v1/mcp?token=redacted®ion=redacted&token=redacted'); + assert.equal(redactUrl('wss://example.test/mcpl'), 'wss://example.test/mcpl'); + // A path segment that looks like a credential goes too: a hosted server's + // per-user secret, a uuid, anything very long. Short and plain-word ones stay. + assert.equal(redactUrl('https://mcp.zapier.com/api/mcp/s/Zt4kQ9xW2vLm8pRs7YbN/mcp'), 'https://mcp.zapier.com/api/mcp/s/redacted/mcp'); + assert.equal(redactUrl('https://mcp.example/server/123e4567-e89b-12d3-a456-426614174000/mcp'), 'https://mcp.example/server/redacted/mcp'); + assert.equal(redactUrl(`https://mcp.example/${'a'.repeat(32)}/mcp`), 'https://mcp.example/redacted/mcp'); + assert.equal(redactUrl('https://mcp.example/v2/2026-07-28/organization-settings/mcp'), 'https://mcp.example/v2/2026-07-28/organization-settings/mcp'); + assert.equal(redactUrl('not a url ?token=abc'), '(an invalid url, not shown)'); + assert.equal(redactUrl(undefined), '(an invalid url, not shown)'); + // The configuration errors that name a url give it redacted, or not at all. + const mismatch = serverConfigProblems(cfg({ url: 'https://u:pw@x/mcp?key=sekret', transport: 'websocket' })).join(); + assert.match(mismatch, /does not match url https:\/\/x\/mcp\?key=redacted/); + const invalid = serverConfigProblems(cfg({ url: 'sekret token not a url' })).join(); + assert.match(invalid, /invalid url/); + for (const text of [mismatch, invalid]) assert.ok(!/sekret|pw@/.test(text), text); +}); + test('every legacy configuration shape passes validation unchanged', () => { const legacy: McplServerConfig[] = [ cfg({ command: 'srv', requestTimeoutMs: 0 }), // zero still disables the legacy watchdog @@ -76,6 +120,17 @@ test('a modern server needs a real deadline', () => { } }); +test('a legacy deadline above the timer limit is refused; every value that worked passes', () => { + for (const bad of [MAX_TIMER_MS + 1, Number.POSITIVE_INFINITY]) { + for (const c of [cfg({ command: 'srv', requestTimeoutMs: bad }), cfg({ url: 'wss://x/mcpl', requestTimeoutMs: bad })]) { + assert.match(serverConfigProblems(c).join(), /requestTimeoutMs must be at most 2147483647/, String(bad)); + } + } + for (const good of [undefined, 0, 1, 60_000, MAX_TIMER_MS]) { + assert.deepEqual(serverConfigProblems(cfg({ command: 'srv', requestTimeoutMs: good })), [], String(good)); + } +}); + test('MCPL policy on a modern server is rejected, empty values are not', () => { const effective: Partial = { enabledFeatureSets: ['a'], @@ -106,6 +161,14 @@ test('MCPL policy on a modern server is rejected, empty values are not', () => { assert.deepEqual(serverConfigProblems(empty), []); }); +test('a credential on a modern stdio server is refused; a legacy stdio server still ignores it', () => { + const provider = async () => 'fresh'; + assert.match(serverConfigProblems(cfg({ command: 'srv', protocol: 'modern', token: 't' })).join(), /"token" applies only to a network \(url\) server/); + assert.match(serverConfigProblems(cfg({ command: 'srv', protocol: 'modern', accessProvider: provider })).join(), /"accessProvider" applies only to a network/); + assert.deepEqual(serverConfigProblems(cfg({ command: 'srv', token: 't', accessProvider: provider })), []); + assert.deepEqual(serverConfigProblems(cfg({ url: 'https://x/mcp', token: 't', accessProvider: provider })), []); +}); + test('a credential sent in cleartext to a host that is not loopback is warned about, never refused', () => { const provider = async () => 'fresh'; const warned = (c: Partial) => serverConfigWarnings(cfg(c)); @@ -123,10 +186,67 @@ test('a credential sent in cleartext to a host that is not loopback is warned ab { url: 'ws://0.0.0.0:8080/mcpl', token: 't' }, { url: 'http://[::1]:8080/mcp', token: 't' }, { url: 'ws://app.localhost/mcpl', token: 't' }, + { url: 'http://[::]:8080/mcp', token: 't' }, + { url: 'http://[::ffff:127.0.0.1]:8080/mcp', token: 't' }, + { url: 'http://[::ffff:127.9.8.7]/mcp', token: 't' }, + { url: 'http://localhost.:8080/mcp', token: 't' }, + { url: 'ws://app.localhost./mcpl', token: 't' }, + { url: 'http://localhost:8080/mcp?token=abc' }, { url: 'http://10.0.0.5:8080/mcp' }, + { url: 'http://10.0.0.5:8080/mcp?region=eu&page=2' }, { command: 'srv', url: 'http://10.0.0.5/mcp', token: 't' }, { url: 'ftp://10.0.0.5/mcp', token: 't' }, ]) assert.deepEqual(warned(quiet), [], JSON.stringify(quiet)); + // Not loopback: a mapped address outside 127/8, and a name that only ends in "localhost". + for (const loud of ['http://[::ffff:10.0.0.5]/mcp', 'http://notlocalhost/mcp']) { + assert.equal(warned({ url: loud, token: 't' }).length, 1, loud); + } +}); + +test('a secret the url carries itself is warned about too, and the warning never quotes it', () => { + const warned = (url: string, extra: Partial = {}) => serverConfigWarnings(cfg({ url, ...extra })); + assert.match(warned('http://user:pw@10.0.0.5/mcp')[0]!, /its url's userinfo goes to 10\.0\.0\.5 unencrypted \(http:\/\/\).*read it;/); + assert.match(warned('ws://mcp.lan/mcpl?api_key=k1')[0]!, /its url's query parameter "api_key" goes to mcp\.lan unencrypted.*use wss:\/\//); + for (const name of ['token', 'Access_Token', 'apikey', 'client_secret', 'password', 'auth', 'X-Amz-Signature']) { + assert.equal(warned(`http://10.0.0.5/mcp?${name}=v`).length, 1, name); + } + const both = warned('http://u:p@10.0.0.5/mcp?token=a&sig_key=b®ion=eu', { token: 't' })[0]!; + assert.match(both, /its credential, its url's userinfo and its url's query parameters "token", "sig_key" go to 10\.0\.0\.5 unencrypted.*read them;/); + for (const text of [both, ...warned('http://user:pw@10.0.0.5/mcp?token=sekret')]) assert.ok(!/sekret|pw|u:p/.test(text), text); + const path = warned('http://10.0.0.5/api/mcp/s/Zt4kQ9xW2vLm8pRs7YbN/mcp')[0]!; + assert.match(path, /a credential-like segment of its url's path goes to 10\.0\.0\.5 unencrypted.*read it;/); + assert.ok(!path.includes('Zt4kQ9xW2vLm8pRs7YbN'), path); + assert.deepEqual(warned('http://10.0.0.5/v2/2026-07-28/mcp'), [], 'a short segment with digits is a name'); + // Encrypted or loopback: nothing. + assert.deepEqual(warned('https://user:pw@10.0.0.5/mcp?token=a'), []); + assert.deepEqual(warned('http://user:pw@127.0.0.1/mcp?token=a'), []); +}); + +test('an access provider that throws is never quoted in the log, on either dial', async () => { + const message = 'refresh token sekret-123 expired'; + const accessProvider = async (): Promise => { throw new Error(message); }; + const logged: string[] = []; + const realError = console.error; + console.error = (...args: unknown[]) => { logged.push(args.map(String).join(' ')); }; + try { + // Nothing listens on port 1: each dial fails after asking the provider. + await assert.rejects(WebSocketTransport.open(cfg({ id: 'ws', url: 'ws://127.0.0.1:1/mcpl', accessProvider }))); + await assert.rejects(ModernMcpConnection.connect(cfg({ id: 'http', url: 'http://127.0.0.1:1/mcp', accessProvider }))); + } finally { + console.error = realError; + } + const fallbacks = logged.filter((line) => line.includes('access provider failed')); + assert.equal(fallbacks.length, 2, logged.join('\n')); + for (const line of fallbacks) assert.ok(line.includes(`access provider failed (Error, ${message.length} characters)`), line); + assert.ok(!logged.some((line) => line.includes('sekret')), logged.join('\n')); +}); + +test('an access provider that throws is logged by its error class and length, not its text', () => { + assert.equal(describeCredentialError(new TypeError('refresh failed for sekret-abc')), 'TypeError, 29 characters'); + class ProviderError extends Error {} + assert.equal(describeCredentialError(new ProviderError('x')), 'ProviderError, 1 character'); + assert.equal(describeCredentialError('sekret'), 'a thrown string'); + assert.equal(describeCredentialError(undefined), 'a thrown undefined'); }); test('both engines admit a configuration through checkServerConfig, which logs its warnings', () => { diff --git a/test/mcpl-legacy-version.test.ts b/test/mcpl-legacy-version.test.ts index 050cd5b65..46e5ae415 100644 --- a/test/mcpl-legacy-version.test.ts +++ b/test/mcpl-legacy-version.test.ts @@ -187,13 +187,24 @@ test('a repeating cursor is an error, not a silently partial inventory', async ( await assert.rejects(connection.sendToolsList(), /repeated tools\/list cursor "same"/); }); -test('a modern configuration is refused by the legacy engine before anything is dialed', async () => { +test('a modern configuration is refused by the legacy engine before anything is dialed or spawned', async () => { await assert.rejects( McplServerConnection.connect({ id: 'modern', url: 'https://example.invalid/mcp' }, HOST_CAPS), /configured for modern MCP \(2026-07-28\); McplServerConnection speaks the legacy family/, ); + const dir = mkdtempSync(join(tmpdir(), 'legacy-version-')); + cleanups.push(() => rmSync(dir, { recursive: true, force: true })); + const startLog = join(dir, 'starts.log'); + await assert.rejects( + McplServerConnection.connect(config('ok', { protocol: 'modern' }, startLog), HOST_CAPS), + /configured for modern MCP/, + ); await assert.rejects( - McplServerConnection.connect({ id: 'modern', command: process.execPath, protocol: 'modern' }, HOST_CAPS), + McplServerConnection.connectWithReconnect(config('ok', { protocol: 'modern', reconnect: true }, startLog), HOST_CAPS), /configured for modern MCP/, ); + assert.equal(existsSync(startLog), false, 'the fixture was never spawned'); + // The same fixture, configured as legacy, does start: the log above can tell. + await connect(config('ok', {}, startLog)); + assert.equal(readFileSync(startLog, 'utf8').split('\n').filter(Boolean).length, 1); }); diff --git a/test/mcpl-list-status.test.ts b/test/mcpl-list-status.test.ts index e30507121..b55eb57c5 100644 --- a/test/mcpl-list-status.test.ts +++ b/test/mcpl-list-status.test.ts @@ -46,6 +46,7 @@ test('listMcplServers exposes the live grant layers and host-owned authority', ( // The stub never handshook, so no revision is established. protocolVersion: null, transport: 'stdio', + refused: null, toolPrefix: 'mcpl--discord', toolCount: 1, policyEstablished: true, @@ -92,3 +93,41 @@ test('listMcplServers distinguishes a disconnected retrying stub from connected' lastNegotiatedAt: null, }); }); + +test('listMcplServers lists a refused configuration as refused, shows urls redacted, and gives a modern server no MCPL policy', () => { + const framework = Object.create(AgentFramework.prototype) as any; + framework.mcplServerConfigs = new Map>([ + ['both', { id: 'both', command: 'node', url: 'wss://u:pw@x.example/mcpl?token=sekret' }], + ['slow', { id: 'slow', url: 'https://x.example/mcp', requestTimeoutMs: 0 }], + ['docs', { id: 'docs', url: 'https://mcp.example/mcp?api_key=sekret®ion=eu' }], + ['chat', { id: 'chat', url: 'wss://user:pw@chat.example/mcpl' }], + ['zap', { id: 'zap', url: 'https://mcp.zapier.com/api/mcp/s/sekretZt4kQ9xW2vLm8/mcp' }], + ]); + framework.mcplServerRegistry = { getServer: () => null }; + framework.mcplTools = []; + framework.mcplPrefixMap = new Map(); + const listed = (framework as AgentFramework).listMcplServers(); + const byId = new Map(listed.map((s) => [s.id, s])); + + // No usable transport: no family or transport is guessed. + const both = byId.get('both')!; + assert.equal(both.family, null); + assert.equal(both.transport, null); + assert.match(both.refused!, /"command" and "url" are both set/); + assert.equal(both.url, 'wss://x.example/mcpl?token=redacted'); + // A binding that resolves, refused for another reason. + const slow = byId.get('slow')!; + assert.equal(slow.family, 'modern'); + assert.equal(slow.transport, 'http'); + assert.match(slow.refused!, /requestTimeoutMs must be an integer from 1/); + // Usable: nothing refused. MCPL policy doesn't apply to a modern server. + const docs = byId.get('docs')!; + assert.deepEqual([docs.family, docs.transport, docs.refused, docs.policyEstablished], ['modern', 'http', null, null]); + assert.equal(docs.url, 'https://mcp.example/mcp?api_key=redacted®ion=redacted'); + const chat = byId.get('chat')!; + assert.deepEqual([chat.family, chat.transport, chat.refused, chat.policyEstablished], ['legacy', 'websocket', null, false]); + assert.equal(chat.url, 'wss://chat.example/mcpl'); + // A per-user secret in the path goes too. + assert.equal(byId.get('zap')!.url, 'https://mcp.zapier.com/api/mcp/s/redacted/mcp'); + assert.ok(!/sekret|pw@/.test(JSON.stringify(listed)), 'no credential in the status'); +}); diff --git a/test/mcpl-ws-transport.test.ts b/test/mcpl-ws-transport.test.ts index 7626eaf76..58cd8415e 100644 --- a/test/mcpl-ws-transport.test.ts +++ b/test/mcpl-ws-transport.test.ts @@ -3,7 +3,7 @@ import assert from 'node:assert/strict'; import { WebSocketServer, type WebSocket } from 'ws'; import type { AddressInfo } from 'node:net'; import { McplServerConnection } from '../src/mcpl/server-connection.js'; -import { buildWebSocketUrl, isWebSocketTransport } from '../src/mcpl/transport.js'; +import { buildWebSocketUrl, isWebSocketTransport, openTransport } from '../src/mcpl/transport.js'; import type { McplHostCapabilities } from '../src/mcpl/types.js'; const HOST_CAPS: McplHostCapabilities = { version: '0.4', pushEvents: true, featureSets: true }; @@ -73,15 +73,32 @@ test('buildWebSocketUrl validates protocol and appends the token as a query para ); assert.throws(() => buildWebSocketUrl({ id: 's', url: 'http://example.com' }), /ws:\/\/ or wss:\/\//); assert.throws(() => buildWebSocketUrl({ id: 's', command: 'x' }), /requires "url"/); + // A url that doesn't parse can't be redacted, so the error doesn't echo it. + assert.throws(() => buildWebSocketUrl({ id: 's', url: 'not a url ?token=sekret' }), + (error: Error) => /invalid WebSocket url/.test(error.message) && !error.message.includes('sekret')); }); test('isWebSocketTransport selects the transport from config shape', () => { assert.equal(isWebSocketTransport({ id: 's', url: 'wss://x/y' }), true); assert.equal(isWebSocketTransport({ id: 's', url: 'wss://x/y', transport: 'websocket' }), true); assert.equal(isWebSocketTransport({ id: 's', command: 'node' }), false); - // Explicit transport wins over shape. assert.equal(isWebSocketTransport({ id: 's', command: 'node', transport: 'stdio' }), false); - assert.equal(isWebSocketTransport({ id: 's', url: 'wss://x', command: 'node' }), false); // url+command → stdio unless transport says otherwise + // The scheme decides, as resolveServerBinding does: an http(s) url is modern MCP, never a WebSocket. + assert.equal(isWebSocketTransport({ id: 's', url: 'https://x/mcp' }), false); + assert.equal(isWebSocketTransport({ id: 's', url: 'http://x/mcp', transport: 'http' }), false); + // With both a command and a url, the explicit transport decides. + assert.equal(isWebSocketTransport({ id: 's', url: 'wss://x', command: 'node', transport: 'websocket' }), true); + assert.equal(isWebSocketTransport({ id: 's', url: 'wss://x', command: 'node', transport: 'stdio' }), false); + // A configuration resolveServerBinding refuses selects nothing. + assert.equal(isWebSocketTransport({ id: 's', url: 'wss://x', command: 'node' }), false); + assert.equal(isWebSocketTransport({ id: 's', url: 'https://x/mcp', transport: 'websocket' }), false); +}); + +test('openTransport refuses a modern or refused configuration before dialing or spawning anything', async () => { + // x.invalid never resolves, so a dial would fail with a connect error, not these. + await assert.rejects(openTransport({ id: 'm', url: 'https://x.invalid/mcp' }), /configured for modern MCP; the legacy transports can't open it/); + await assert.rejects(openTransport({ id: 'm', command: 'srv', protocol: 'modern' }), /configured for modern MCP; the legacy transports can't open it/); + await assert.rejects(openTransport({ id: 'b', command: 'srv', url: 'wss://x.invalid/mcpl' }), /"command" and "url" are both set/); }); test('connects over WebSocket, negotiates capabilities, lists tools, and round-trips a call', async () => { @@ -118,6 +135,17 @@ test('a failed WebSocket dial rejects with a clear error (no reconnect)', async ); }); +test("a failed dial's error names the endpoint, never a credential the url or the token carries", async () => { + await assert.rejects( + McplServerConnection.connect({ id: 'dead', url: 'ws://u:pw@127.0.0.1:1/mcpl/s/sekret4kQ9xW2vLm8pRs/x?api_key=sekret&v=2', token: 'tok-sekret' }, HOST_CAPS), + (error: Error) => { + assert.match(error.message, /WebSocket connect (failed|timed out).*\(ws:\/\/127\.0\.0\.1:1\/mcpl\/s\/redacted\/x\?api_key=redacted&v=redacted\)/); + assert.ok(!/sekret|pw@/.test(error.message), error.message); + return true; + }, + ); +}); + // A tiny stdio MCPL server: reads NDJSON on stdin, answers the same three // methods on stdout, and writes one line to stderr on boot (to prove stderr // still surfaces after the transport refactor). Kept inline so the test is From f1a6e8f84e403003b045a935899b8dd3e0f72969 Mon Sep 17 00:00:00 2001 From: Olive-1011 Date: Sun, 11 Oct 2026 17:22:48 -0400 Subject: [PATCH 0122/1112] fix(mcp): the modern engine classifies refused sends and HTTP refusals by what reached the server; a revision refusal is never retried A review of this pull request found places where the modern engine (ModernMcpConnection) reported a request outcome the wire didn't support, retried what can't succeed, or let close() resolve before a connect's own cleanup had finished, and test checks that judged behaviour by the wall clock. This commit answers them. Request outcomes (ObservedTransport, requestFailure): - A send Connectome's stdio transport refuses before writing anything (its line is closed) now throws a typed NotWrittenError, and ObservedTransport puts the request's record back as it was before that leg: nothing reached the server, so the outcome is not-sent. A throw from the HTTP transport's send stays handed off, since part of a request may have gone out. - requestFailure applies its rules in the documented order: nothing handed off is not-sent, whatever was raised; then the server's answer on the most recent leg (a JSON-RPC error, a complete result, or an answer that didn't finish the call); only then an HTTP 401/403 refusal of the current leg. The record now remembers that an earlier leg asked for another round (`continued`), which a later leg's send doesn't clear, so a 401/403 on a continuation leg is no-response, since the server may already have acted. Before, the HTTP check came first: a 401 raised before anything was handed off read as error-response, and so did one on a later round. - A tools/list failure says what a read is: at its deadline, the list didn't arrive in time, and listing changes nothing on the server, so retrying is safe. It used the tool-call warning about duplicating a side effect. Lifetime: - A server that shares no revision with this client (the SDK's UnsupportedProtocolVersionError: no common revision in server/discover, or an unsupported-version error) is a configuration verdict, as for the legacy engine, and in the same type, McplProtocolVersionError (kind rejected, the server's supported list, code -32022, offered 2026-07-28), so every reader treats both engines alike. The first connect throws it even with reconnect set, and a reconnect that meets it emits reconnect-failed with permanent: true; either way reconnecting stops. Before, it was retried forever. When the server lists the legacy revision, the message says what reaches it: for a stdio server, its configuration without protocol: 'modern'; for an http(s) url, which is modern by its scheme, that legacy MCP is reached over stdio or WebSocket. McplProtocolVersionError takes the offered revision as an optional last constructor parameter, defaulting to the legacy one, so legacy call sites don't change. - A new connect generation resets the list-change reopen counter, so its first reopen backs off from the base interval rather than from the previous generation's failures. - The reconnect and reopen backoff is clamped to the timer limit (MAX_TIMER_MS): with jitter it could pass 2^31-1 ms, which Node fires at once, turning the backoff into a tight loop. - open()'s `done`, which close() waits on, now settles only after that connect's own cleanup, its second close of the generation included, on every path (establish() holds the connect and its cleanup). close() then reports a child that cleanup couldn't reap, instead of resolving before it and leaving the failure behind. - The client.onerror comment says what happens: those errors are emitted as the connection's 'error' and reach the host's listeners. Tests (mcp-modern-outcomes, with two fixture flags and a tool): - a send the stdio line refused before writing is not-sent, with nothing on the wire; - a plain call whose child exits before answering is no-response; - the classification order: an HTTP refusal before anything was handed off is not-sent, on a first leg error-response with the status, and after an earlier round no-response; a tools/list deadline reads as a safe-to-retry read; - the backoff never exceeds the timer limit; - a new generation's first reopen is scheduled with the counter reset; - a server listing only another revision fails the first connect with the verdict, nothing will retry it, and one launch ran; a reconnect that meets it emits one permanent reconnect-failed and schedules nothing; a legacy-only server's verdict points a stdio server to its configuration and an http url to stdio or WebSocket; - close() during a connect waits for that connect's own cleanup and reports what it couldn't reap; - the concurrent-close test counts the signals teardown sends, so a second close running its own teardown fails it; - the reconnect-awareness test snapshots once the reopened subscription has been answered (the refresh it announces), not after a fixed 50 ms; - the deadline-between-rounds test holds the round explicitly: leg 1 asks for elicitation input, and an input handler registered on the session's client holds until the deadline aborts it, as the SDK's own pause ends. It no longer races leg 1 against the SDK's internal 250 ms pause, and it shows only leg 1 reached the server by a later call's arrival on the same ordered wire, not a fixed 100 ms wait. Wall-clock ceilings: the modern deadline test runs a 20 s tool and checks that the call rejected before the tool finished and that the server saw the cancellation, instead of a 1.5 s ceiling; the "promptly" checks in mcp-server-ownership and mcpl-legacy-lifetime keep a bound, raised to 20 s, well below the 30 s timeouts they rule out. Each of these fixes, undone alone, fails mcp-modern-outcomes: the pre-write undo; the later-round HTTP rule; the relisten reset; the first connect's verdict; the reconnect's verdict; the verdict's wrapping; the clamp; the listing's wording; close() reporting the connect's own cleanup; the transport-aware hint. --- src/mcpl/modern-connection.ts | 195 +++++++++++++++++++----- src/mcpl/server-connection.ts | 24 +-- test/fixtures/modern-raw-server.mjs | 16 +- test/mcp-modern-connection.test.ts | 8 +- test/mcp-modern-outcomes.test.ts | 228 ++++++++++++++++++++++++++-- test/mcp-server-ownership.test.ts | 3 +- test/mcpl-legacy-lifetime.test.ts | 5 +- 7 files changed, 411 insertions(+), 68 deletions(-) diff --git a/src/mcpl/modern-connection.ts b/src/mcpl/modern-connection.ts index 7af667292..cafc4ddf3 100644 --- a/src/mcpl/modern-connection.ts +++ b/src/mcpl/modern-connection.ts @@ -26,7 +26,9 @@ * `input_required` answer asks for another round, so a call that ends * after one has no answer. * - **Lifetime.** On a lost transport the connection restarts with the same - * backoff settings as legacy, when `reconnect` is set. List-change + * backoff settings as legacy, when `reconnect` is set. A server that shares + * no revision with this client is a configuration verdict, as for legacy + * (`McplProtocolVersionError`): it is never retried. List-change * awareness is kept while the connection lives: a subscription the server * advertises but that didn't open, or that ended, is reopened. Each connect * is a generation, so a superseded client never touches a newer one. @@ -45,6 +47,7 @@ import { SdkErrorCode, SdkHttpError, StreamableHTTPClientTransport, + UnsupportedProtocolVersionError, parseJSONRPCMessage, type AuthProvider, type CallToolResult, @@ -56,8 +59,8 @@ import { } from '@modelcontextprotocol/client'; import { StdioTransport, type McplTransport } from './transport.js'; -import { McplRequestError } from './server-connection.js'; -import { MODERN_MCP_PROTOCOL_VERSION, checkServerConfig, describeCredentialError } from './protocol-family.js'; +import { McplProtocolVersionError, McplRequestError } from './server-connection.js'; +import { LEGACY_MCP_PROTOCOL_VERSION, MAX_TIMER_MS, MODERN_MCP_PROTOCOL_VERSION, checkServerConfig, describeCredentialError } from './protocol-family.js'; import type { McplServerConfig, McpToolDefinition } from './types.js'; const DEFAULT_REQUEST_TIMEOUT_MS = 60_000; @@ -91,8 +94,24 @@ interface WireRecord { readonly method: string; handedOff: boolean; currentId?: string | number; + /** The current leg's answer. */ answer: 'none' | 'error' | 'complete' | 'input-required' | 'unrecognized'; error?: { code: number; message: string; data?: unknown }; + /** An earlier leg's answer asked for another round (or wasn't a complete + * result): the server may have acted on the call, whatever a later leg + * meets. A new leg's send never clears it. */ + continued: boolean; +} + +/** + * A send that Connectome's own transport refused before writing anything (its + * stdio line is closed): nothing of that request reached the server. + */ +class NotWrittenError extends Error { + constructor(message: string) { + super(message); + this.name = 'NotWrittenError'; + } } /** The logical request a send belongs to. The SDK hands each request to the @@ -146,15 +165,30 @@ class ObservedTransport implements Transport { async send(message: JSONRPCMessage, options?: TransportSendOptions): Promise { const record = wireScope.getStore(); + let undo: (() => void) | undefined; if (record && 'method' in message && 'id' in message && message.method === record.method) { - // Handed off from here on: past this point nothing proves it unsent. + const prior = { handedOff: record.handedOff, currentId: record.currentId, answer: record.answer, error: record.error }; + // Handed off from here on: past this point nothing proves it unsent, + // unless the transport says it wrote nothing. record.handedOff = true; record.currentId = message.id; record.answer = 'none'; record.error = undefined; this.awaiting.set(message.id, record); + undo = () => { + this.awaiting.delete(message.id); + Object.assign(record, prior); + }; + } + try { + await this.inner.send(message, options); + } catch (error) { + // Refused before any write: this leg never reached the server, so the + // record is as it was before it. Any other throw (an HTTP send can fail + // after part of the request went out) leaves it handed off. + if (error instanceof NotWrittenError) undo?.(); + throw error; } - await this.inner.send(message, options); } async close(): Promise { @@ -183,6 +217,7 @@ class ObservedTransport implements Transport { const result = (message as { result?: unknown }).result; const type = typeof result === 'object' && result !== null ? (result as { resultType?: unknown }).resultType : undefined; record.answer = type === 'complete' ? 'complete' : type === 'input_required' ? 'input-required' : 'unrecognized'; + if (record.answer !== 'complete') record.continued = true; } return record; } @@ -239,7 +274,7 @@ class SpawnerStdioTransport implements Transport { } async send(message: JSONRPCMessage): Promise { - if (!this.line || this.line.closed) throw new Error(`MCP server "${this.config.id}" stdio transport is closed`); + if (!this.line || this.line.closed) throw new NotWrittenError(`MCP server "${this.config.id}" stdio transport is closed`); this.line.writeLine(JSON.stringify(message)); } @@ -297,7 +332,7 @@ export class ModernMcpConnection extends EventEmitter { private opening: { session: Session; done: Promise } | null = null; private generation = 0; private closedByHost = false; - private readonly reconnectEnabled: boolean; + private reconnectEnabled: boolean; private readonly reconnectIntervalMs: number; private readonly reconnectMaxIntervalMs: number; private reconnectTimer: ReturnType | null = null; @@ -416,6 +451,12 @@ export class ModernMcpConnection extends EventEmitter { await this.open(); } catch (error) { if (this.closedByHost) return; + // No common revision is a configuration fact, not an outage: a retry + // would restart the server to hear the same answer. + if (error instanceof McplProtocolVersionError) { + this.reconnectEnabled = false; + throw error; + } if (!this.reconnectEnabled) throw error; const halted = this.unreapedFailure(); if (halted) { @@ -446,6 +487,10 @@ export class ModernMcpConnection extends EventEmitter { if (this.closedByHost) return; const generation = ++this.generation; this.cancelRelisten(); + // A new generation's subscription has failed nothing yet: its first + // reopen backs off from the base interval, not from the previous + // generation's failures. + this.relistenAttempts = 0; const inner: Transport = this.transportKind === 'stdio' ? new SpawnerStdioTransport(this.config, (line) => { if (this.isCurrent(generation)) this.deliverStderr(line); }) : new StreamableHTTPClientTransport(new URL(this.config.url!), { authProvider: bearerAuth(this.config) }); @@ -470,8 +515,10 @@ export class ModernMcpConnection extends EventEmitter { client.onerror = (error: Error) => { // The SDK reports late responses to deadline-abandoned calls here // ("unknown message ID"), and a list-change subscription it could not - // open at connect, among other out-of-band errors. They are logged, - // never delivered: the call's outcome was already reported. + // open at connect, among other out-of-band errors. They are emitted as + // this connection's 'error', so they reach the host's error listeners + // (and are logged when there are none). They never change a call's + // outcome, which was already reported. if (this.isCurrent(generation)) this.emit('error', error); }; client.onclose = () => { @@ -479,18 +526,29 @@ export class ModernMcpConnection extends EventEmitter { }; const session: Session = { generation, client, wire }; - const done = client.connect(wire, { timeout: CONNECT_TIMEOUT_MS }); - this.opening = { session, done }; + // `done` settles only once this connect's own cleanup has finished, on + // every path: close() waits on it, and must not resolve while that + // cleanup could still find a child it can't reap. + let settle!: () => void; + this.opening = { session, done: new Promise((resolve) => { settle = resolve; }) }; try { - await done; + await this.establish(session); + } finally { + if (this.opening?.session === session) this.opening = null; + settle(); + } + } + + /** Connect one generation's client, and adopt it if it's still current. */ + private async establish(session: Session): Promise { + try { + await session.client.connect(session.wire, { timeout: CONNECT_TIMEOUT_MS }); } catch (error) { const failure = await ModernMcpConnection.closeSession(session); if (failure) this.holdUnreaped(session, failure); - throw error; - } finally { - if (this.opening?.session === session) this.opening = null; + throw this.versionVerdict(error) ?? error; } - if (!this.isCurrent(generation)) { + if (!this.isCurrent(session.generation)) { // Superseded or closed while connecting: this generation never serves. const failure = await ModernMcpConnection.closeSession(session); if (failure) this.holdUnreaped(session, failure); @@ -499,7 +557,40 @@ export class ModernMcpConnection extends EventEmitter { this.session = session; this.protocolVersion = MODERN_MCP_PROTOCOL_VERSION; this.reconnectAttempts = 0; - this.keepListening(session, client.autoOpenedSubscription); + this.keepListening(session, session.client.autoOpenedSubscription); + } + + /** + * The server refused the pinned revision (no common one in its + * `server/discover` answer, or an unsupported-version error): a + * configuration verdict, as for the legacy engine, in the same type, so + * every reader treats both engines alike. Retrying can't change it. + */ + private versionVerdict(error: unknown): McplProtocolVersionError | null { + if (!(error instanceof UnsupportedProtocolVersionError)) return null; + const data = (error as { data?: unknown }).data as { supported?: unknown } | undefined; + const supported = Array.isArray(data?.supported) && data.supported.every((v) => typeof v === 'string') + ? (data.supported as string[]) + : null; + // Legacy MCP is reached over stdio or WebSocket. A stdio server's + // family is its configuration's choice; an http(s) url is modern by its + // scheme, so there is nothing to reconfigure on it. + const legacyHint = !supported?.includes(LEGACY_MCP_PROTOCOL_VERSION) + ? '' + : this.transportKind === 'stdio' + ? ` It speaks legacy MCP: configure it without protocol: 'modern'.` + : ` It speaks only legacy MCP, which this client reaches over stdio or WebSocket (ws://, wss://), not http(s).`; + return new McplProtocolVersionError( + `MCP server "${this.id}" does not support MCP ${MODERN_MCP_PROTOCOL_VERSION}` + + (supported ? ` (it supports: ${supported.join(', ') || 'none listed'}).` : ` (${error.message}).`) + + legacyHint, + 'rejected', + null, + supported, + (error as { code?: unknown }).code === undefined ? undefined : Number((error as { code?: unknown }).code), + (error as { data?: unknown }).data, + MODERN_MCP_PROTOCOL_VERSION, + ); } /** @@ -570,11 +661,12 @@ export class ModernMcpConnection extends EventEmitter { } /** The legacy engine's backoff: doubling from the base interval, capped, - * with ±25% jitter. */ + * with ±25% jitter, and never past the timer limit (a longer delay would + * fire at once, turning the backoff into a tight loop). */ private backoffDelay(failures: number): number { const exponent = Math.max(0, Math.min(failures - 1, 30)); const capped = Math.min(this.reconnectIntervalMs * 2 ** exponent, this.reconnectMaxIntervalMs); - return capped * (0.75 + Math.random() * 0.5); + return Math.min(capped * (0.75 + Math.random() * 0.5), MAX_TIMER_MS); } private scheduleReconnect(): void { @@ -601,6 +693,13 @@ export class ModernMcpConnection extends EventEmitter { this.emit('reconnect-failed', { error: `${(error as Error).message}; ${halted}`, attempt, permanent: true }); return; } + // No common revision: every further attempt would hear the same + // answer. Stop; only the configuration can fix it. + if (error instanceof McplProtocolVersionError) { + this.reconnectEnabled = false; + this.emit('reconnect-failed', { error: error.message, attempt, permanent: true }); + return; + } this.emit('reconnect-failed', { error: (error as Error).message, attempt }); this.scheduleReconnect(); } @@ -657,7 +756,7 @@ export class ModernMcpConnection extends EventEmitter { // Connectome refuses before anything reaches the SDK: provably not sent. throw new McplRequestError(`Cannot send ${what}: connection to "${this.id}" is not established`, 'not-sent'); } - const record: WireRecord = { method, handedOff: false, answer: 'none' }; + const record: WireRecord = { method, handedOff: false, answer: 'none', continued: false }; try { return await wireScope.run(record, () => run(session.client)); } catch (error) { @@ -726,7 +825,13 @@ export class ModernMcpConnection extends EventEmitter { if (!this.unreaped.some((held) => held.session === target)) this.unreaped.push({ session: target, failure }); } } - if (opening) await opening.done.catch(() => {}); + if (opening) { + // The connect's own cleanup closes its generation again, after the + // close above; `done` settles once it has. A child that cleanup + // couldn't reap is this close's failure too. + await opening.done; + for (const held of this.unreaped) if (!failures.includes(held.failure)) failures.push(held.failure); + } if (session) this.emit('close', { reason: 'closed by host' }); if (failures.length > 0) { throw new Error(`MCP server "${this.id}" did not close cleanly: ${failures.map((f) => f.message).join('; ')}`); @@ -736,13 +841,12 @@ export class ModernMcpConnection extends EventEmitter { /** * Map a failure onto the request-outcome contract by what crossed the * transport boundary for this request, claiming no more than that shows. - * The call's answer is the server's final one: an error or a `complete` - * result. - * - nothing handed to the transport: `not-sent`, whatever the SDK raised - * (a pre-dispatch validation, a signal already aborted, no transport); - * - an HTTP 401/403 refusal: `error-response`. The server answered, at the - * HTTP layer, so the status is in `data.httpStatus` and `code` stays the - * JSON-RPC code space's (absent); + * The rules apply in this order. The call's answer is the server's final + * one: an error or a `complete` result. + * - nothing handed to the transport: `not-sent`, whatever was raised (a + * pre-dispatch validation, a signal already aborted, no transport, a send + * Connectome's stdio line refused before writing, an HTTP refusal before + * any request went out); * - the most recent leg answered with a JSON-RPC error: `error-response` * with that error's code and data; * - the most recent leg answered with a complete result: the server @@ -756,16 +860,22 @@ export class ModernMcpConnection extends EventEmitter { * rounds cap). Nothing is in flight then, so nothing is cancelled. A * result of a type this client doesn't recognize ends the call where it * is; + * - an HTTP 401/403 refusal of the current leg: `error-response`, the + * status in `data.httpStatus` with `code` absent (the server answered at + * the HTTP layer, outside the JSON-RPC code space). After an earlier leg + * asked for another round it is `no-response` instead: the server may + * already have acted on the call; * - handed off and unanswered: `no-response`. At the deadline, cancellation * was requested and the outcome is unknown. + * A `tools/list` failure says so: listing changes nothing on the server, so + * a retry is safe, where a tool call's retry may duplicate a side effect. */ private requestFailure(what: string, record: WireRecord, error: unknown, timeoutMs?: number): Error { const prefix = `MCP server "${this.id}"`; const message = error instanceof Error ? error.message : String(error); - if (error instanceof SdkHttpError && (error.code === SdkErrorCode.ClientHttpAuthentication || error.code === SdkErrorCode.ClientHttpForbidden)) { - const data = error.data && typeof error.data === 'object' ? error.data : {}; - return new McplRequestError(`${prefix} refused ${what}: HTTP ${error.status} (${message})`, 'error-response', undefined, { ...data, httpStatus: error.status }); - } + const advice = record.method === 'tools/list' + ? 'Listing changes nothing on the server, so it is safe to retry.' + : 'Verify state before retrying; a blind retry of a side-effecting tool may duplicate it.'; if (!record.handedOff) { return new McplRequestError(`${prefix} ${what} was not sent: ${message}`, 'not-sent'); } @@ -787,16 +897,27 @@ export class ModernMcpConnection extends EventEmitter { : `it asked for another round (input_required), and the call ended before that round was sent: ${reason}`; return new McplRequestError( `${prefix} did not finish ${what}${within}: ${ending} ` + - `The outcome is unknown: the server may already have acted on the call. Verify state before retrying; ` + - `a blind retry of a side-effecting tool may duplicate it.`, + `The outcome is unknown: the server may already have acted on the call. ${advice}`, 'no-response', ); } + if (error instanceof SdkHttpError && (error.code === SdkErrorCode.ClientHttpAuthentication || error.code === SdkErrorCode.ClientHttpForbidden)) { + if (record.continued) { + return new McplRequestError( + `${prefix} did not finish ${what}: it asked for another round (input_required), and the server refused a later one: ` + + `HTTP ${error.status} (${message}). The outcome is unknown: the server may already have acted on the call. ${advice}`, + 'no-response', + ); + } + const data = error.data && typeof error.data === 'object' ? error.data : {}; + return new McplRequestError(`${prefix} refused ${what}: HTTP ${error.status} (${message})`, 'error-response', undefined, { ...data, httpStatus: error.status }); + } if (deadline) { + const outcome = record.method === 'tools/list' + ? 'the list did not arrive in time.' + : 'the outcome is unknown: the tool may still complete server-side.'; return new McplRequestError( - `${prefix} did not answer ${what}${within}. Cancellation was requested; ` + - `the outcome is unknown: the tool may still complete server-side. Verify state before retrying; ` + - `a blind retry of a side-effecting tool may duplicate it.`, + `${prefix} did not answer ${what}${within}. Cancellation was requested; ${outcome} ${advice}`, 'no-response', ); } diff --git a/src/mcpl/server-connection.ts b/src/mcpl/server-connection.ts index e1c5b1463..7a1eefd31 100644 --- a/src/mcpl/server-connection.ts +++ b/src/mcpl/server-connection.ts @@ -77,19 +77,22 @@ const UNSUPPORTED_PROTOCOL_VERSION_CODE = -32022; const MAX_TOOL_LIST_PAGES = 100; /** - * The server and this legacy engine have no protocol revision in common, so - * the connection was never established. This is a protocol-negotiation fact, - * not a request outcome. Retrying cannot change it: the engine offers one + * The server and this engine have no protocol revision in common, so the + * connection was never established. This is a protocol-negotiation fact, not + * a request outcome. Retrying cannot change it: each engine offers one * revision, and only configuration can fix the mismatch. A connection that - * gets this error does not schedule reconnects. + * gets this error does not schedule reconnects. The legacy engine and the + * modern one (ModernMcpConnection, whose pinned revision the server refused) + * both throw it, so its readers treat them alike. * - * - `rejected`: the server answered `initialize` with error -32022. - * `supported` is the server's own list, when it gave one. - * - `mismatch`: the server answered `initialize` with a revision other than - * 2024-11-05, or with none. + * - `rejected`: the server refused the offered revision: the legacy engine's + * `initialize` answered with error -32022, or a modern server's discovery + * or error naming no common revision. `supported` is the server's own list, + * when it gave one. + * - `mismatch`: the server answered the legacy engine's `initialize` with a + * revision other than 2024-11-05, or with none. */ export class McplProtocolVersionError extends Error { - readonly offered = MCP_PROTOCOL_VERSION; constructor( message: string, readonly kind: 'rejected' | 'mismatch', @@ -100,6 +103,9 @@ export class McplProtocolVersionError extends Error { /** The JSON-RPC error's code and data, verbatim (`rejected`). */ readonly code?: number, readonly data?: unknown, + /** The revision this client offered: the legacy engine's, unless a + * modern connection's. */ + readonly offered: string = MCP_PROTOCOL_VERSION, ) { super(message); this.name = 'McplProtocolVersionError'; diff --git a/test/fixtures/modern-raw-server.mjs b/test/fixtures/modern-raw-server.mjs index da7abbfb5..2044d8889 100644 --- a/test/fixtures/modern-raw-server.mjs +++ b/test/fixtures/modern-raw-server.mjs @@ -11,6 +11,8 @@ // reject-discover-later launches after the first answer server/discover with an error // reject-discover every launch answers server/discover with an error // ignore-sigterm the process survives SIGTERM (only SIGKILL ends it) +// other-revision every launch's server/discover lists only a revision the client doesn't speak +// other-revision-later launches after the first do // Tools: op (outputSchema {value: integer}, answers with a string: invalid // structured content), plain (text), err (a JSON-RPC error), touch (announces // a tool list change on the open subscription), nullframe (writes JSON lines @@ -19,7 +21,8 @@ // 'error', 'hang', 'ok', or 'again' for another state-only round every time. // arguments.leg1 'exit' exits just after the first leg's answer, and // 'elicit' asks for elicitation input instead), task (a result type this -// revision doesn't define), bare (a result with no resultType), die (exits). +// revision doesn't define), bare (a result with no resultType), die (exits), +// vanish (exits without answering). import { createInterface } from 'node:readline'; import { appendFileSync, existsSync, readFileSync } from 'node:fs'; @@ -45,6 +48,15 @@ createInterface({ input: process.stdin }).on('line', (line) => { note({ event: m.id === undefined ? 'notification' : 'request', method: m.method, id: m.id ?? null, ...(m.method === 'tools/call' ? { tool: m.params?.name } : {}) }); switch (m.method) { case 'server/discover': + if (flags.has('other-revision') || (flags.has('other-revision-later') && launch > 1)) { + return result(m.id, { + supportedVersions: ['2099-01-01'], + capabilities: { tools: {} }, + ttlMs: 60000, + cacheScope: 'private', + _meta: { 'io.modelcontextprotocol/serverInfo': { name: 'raw-fixture', version: '1' } }, + }); + } if (flags.has('hang-discover-later') && launch > 1) return; if (flags.has('hang-discover')) return; if (flags.has('reject-discover-later') && launch > 1) return error(m.id, -32603, 'not today'); @@ -88,6 +100,7 @@ createInterface({ input: process.stdin }).on('line', (line) => { { name: 'task', inputSchema: { type: 'object' } }, { name: 'bare', inputSchema: { type: 'object' } }, { name: 'die', inputSchema: { type: 'object' } }, + { name: 'vanish', inputSchema: { type: 'object' } }, ], }); case 'tools/call': { @@ -101,6 +114,7 @@ createInterface({ input: process.stdin }).on('line', (line) => { return result(m.id, { content: [{ type: 'text', text: subscription === null ? 'no subscription' : 'touched' }] }); } if (name === 'die') { setTimeout(() => process.exit(5), 10); return result(m.id, { content: [{ type: 'text', text: 'bye' }] }); } + if (name === 'vanish') process.exit(6); if (name === 'task') return send({ jsonrpc: '2.0', id: m.id, result: { resultType: 'task', taskId: 'task-1' } }); if (name === 'bare') return send({ jsonrpc: '2.0', id: m.id, result: { content: [{ type: 'text', text: 'no resultType' }] } }); if (name === 'nullframe') { diff --git a/test/mcp-modern-connection.test.ts b/test/mcp-modern-connection.test.ts index 28539665b..148adca4c 100644 --- a/test/mcp-modern-connection.test.ts +++ b/test/mcp-modern-connection.test.ts @@ -107,15 +107,17 @@ test('a deadline requests cancellation and reports no-response, outcome unknown' const dir = scratch(); const connection = await connect(config(dir, { requestTimeoutMs: 300 })); await connection.callTool('echo', { text: 'warm' }); // the slow call is not the connection's first request - const started = Date.now(); - await assert.rejects(connection.callTool('slow', { ms: 2000 }), (err: unknown) => { + // The tool would run far past the deadline, so the call rejecting while + // the tool hasn't finished shows it ended at the deadline, whatever the + // runner's load did to the wall clock. + await assert.rejects(connection.callTool('slow', { ms: 20_000 }), (err: unknown) => { assert.ok(err instanceof McplRequestError); assert.equal(err.outcome, 'no-response'); assert.match(err.message, /did not answer tools\/call "slow" within 300ms/); assert.match(err.message, /Cancellation was requested; the outcome is unknown/); return true; }); - assert.ok(Date.now() - started < 1500, 'rejected at the deadline, not at the tool\'s end'); + assert.ok(!lines(join(dir, 'events.log')).includes('slow-finished'), 'rejected at the deadline, not at the tool\'s end'); // The server saw the cancellation: the request was cancelled, not merely abandoned. await until(() => lines(join(dir, 'events.log')).includes('slow-aborted')); }); diff --git a/test/mcp-modern-outcomes.test.ts b/test/mcp-modern-outcomes.test.ts index bd555d669..62b06cbc1 100644 --- a/test/mcp-modern-outcomes.test.ts +++ b/test/mcp-modern-outcomes.test.ts @@ -22,9 +22,11 @@ import { tmpdir } from 'node:os'; import { join, dirname } from 'node:path'; import { fileURLToPath } from 'node:url'; +import { SdkError, SdkErrorCode, SdkHttpError, UnsupportedProtocolVersionError } from '@modelcontextprotocol/client'; import { ModernMcpConnection } from '../src/mcpl/modern-connection.js'; import { StdioTransport } from '../src/mcpl/transport.js'; -import { McplRequestError } from '../src/mcpl/server-connection.js'; +import { McplProtocolVersionError, McplRequestError } from '../src/mcpl/server-connection.js'; +import { MAX_TIMER_MS } from '../src/mcpl/protocol-family.js'; import type { McplServerConfig } from '../src/mcpl/types.js'; const here = dirname(fileURLToPath(import.meta.url)); @@ -123,15 +125,18 @@ test('a subscription refused at connect is reopened, and change awareness return test('awareness survives a reconnect whose first listen is refused again', async () => { const log = scratchLog(); const connection = await connect('listen-fail-first', log, { reconnect: true, reconnectIntervalMs: 30, reconnectMaxIntervalMs: 60 }); - let reconnected = false; - connection.on('reconnect', () => { reconnected = true; }); - await connection.callTool('die', {}); - await until(() => reconnected, 'reconnect'); - // Launch 2 refuses its first listen too; the newer generation reopens it. - await until(() => wire(log).filter((e) => e.method === 'subscriptions/listen' && e.launch === 2).length >= 2, 'a second listen on launch 2'); let changes = 0; connection.on('tools-list-changed', () => changes++); - await new Promise((r) => setTimeout(r, 50)); + let atReconnect: number | null = null; + connection.on('reconnect', () => { atReconnect = changes; }); + await connection.callTool('die', {}); + await until(() => atReconnect !== null, 'reconnect'); + // Launch 2 refuses its first listen too; the newer generation reopens it. + // A reopened subscription is answered before the connection announces the + // refresh it makes for changes it may have missed, so that announcement + // marks the subscription accepted. + await until(() => changes > atReconnect!, 'the reopened subscription answered on launch 2'); + assert.ok(wire(log).filter((e) => e.method === 'subscriptions/listen' && e.launch === 2).length >= 2, 'a second listen on launch 2'); const before = changes; await connection.callTool('touch', {}); await until(() => changes > before, 'the announced change on launch 2'); @@ -172,22 +177,41 @@ test('an input_required continuation keeps its evidence: the second leg decides }); test('a deadline between continuation rounds is no-response, and claims no cancellation', async () => { - // The SDK pauses 250 ms before a state-only round. With a shorter budget, - // the deadline passes inside that pause whenever leg 1 answers in time. const log = scratchLog(); - const connection = await connect('', log, { requestTimeoutMs: 240 }); + const connection = await connect('', log, { requestTimeoutMs: 1500 }); await connection.listTools(); - await assert.rejects(connection.callTool('cont', { leg2: 'ok' }), (err: unknown) => { + // Hold the round between leg 1's answer and leg 2 explicitly, so the + // deadline passes with nothing in flight by construction. Leg 1 asks for + // elicitation input, and an input handler registered on the session's + // client holds until the deadline aborts it, ending as the SDK's own pause + // does. Don't replace this with a deadline timed against the SDK's internal + // pause before a state-only round: that races leg 1 under load. + let held = false; + const client = (connection as unknown as { session: { client: { _requestHandlers: Map } } }).session.client; + client._requestHandlers.set('elicitation/create', (_request: unknown, ctx: { mcpReq: { signal: AbortSignal } }) => { + held = true; + const signal = ctx.mcpReq.signal; + return new Promise((_, reject) => { + const stop = () => reject(signal.reason instanceof SdkError ? signal.reason : new SdkError(SdkErrorCode.RequestTimeout, String(signal.reason))); + if (signal.aborted) stop(); + else signal.addEventListener('abort', stop, { once: true }); + }); + }); + await assert.rejects(connection.callTool('cont', { leg1: 'elicit' }), (err: unknown) => { assert.ok(err instanceof McplRequestError, String(err)); assert.equal(err.outcome, 'no-response'); assert.match( err.message, - /did not finish tools\/call "cont" within 240ms: it asked for another round \(input_required\), and the deadline passed before that round was sent, so nothing was in flight to cancel\. The outcome is unknown/, + /did not finish tools\/call "cont" within 1500ms: it asked for another round \(input_required\), and the deadline passed before that round was sent, so nothing was in flight to cancel\. The outcome is unknown/, ); assert.doesNotMatch(err.message, /Cancellation was requested|answered/); return true; }); - await new Promise((r) => setTimeout(r, 100)); + assert.ok(held, 'the round was held'); + // The wire is ordered: once a later call has reached the server, any + // second leg sent before it would have too. + await connection.callTool('plain', {}); + assert.equal(calls(log, 'plain'), 1); assert.equal(calls(log, 'cont'), 1, 'only leg 1 reached the wire'); assert.equal(wire(log).filter((e) => e.method === 'notifications/cancelled').length, 0, 'nothing was cancelled'); }); @@ -346,11 +370,13 @@ test('concurrent close() calls share one teardown and its verdict; a later close const log = scratchLog(); const realSpawn = StdioTransport.spawn; let stuck: { pid: number; kill: (signal?: string) => boolean } | null = null; + // Every signal teardown sends: one teardown escalates once, SIGTERM then SIGKILL. + const signals: Array = []; (StdioTransport as unknown as { spawn: typeof StdioTransport.spawn }).spawn = (config) => { const transport = realSpawn.call(StdioTransport, config); const child = (transport as unknown as { child: { pid: number; kill: (signal?: string) => boolean } }).child; const realKill = child.kill.bind(child); - child.kill = () => true; // teardown can't reap it + child.kill = (signal?: string) => { signals.push(signal); return true; }; // teardown can't reap it stuck = { pid: child.pid, kill: realKill }; return transport; }; @@ -366,6 +392,7 @@ test('concurrent close() calls share one teardown and its verdict; a later close assert.deepEqual(settled, [], 'neither resolves while teardown is pending'); await Promise.all([first, second]); assert.deepEqual(settled, ['first:failed', 'second:failed'], 'one verdict, in order'); + assert.deepEqual(signals, ['SIGTERM', 'SIGKILL'], 'one teardown, shared: the second close sent no signals of its own'); // Once the child is gone, a later close re-checks and settles. stuck!.kill('SIGKILL'); await until(() => { try { process.kill(stuck!.pid, 0); return false; } catch { return true; } }, 'the child gone'); @@ -388,3 +415,174 @@ test('start() runs once: a second call shares the first connect, and no second s const alive = (p: number) => { try { process.kill(p, 0); return true; } catch { return false; } }; assert.equal(alive(launches[0]!.pid!), false, 'close() reaped the only child'); }); + +test('a send the stdio line refused before writing anything is not-sent', async () => { + const log = scratchLog(); + const connection = await connect('', log); + await connection.listTools(); + // The line is closed, while the session still stands: the transport + // refuses the write, so nothing reached the server. + const line = (connection as unknown as { session: { wire: { inner: { line: { _closed: boolean } } } } }).session.wire.inner.line; + line._closed = true; + try { + await assert.rejects(connection.callTool('plain', {}), (err: unknown) => { + assert.ok(err instanceof McplRequestError, String(err)); + assert.equal(err.outcome, 'not-sent'); + assert.match(err.message, /tools\/call "plain" was not sent: .*stdio transport is closed/); + return true; + }); + } finally { + line._closed = false; + } + assert.equal(calls(log, 'plain'), 0); +}); + +test('a plain call whose child dies before answering is no-response', async () => { + const log = scratchLog(); + const connection = await connect('', log); + connection.on('error', () => {}); + await connection.listTools(); + await assert.rejects(connection.callTool('vanish', {}), (err: unknown) => { + assert.ok(err instanceof McplRequestError, String(err)); + assert.equal(err.outcome, 'no-response'); + assert.match(err.message, /MCP server "raw" tools\/call "vanish" failed: /); + return true; + }); + assert.equal(calls(log, 'vanish'), 1, 'the call reached the server'); +}); + +test('failures are classified in order: nothing sent first, then the answers, then an HTTP refusal', () => { + const connection = ModernMcpConnection.create({ id: 'raw', command: process.execPath, args: [SERVER, '', scratchLog()], protocol: 'modern' }); + const classify = (record: Record, error: unknown, method = 'tools/call') => + (connection as unknown as { + requestFailure: (what: string, record: unknown, error: unknown, timeoutMs?: number) => Error; + }).requestFailure(method === 'tools/list' ? 'tools/list' : 'tools/call "op"', { method, handedOff: true, answer: 'none', continued: false, ...record }, error, 1000); + const refused = () => new SdkHttpError(SdkErrorCode.ClientHttpAuthentication, 'Unauthorized', { status: 401 } as never); + const outcome = (error: Error) => (error instanceof McplRequestError ? error.outcome : 'plain'); + + const unsent = classify({ handedOff: false }, refused()); + assert.equal(outcome(unsent), 'not-sent', 'a refusal before anything was handed off'); + + const first = classify({}, refused()); + assert.equal(outcome(first), 'error-response'); + assert.equal(((first as McplRequestError).data as { httpStatus?: number }).httpStatus, 401); + + const later = classify({ continued: true }, refused()); + assert.equal(outcome(later), 'no-response', 'after an earlier leg asked for another round'); + assert.match(later.message, /asked for another round \(input_required\), and the server refused a later one: HTTP 401/); + + const listing = classify({}, new SdkError(SdkErrorCode.RequestTimeout, 'Request timed out'), 'tools/list'); + assert.equal(outcome(listing), 'no-response'); + assert.match(listing.message, /did not answer tools\/list within 1000ms\. Cancellation was requested; the list did not arrive in time\. Listing changes nothing on the server, so it is safe to retry\./); + assert.doesNotMatch(listing.message, /side-effecting/); +}); + +test('a reconnect backoff never exceeds the timer limit', () => { + const connection = ModernMcpConnection.create({ + id: 'raw', command: process.execPath, args: [SERVER, '', scratchLog()], protocol: 'modern', + reconnect: true, reconnectIntervalMs: MAX_TIMER_MS, reconnectMaxIntervalMs: MAX_TIMER_MS, + }); + const backoff = (connection as unknown as { backoffDelay: (failures: number) => number }).backoffDelay.bind(connection); + const random = Math.random; + Math.random = () => 0.999; // the jitter's top end + try { + assert.equal(backoff(1), MAX_TIMER_MS); + assert.equal(backoff(30), MAX_TIMER_MS); + } finally { + Math.random = random; + } +}); + +test('a new generation backs off its first reopen from the base interval', async () => { + const log = scratchLog(); + const connection = await connect('listen-fail-first', log, { reconnect: true, reconnectIntervalMs: 30, reconnectMaxIntervalMs: 60_000 }); + const internals = connection as unknown as { + relistenAttempts: number; + generation: number; + scheduleRelisten: (generation: number) => void; + }; + // The previous generation had failed to reopen many times. + internals.relistenAttempts = 9; + const seen: Array<{ generation: number; attempts: number }> = []; + const schedule = internals.scheduleRelisten.bind(connection); + internals.scheduleRelisten = (generation: number) => { + seen.push({ generation, attempts: internals.relistenAttempts }); + schedule(generation); + }; + let reconnected = false; + connection.on('reconnect', () => { reconnected = true; }); + await connection.callTool('die', {}); + await until(() => reconnected, 'reconnect'); + // Launch 2 refuses its first listen, so the new generation schedules a reopen. + await until(() => seen.some((s) => s.generation === internals.generation), 'the new generation scheduling its reopen'); + assert.equal(seen.find((s) => s.generation === internals.generation)!.attempts, 0); +}); + +test('a server that shares no revision with this client is a configuration verdict: no retry', async () => { + const log = scratchLog(); + const connection = ModernMcpConnection.create({ + id: 'raw', command: process.execPath, args: [SERVER, 'other-revision', log], protocol: 'modern', + reconnect: true, reconnectIntervalMs: 30, reconnectMaxIntervalMs: 60, + }); + cleanups.push(() => connection.close()); + await assert.rejects(connection.start(), (err: unknown) => { + assert.ok(err instanceof McplProtocolVersionError, String(err)); + assert.equal(err.kind, 'rejected'); + assert.equal(err.offered, '2026-07-28'); + assert.deepEqual(err.supported, ['2099-01-01']); + assert.match(err.message, /MCP server "raw" does not support MCP 2026-07-28 \(it supports: 2099-01-01\)\./); + return true; + }); + assert.equal(connection.willReconnect, false, 'nothing will retry it'); + assert.equal((connection as unknown as { reconnectTimer: unknown }).reconnectTimer, null); + assert.equal(wire(log).filter((e) => e.event === 'start').length, 1); +}); + +test('a legacy-only server is pointed to what reaches it: its configuration over stdio, stdio or WebSocket for an http url', () => { + const refusal = new UnsupportedProtocolVersionError({ supported: ['2024-11-05'], requested: '2026-07-28' }); + const verdictFor = (config: Parameters[0]) => + (ModernMcpConnection.create(config) as unknown as { versionVerdict: (error: unknown) => McplProtocolVersionError | null }).versionVerdict(refusal)!; + const stdio = verdictFor({ id: 'raw', command: process.execPath, args: [SERVER, '', scratchLog()], protocol: 'modern' }); + assert.match(stdio.message, /\(it supports: 2024-11-05\)\. It speaks legacy MCP: configure it without protocol: 'modern'\.$/); + const http = verdictFor({ id: 'remote', url: 'http://127.0.0.1:1/mcp' }); + assert.match(http.message, /\(it supports: 2024-11-05\)\. It speaks only legacy MCP, which this client reaches over stdio or WebSocket \(ws:\/\/, wss:\/\/\), not http\(s\)\.$/); + assert.doesNotMatch(http.message, /protocol: 'modern'/); +}); + +test('a reconnect that meets a revision the client does not share stops for good', async () => { + const log = scratchLog(); + const connection = await connect('other-revision-later', log, { reconnect: true, reconnectIntervalMs: 30, reconnectMaxIntervalMs: 60 }); + const failures: Array<{ error: string; permanent?: boolean }> = []; + connection.on('reconnect-failed', (f) => failures.push(f)); + await connection.callTool('die', {}); + await until(() => failures.length > 0, 'the reconnect failing'); + assert.equal(failures.length, 1); + assert.equal(failures[0]!.permanent, true); + assert.match(failures[0]!.error, /does not support MCP 2026-07-28 \(it supports: 2099-01-01\)/); + assert.equal(connection.willReconnect, false); + assert.equal((connection as unknown as { reconnectTimer: unknown }).reconnectTimer, null, 'no further attempt scheduled'); + assert.equal(wire(log).filter((e) => e.event === 'start').length, 2); +}); + +test('close() during a connect waits for that connect\'s own cleanup, and reports what it could not reap', async () => { + const log = scratchLog(); + const connection = ModernMcpConnection.create({ id: 'raw', command: process.execPath, args: [SERVER, 'hang-discover', log], protocol: 'modern' }); + connection.on('error', () => {}); + const statics = ModernMcpConnection as unknown as { closeSession: (session: unknown) => Promise }; + const real = statics.closeSession; + let closes = 0; + // close() closes the generation first; the connect's own cleanup closes it + // again, and that second close is the one that can't confirm the exit. + statics.closeSession = async (session: unknown) => { + const verdict = await real.call(ModernMcpConnection, session); + return ++closes === 2 ? new Error('the child could not be confirmed gone (simulated)') : verdict; + }; + cleanups.push(() => { statics.closeSession = real; }); + const starting = connection.start(); + await until(() => wire(log).some((e) => e.method === 'server/discover'), 'the launch waiting on discover'); + await assert.rejects(connection.close(), /did not close cleanly: the child could not be confirmed gone \(simulated\)/); + assert.equal(closes, 2, 'close() resolved only after the connect\'s own cleanup'); + await starting; + statics.closeSession = real; + await connection.close(); +}); diff --git a/test/mcp-server-ownership.test.ts b/test/mcp-server-ownership.test.ts index 65def7234..875c0d85e 100644 --- a/test/mcp-server-ownership.test.ts +++ b/test/mcp-server-ownership.test.ts @@ -196,7 +196,8 @@ test('a disconnect during a modern connect ends it at once', async () => { const began = Date.now(); await framework.disconnectMcplServer('m'); await assert.rejects(pending, /MCP server "m" was not connected: it was disconnected while connecting/); - assert.ok(Date.now() - began < 5_000, 'not after the 30 s connect timeout'); + // A generous bound, well below the 30 s connect timeout it rules out. + assert.ok(Date.now() - began < 20_000, 'not after the 30 s connect timeout'); assert.equal(status(framework, 'm'), undefined); assert.equal(alive(launches(log)[0]!), false, 'its child was reaped'); }); diff --git a/test/mcpl-legacy-lifetime.test.ts b/test/mcpl-legacy-lifetime.test.ts index 0ee93a278..31abe8a38 100644 --- a/test/mcpl-legacy-lifetime.test.ts +++ b/test/mcpl-legacy-lifetime.test.ts @@ -200,7 +200,8 @@ test('close() during a reconnect handshake ends that launch before resolving, an const second = starts(log)[1]!; const began = Date.now(); await connection.close(); - assert.ok(Date.now() - began < 5_000, 'promptly: the handshake was ended, not waited out (30 s)'); + // A generous bound, well below the 30 s handshake timeout it rules out. + assert.ok(Date.now() - began < 20_000, 'promptly: the handshake was ended, not waited out (30 s)'); assert.equal(alive(second), false, 'the launch in flight was reaped before close() resolved'); assert.equal(reconnected, false); assert.equal(connection.isConnected, false); @@ -221,7 +222,7 @@ test('a recycle during a reconnect in flight starts no second attempt, and close assert.equal(starts(log).length, 2, 'no second attempt beside the one in flight'); const began = Date.now(); await connection.close(); - assert.ok(Date.now() - began < 5_000, 'promptly'); + assert.ok(Date.now() - began < 20_000, 'promptly: well before the 30 s handshake timeout'); assert.deepEqual(starts(log).filter(alive), [], 'every launch reaped when close() resolved'); }); From 559ecaaff5c79b046bf396279b6623c5b1360f61 Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Sun, 11 Oct 2026 17:43:14 -0400 Subject: [PATCH 0123/1112] fix(mcp): hold a server's id until its child is confirmed exited, everywhere, and keep the tool list and traces true A review of this change found that the lifetime contract it states (a launch whose child can't be confirmed exited stays registered, holding its id, and a later disconnect retries the reap) didn't hold on every path, and that the tool list and traces could disagree with what the servers were doing. The id is held until the exit is confirmed: - The registry's removeServer keeps the connection registered while its close runs, and frees the id only once the close succeeds. Before, the id was free during the reap, so a concurrent addServer could launch a second child beside the first. - addServer refuses a second add for an id whose first is still connecting, so a failed launch's closed connection can always be registered as the owner of a child it couldn't reap. - A connection that still owns a child it couldn't reap (holdsUnreapedChild) isn't dropped by the registry when it emits 'close', so its own close() keeps the id held too. - closeAll closes every connection, removes the ones it reaped, keeps each one it couldn't, and reports every failure, not only the first. - The modern connections close the same way at stop() and after a failed startup: one whose close fails stays in the map. - A failed startup's cleanup closes both engines' connections, each whatever the other's close did, logs what it couldn't close, and rethrows the startup failure itself, so a cleanup failure never takes its place. - stop() finishes every part of its teardown and then rejects, naming each child it couldn't reap, so its caller learns that a child may have outlived the framework. Other lifetime fixes: - close() during a WebSocket reconnect dial ends the dial at once: the handshake's abort signal reaches the dial (openTransport and WebSocketTransport.open take it), instead of close() waiting out the 15 s open timeout. - restartMcplServer checks the new configuration before it tears the running server down, and refuses without touching it. - A configuration refused at startup is reported with willRetry false: nothing retries it, reconnect or not. - A stdio server switched to protocol: 'modern' is a new coalescing binding. The marker is added only for that case, so every binding stored for a legacy configuration stays the same. - A null nextCursor ends a tools/list inventory, as an absent one does and as before this change; only a non-string, non-null cursor is malformed. - The reconnect comment no longer promises a resume after an unreapable launch: close() is terminal. The tool list and traces: - A refresh publishes nothing for a server removed while it awaited the lists, so a refresh that finishes after a disconnect can't bring the removed server's tools back. - A server lost for good (no reconnect), legacy or modern, leaves the agents' tool list, as a disconnect's tools do. - An MCP tool result the server marked an error is traced as tool:failed, on both engines, as the other tool paths trace one. - mcpToolResult's doc says the direct path and a structured script result get the server's result itself. The changelog fragments say so: stop() rejecting after its teardown (a breaking note for hosts, which should catch it where they chain work on stop()), the registry's id hold, close() ending a WebSocket dial, a server lost for good leaving the tool list, and tool:failed for an error result. Tests: - registry: an id held through its reap; a second add refused while one is connecting; a connection whose reap fails staying registered through its own close(), closeAll() and a failed removeServer, until a retry succeeds; closeAll naming each failure and removing the rest. - a reconnect whose previous child can't be reaped halting; close() during a WebSocket reconnect dial returning promptly (a real server whose second upgrade hangs); a dial abandoned on its signal; a stdin EPIPE never thrown. - stop() rejecting after keeping both engines' unreaped connections; the failed-startup cleanup closing the modern connection though the legacy close failed; a refused restart leaving the server running; a refresh in flight during a disconnect publishing none of its tools; a server lost for good leaving the list; the coalescing binding. - legacy protocol and inventory: a -32022 without the modern revision naming no modern remedy; a version verdict launched once, its child reaped, never retried; a non-version initialize error staying plain and retryable; a modern stdio configuration refused before any spawn; a list that never ends refused at its page bound; a malformed later page; a null cursor ending the list; willRetry and the ops alert at startup for a verdict, failures with and without reconnect, and a refused configuration. - modern: a lost child relisted from the new launch (what only the old launch listed is gone); quiesce beginning while a call is provably in flight; a tool error traced as tool:failed. - the negative waits now say which timer they outlast, and labels say what each wait actually sees. Red checks on dist, each change removed alone (16): the id freed before the reap, no in-flight add refusal, close forgetting an unreaped connection, closeAll clearing first, a null cursor as malformed, the dial ignoring its signal, the handshake dialing without it, the startup cleanup run in sequence, the modern close forgetting the unreaped, stop() swallowing reap failures, restart tearing down first, a refresh republishing removed servers, a lost server keeping its tools, an error result traced as completed, no family in the binding, and a refused configuration reported as retrying: each fails the test aimed at it. The full suite at this tree, each test file in its own process on the VPS, with the three commits before it: 1530/1530 across 136 files on Node 20.20.2. Node 24.21.0 counted 1527, because --test-force-exit cut mcpl-tool-lifecycle-emitter to 23 of its 26 tests; alone and unforced it passes 26/26. --- .../legacy-mcp-protocol-boundary.breaking.md | 6 + .../mcp-standard-tool-results.changed.md | 11 +- changelog.d/modern-mcp-client.added.md | 3 + src/framework.ts | 152 +++++++++++++--- src/mcpl/server-connection.ts | 29 ++- src/mcpl/server-registry.ts | 50 ++++-- src/mcpl/transport.ts | 33 ++-- test/fixtures/legacy-version-server.mjs | 12 +- test/mcp-modern-framework.test.ts | 60 ++++++- test/mcp-server-ownership.test.ts | 134 +++++++++++++- test/mcpl-legacy-lifetime.test.ts | 165 +++++++++++++++++- test/mcpl-legacy-version.test.ts | 121 ++++++++++++- 12 files changed, 689 insertions(+), 87 deletions(-) diff --git a/changelog.d/legacy-mcp-protocol-boundary.breaking.md b/changelog.d/legacy-mcp-protocol-boundary.breaking.md index fc815cfa6..d232dc8e1 100644 --- a/changelog.d/legacy-mcp-protocol-boundary.breaking.md +++ b/changelog.d/legacy-mcp-protocol-boundary.breaking.md @@ -27,3 +27,9 @@ `requestTimeoutMs` above 2^31−1 is refused too: Node ran it as a timer of about 1 ms, so every request timed out at once. Every value that worked, `0` included, passes. +- **Hosts:** `stop()` now rejects, after its whole teardown, when an MCP + server's child couldn't be confirmed exited, naming each such child. + Before, closing a stdio server returned as soon as a signal was sent, so + `stop()` never failed for this. Code that chains work on `stop()`, such as + exiting a TUI or switching sessions, should catch that rejection: the + framework is stopped either way. diff --git a/changelog.d/mcp-standard-tool-results.changed.md b/changelog.d/mcp-standard-tool-results.changed.md index 4bb318b76..c6a74c414 100644 --- a/changelog.d/mcp-standard-tool-results.changed.md +++ b/changelog.d/mcp-standard-tool-results.changed.md @@ -40,8 +40,9 @@ - Closing a stdio MCP/MCPL server now waits for the child to exit, sending SIGKILL after 2 s. A child still alive 2 s after SIGKILL fails `close()` with an explicit error. Before, close returned as soon as a signal was - sent. `stop()` reports such a failure and still finishes shutting - everything else down. Concurrent `close()` calls share one teardown, and a + sent. `stop()` still finishes shutting everything else down after such a + failure, and then rejects (see the breaking notes). Concurrent `close()` + calls share one teardown, and a later call re-checks a child that couldn't be reaped. A launch whose exit can't be confirmed is never orphaned, and nothing launches beside it: - `McplServerConnection.connect()` throws the new `McplUnreapedLaunchError`, @@ -52,7 +53,11 @@ (`reconnect-failed`, `permanent: true`). Reconnect attempts run one at a time, and never start before the previous child has exited. - `close()` during a reconnect handshake ends that launch before it - resolves. + resolves, a WebSocket dial still opening included. + - `McplServerRegistry` holds an id until its child is confirmed exited: + `removeServer` frees it only once the close succeeds, a second + `addServer` for an id still connecting is refused, and `closeAll()` + keeps each connection it couldn't reap, naming every failure. - In the framework, a disconnect or failed connect whose cleanup can't reap the child keeps the server registered, closed and listed as disconnected, and its error says so. A connect for that id is refused diff --git a/changelog.d/modern-mcp-client.added.md b/changelog.d/modern-mcp-client.added.md index 8d4a38ac6..8f9af2a81 100644 --- a/changelog.d/modern-mcp-client.added.md +++ b/changelog.d/modern-mcp-client.added.md @@ -32,3 +32,6 @@ call's promise. - **Status:** `listMcplServers()` entries gain `family`, `protocolVersion` and `transport`. +- A server lost for good (it closed and won't reconnect), MCPL or modern, + now leaves the agents' tool list, as a disconnected one does. An MCP tool + result marked as an error is traced as `tool:failed`, on both engines. diff --git a/src/framework.ts b/src/framework.ts index 45f7be15a..0d62e8f87 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -477,6 +477,15 @@ interface DeferredWrite { } const bySeq = (a: { seq: number }, b: { seq: number }): number => a.seq - b.seq; +/** A server configuration refused before anything was spawned or dialed: + * only a corrected configuration helps, so nothing retries it. */ +class McpConfigurationRefusal extends Error { + constructor(message: string) { + super(message); + this.name = 'McpConfigurationRefusal'; + } +} + /** Stamp a deferred write's durable id into the metadata it is stored with * (boot recovery dedups replays by it). Idempotent for an already-stamped * message. */ @@ -2045,9 +2054,13 @@ export class AgentFramework { shutdownPromises.push(this.closeModernMcpConnections()); // Every part shuts down even if one fails (a child that could not be // reaped, say); each failure is reported, and the rest of stop() runs. + // stop() then rejects with them, once everything else is done, so its + // caller learns that a child may have outlived the framework. + const shutdownFailures: string[] = []; for (const outcome of await Promise.allSettled(shutdownPromises)) { if (outcome.status === 'rejected') { const reason = outcome.reason instanceof Error ? outcome.reason.message : String(outcome.reason); + shutdownFailures.push(reason); console.error(`[shutdown] ${reason}`); this.emitTrace({ type: 'mcpl:server-error', serverId: 'shutdown', error: reason }); } @@ -2073,6 +2086,10 @@ export class AgentFramework { if (this.ownsStore) { this.store.close(); } + + if (shutdownFailures.length > 0) { + throw new Error(`stop() finished, but not cleanly: ${shutdownFailures.join('; ')}`); + } } /** @@ -7653,7 +7670,14 @@ export class AgentFramework { */ private coalescingBinding(serverId: string): string { const config = this.mcplServerConfigs.get(serverId); - return createHash('sha256').update(JSON.stringify([serverId, config?.url ?? null, config?.command ?? null, config?.args ?? null])).digest('hex').slice(0, 16); + // A stdio server switched to the modern family is another peer, so its + // binding changes. The marker is added only for protocol: 'modern', so + // every binding stored for a legacy configuration stays the same. A + // network server's family follows its url's scheme, which is hashed. + const family = config?.protocol === 'modern' ? ['modern'] : []; + return createHash('sha256') + .update(JSON.stringify([serverId, config?.url ?? null, config?.command ?? null, config?.args ?? null, ...family])) + .digest('hex').slice(0, 16); } private initializePushCoalescer(): void { @@ -13368,8 +13392,7 @@ export class AgentFramework { await this.connectMcplServerInternal(config, true); } catch (error) { if (error instanceof DiscordAwarenessAccountingError) { - await this.mcplServerRegistry.closeAll(); - await this.closeModernMcpConnections(); + await this.closeMcpConnectionsAfterFailedStartup(); throw error; } // Fail-open: log and continue with remaining servers @@ -13385,8 +13408,10 @@ export class AgentFramework { error: err.message, attempt: 0, // A protocol-version verdict is never retried (no common revision; - // only configuration can fix it). - willRetry: config.reconnect === true && !(error instanceof McplProtocolVersionError), + // only configuration can fix it), and neither is a configuration + // refused before anything was launched. + willRetry: config.reconnect === true && !(error instanceof McplProtocolVersionError) + && !(error instanceof McpConfigurationRefusal), }); // So the server doesn't vanish quietly: alert at once, as the // reconnect path does for the same verdict, since no attempt follows. @@ -13405,8 +13430,7 @@ export class AgentFramework { await startupBarrier.promise; this.completeMcplDataPlaneGate(startupBarrier); } catch (error) { - await this.mcplServerRegistry.closeAll(); - await this.closeModernMcpConnections(); + await this.closeMcpConnectionsAfterFailedStartup(); throw error; } @@ -13465,16 +13489,41 @@ export class AgentFramework { } /** Close every modern-MCP connection, including any connect still in - * flight, so none outlives a failed startup or stop(). */ + * flight, so none outlives a failed startup or stop(). One whose close + * can't confirm its child exited stays registered, holding its id, as a + * failed connect's does; every failure is thrown together. */ private async closeModernMcpConnections(): Promise { - const modern = [...this.modernServers().values()]; - this.modernServers().clear(); - const failures = (await Promise.allSettled(modern.map((connection) => connection.close()))) - .filter((o): o is PromiseRejectedResult => o.status === 'rejected') - .map((o) => (o.reason instanceof Error ? o.reason.message : String(o.reason))); + const modern = [...this.modernServers().entries()]; + const outcomes = await Promise.allSettled(modern.map(([, connection]) => connection.close())); + const failures: string[] = []; + outcomes.forEach((outcome, index) => { + const [id, connection] = modern[index]!; + if (outcome.status === 'fulfilled') { + if (this.modernServers().get(id) === connection) this.modernServers().delete(id); + } else { + failures.push(outcome.reason instanceof Error ? outcome.reason.message : String(outcome.reason)); + } + }); if (failures.length > 0) throw new Error(failures.join('; ')); } + /** After a failed startup: close both engines' connections, each whatever + * the other's close did, and log what couldn't be closed. The caller then + * rethrows the startup failure itself, so a cleanup failure never takes + * its place. */ + private async closeMcpConnectionsAfterFailedStartup(): Promise { + const outcomes = await Promise.allSettled([ + this.mcplServerRegistry?.closeAll() ?? Promise.resolve(), + this.closeModernMcpConnections(), + ]); + for (const outcome of outcomes) { + if (outcome.status === 'rejected') { + console.error('[mcpl] after a failed startup, some MCP connections could not be closed:', + outcome.reason instanceof Error ? outcome.reason.message : outcome.reason); + } + } + } + /** Reconcile the durable ledger with Chronicle, then deliver every server's work. */ async syncDiscordAwarenessMarkers(_onlyServerId?: string): Promise { if (!this.discordAwarenessOutbox) return; @@ -13840,16 +13889,7 @@ export class AgentFramework { // 'websocket', transport 'http' with a ws url or with no url at all), // protocol on a URL, a modern deadline of 0, MCPL policy on a modern // server. Every valid legacy shape resolves exactly as before. - let family: 'legacy' | 'modern'; - try { - family = resolveServerBinding(config).family; - } catch (error) { - throw new Error(`MCP server "${config.id}" configuration: ${error instanceof Error ? error.message : String(error)}`); - } - const problems = serverConfigProblems(config); - if (problems.length > 0) { - throw new Error(`MCP server "${config.id}" configuration: ${problems.join('; ')}`); - } + const family = AgentFramework.usableConfigFamily(config); // One owner per id, across both engines (holdMcpServerId): a connect or // disconnect in flight holds it, and so does a registered connection, // including one kept because its launch couldn't be confirmed exited. @@ -14453,10 +14493,29 @@ export class AgentFramework { if (!config) { throw new Error(`MCPL server "${id}" is not configured`); } + // A configuration the connect would refuse is refused here, before the + // running server is torn down for it. + AgentFramework.usableConfigFamily(config); await this.disconnectMcplServer(id); await this.connectMcplServer(config); } + /** The protocol family a configuration resolves to, or a refusal naming + * every problem with it, before anything is spawned, dialed or torn down. */ + private static usableConfigFamily(config: import('./mcpl/types.js').McplServerConfig): 'legacy' | 'modern' { + let family: 'legacy' | 'modern'; + try { + family = resolveServerBinding(config).family; + } catch (error) { + throw new McpConfigurationRefusal(`MCP server "${config.id}" configuration: ${error instanceof Error ? error.message : String(error)}`); + } + const problems = serverConfigProblems(config); + if (problems.length > 0) { + throw new McpConfigurationRefusal(`MCP server "${config.id}" configuration: ${problems.join('; ')}`); + } + return family; + } + /** * List configured MCPL servers with live connection status and the number * of tools each currently contributes. @@ -14899,6 +14958,11 @@ export class AgentFramework { willReconnect: connection.willReconnect, }); this.emitTrace({ type: 'module:removed', moduleName: `mcpl:${connection.id}` }); + // Lost for good (no reconnect, and the registry has dropped it): its + // tools leave the agents' list, as a disconnect's do. + if (!connection.willReconnect && this.mcplServerRegistry?.getServer(connection.id) !== connection) { + this.handleToolsListChanged(connection.id); + } }); } @@ -14959,10 +15023,23 @@ export class AgentFramework { // closes it removes it, so the id stays held meanwhile. if (!connection.isClosed && !connection.willReconnect && this.modernServers().get(connection.id) === connection) { this.modernServers().delete(connection.id); + // Gone for good: its tools leave the agents' list, as a disconnect's do. + this.handleToolsListChanged(connection.id); } }); } + /** An MCP tool call's trace: a result the server marked an error (or one + * the reading couldn't complete) is tool:failed, as on the other tool + * paths; anything else is tool:completed. */ + private traceMcpToolOutcome(serverId: string, tool: string, callId: string, result: ToolResult, durationMs: number): void { + if (result.isError) { + this.emitTrace({ type: 'tool:failed', module: `mcpl:${serverId}`, tool, callId, error: result.error ?? 'tool returned an error' }); + } else { + this.emitTrace({ type: 'tool:completed', module: `mcpl:${serverId}`, tool, callId, durationMs }); + } + } + /** * Discover tools from all connected MCPL servers and cache them. * Tools are namespaced as `{toolPrefix}--{toolName}` per server config. @@ -15051,7 +15128,25 @@ export class AgentFramework { } } - this.mcplTools = tools; + // A server removed while this refresh awaited its lists (a disconnect, + // or a server lost for good) publishes nothing: whichever refresh + // finishes last must not bring back a removed server's tools. + // Asked of the same listing the refresh began from, now. + const live = new Set([ + ...(this.mcplServerRegistry?.getAllServers() ?? []).map((server) => server.id), + ...this.modernServers().keys(), + ]); + const gone = new Set([...new Set(toolServers.values())].filter((serverId) => !live.has(serverId))); + if (gone.size > 0) { + for (const [name, serverId] of [...toolServers]) { + if (!gone.has(serverId)) continue; + toolServers.delete(name); + toolFeatureSets.delete(name); + toolClasses.delete(name); + } + for (const serverId of gone) listedServers.delete(serverId); + } + this.mcplTools = gone.size > 0 ? tools.filter((tool) => toolServers.has(tool.name)) : tools; this.mcplToolFeatureSets = toolFeatureSets; this.mcplToolClasses = toolClasses; this.mcplToolServers = toolServers; @@ -15269,7 +15364,7 @@ export class AgentFramework { modern.callTool(toolName, args) .then((result) => reading(result, false)) .then((result) => { - this.emitTrace({ type: 'tool:completed', module: `mcpl:${serverId}`, tool: toolName, callId: call.id, durationMs: Date.now() - startTime }); + this.traceMcpToolOutcome(serverId, toolName, call.id, result, Date.now() - startTime); this.pushEvent({ type: 'tool-result', callId: call.id, agentName, moduleName: `mcpl:${serverId}`, result }); }) .catch(failDispatch); @@ -15340,7 +15435,7 @@ export class AgentFramework { }) .then(({ result, toolResult }) => { const durationMs = Date.now() - startTime; - this.emitTrace({ type: 'tool:completed', module: `mcpl:${serverId}`, tool: toolName, callId: call.id, durationMs }); + this.traceMcpToolOutcome(serverId, toolName, call.id, toolResult, durationMs); // Record checkpoint from stateful tool response (Step 8). // Explicit response attribution is authoritative. Otherwise use the @@ -15449,7 +15544,10 @@ export class AgentFramework { * payloads, and `structured` kept by presence. An MCPL peer keeps the * RFC-005 reading. Its content array passes through when it holds * non-text blocks, so references and images render natively. It gets - * `structured` too, with the JSON view when its content is empty. + * `structured` too, with the JSON view when its content is empty. The + * direct path (a module's callTool, a channel's tool call) doesn't come + * here: it returns the server's result itself (directToolResult), for + * either engine, as does a script's call whose result is structured. */ private async mcpToolResult( result: { content?: unknown; isError?: boolean; structuredContent?: unknown }, diff --git a/src/mcpl/server-connection.ts b/src/mcpl/server-connection.ts index 7a1eefd31..93917754b 100644 --- a/src/mcpl/server-connection.ts +++ b/src/mcpl/server-connection.ts @@ -398,6 +398,13 @@ export class McplServerConnection extends EventEmitter { return this.reconnectEnabled; } + /** Whether a child this connection launched couldn't be confirmed exited. + * While it's true the connection owns that child, so whatever registered + * it keeps its id held; close() retries the reap. */ + get holdsUnreapedChild(): boolean { + return this.unreaped.length > 0; + } + /** * Private constructor. Use `McplServerConnection.connect()` instead. */ @@ -692,8 +699,8 @@ export class McplServerConnection extends EventEmitter { * open socket never leaks. If that close can't confirm the child exited, * the failure is a {@link HandshakeCleanupError} carrying the transport, so * the caller keeps owning it. `signal` ends a handshake whose connection is - * closed meanwhile: it is checked once the transport is open (a WebSocket - * dial isn't cut short) and races the initialize answer. + * closed meanwhile: it cuts a WebSocket dial short, is checked once the + * transport is open, and races the initialize answer. */ private static async handshake( config: McplServerConfig, @@ -702,7 +709,13 @@ export class McplServerConnection extends EventEmitter { ): Promise<{ transport: McplTransport; capabilities: McplCapabilities | null; droppedCapabilities: ReadonlySet; mcpToolsAdvertised: boolean; protocolVersion: string }> { const closedWhileConnecting = () => new Error(`MCPL server "${config.id}" was closed while connecting`); if (signal?.aborted) throw closedWhileConnecting(); - const transport = await openTransport(config); + let transport: McplTransport; + try { + transport = await openTransport(config, signal); + } catch (error) { + if (signal?.aborted) throw closedWhileConnecting(); + throw error; + } try { if (signal?.aborted) throw closedWhileConnecting(); @@ -1052,8 +1065,9 @@ export class McplServerConnection extends EventEmitter { /** * Send `tools/list` and return the server's complete tool inventory, * following `nextCursor` across pages (2024-11-05 pagination). Cursors are - * opaque strings, `''` included: only an absent `nextCursor` ends the - * inventory. A page that isn't well formed (no `tools` array, a cursor + * opaque strings, `''` included: only an absent or null `nextCursor` ends + * the inventory (a null one is how some servers say "no more", and main + * read it as a complete page). A page that isn't well formed (no `tools` array, a cursor * that isn't a string), or cursors that don't terminate (a repeat, or more * than {@link MAX_TOOL_LIST_PAGES} pages), are errors, never a partial or * empty inventory published as complete. @@ -1072,7 +1086,7 @@ export class McplServerConnection extends EventEmitter { } first ??= result; tools.push(...(result.tools as McpToolDefinition[])); - if (result.nextCursor === undefined) { + if (result.nextCursor === undefined || result.nextCursor === null) { // The first page's other fields are kept as they came, so a // single-page answer is returned exactly as before. const { nextCursor: _done, ...rest } = first as { nextCursor?: unknown }; @@ -1299,7 +1313,8 @@ export class McplServerConnection extends EventEmitter { // No common protocol revision: every further attempt would restart the // server to hear the same answer. Stop retrying; the configuration is // what has to change. A launch that couldn't be reaped halts retrying - // too, until close() confirms its exit. Every other failure keeps the + // for good: close() can confirm its exit, but close() is terminal, so + // nothing resumes this connection. Every other failure keeps the // backoff loop. if (error instanceof McplProtocolVersionError) this.reconnectEnabled = false; const permanent = unreaped || error instanceof McplProtocolVersionError; diff --git a/src/mcpl/server-registry.ts b/src/mcpl/server-registry.ts index fc773df9f..3490e1c5d 100644 --- a/src/mcpl/server-registry.ts +++ b/src/mcpl/server-registry.ts @@ -25,6 +25,10 @@ export type McplCapabilityQuery = */ export class McplServerRegistry { private servers = new Map(); + /** Ids whose addServer is still connecting. A second add for one is + * refused, so a failed launch's closed connection can always be registered + * as the owner of a child it couldn't reap. */ + private adding = new Set(); /** * Connect to an MCPL server and register it. @@ -39,20 +43,21 @@ export class McplServerRegistry { config: McplServerConfig, hostCapabilities: McplHostCapabilities, ): Promise { - if (this.servers.has(config.id)) { + if (this.servers.has(config.id) || this.adding.has(config.id)) { throw new Error(`MCPL server "${config.id}" is already registered`); } + this.adding.add(config.id); let connection: McplServerConnection; try { connection = config.reconnect ? await McplServerConnection.connectWithReconnect(config, hostCapabilities) : await McplServerConnection.connect(config, hostCapabilities); } catch (error) { - if (error instanceof McplUnreapedLaunchError && !this.servers.has(config.id)) { - this.register(config, error.connection); - } + if (error instanceof McplUnreapedLaunchError) this.register(config, error.connection); throw error; + } finally { + this.adding.delete(config.id); } this.register(config, connection); return connection; @@ -61,9 +66,11 @@ export class McplServerRegistry { private register(config: McplServerConfig, connection: McplServerConnection): void { this.servers.set(config.id, connection); - // Auto-remove on unexpected close (unless reconnect will re-add) + // Auto-remove on unexpected close (unless reconnect will re-add). A + // connection that still owns a child it couldn't reap keeps its id, so + // nothing launches beside that child; removeServer retries the reap. connection.on('close', () => { - if (!config.reconnect && this.servers.get(config.id) === connection) { + if (!config.reconnect && this.servers.get(config.id) === connection && !connection.holdsUnreapedChild) { this.servers.delete(config.id); } }); @@ -86,13 +93,10 @@ export class McplServerRegistry { if (!connection) { return; } - this.servers.delete(id); - try { - await connection.close(); - } catch (error) { - if (!this.servers.has(id)) this.servers.set(id, connection); - throw error; - } + // The id stays held until the close settles, so nothing launches beside + // a child still being reaped; it's freed only once the exit is confirmed. + await connection.close(); + if (this.servers.get(id) === connection) this.servers.delete(id); } /** @@ -156,11 +160,23 @@ export class McplServerRegistry { } /** - * Close all server connections and clear the registry. + * Close every server connection. Each one whose close confirms its exit + * is removed. One whose child couldn't be confirmed exited stays + * registered, holding its id, and every such failure is thrown together; + * removeServer retries a reap. */ async closeAll(): Promise { - const connections = Array.from(this.servers.values()); - this.servers.clear(); - await Promise.all(connections.map((c) => c.close())); + const entries = Array.from(this.servers.entries()); + const outcomes = await Promise.allSettled(entries.map(([, connection]) => connection.close())); + const failures: string[] = []; + outcomes.forEach((outcome, index) => { + const [id, connection] = entries[index]!; + if (outcome.status === 'fulfilled') { + if (this.servers.get(id) === connection) this.servers.delete(id); + } else { + failures.push(outcome.reason instanceof Error ? outcome.reason.message : String(outcome.reason)); + } + }); + if (failures.length > 0) throw new Error(failures.join('; ')); } } diff --git a/src/mcpl/transport.ts b/src/mcpl/transport.ts index 09ba3de81..529b5feb2 100644 --- a/src/mcpl/transport.ts +++ b/src/mcpl/transport.ts @@ -305,9 +305,11 @@ export class WebSocketTransport extends McplTransport { /** * Dial `config.url` and resolve once the socket is OPEN. Appends `?token=` * when `config.token` is set. Rejects on connect error / timeout (and tears - * the socket down) so a failed dial never leaks an open handle. + * the socket down) so a failed dial never leaks an open handle. `signal` + * ends a dial whose connection is closed meanwhile, at once, rather than + * at the open timeout. */ - static async open(config: McplServerConfig): Promise { + static async open(config: McplServerConfig, signal?: AbortSignal): Promise { // Resolve the per-dial credential first: the provider runs on every // connect AND reconnect, so rotation happens at the transport layer and // nothing above it ever holds a credential. @@ -341,31 +343,36 @@ export class WebSocketTransport extends McplTransport { return new Promise((resolve, reject) => { let settled = false; - const timer = setTimeout(() => { + const abandon = (error: Error): void => { if (settled) return; settled = true; + clearTimeout(timer); + signal?.removeEventListener('abort', onAbort); ws.removeAllListeners(); ws.on('error', swallowLateErrors); try { ws.terminate(); } catch { /* already gone */ } - reject(new Error(`MCPL server "${config.id}" WebSocket connect timed out after ${WS_OPEN_TIMEOUT_MS}ms (${redacted})`)); + reject(error); + }; + const timer = setTimeout(() => { + abandon(new Error(`MCPL server "${config.id}" WebSocket connect timed out after ${WS_OPEN_TIMEOUT_MS}ms (${redacted})`)); }, WS_OPEN_TIMEOUT_MS); + const onAbort = (): void => { + abandon(new Error(`MCPL server "${config.id}" WebSocket connect abandoned: the connection was closed (${redacted})`)); + }; + if (signal?.aborted) onAbort(); + else signal?.addEventListener('abort', onAbort, { once: true }); ws.once('open', () => { if (settled) return; settled = true; clearTimeout(timer); + signal?.removeEventListener('abort', onAbort); ws.removeAllListeners('error'); // the instance re-attaches its own handlers resolve(new WebSocketTransport(ws)); }); ws.once('error', (err: Error) => { - if (settled) return; - settled = true; - clearTimeout(timer); - ws.removeAllListeners(); - ws.on('error', swallowLateErrors); - try { ws.terminate(); } catch { /* already gone */ } - reject(new Error(`MCPL server "${config.id}" WebSocket connect failed: ${err.message} (${redacted})`)); + abandon(new Error(`MCPL server "${config.id}" WebSocket connect failed: ${err.message} (${redacted})`)); }); }); } @@ -444,10 +451,10 @@ export function isWebSocketTransport(config: McplServerConfig): boolean { * The returned transport is connected at the byte level but has NOT * performed the MCPL handshake. */ -export async function openTransport(config: McplServerConfig): Promise { +export async function openTransport(config: McplServerConfig, signal?: AbortSignal): Promise { const binding = resolveServerBinding(config); if (binding.family !== 'legacy') { throw new Error(`MCP server "${config.id}" is configured for modern MCP; the legacy transports can't open it`); } - return binding.transport === 'websocket' ? WebSocketTransport.open(config) : StdioTransport.spawn(config); + return binding.transport === 'websocket' ? WebSocketTransport.open(config, signal) : StdioTransport.spawn(config); } diff --git a/test/fixtures/legacy-version-server.mjs b/test/fixtures/legacy-version-server.mjs index e2968ff90..7a12ee592 100644 --- a/test/fixtures/legacy-version-server.mjs +++ b/test/fixtures/legacy-version-server.mjs @@ -4,6 +4,7 @@ // Modes (how `initialize` is answered): // ok protocolVersion 2024-11-05 // reject error -32022 with data { supported: ['2026-07-28'], requested } +// reject-old error -32022 with data { supported: ['2025-03-26'], requested } // reject-nodata error -32022 without data // mismatch protocolVersion 2025-06-18 // missing no protocolVersion at all @@ -17,7 +18,9 @@ // tools/list pages: by default three pages (cursors p2, p3). MODE_LIST picks // an edge case: loop (always nextCursor "same"), empty (nextCursor "" then a // second page), notools (no tools array), badcursor (nextCursor 5), -// nullcursor (nextCursor null). Tool `die` exits the process. +// nullcursor (nextCursor null), endless (a new cursor on every page, so the +// list never ends), badlater (a well-formed first page, then a page with no +// tools array). Tool `die` exits the process. import { appendFileSync, existsSync, readFileSync } from 'node:fs'; let mode = process.argv[2] ?? 'ok'; @@ -49,6 +52,7 @@ process.stdin.on('data', (chunk) => { if (m.method === 'initialize') { const requested = m.params?.protocolVersion; if (mode === 'reject') error(m.id, -32022, `Unsupported protocol version: ${requested}`, { supported: ['2026-07-28'], requested }); + else if (mode === 'reject-old') error(m.id, -32022, `Unsupported protocol version: ${requested}`, { supported: ['2025-03-26'], requested }); else if (mode === 'reject-nodata') error(m.id, -32022, `Unsupported protocol version: ${requested}`); else if (mode === 'mismatch') reply(m.id, { protocolVersion: '2025-06-18', capabilities: { tools: {} } }); else if (mode === 'missing') reply(m.id, { capabilities: { tools: {} } }); @@ -65,6 +69,12 @@ process.stdin.on('data', (chunk) => { else if (listMode === 'notools') reply(m.id, {}); else if (listMode === 'badcursor') reply(m.id, { tools: [tool('x')], nextCursor: 5 }); else if (listMode === 'nullcursor') reply(m.id, { tools: [tool('x')], nextCursor: null }); + else if (listMode === 'endless') { + const n = cursor === undefined ? 1 : Number(String(cursor).slice(1)) + 1; + reply(m.id, { tools: [tool(`t${n}`)], nextCursor: `c${n}` }); + } + else if (listMode === 'badlater' && cursor === undefined) reply(m.id, { tools: [tool('first')], nextCursor: 'p2' }); + else if (listMode === 'badlater') reply(m.id, {}); else if (cursor === undefined) reply(m.id, { tools: [tool('a1'), tool('a2')], nextCursor: 'p2' }); else if (cursor === 'p2') reply(m.id, { tools: [tool('b1')], nextCursor: 'p3' }); else if (cursor === 'p3') reply(m.id, { tools: [tool('c1'), tool('die')] }); diff --git a/test/mcp-modern-framework.test.ts b/test/mcp-modern-framework.test.ts index 0eac21d63..0c827aa97 100644 --- a/test/mcp-modern-framework.test.ts +++ b/test/mcp-modern-framework.test.ts @@ -1,8 +1,7 @@ /** - * A modern-MCP server inside a real framework (room-284, shelf-487): tool - * listing and status, both dispatch paths, the script view of structured - * results, an MCP-only legacy peer read the same standard way, and the host - * lifetime contract (Tessa #53152, Agnes/Kit #53184, Theo #53178): + * A modern-MCP server inside a real framework: tool listing and status, both + * dispatch paths, the script view of structured results, an MCP-only legacy + * peer read the same standard way, and the host lifetime contract: * - a modern call already in flight completes while the host quiesces; * - a list change while quiesced refreshes the inventory, and its wake parks * until resume; @@ -188,8 +187,8 @@ test('scripts get structured results whole, as one JSON object', async () => { }); }); -// The script cap measures the serialized object's length (Iris-1827, -// room-293): `sized` pads structuredContent so that length is exact. +// The script cap measures the serialized object's length: `sized` pads +// structuredContent so that length is exact. const SCRIPT_CAP = 5_000_000; const sizedObject = (pad: number, isError = false) => ({ content: [], @@ -398,18 +397,22 @@ test('quiesce: a modern call already in flight completes, and its result reaches await withModern(async ({ framework, membrane, dir }) => { framework.start(); membrane.pushResponse(createMockResponse([ - { type: 'tool_use', id: 'toolu_01MODERNSLOW0000000000000', name: 'mcpl--modern--slow', input: { ms: 600 } }, + { type: 'tool_use', id: 'toolu_01MODERNSLOW0000000000000', name: 'mcpl--modern--slow', input: { ms: 1500 } }, ] as never, 'tool_use')); membrane.pushResponse(createMockResponse([{ type: 'text', text: 'Done.' }])); framework.pushEvent({ type: 'external-message', source: 'test', content: 'go', metadata: {} } as never); await waitFor('the slow call to start server-side', () => lines(join(dir, 'events.log')).includes('slow-start')); - const status = await framework.quiesce({ reason: 'maintenance', timeoutMs: 10_000 }); + // Quiesce begins while the call is provably still running server-side, + // so its resolving after 'slow-finished' shows it waited for the call. + const quiescing = framework.quiesce({ reason: 'maintenance', timeoutMs: 10_000 }); + assert.ok(!lines(join(dir, 'events.log')).includes('slow-finished'), 'the call is still in flight as quiesce begins'); + const status = await quiescing; assert.equal(status.quiesced, true); assert.equal(status.drained, true, 'the turn drained: its modern call completed'); assert.ok(lines(join(dir, 'events.log')).includes('slow-finished')); assert.ok(!lines(join(dir, 'events.log')).includes('slow-aborted'), 'quiesce did not cancel it'); - assert.match(JSON.stringify(membrane.lastStream!.receivedToolResults[0]), /slept 600/); + assert.match(JSON.stringify(membrane.lastStream!.receivedToolResults[0]), /slept 1500/); await framework.resume(); }); }); @@ -485,3 +488,42 @@ test('the framework owns a modern connect in flight: stop() ends it, and a secon rmSync(dir, { recursive: true, force: true }); } }); + +test('a lost modern child is relisted from the new launch: what only the old one listed is gone', async () => { + await withModern(async ({ framework, dir }) => { + framework.start(); + // The first launch registers a tool at runtime; a fresh launch won't have it. + await framework.executeToolCall(call('mcpl--modern--add_tool')); + await waitFor('the first launch\'s added tool', () => framework.getAllTools().some((t) => t.name === 'mcpl--modern--added_1')); + await framework.executeToolCall(call('mcpl--modern--die')); + await waitFor('a second launch', () => lines(join(dir, 'starts.log')).length === 2); + await waitFor('the relist from the new launch', () => + framework.listMcplServers()[0]!.connected + && framework.getAllTools().some((t) => t.name === 'mcpl--modern--echo') + && !framework.getAllTools().some((t) => t.name === 'mcpl--modern--added_1')); + }); +}); + +test("a modern tool's error result is traced as tool:failed, not tool:completed", async () => { + const traces: Array<{ type: string; tool?: string; error?: string }> = []; + const watcher: Module = { + name: 'trace-watcher', + async start(ctx: ModuleContext) { ctx.onTrace((event) => traces.push(event as never)); }, + async stop() {}, + getTools: (): ToolDefinition[] => [], + handleToolCall: async (): Promise => ({ success: false, error: 'no tools', isError: true }), + onProcess: async () => ({}), + }; + await withModern(async ({ framework, membrane }) => { + framework.start(); + membrane.pushResponse(createMockResponse([ + { type: 'tool_use', id: 'toolu_01MODERNFAIL0000000000000', name: 'mcpl--modern--fail', input: {} }, + ] as never, 'tool_use')); + membrane.pushResponse(createMockResponse([{ type: 'text', text: 'Noted.' }])); + framework.pushEvent({ type: 'external-message', source: 'test', content: 'go', metadata: {} } as never); + await waitFor('the tool trace', () => traces.some((t) => (t.type === 'tool:failed' || t.type === 'tool:completed') && t.tool === 'fail')); + const outcome = traces.find((t) => (t.type === 'tool:failed' || t.type === 'tool:completed') && t.tool === 'fail')!; + assert.equal(outcome.type, 'tool:failed'); + assert.match(outcome.error ?? '', /the tool failed on purpose/); + }, [], [watcher]); +}); diff --git a/test/mcp-server-ownership.test.ts b/test/mcp-server-ownership.test.ts index 875c0d85e..e87aad601 100644 --- a/test/mcp-server-ownership.test.ts +++ b/test/mcp-server-ownership.test.ts @@ -1,7 +1,6 @@ /** - * One owner per MCP server id across both engines (room-293, greptile's - * review of PR #262 and Iris-1827's invariant): from a connect's admission - * until its teardown is done, exactly one owner holds an id. + * One owner per MCP server id across both engines: from a connect's + * admission until its teardown is done, exactly one owner holds an id. * - a connect is refused while a connect or disconnect for the id is in * flight, a legacy handshake included; * - a disconnect waits for what held the id before it, so nothing it removes @@ -287,3 +286,132 @@ test('legacy: a failed connect whose cleanup cannot reap stays registered until assert.equal(status(framework, 'l')?.connected, true); }); }); + +test('stop() closes every connection, keeps the ones it cannot reap registered, and then rejects naming them', async () => { + const dir = mkdtempSync(join(tmpdir(), 'mcp-ownership-')); + cleanups.push(() => rmSync(dir, { recursive: true, force: true })); + const framework = await AgentFramework.create({ + storePath: join(dir, 'store'), + membrane: new MockMembrane().asMembrane(), + agents: [{ name: 'agent', model: 'test-model', systemPrompt: 'test' }], + modules: [], + }); + const reaping = unreapable(); + await framework.connectMcplServer(modern('m', '', join(dir, 'raw.jsonl'))); + await framework.connectMcplServer(legacy('l', 'ok', join(dir, 'legacy.log'))); + await assert.rejects(framework.stop(), /stop\(\) finished, but not cleanly: .*could not be reaped/); + const internals = framework as unknown as { + modernServers(): Map; + mcplServerRegistry: { getServer(id: string): unknown }; + }; + assert.ok(internals.modernServers().has('m'), 'the modern one it could not reap keeps its id'); + assert.ok(internals.mcplServerRegistry.getServer('l'), 'and so does the legacy one'); + for (const stuck of reaping.stuck) await killAndWait(stuck); +}); + +test("a failed startup's cleanup closes both engines' connections, each whatever the other's close did", async () => { + const dir = mkdtempSync(join(tmpdir(), 'mcp-ownership-')); + cleanups.push(() => rmSync(dir, { recursive: true, force: true })); + const framework = await AgentFramework.create({ + storePath: join(dir, 'store'), + membrane: new MockMembrane().asMembrane(), + agents: [{ name: 'agent', model: 'test-model', systemPrompt: 'test' }], + modules: [], + }); + const reaping = unreapable(); + try { + await framework.connectMcplServer(legacy('l', 'ok', join(dir, 'legacy.log'))); + reaping.active = false; + await framework.connectMcplServer(modern('m', '', join(dir, 'raw.jsonl'))); + const errors: unknown[] = []; + const realError = console.error; + console.error = (...args: unknown[]) => { errors.push(args); }; + try { + await (framework as unknown as { closeMcpConnectionsAfterFailedStartup(): Promise }).closeMcpConnectionsAfterFailedStartup(); + } finally { + console.error = realError; + } + const internals = framework as unknown as { + modernServers(): Map; + mcplServerRegistry: { getServer(id: string): unknown }; + }; + assert.equal(internals.modernServers().has('m'), false, 'the modern connection was closed and let go, though the legacy close failed'); + assert.equal(status(framework, 'm')?.connected, false); + assert.ok(internals.mcplServerRegistry.getServer('l'), 'the one it could not reap stays registered'); + assert.equal(status(framework, 'l')?.connected, false, 'closed'); + assert.match(JSON.stringify(errors), /could not be reaped/, 'and the failure is logged, not thrown'); + } finally { + // Whatever the assertions found, end the stuck child so stop() can reap it. + for (const stuck of reaping.stuck) await killAndWait(stuck); + await framework.stop(); + } +}); + +test('a restart whose new configuration is refused leaves the running server as it was', async () => { + await withFramework(async ({ framework, dir }) => { + const log = join(dir, 'raw.jsonl'); + await framework.connectMcplServer(modern('m', '', log)); + await assert.rejects( + framework.restartMcplServer('m', { id: 'm', url: 'ftp://127.0.0.1/x' }), + /MCP server "m" configuration: .*url must be/, + ); + assert.equal(status(framework, 'm')?.connected, true, 'still connected'); + assert.equal(launches(log).length, 1, 'never torn down or relaunched'); + assert.ok(toolsOf(framework, 'm').length > 0, 'its tools still listed'); + }); +}); + +test("a refresh in flight when a server is disconnected doesn't bring its tools back", async () => { + await withFramework(async ({ framework, dir }) => { + await framework.connectMcplServer(modern('m', '', join(dir, 'raw.jsonl'))); + assert.ok(toolsOf(framework, 'm').length > 0); + const internals = framework as unknown as { + modernServers(): Map }>; + refreshMcplTools(): Promise; + }; + const connection = internals.modernServers().get('m')!; + const realList = connection.listTools.bind(connection); + let release!: () => void; + const gate = new Promise((resolve) => { release = resolve; }); + let listed = false; + connection.listTools = async () => { const tools = await realList(); listed = true; await gate; return tools; }; + const refreshing = internals.refreshMcplTools(); + await until(() => listed, 'the refresh holds the server\'s list'); + await framework.disconnectMcplServer('m'); + release(); + await refreshing; + assert.deepEqual(toolsOf(framework, 'm'), [], 'the earlier refresh, finishing last, published none of its tools'); + }); +}); + +test('a server lost for good, legacy or modern, leaves the tool list', async () => { + await withFramework(async ({ framework, dir }) => { + await framework.connectMcplServer(legacy('l', 'ok', join(dir, 'legacy.log'))); + await framework.connectMcplServer(modern('m', '', join(dir, 'raw.jsonl'))); + assert.ok(toolsOf(framework, 'l').length > 0 && toolsOf(framework, 'm').length > 0); + await framework.executeToolCall(call('mcpl--l--die')).catch(() => {}); + await framework.executeToolCall(call('mcpl--m--die')).catch(() => {}); + await until(() => toolsOf(framework, 'l').length === 0, 'the legacy server\'s tools gone'); + await until(() => toolsOf(framework, 'm').length === 0, 'the modern server\'s tools gone'); + }); +}); + +test("a stdio server switched to the modern family is a new coalescing binding; a legacy configuration's binding is unchanged", async () => { + await withFramework(async ({ framework }) => { + const internals = framework as unknown as { + mcplServerConfigs: Map; + coalescingBinding(id: string): string; + }; + internals.mcplServerConfigs.set('s', { id: 's', command: 'srv', args: ['a'] }); + const legacyBinding = internals.coalescingBinding('s'); + const { createHash } = await import('node:crypto'); + assert.equal( + legacyBinding, + createHash('sha256').update(JSON.stringify(['s', null, 'srv', ['a']])).digest('hex').slice(0, 16), + 'the hash a stored legacy binding was made with', + ); + internals.mcplServerConfigs.set('s', { id: 's', command: 'srv', args: ['a'], protocol: 'modern' }); + assert.notEqual(internals.coalescingBinding('s'), legacyBinding); + internals.mcplServerConfigs.delete('s'); + }); +}); diff --git a/test/mcpl-legacy-lifetime.test.ts b/test/mcpl-legacy-lifetime.test.ts index 31abe8a38..aea5ce5c5 100644 --- a/test/mcpl-legacy-lifetime.test.ts +++ b/test/mcpl-legacy-lifetime.test.ts @@ -1,6 +1,6 @@ /** - * The legacy engine's lifetime contract (room-293, greptile's review of - * PR #262): one owner for every launch, the rule the modern engine keeps. + * The legacy engine's lifetime contract: one owner for every launch, the + * rule the modern engine keeps. * - a configuration it can't use is refused before anything is dialed, and * never turned into a retry stub; * - concurrent close() calls share one teardown and its verdict; @@ -20,7 +20,11 @@ import { fileURLToPath } from 'node:url'; import { WebSocketServer } from 'ws'; import { McplServerConnection, McplUnreapedLaunchError } from '../src/mcpl/server-connection.js'; -import { StdioTransport } from '../src/mcpl/transport.js'; +import { McplServerRegistry } from '../src/mcpl/server-registry.js'; +import { StdioTransport, openTransport } from '../src/mcpl/transport.js'; +import { createServer, type Socket } from 'node:net'; +import { createServer as createHttpServer } from 'node:http'; +import { EventEmitter } from 'node:events'; import type { McplHostCapabilities, McplServerConfig } from '../src/mcpl/types.js'; const here = dirname(fileURLToPath(import.meta.url)); @@ -102,6 +106,8 @@ test('a configuration the engine cannot use is refused before anything is dialed McplServerConnection.connectWithReconnect({ id: 'ftp', url: 'ftp://127.0.0.1/x', ...RECONNECT }, HOST_CAPS), /url must be ws:\/\/ or wss:\/\//, ); + // A refusal is synchronous with the connect call; 200 ms covers several + // reconnect intervals (20–40 ms) in which a stub would have dialed. await new Promise((r) => setTimeout(r, 200)); assert.equal(dials, 0, 'nothing was dialed'); }); @@ -127,6 +133,8 @@ test('a close that cannot reap fails, and so do concurrent ones; a later close r const outcome = (name: string) => [() => settled.push(`${name}:ok`), (e: Error) => settled.push(`${name}:${e.message}`)] as const; const first = connection.close().then(...outcome('first')); const second = connection.close().then(...outcome('second')); + // The teardown sends SIGTERM, waits 2 s, sends SIGKILL and waits 2 s more + // before it gives up, so at 1 s it is still pending. await new Promise((r) => setTimeout(r, 1000)); assert.deepEqual(settled, [], 'neither resolves while the teardown is pending'); await Promise.all([first, second]); @@ -164,6 +172,7 @@ test('with reconnect, a failed first launch that cannot be reaped halts instead assert.equal(failures[0]!.permanent, true); assert.match(failures[0]!.error, /initialize failed on purpose; the previous launch could not be reaped .*reconnecting is halted/); assert.equal(connection.willReconnect, false); + // 300 ms is several reconnect intervals (20–40 ms): a retry would have launched. await new Promise((r) => setTimeout(r, 300)); assert.equal(starts(log).length, 1, 'no launch beside the unreaped one'); await killAndWait(stuck[0]!); @@ -182,6 +191,7 @@ test('a reconnect launch that cannot be reaped halts reconnecting; close() settl await until(() => failures.some((f) => f.permanent), 'a permanent reconnect failure'); assert.match(failures.find((f) => f.permanent)!.error, /initialize failed on purpose; the previous launch could not be reaped .*reconnecting is halted/); assert.equal(connection.willReconnect, false); + // 300 ms is several reconnect intervals (20–40 ms): a retry would have launched. await new Promise((r) => setTimeout(r, 300)); assert.equal(starts(log).length, 2, 'no third launch while the second is unreaped'); await assert.rejects(connection.close(), /could not be reaped/); @@ -196,7 +206,7 @@ test('close() during a reconnect handshake ends that launch before resolving, an let reconnected = false; connection.on('reconnect', () => { reconnected = true; }); await assert.rejects(connection.sendToolsCall('die', {})); - await until(() => starts(log).length === 2, 'the reconnect launch, waiting on initialize'); + await until(() => starts(log).length === 2, 'the reconnect launch has started (its start line)'); const second = starts(log)[1]!; const began = Date.now(); await connection.close(); @@ -205,6 +215,7 @@ test('close() during a reconnect handshake ends that launch before resolving, an assert.equal(alive(second), false, 'the launch in flight was reaped before close() resolved'); assert.equal(reconnected, false); assert.equal(connection.isConnected, false); + // Several reconnect intervals (20–40 ms) later, still nothing. await new Promise((r) => setTimeout(r, 200)); assert.equal(starts(log).length, 2, 'and no later launch'); }); @@ -216,8 +227,9 @@ test('a recycle during a reconnect in flight starts no second attempt, and close const connection = await McplServerConnection.connectWithReconnect(config('ok-then-hang', log, RECONNECT), HOST_CAPS); connection.ready(); await assert.rejects(connection.sendToolsCall('die', {})); - await until(() => starts(log).length === 2, 'the reconnect launch, waiting on initialize'); + await until(() => starts(log).length === 2, 'the reconnect launch has started (its start line)'); await connection.reconnectAfterFailure(); + // Several reconnect intervals (20–40 ms) later, still only the one in flight. await new Promise((r) => setTimeout(r, 400)); assert.equal(starts(log).length, 2, 'no second attempt beside the one in flight'); const began = Date.now(); @@ -245,3 +257,146 @@ test('a reconnect never launches while the previous child may still be running', await recycled; await until(() => reconnected, 'reconnected'); }); + +test('registry: an id stays held through its reap, and a second add is refused while one is connecting', async () => { + const log = scratchLog(); + const registry = new McplServerRegistry(); + // A child that ignores SIGTERM keeps its reap going for about 2 s. + const removing = (async () => { + await registry.addServer(config('ok', log, { env: { IGNORE_SIGTERM: '1' } }), HOST_CAPS); + return registry.removeServer('legacy'); + })(); + await until(() => registry.getServer('legacy') !== null, 'registered'); + await until(() => starts(log).length === 1, 'launched'); + await new Promise((r) => setTimeout(r, 100)); + await assert.rejects(registry.addServer(config('ok', log), HOST_CAPS), /already registered/, 'held while its reap runs'); + await removing; + assert.equal(registry.getServer('legacy'), null, 'freed once the exit is confirmed'); + assert.equal(starts(log).length, 1, 'nothing launched beside it'); + + // Two adds at once: the second is refused while the first is connecting. + const slow = registry.addServer(config('slow', log, { env: { SLOW_INIT_MS: '300' } }), HOST_CAPS); + await assert.rejects(registry.addServer(config('ok', log), HOST_CAPS), /already registered/); + await slow; + await registry.removeServer('legacy'); +}); + +test('registry: a connection whose reap fails stays registered, through close() and closeAll(), until a retry confirms the exit', async () => { + const log = scratchLog(); + const registry = new McplServerRegistry(); + const stuck = unreapableFrom(1); + const connection = await registry.addServer(config('ok', log), HOST_CAPS); + // Its own close() can't reap: the registry keeps it, holding the id. + await assert.rejects(connection.close(), /could not be reaped/); + assert.equal(registry.getServer('legacy'), connection, 'still registered after its own close'); + // closeAll() reports it, and keeps it. + await assert.rejects(registry.closeAll(), /could not be reaped/); + assert.equal(registry.getServer('legacy'), connection, 'still registered after closeAll'); + await assert.rejects(registry.addServer(config('ok', log), HOST_CAPS), /already registered/); + // removeServer retries the reap; once the child is gone it succeeds. + await assert.rejects(registry.removeServer('legacy'), /could not be reaped/); + assert.equal(registry.getServer('legacy'), connection); + await killAndWait(stuck[0]!); + await registry.removeServer('legacy'); + assert.equal(registry.getServer('legacy'), null); +}); + +test('registry: closeAll closes every connection, removes the ones it reaped, and names each it could not', async () => { + const log = scratchLog(); + const registry = new McplServerRegistry(); + const stuck = unreapableFrom(2); + const fine = await registry.addServer({ ...config('ok', log), id: 'fine' }, HOST_CAPS); + const held = await registry.addServer({ ...config('ok', log), id: 'held' }, HOST_CAPS); + await assert.rejects(registry.closeAll(), /"held" did not close cleanly/); + assert.equal(registry.getServer('fine'), null, 'the one it reaped is removed'); + assert.equal(fine.isConnected, false); + assert.equal(registry.getServer('held'), held, 'the one it could not reap stays'); + await killAndWait(stuck[0]!); + await registry.removeServer('held'); +}); + +test('a reconnect whose previous child cannot be reaped holds it and halts reconnecting', async () => { + const log = scratchLog(); + const stuck = unreapableFrom(1); + const connection = await McplServerConnection.connectWithReconnect(config('ok', log, RECONNECT), HOST_CAPS); + cleanups.push(() => connection.close().catch(() => {})); + connection.ready(); + const failures: Array<{ error: string; attempt: number; permanent?: boolean }> = []; + connection.on('reconnect-failed', (f) => failures.push(f)); + // Recycle it as the framework does: the host closes the transport, whose + // child can't be reaped, and the loop then finds that child still held. + await connection.reconnectAfterFailure().catch(() => {}); + await until(() => failures.some((f) => f.permanent), 'a permanent reconnect failure'); + assert.match(failures.find((f) => f.permanent)!.error, /could not be reaped/); + assert.equal(connection.willReconnect, false); + // Several reconnect intervals (20–40 ms) later, nothing launched beside it. + await new Promise((r) => setTimeout(r, 300)); + assert.equal(starts(log).length, 1); + await killAndWait(stuck[0]!); + await connection.close(); +}); + +test('a WebSocket dial whose connection is closed meanwhile ends at once, not at the open timeout', async () => { + // A listener that accepts and never answers the upgrade. + const sockets: Socket[] = []; + const server = createServer((socket) => { sockets.push(socket); }); + await new Promise((r) => server.listen(0, '127.0.0.1', () => r())); + cleanups.push(() => new Promise((r) => { for (const s of sockets) s.destroy(); server.close(() => r()); })); + const url = `ws://127.0.0.1:${(server.address() as AddressInfo).port}/mcpl`; + const abort = new AbortController(); + const began = Date.now(); + const dial = openTransport({ id: 'ws', url }, abort.signal); + await until(() => sockets.length === 1, 'the dial reached the listener'); + abort.abort(); + await assert.rejects(dial, /WebSocket connect abandoned: the connection was closed/); + assert.ok(Date.now() - began < 5_000, 'well before the 15 s open timeout'); +}); + +test("a write to a stdio child that has just exited is reported, never thrown", (t) => { + const child = Object.assign(new EventEmitter(), { stdin: new EventEmitter(), stderr: new EventEmitter(), killed: false, exitCode: null }); + const rl = new EventEmitter(); + const transport = new (StdioTransport as unknown as new (c: unknown, r: unknown) => StdioTransport)(child, rl); + const logged: unknown[] = []; + t.mock.method(console, 'error', (...args: unknown[]) => { logged.push(args); }); + const epipe = Object.assign(new Error('write EPIPE'), { code: 'EPIPE' }); + assert.doesNotThrow(() => child.stdin.emit('error', epipe), 'with nothing listening'); + const reported: unknown[] = []; + transport.on('error', (error) => reported.push(error)); + child.stdin.emit('error', epipe); + assert.deepEqual(reported, [epipe], "it's the transport's error once someone listens"); +}); + +test('close() during a WebSocket reconnect dial returns promptly, ending the dial', async () => { + // The first upgrade gets a minimal MCPL server; every later one hangs, so + // the reconnect's dial would wait out its 15 s open timeout. + const http = createHttpServer(); + const wss = new WebSocketServer({ noServer: true }); + const hung: Socket[] = []; + let upgrades = 0; + http.on('upgrade', (request, socket, head) => { + if (++upgrades === 1) wss.handleUpgrade(request, socket as Socket, head, (ws) => wss.emit('connection', ws, request)); + else hung.push(socket as Socket); + }); + let first: import('ws').WebSocket | undefined; + wss.on('connection', (ws) => { + first = ws; + ws.on('message', (data) => { + const message = JSON.parse(String(data)) as { id?: number; method?: string }; + if (message.id === undefined) return; + const result = message.method === 'initialize' ? { protocolVersion: '2024-11-05', capabilities: {} } : {}; + ws.send(JSON.stringify({ jsonrpc: '2.0', id: message.id, result })); + }); + }); + await new Promise((r) => http.listen(0, '127.0.0.1', () => r())); + cleanups.push(() => new Promise((r) => { for (const s of hung) s.destroy(); wss.close(); http.close(() => r()); })); + const url = `ws://127.0.0.1:${(http.address() as AddressInfo).port}/mcpl`; + + const connection = await McplServerConnection.connectWithReconnect({ id: 'ws', url, ...RECONNECT }, HOST_CAPS); + connection.ready(); + first!.close(); + await until(() => upgrades === 2, 'the reconnect is dialing'); + const began = Date.now(); + await connection.close(); + assert.ok(Date.now() - began < 5_000, 'promptly: the dial was ended, not waited out (15 s)'); + assert.equal(connection.isConnected, false); +}); diff --git a/test/mcpl-legacy-version.test.ts b/test/mcpl-legacy-version.test.ts index 46e5ae415..c5f51cfcf 100644 --- a/test/mcpl-legacy-version.test.ts +++ b/test/mcpl-legacy-version.test.ts @@ -1,5 +1,5 @@ /** - * The legacy engine's protocol boundary and inventory (room-284, shelf-487): + * The legacy engine's protocol boundary and inventory: * - it offers and accepts exactly MCP 2024-11-05; anything else is a typed * McplProtocolVersionError carrying what each side said; * - such a verdict is never retried (no reconnect stub, no backoff loop), and @@ -175,13 +175,18 @@ test('a malformed page is an error, never a partial or empty inventory', async ( for (const [mode, pattern] of [ ['notools', /malformed tools\/list page 1: no tools array/], ['badcursor', /malformed tools\/list page 1: nextCursor must be a string, got 5/], - ['nullcursor', /malformed tools\/list page 1: nextCursor must be a string, got null/], ] as const) { const connection = await connect(config('ok', { env: { MODE_LIST: mode } })); await assert.rejects(connection.sendToolsList(), pattern, mode); } }); +test('a null nextCursor ends the inventory, as an absent one does', async () => { + const connection = await connect(config('ok', { env: { MODE_LIST: 'nullcursor' } })); + const { tools } = await connection.sendToolsList(); + assert.deepEqual(tools.map((t) => t.name), ['x']); +}); + test('a repeating cursor is an error, not a silently partial inventory', async () => { const connection = await connect(config('ok', { env: { MODE_LIST: 'loop' } })); await assert.rejects(connection.sendToolsList(), /repeated tools\/list cursor "same"/); @@ -208,3 +213,115 @@ test('a modern configuration is refused by the legacy engine before anything is await connect(config('ok', {}, startLog)); assert.equal(readFileSync(startLog, 'utf8').split('\n').filter(Boolean).length, 1); }); + +test('a -32022 whose supported list lacks the modern revision names no modern remedy', async () => { + await assert.rejects(McplServerConnection.connect(config('reject-old'), HOST_CAPS), (err: unknown) => { + assert.ok(err instanceof McplProtocolVersionError); + assert.deepEqual(err.supported, ['2025-03-26']); + assert.match(err.message, /it supports: 2025-03-26/); + assert.doesNotMatch(err.message, /protocol: 'modern'/); + return true; + }); +}); + +function startsIn(startLog: string): string[] { + return existsSync(startLog) ? readFileSync(startLog, 'utf8').split('\n').filter(Boolean) : []; +} + +function alive(pid: number): boolean { + try { process.kill(pid, 0); return true; } catch { return false; } +} + +test('a version verdict is never retried: one launch, and that child is reaped', async () => { + const dir = mkdtempSync(join(tmpdir(), 'legacy-version-')); + cleanups.push(() => rmSync(dir, { recursive: true, force: true })); + for (const mode of ['reject', 'mismatch']) { + const startLog = join(dir, `${mode}.log`); + await assert.rejects( + McplServerConnection.connectWithReconnect(config(mode, { reconnect: true, reconnectIntervalMs: 20, reconnectMaxIntervalMs: 40 }, startLog), HOST_CAPS), + (err: unknown) => err instanceof McplProtocolVersionError, + ); + const starts = startsIn(startLog); + assert.equal(starts.length, 1, `${mode}: launched once`); + // The rejection waits for the handshake's own cleanup, so its child is gone. + const pid = Number(starts[0]!.split(' ')[1]); + assert.equal(alive(pid), false, `${mode}: its child was reaped`); + // Past several reconnect intervals, nothing has launched again. + await new Promise((r) => setTimeout(r, 200)); + assert.equal(startsIn(startLog).length, 1, `${mode}: never retried`); + } +}); + +test('an initialize error that is not a version verdict stays a plain, retryable error', async () => { + await assert.rejects(McplServerConnection.connect(config('error'), HOST_CAPS), (err: unknown) => { + assert.ok(err instanceof Error); + assert.ok(!(err instanceof McplProtocolVersionError)); + assert.match(err.message, /initialize failed on purpose/); + return true; + }); + // With reconnect, it leaves a stub that keeps trying, as any transient failure does. + const stub = await McplServerConnection.connectWithReconnect(config('error', { reconnect: true, reconnectIntervalMs: 20 }), HOST_CAPS); + cleanups.push(() => stub.close()); + assert.equal(stub.willReconnect, true); +}); + +test('a modern stdio configuration is refused before anything is spawned', async () => { + const dir = mkdtempSync(join(tmpdir(), 'legacy-version-')); + cleanups.push(() => rmSync(dir, { recursive: true, force: true })); + const startLog = join(dir, 'starts.log'); + await assert.rejects( + McplServerConnection.connect(config('ok', { protocol: 'modern' }, startLog), HOST_CAPS), + /configured for modern MCP/, + ); + assert.deepEqual(startsIn(startLog), [], 'nothing was spawned'); +}); + +test('a list that never ends is refused after its page bound', async () => { + const connection = await connect(config('ok', { env: { MODE_LIST: 'endless' } })); + await assert.rejects(connection.sendToolsList(), /did not finish within 100 pages/); +}); + +test('a page after the first is checked as the first is', async () => { + const connection = await connect(config('ok', { env: { MODE_LIST: 'badlater' } })); + await assert.rejects(connection.sendToolsList(), /malformed tools\/list page 2: no tools array/); +}); + +async function startupTraces(dir: string, server: McplServerConfig): Promise { + const traces: TraceEvent[] = []; + const watcher: Module = { + name: 'trace-watcher', + async start(ctx: ModuleContext) { ctx.onTrace((event) => traces.push(event)); }, + async stop() {}, + getTools: (): ToolDefinition[] => [], + handleToolCall: async (): Promise => ({ success: false, error: 'no tools', isError: true }), + onProcess: async () => ({}), + }; + const framework = await AgentFramework.create({ + storePath: join(dir, 'store'), + membrane: { complete: async () => { throw new Error('not used'); } } as unknown as import('@animalabs/membrane').Membrane, + agents: [{ name: 'agent', model: 'test-model', systemPrompt: 'test' }], + modules: [watcher], + mcplServers: [server], + }); + cleanups.push(() => framework.stop().catch(() => {})); + return traces; +} + +test('at startup, willRetry follows reconnect and the verdict, and only a verdict raises the alert', async () => { + const cases: Array<[string, McplServerConfig, boolean, boolean]> = [ + // [label, config, willRetry, alert] + ['verdict without reconnect', config('missing'), false, true], + ['failure with reconnect', config('error', { reconnect: true }), true, false], + ['failure without reconnect', config('error'), false, false], + // Refused before anything launches: nothing will retry it, reconnect or not. + ['configuration refused, with reconnect', { id: 'legacy', url: 'ftp://127.0.0.1/x', reconnect: true }, false, false], + ]; + for (const [label, server, willRetry, alert] of cases) { + const dir = mkdtempSync(join(tmpdir(), 'legacy-version-')); + cleanups.push(() => rmSync(dir, { recursive: true, force: true })); + const traces = await startupTraces(dir, server); + const failed = traces.find((t) => t.type === 'mcpl:server-connect-failed') as { willRetry?: boolean } | undefined; + assert.equal(failed?.willRetry, willRetry, label); + assert.equal(traces.some((t) => t.type === 'ops:alert'), alert, `${label}: alert`); + } +}); From 5abaf9f84d584185426e148556aecddd42eab0cb Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Mon, 12 Oct 2026 05:06:29 -0400 Subject: [PATCH 0124/1112] fix(code-execution): a script's structured result gives an MCPL peer's blocks the treatment the model's lanes give them, and a reference its id and saved path A script whose tool call returns structuredContent gets the server's content array in its JSON object, so images and other payloads survive. From an MCPL peer, an RFC-005 reference went there as the server sent it, with its URI, which can be a bearer credential, and without the workspace path that the script-demand fetch had just saved its payload to. So did a block the classifier rejects, and inline data that claims a bulk disposition, both of which the model's lanes withhold. What a script prints comes back to the model. The dispatch knows whether the server is an MCPL peer, so it now gives a script's call its reading: - For an MCPL peer, autoFetchReferences fetches every reference a script will read, after the eager pass, and the script-side fetch loop is gone. A script reads the blocks unless the result is an error without structured content. - An MCPL peer's structured content goes through referenceBlockForScript (references.ts), which makes referenceStubOrNull's decision for each block. Text and plain inline data stay as sent. Inline data that claims a bulk disposition becomes the withheld notice. A reference becomes { type, refId, name?, mimeType?, sizeBytes?, digest?, expiresAt?, savedTo }: the registry's id, that block's own testimony with its text fields stripped of control and bidi characters as in the model's stub, and the path the fetch saved its payload to (null when it didn't; fetch_reference takes the id as its ref_id argument). A block the classifier rejects becomes the unrecognized-content notice the model's lanes show. - A server that isn't an MCPL peer sends no references, so its structured content reaches the script as sent. The changelog fragment says what a script reads from each kind of server, and credits #181, which first found scripts losing structuredContent and kept the saved paths of references by falling back to the text rendering. It and the normalizer say a resource_link is shown as a link, with its name and URI. Tests, through the real dispatch with an MCPL peer and an MCP-only peer (script-structured-references): - from the MCPL peer, a structured result holds one reference in two blocks with different testimony, a reference the fetch doesn't save, inline data with a disposition, inline data with a uri beside it, plain inline data, three rejected blocks, and labels with control and bidi characters. The script gets no URI and each block as the model's lanes treat it, and both references are fetched before it reads them; - the same blocks from the MCP-only peer reach the script as sent, and nothing is fetched; - without structuredContent, the stubs name the same reference id and saved path; - past the script cap, the saved file holds the treated blocks and no URI; - the model's call, and an error a script reads as text, get no script-demand fetch, and the eager pass still runs for both; - an error with structured content is fetched for the script, and its blocks reach it treated, with its error text and structured content. In mcp-modern-framework, a modern server's resource link, embedded resources and audio reach a script as sent. --- .../mcp-standard-tool-results.changed.md | 31 ++- src/framework.ts | 98 ++++--- src/mcpl/references.ts | 44 +++- src/mcpl/tool-result-normalize.ts | 5 +- test/fixtures/modern-tools.mjs | 21 +- test/mcp-modern-framework.test.ts | 16 ++ test/script-structured-references.test.ts | 248 ++++++++++++++++++ 7 files changed, 401 insertions(+), 62 deletions(-) create mode 100644 test/script-structured-references.test.ts diff --git a/changelog.d/mcp-standard-tool-results.changed.md b/changelog.d/mcp-standard-tool-results.changed.md index c6a74c414..4d809347f 100644 --- a/changelog.d/mcp-standard-tool-results.changed.md +++ b/changelog.d/mcp-standard-tool-results.changed.md @@ -12,7 +12,8 @@ and the host's log says why the save failed; - a failed result is read as its text alone, so there an inline image is saved and stubbed like any other payload; - - a `resource_link` is shown as a reference and never fetched; + - a `resource_link` is shown as a link, with its name and URI, and never + fetched; - an embedded text resource shows its text; - an unknown block is shown as JSON, bounded, saying how much was left out. A malformed entry is named and left out, and content that isn't a @@ -26,12 +27,28 @@ reading and gain `structured`. - A script whose tool call returns `structuredContent` receives the server's result itself, as one JSON object: `{ content, structuredContent, - isError, error? }`, with `content` the blocks as sent, so images and other - payloads survive. If it would exceed the 5 MB script cap, it is saved to the - workspace and the script gets `{ isError, oversized: { chars, savedTo } }`. - If it can't be saved, the script gets a failure that says the tool may have - completed. Results without structured content keep the existing script - contract. + isError, error? }`. From a server that isn't an MCPL peer, `content` is the + blocks as sent, so images and other payloads survive. `structuredContent` + is the tool's own JSON, as sent. +- An MCPL peer's blocks in a script's `content` get the treatment every + lane the model reads gives them, since what a script prints comes back to + the model. An RFC-005 reference arrives without its URI, as the host's + reference id (`refId`), what the server said about it in that block + (`name`, `mimeType`, `sizeBytes`, `digest`, `expiresAt`, with control and + bidi characters stripped from the text fields, as in the model's stub), + and `savedTo`, the workspace path the framework fetched its payload to for + the script (`null` when it couldn't; `fetch_reference` takes the id as its + `ref_id` argument). A block the classifier rejects arrives as the model's + `[unrecognized content block: …]` notice, and inline data that also claims + a bulk disposition as the withheld notice. #181, by @Meganeuridae, first + found scripts losing `structuredContent`, and it kept the saved paths of + RFC-005 payloads for them by falling back to the text rendering. Here the + paths are in the structured object, as `savedTo`. +- If a script's structured result would exceed the 5 MB script cap, the + framework saves it to the workspace and the script gets + `{ isError, oversized: { chars, savedTo } }`. If it can't be saved, the + script gets a failure that says the tool may have completed. Results + without structured content keep the existing script contract. - The direct tool path (`executeToolCall`, `ModuleContext.callTool`) now treats an MCP tool error (`isError: true`) as a failure, with `success: false` and the error text; before, every answer counted as diff --git a/src/framework.ts b/src/framework.ts index 0d62e8f87..5c3eaf2e2 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -1,6 +1,6 @@ import { ToolPresentation, presentationTools, isPresentationTool, renderCatalogue, type PresentationSnapshot } from "./tool-presentation.js"; import { dirname, join } from 'node:path'; -import { INLINE_WITHHELD_TEXT, classifyBlock, isInlineContradiction, referenceRegistry, referenceStubOrNull } from './mcpl/references.js'; +import { INLINE_WITHHELD_TEXT, classifyBlock, isInlineContradiction, referenceBlockForScript, referenceRegistry, referenceStubOrNull } from './mcpl/references.js'; import { ReferenceFetcher, DEFAULT_FETCH_MAX_BYTES, EAGER_FETCH_TIMEOUT_MS } from './mcpl/reference-fetcher.js'; import { appendFileSync, existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs'; import { JsStore } from '@animalabs/chronicle'; @@ -2573,28 +2573,39 @@ export class AgentFramework { return this.referenceFetcherInstance; } - /** Register every RFC-005 reference in a tool-result content array against - * its server, then eagerly fetch the ones inside the autofetch envelope - * (below the eager ceiling, connection origin — never worse than the - * server having inlined the bytes). Failures are non-fatal: the stub then - * says fetch_reference instead of a path. */ - private async autoFetchReferences(content: unknown[] | undefined, serverId: string, eager: boolean): Promise { - if (!content) return; + /** Register every RFC-005 reference in an MCPL peer's tool-result content + * against its server, then eagerly fetch the ones inside the autofetch + * envelope (below the eager ceiling, connection origin — never worse than + * the server having inlined the bytes). Failures are non-fatal: the stub + * then says fetch_reference instead of a path. + * + * For a script that will read the blocks, every reference is then fetched + * ("eagerly lazy"): a script receiving a result is the strongest + * available signal that the bytes are wanted (scripts process, they do + * not browse), so the stub it receives carries a workspace path it can + * open directly. */ + private async autoFetchReferences(content: unknown, serverId: string, forScript: boolean): Promise { + if (!Array.isArray(content)) return; const fetcher = this.getReferenceFetcher(); - const eligible: import('./mcpl/references.js').ReferenceRecord[] = []; + const records: import('./mcpl/references.js').ReferenceRecord[] = []; for (const block of content) { const c = classifyBlock(block); - if (c.kind !== 'reference') continue; - const record = referenceRegistry.register(c.testimony, serverId); - if (eager && fetcher.isEagerEligible(record)) eligible.push(record); + if (c.kind === 'reference') records.push(referenceRegistry.register(c.testimony, serverId)); } // Parallel with a count cap: N small fetches cost one round-trip of // wall-clock, and anything past the cap stays lazy (stub says so). const MAX_EAGER_PER_RESULT = 8; + const eligible = records.filter((record) => fetcher.isEagerEligible(record)); await Promise.all(eligible.slice(0, MAX_EAGER_PER_RESULT).map(async (record) => { const outcome = await fetcher.fetch(record, { timeoutMs: EAGER_FETCH_TIMEOUT_MS }); if (!outcome.ok) console.error(`[rfc005] eager fetch of ${record.refId} failed: ${outcome.error}`); })); + if (!forScript) return; + // A record the eager pass saved returns at once. + for (const record of records) { + const outcome = await fetcher.fetch(record); + if (!outcome.ok) console.error(`[rfc005] script-demand fetch of ${record.refId} failed: ${outcome.error}`); + } } /** Core agent_settings keys — extension keys must not collide with these. */ @@ -11885,21 +11896,9 @@ export class AgentFramework { return `Error: ${result.error ?? 'tool call failed'}`; } if (result.data === undefined && !structured) return ''; - // RFC-005 "eagerly lazy": a script receiving a result is the strongest - // available signal that the bytes are wanted (scripts process, they do - // not browse) — materialize every referenced payload now, so the stub - // the script receives carries a workspace path it can open directly. - if (Array.isArray(result.data)) { - const fetcher = this.getReferenceFetcher(); - for (const block of result.data) { - const c = classifyBlock(block); - if (c.kind !== 'reference') continue; - const record = referenceRegistry.findByUri(c.testimony.uri) - ?? referenceRegistry.register(c.testimony); - const outcome = await fetcher.fetch(record); - if (!outcome.ok) console.error(`[rfc005] script-demand fetch of ${record.refId} failed: ${outcome.error}`); - } - } + // An MCPL peer's references were fetched for the script at dispatch + // (autoFetchReferences), so each one's stub carries the workspace path + // its payload was saved to. // Scripts get (nearly) full data — the script environment IS the spill: // filtering big results in code is the whole point. 5MB protocol safety // cap only; no context-cap truncation, no file spill. @@ -11914,11 +11913,14 @@ export class AgentFramework { * A structured MCP result as a script receives it: one JSON object holding * both of the server's views, `{ content, structuredContent, isError }`, * plus `error` (the text) on a tool error. `content` is the content array - * as the server sent it (the dispatch gave this call the direct reading), - * so payloads and images survive. Past the size cap the result stays - * valid JSON: the whole object is saved to the workspace, and the script - * gets `{ isError, oversized: { chars, savedTo } }`. If it can't be saved, - * the script gets a failure to materialize the result, saying the tool may + * the dispatch gave this call (the direct reading): the server's blocks as + * sent, so payloads and images survive, except that an MCPL peer's blocks + * get the treatment RFC-005 gives them on every lane the model reads + * (referenceBlockForScript). `structuredContent` is the tool's own JSON, as + * sent. Past the size cap the result stays valid JSON: the whole object is + * saved to the workspace, and the script gets + * `{ isError, oversized: { chars, savedTo } }`. If it can't be saved, the + * script gets a failure to materialize the result, saying the tool may * already have completed, never a success missing its result. */ private async scriptStructuredResult(result: ToolResult): Promise { @@ -15352,12 +15354,18 @@ export class AgentFramework { // A script's call that returns structured content gets the server's // result itself (the direct reading): the script receives both views // whole, so nothing the model reading lowers (stubs, image placeholders, - // generated JSON) stands in for what the server sent. + // generated JSON) stands in for what the server sent. An MCPL peer's + // content blocks are the exception: what a script prints comes back to + // the model, so each gets the treatment RFC-005 gives it on every lane + // the model reads. A standard peer's blocks reach the script as sent. const forScript = this.scriptToolWaiters?.has(call.id) ?? false; - const reading = (result: { content?: unknown; isError?: boolean; structuredContent?: unknown }, mcpl: boolean) => - forScript && Object.prototype.hasOwnProperty.call(result, 'structuredContent') - ? Promise.resolve(AgentFramework.directToolResult(result)) - : this.mcpToolResult(result, call.id, mcpl); + const reading = async (result: { content?: unknown; isError?: boolean; structuredContent?: unknown }, mcpl: boolean): Promise => { + if (!forScript || !Object.prototype.hasOwnProperty.call(result, 'structuredContent')) { + return this.mcpToolResult(result, call.id, mcpl); + } + const direct = AgentFramework.directToolResult(result); + return mcpl && Array.isArray(direct.data) ? { ...direct, data: direct.data.map(referenceBlockForScript) } : direct; + }; if (modern) { // Modern MCP: no state exchange, no channels, no RFC-005 references. @@ -15424,11 +15432,16 @@ export class AgentFramework { legacy.sendToolsCall(toolName, args, stateParams) .then(async (result) => { - // RFC-005: register references and eagerly fetch the small ones - // before serialization, so their stubs carry a saved path. Only an - // MCPL peer's blocks are references; an MCP-only peer's are standard. + // RFC-005: register references and fetch the small ones, and every + // one a script will read, before serialization, so their stubs carry + // a saved path. Only an MCPL peer's blocks are references; an + // MCP-only peer's are standard. A script reads the blocks unless the + // result is an error without structured content, which reaches it as + // its error text alone. if (mcplPeer) { - await this.autoFetchReferences(result.content, serverId, true) + const scriptReads = forScript + && (!result.isError || Object.prototype.hasOwnProperty.call(result, 'structuredContent')); + await this.autoFetchReferences(result.content, serverId, scriptReads) .catch((e) => console.error('[rfc005] autofetch error:', (e as Error).message)); } return { result, toolResult: await reading(result, mcplPeer) }; @@ -15547,7 +15560,8 @@ export class AgentFramework { * `structured` too, with the JSON view when its content is empty. The * direct path (a module's callTool, a channel's tool call) doesn't come * here: it returns the server's result itself (directToolResult), for - * either engine, as does a script's call whose result is structured. + * either engine, as does a script's call whose result is structured, with + * an MCPL peer's blocks given RFC-005's treatment (referenceBlockForScript). */ private async mcpToolResult( result: { content?: unknown; isError?: boolean; structuredContent?: unknown }, diff --git a/src/mcpl/references.ts b/src/mcpl/references.ts index acadd26ae..c91d232de 100644 --- a/src/mcpl/references.ts +++ b/src/mcpl/references.ts @@ -295,6 +295,48 @@ export function referenceStubOrNull(block: unknown, provenance?: string, serverI case 'inline': return c.contradiction ? INLINE_WITHHELD_TEXT : null; case 'reference': return buildReferenceStub(c.testimony, provenance, serverId); - case 'invalid': return `[unrecognized content block: ${sanitizeLabel(c.reason, STUB_FIELD_CHARS)}]`; + case 'invalid': return unrecognizedBlockNotice(c.reason); + } +} + +/** What every lane shows in place of a block the classifier rejects. */ +function unrecognizedBlockNotice(reason: string): string { + return `[unrecognized content block: ${sanitizeLabel(reason, STUB_FIELD_CHARS)}]`; +} + +/** + * RFC-005 treatment for one MCPL wire block, for a script's structured + * result, which carries blocks rather than text. It makes the decision + * referenceStubOrNull makes for the model's lanes, since what a script prints + * comes back to the model. A reference becomes the stub's facts as data, + * without its URI: the host's reference id, what the server said about it in + * this block (its labels stripped and bounded as in the stub), and `savedTo`, + * the workspace path its payload was fetched to, or null when it wasn't + * (`fetch_reference` takes the id as `ref_id`). A block the classifier + * rejects becomes the notice the model's lanes show, and inline data that + * also claims a bulk disposition the withheld notice, each as a text block. + * A text block, or inline data that claims nothing, is returned as sent. + */ +export function referenceBlockForScript(block: unknown): unknown { + const c = classifyBlock(block); + switch (c.kind) { + case 'text': return block; + case 'inline': + return c.contradiction ? { type: 'text', text: INLINE_WITHHELD_TEXT } : block; + case 'invalid': return { type: 'text', text: unrecognizedBlockNotice(c.reason) }; + case 'reference': { + const t = c.testimony; + const record = referenceRegistry.register(t); + return { + type: (block as { type: string }).type, + refId: record.refId, + ...(t.name !== undefined ? { name: sanitizeLabel(t.name, STUB_FIELD_CHARS) } : {}), + ...(t.mimeType !== undefined ? { mimeType: sanitizeLabel(t.mimeType, STUB_FIELD_CHARS) } : {}), + ...(t.sizeBytes !== undefined ? { sizeBytes: t.sizeBytes } : {}), + ...(t.digest !== undefined ? { digest: t.digest } : {}), + ...(t.expiresAt !== undefined ? { expiresAt: sanitizeLabel(t.expiresAt, STUB_FIELD_CHARS) } : {}), + savedTo: record.fetchedPath ?? null, + }; + } } } diff --git a/src/mcpl/tool-result-normalize.ts b/src/mcpl/tool-result-normalize.ts index a0adf48e7..b38f20b59 100644 --- a/src/mcpl/tool-result-normalize.ts +++ b/src/mcpl/tool-result-normalize.ts @@ -12,8 +12,9 @@ * to the workspace, and a bounded stub says where it went. A stub tells the * reader where the payload is; nothing is dropped silently. If a payload * can't be saved, the result is a failure to materialize it, not a - * success. A `resource_link` is shown as a reference and never fetched. An - * embedded resource shows its text, or is saved like a blob. + * success. A `resource_link` is shown as a link, with its name and URI, + * and never fetched. An embedded resource shows its text, or is saved like + * a blob. * - **`structuredContent` is the machine-readable view.** It is kept by * presence on `ToolResult.structured`. The model sees it rendered as JSON * only when the server's content has no text block of its own, because diff --git a/test/fixtures/modern-tools.mjs b/test/fixtures/modern-tools.mjs index 5fb435165..242f7b9a6 100644 --- a/test/fixtures/modern-tools.mjs +++ b/test/fixtures/modern-tools.mjs @@ -41,17 +41,18 @@ export function createServer(opts = {}) { content: [{ type: 'image', data: 'iVBORw0KGgo=', mimeType: 'image/png' }], structuredContent: { score: 0.9, label: 'cat' }, })); + const media = [ + { type: 'text', text: 'before' }, + { type: 'resource_link', uri: 'file:///srv/report.pdf', name: 'report.pdf', mimeType: 'application/pdf' }, + { type: 'resource', resource: { uri: 'memo://note', mimeType: 'text/plain', text: 'embedded note text' } }, + { type: 'resource', resource: { uri: 'memo://bin', mimeType: 'application/octet-stream', blob: Buffer.from('binary!').toString('base64') } }, + { type: 'audio', data: Buffer.from('RIFF0000WAVEfmt ').toString('base64'), mimeType: 'audio/wav' }, + { type: 'image', data: 'iVBORw0KGgo=', mimeType: 'image/png' }, + ]; server.registerTool('media', { description: 'Mixed standard content', inputSchema: obj() }, - async () => ({ - content: [ - { type: 'text', text: 'before' }, - { type: 'resource_link', uri: 'file:///srv/report.pdf', name: 'report.pdf', mimeType: 'application/pdf' }, - { type: 'resource', resource: { uri: 'memo://note', mimeType: 'text/plain', text: 'embedded note text' } }, - { type: 'resource', resource: { uri: 'memo://bin', mimeType: 'application/octet-stream', blob: Buffer.from('binary!').toString('base64') } }, - { type: 'audio', data: Buffer.from('RIFF0000WAVEfmt ').toString('base64'), mimeType: 'audio/wav' }, - { type: 'image', data: 'iVBORw0KGgo=', mimeType: 'image/png' }, - ], - })); + async () => ({ content: media })); + server.registerTool('structured_media', { description: 'Mixed standard content with structured data', inputSchema: obj() }, + async () => ({ content: media, structuredContent: { blocks: media.length } })); server.registerTool('fail', { description: 'Reports a tool error', inputSchema: obj() }, async () => ({ content: [{ type: 'text', text: 'the tool failed on purpose' }], isError: true })); server.registerTool('slow', { description: 'Sleeps, then answers', inputSchema: obj({ ms: { type: 'number' } }) }, diff --git a/test/mcp-modern-framework.test.ts b/test/mcp-modern-framework.test.ts index 0c827aa97..666424e3f 100644 --- a/test/mcp-modern-framework.test.ts +++ b/test/mcp-modern-framework.test.ts @@ -181,6 +181,22 @@ test('scripts get structured results whole, as one JSON object', async () => { structuredContent: { score: 0.9, label: 'cat' }, isError: false, }); + // A resource link, embedded resources and audio are ordinary MCP blocks + // from a modern server, not RFC-005 references: the script reads them as + // the server sent them. + const media = JSON.parse(await handle('agent', 'mcpl--modern--structured_media', {})); + assert.deepEqual(media, { + content: [ + { type: 'text', text: 'before' }, + { type: 'resource_link', uri: 'file:///srv/report.pdf', name: 'report.pdf', mimeType: 'application/pdf' }, + { type: 'resource', resource: { uri: 'memo://note', mimeType: 'text/plain', text: 'embedded note text' } }, + { type: 'resource', resource: { uri: 'memo://bin', mimeType: 'application/octet-stream', blob: Buffer.from('binary!').toString('base64') } }, + { type: 'audio', data: Buffer.from('RIFF0000WAVEfmt ').toString('base64'), mimeType: 'audio/wav' }, + { type: 'image', data: 'iVBORw0KGgo=', mimeType: 'image/png' }, + ], + structuredContent: { blocks: 6 }, + isError: false, + }); // Without structured content the script contract is unchanged (pinned): // the history rendering, which JSON-encodes a text result. assert.equal(await handle('agent', 'mcpl--modern--echo', { text: 'x' }), JSON.stringify('echo:x')); diff --git a/test/script-structured-references.test.ts b/test/script-structured-references.test.ts new file mode 100644 index 000000000..55a6b8f30 --- /dev/null +++ b/test/script-structured-references.test.ts @@ -0,0 +1,248 @@ +/** + * What a script receives from a tool whose result carries structuredContent, + * through the real dispatch. An MCPL peer's content blocks get the treatment + * RFC-005 gives them on every lane the model reads, since what a script prints + * comes back to the model: a reference arrives without its URI, as the host's + * reference id, what the server said about it in that block, and `savedTo`, + * the workspace path the dispatch fetched its payload to for the script; a + * block the classifier rejects arrives as the notice the model's lanes show, + * and inline data that claims a bulk disposition as the withheld notice. A + * standard peer's blocks arrive as the server sent them. + */ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { randomUUID } from 'node:crypto'; +import { mkdirSync, mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import { AgentFramework, WorkspaceModule } from '../src/index.js'; +import type { ToolResult } from '../src/index.js'; +import { INLINE_WITHHELD_TEXT } from '../src/mcpl/references.js'; +import type { ReferenceRecord } from '../src/mcpl/references.js'; +import { MockMembrane } from './helpers/mock-membrane.js'; + +const SAVED = 'board/references/report.pdf'; +const DIGEST = 'sha256:47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU'; + +/** The URIs of one test's references. The reference registry lives as long + * as the process, so each test names its own. */ +const uris = (test: string) => ({ + report: `https://files.example.invalid/${test}/report.pdf?sig=secret-token`, + big: `https://files.example.invalid/${test}/big.png?sig=other-token`, +}); + +/** A block of each kind the dispatch decides about. */ +const blocks = (test: string) => [ + // A reference whose payload the fetch saves. Its labels carry a bidi + // override, a bidi mark and a control character. + { type: 'resource', uri: uris(test).report, name: 'report\u202E.pdf\u0007', mimeType: 'application/pdf\u200F', sizeBytes: 1234, + digest: DIGEST, expiresAt: '2026-12-31T00:00:00Z' }, + // The same reference again, saying something else about itself. + { type: 'resource', uri: uris(test).report, name: 'report-copy.pdf' }, + // A reference the fetch doesn't save. + { type: 'image', uri: uris(test).big, mimeType: 'image/png', disposition: 'ref', expiresAt: '2027-01-01T00:00:00Z\u202E' }, + // Inline data that claims a bulk disposition, and inline data with a uri + // beside it: withheld. + { type: 'image', data: 'iVBORw0KGgo=', mimeType: 'image/png', disposition: 'ref' }, + { type: 'audio', data: 'UklGRg==', mimeType: 'audio/wav', uri: 'https://files.example.invalid/beside?sig=beside-token' }, + // Inline data that claims nothing. + { type: 'image', data: 'iVBORw0KGgo=', mimeType: 'image/png' }, + // Blocks the classifier rejects: a uri longer than its limit (4096 UTF-16 + // units), an MCP embedded resource (its uri nested), and a type RFC-005 + // doesn't define. + { type: 'resource', uri: `https://files.example.invalid/long?sig=${'s'.repeat(4096)}`, name: 'long' }, + { type: 'resource', resource: { uri: 'https://files.example.invalid/embedded?sig=embedded-token', mimeType: 'text/plain', text: 'embedded' } }, + { type: 'resource_link', uri: 'https://files.example.invalid/link?sig=link-token', name: 'link' }, + { type: 'text', text: 'two attachments' }, +]; + +/** A tools/call result as a server may send it, nonconformant blocks included. */ +interface Answer { content: unknown[]; structuredContent?: unknown; isError?: boolean } + +interface Harness { + /** Sets what every tools/call answers from now on. */ + answer: (result: Answer) => void; + /** A script's call to a tool of the MCPL peer `files`, or of the MCP-only peer `plain`. */ + script: (tool: string) => Promise; + /** A call the model makes to a tool of `files`, through the same dispatch. */ + modelCall: (tool: string) => Promise; + /** The URIs the fetcher was asked for, saved records aside. */ + fetched: string[]; + workspace: WorkspaceModule; +} + +async function withPeers(fn: (h: Harness) => Promise, opts: { eager?: boolean } = {}): Promise { + const dir = mkdtempSync(join(tmpdir(), 'script-structured-references-')); + const board = join(dir, 'board'); + mkdirSync(board); + const workspace = new WorkspaceModule({ mounts: [{ name: 'board', path: board, mode: 'read-write', watch: 'never' }] }); + const framework = await AgentFramework.create({ + storePath: join(dir, 'store'), membrane: new MockMembrane().asMembrane(), + agents: [{ name: 'agent', model: 'test-model', systemPrompt: 'test' }], modules: [workspace], + }); + workspace.initStore(framework.getStore()); + let answer: Answer = { content: [] }; + // Two legacy-engine peers: `files` negotiated MCPL, `plain` is MCP-only. + const peer = (id: string, capabilities: Record | null) => ({ id, capabilities, sendToolsCall: async () => answer }); + const peers = new Map([['files', peer('files', {})], ['plain', peer('plain', null)]]); + const fetched: string[] = []; + const internals = framework as unknown as { + mcplServerRegistry: unknown; + mcplPrefixMap: Map; + getReferenceFetcher: () => unknown; + handleScriptToolCall: (agent: string, tool: string, args: Record) => Promise; + dispatchMcplToolCall: (agent: string, call: { id: string; name: string; input: unknown }, serverId: string, prefix: string) => void; + pushEvent: (event: { type: string; callId?: string; result?: ToolResult }) => void; + }; + internals.mcplServerRegistry = { getServer: (id: string) => peers.get(id) ?? null, getAllServers: () => [], closeAll: async () => {} }; + internals.mcplPrefixMap.set('mcpl--files', 'files'); + internals.mcplPrefixMap.set('mcpl--plain', 'plain'); + // Stands in for the host-mediated fetcher: every reference is eager or + // none is, as the test asks, the report is saved and anything else + // refused, and a saved record returns at once. + internals.getReferenceFetcher = () => ({ + isEagerEligible: () => opts.eager === true, + fetch: async (record: ReferenceRecord) => { + if (record.fetchedPath) return { ok: true, path: record.fetchedPath }; + fetched.push(record.testimony.uri); + if (!record.testimony.uri.includes('report.pdf')) return { ok: false, error: 'refused by the test' }; + record.fetchedPath = SAVED; + return { ok: true, path: SAVED }; + }, + }); + const modelCall = (tool: string) => new Promise((resolve) => { + const id = `model-${randomUUID()}`; + const push = internals.pushEvent; + internals.pushEvent = (event) => { + if (event.type !== 'tool-result' || event.callId !== id) return push.call(framework, event); + internals.pushEvent = push; + resolve(event.result!); + }; + internals.dispatchMcplToolCall('agent', { id, name: `mcpl--files--${tool}`, input: {} }, 'files', 'mcpl--files'); + }); + framework.start(); + try { + await fn({ + answer: (result) => { answer = result; }, + script: (tool) => internals.handleScriptToolCall.call(framework, 'agent', tool, {}), + modelCall, + fetched, + workspace, + }); + } finally { + await framework.stop(); + rmSync(dir, { recursive: true, force: true }); + } +} + +/** What a script reads for those blocks from an MCPL peer, given each reference's id. */ +const treated = (report: string, big: string) => [ + { type: 'resource', refId: report, name: 'report.pdf', mimeType: 'application/pdf', sizeBytes: 1234, digest: DIGEST, + expiresAt: '2026-12-31T00:00:00Z', savedTo: SAVED }, + { type: 'resource', refId: report, name: 'report-copy.pdf', savedTo: SAVED }, + { type: 'image', refId: big, mimeType: 'image/png', expiresAt: '2027-01-01T00:00:00Z', savedTo: null }, + { type: 'text', text: INLINE_WITHHELD_TEXT }, + { type: 'text', text: INLINE_WITHHELD_TEXT }, + { type: 'image', data: 'iVBORw0KGgo=', mimeType: 'image/png' }, + { type: 'text', text: '[unrecognized content block: uri exceeds schema maximum]' }, + { type: 'text', text: '[unrecognized content block: reference without string uri]' }, + { type: 'text', text: '[unrecognized content block: unknown block type: resource_link]' }, + { type: 'text', text: 'two attachments' }, +]; + +test("an MCPL peer's structured result reaches a script with each block treated as the model's lanes treat it", async () => { + await withPeers(async ({ answer, script, fetched }) => { + answer({ content: blocks('treated'), structuredContent: { attachments: 2 } }); + const out = await script('mcpl--files--attach'); + assert.ok(!out.includes('example.invalid'), `no URI reaches the script: ${out.slice(0, 2000)}`); + assert.deepEqual(fetched, [uris('treated').report, uris('treated').big], 'every reference was fetched for the script before it read the result'); + const object = JSON.parse(out) as { content: Array<{ refId?: string }>; structuredContent: unknown; isError: boolean }; + const [report, , big] = object.content; + assert.match(String(report!.refId), /^ref_/); + assert.match(String(big!.refId), /^ref_/); + assert.notEqual(report!.refId, big!.refId); + assert.deepEqual(object, { content: treated(report!.refId!, big!.refId!), structuredContent: { attachments: 2 }, isError: false }); + }); +}); + +test("an MCP-only peer's structured result reaches a script as the server sent it", async () => { + await withPeers(async ({ answer, script, fetched }) => { + answer({ content: blocks('plain'), structuredContent: { attachments: 2 } }); + const object = JSON.parse(await script('mcpl--plain--attach')); + assert.deepEqual(object, { content: blocks('plain'), structuredContent: { attachments: 2 }, isError: false }, + 'resource links, embedded resources, uri-form media and inline data are ordinary MCP blocks there'); + assert.deepEqual(fetched, [], 'nothing is a reference, so nothing is fetched'); + }); +}); + +test('without structured content, an MCPL peer\'s result reaches a script as stubs naming the same reference id and saved path', async () => { + await withPeers(async ({ answer, script }) => { + // Only blocks the text lane recognizes: one it doesn't sends the whole + // array to the JSON fallback, which is older than this script path. + const all = blocks('unstructured'); + const content = [all[0], all[2], all[9]]; + answer({ content, structuredContent: {} }); + const structuredId = String((JSON.parse(await script('mcpl--files--attach')) as { content: Array<{ refId?: string }> }).content[0]!.refId); + answer({ content }); + const out = await script('mcpl--files--attach'); + assert.ok(!out.includes('example.invalid'), out); + assert.ok(out.includes(`[${structuredId}]`), `the same reference id: ${out}`); + assert.ok(out.includes(`saved: ${SAVED}`), `the same saved path: ${out}`); + }); +}); + +test("an MCPL peer's structured result past the script cap is saved with its references as their ids, no URI in the file", async () => { + await withPeers(async ({ answer, script, workspace }) => { + answer({ content: blocks('oversized'), structuredContent: { pad: 'x'.repeat(5_000_000) } }); + const out = JSON.parse(await script('mcpl--files--attach')) as { isError: boolean; oversized: { chars: number; savedTo: string } }; + assert.equal(out.isError, false); + const read = await workspace.readBinary(out.oversized.savedTo); + assert.ok('data' in read, `the saved result is readable: ${out.oversized.savedTo}`); + const saved = read.data.toString('utf8'); + assert.ok(!saved.includes('example.invalid'), 'no URI in the saved file'); + const object = JSON.parse(saved) as { content: Array<{ refId?: string }> }; + assert.deepEqual(object.content, treated(object.content[0]!.refId!, object.content[2]!.refId!)); + }); +}); + +test("only a script that reads the blocks gets the script-demand fetch: not the model's call, nor an error a script reads as text", async () => { + await withPeers(async ({ answer, script, modelCall, fetched }) => { + answer({ content: blocks('model'), structuredContent: { attachments: 2 } }); + const model = await modelCall('attach'); + assert.equal(model.success, true); + assert.deepEqual(fetched, [], 'nothing here is eager-eligible, and the model\'s call gets no more than the eager pass'); + + answer({ content: [blocks('error')[0], { type: 'text', text: 'the attach failed' }], isError: true }); + assert.equal(await script('mcpl--files--attach'), 'Error: the attach failed'); + assert.deepEqual(fetched, [], 'a script that reads an error as its text alone gets no more than the eager pass'); + }); +}); + +test("the eager pass still runs for the model's call and for an error a script reads as text", async () => { + await withPeers(async ({ answer, script, modelCall, fetched }) => { + answer({ content: [blocks('eager-model')[0]], structuredContent: { attachments: 1 } }); + assert.equal((await modelCall('attach')).success, true); + assert.deepEqual(fetched, [uris('eager-model').report]); + + answer({ content: [blocks('eager-error')[0], { type: 'text', text: 'the attach failed' }], isError: true }); + assert.equal(await script('mcpl--files--attach'), 'Error: the attach failed'); + assert.deepEqual(fetched, [uris('eager-model').report, uris('eager-error').report]); + }, { eager: true }); +}); + +test('an error with structured content is fetched for the script, and reaches it treated, with its error text and structured content', async () => { + await withPeers(async ({ answer, script, fetched }) => { + answer({ content: [blocks('structured-error')[0], { type: 'text', text: 'the attach failed' }], structuredContent: { partial: true }, isError: true }); + const out = await script('mcpl--files--attach'); + assert.ok(!out.includes('example.invalid'), out); + const object = JSON.parse(out) as { content: Array<{ refId?: string }>; structuredContent: unknown; isError: boolean; error: string }; + assert.deepEqual(object, { + content: [treated(object.content[0]!.refId!, '')[0], { type: 'text', text: 'the attach failed' }], + structuredContent: { partial: true }, + isError: true, + error: 'the attach failed', + }); + assert.deepEqual(fetched, [uris('structured-error').report], 'the script reads the blocks, so its reference was fetched first'); + }); +}); From 5082128b2f0d26f39e6768bba3de7cf6dec40cbc Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Mon, 12 Oct 2026 06:42:16 -0400 Subject: [PATCH 0125/1112] fix(chronicle): an append that throws inside the store blocks the journal's writes until the store is reopened Chronicle's append (v0.4.0, src/records/log.rs and src/store.rs) assigns the record's id, writes the record to its log and advances the file size, syncs the log every sync_interval writes, and only then adds the record to the index that getRecordIdsByType reads. When that periodic sync fails, the append throws with the record in the log but not in this process's view of the store. RecordJournal treated an append that threw like a failed barrier: load() cleared the unreconciled state, but the record wasn't in what load() read, so a later checkpoint covered it, and the reopened store replayed it as covered, never reduced. Nothing in the store's JS API rereads the log short of reopening the store, so an append that throws now leaves the journal unwritable for the rest of its life: every append and checkpoint throws, saying nothing more is written until the store is reopened, as at the next start. A journal on the reopened store loads that record with its tail, for the caller to reduce. needsReconcile stays false for it, since no load() resolves it. A failure after the append returned (a durable barrier, or reading the id it returned) is resolved by load() as before, and a failure before the record reaches the store (the afterCommittedState sync) still leaves the journal usable. The source change is the same as on agent-framework #248, where this journal's commits are shared. Test (record-journal, 9): an append that throws after writing its record, modelled as Chronicle's failed periodic sync (the record hidden from this view of the store until it's reopened), blocks append and checkpoint, before and after load(); a journal on the reopened store loads it and writes again; an append that throws before writing anything blocks the same way. A native probe outside the suite injected one EIO at the log's periodic fsync (an LD_PRELOAD shim) on Linux, Node 24.21.0, Chronicle 0.4.0. The 100th append threw with its record in the log. The journal then refused the next append and checkpoint, and after a reopen the tail held all 100 entries, the failed one included. With the old rule, the next checkpoint covered it. --- src/record-journal.ts | 57 ++++++++++++++++++++++++++++--------- test/record-journal.test.ts | 52 +++++++++++++++++++++++++++++++++ 2 files changed, 95 insertions(+), 14 deletions(-) diff --git a/src/record-journal.ts b/src/record-journal.ts index 0565e5422..ee6544ee0 100644 --- a/src/record-journal.ts +++ b/src/record-journal.ts @@ -32,16 +32,25 @@ * one THIS journal loaded or appended, so entries another instance appended * to the same type in between would be covered without being reduced. * - * An ambiguous write must be reconciled before the journal writes again. Once - * `append` or `checkpoint` has handed a record to the store, any failure from - * there on (the append itself, or a barrier after it) leaves it unknown - * whether that record exists, and the caller didn't reduce it. A later - * checkpoint would then cover it without its ever entering the reducer. So - * such a failure marks the journal unreconciled: every further `append` and - * `checkpoint` throws until `load()` has replayed the tail — the possibly - * written record included — for the caller to reduce. A failure BEFORE the - * record reaches the store (the `afterCommittedState` sync) wrote nothing and - * leaves the journal usable. + * An ambiguous write must be resolved before the journal writes again. Once + * `append` or `checkpoint` has handed a record to the store, a failure leaves + * a record the caller didn't reduce, or may leave one, and a later checkpoint + * would cover it without its ever entering the reducer. There are two kinds: + * - The store's append itself throws. Its record may still be in the store's + * log, outside this process's view of the store: the store writes a + * record to its log, syncs the log every so often, and only then indexes + * the record, which is what `getRecordIdsByType` reads. So a failed sync + * leaves a record that reopening the store will find and no `load()` on + * this store can. Nothing but a reopen rereads the log, so from then on + * every `append` and `checkpoint` throws, for the life of the journal. A + * journal on the store reopened, as at the next start, loads that record + * with its tail for the caller to reduce. + * - A failure after the append returned: a `durable` barrier, or reading + * the id it returned. The record is in this process's view, so every + * further `append` and `checkpoint` throws until `load()` has replayed the + * tail, that record included, for the caller to reduce. + * A failure BEFORE the record reaches the store (the `afterCommittedState` + * sync) wrote nothing and leaves the journal usable. */ import type { JsStore } from '@animalabs/chronicle'; @@ -78,6 +87,8 @@ interface CheckpointPayload { snapshot: Snapshot; } +const asError = (err: unknown): Error => (err instanceof Error ? err : new Error(String(err))); + /** Record ids are decimal strings from one store-wide counter. */ function idValue(id: string): bigint { if (!/^\d+$/.test(id)) throw new Error(`RecordJournal: unexpected chronicle record id "${id}"`); @@ -99,8 +110,11 @@ export class RecordJournal { /** Highest entry id this journal has loaded or appended (0 when none). */ private lastEntryId = 0n; private sinceCheckpoint = 0; - /** Set by a failure after a record reached the store; cleared by load(). */ + /** Set by a failure after the store's append returned; cleared by load(). */ private unreconciled: Error | null = null; + /** Set when the store's append itself threw: its record may be in the log + * but not in this process's view of the store. Never cleared. */ + private unreadable: Error | null = null; constructor(store: JsStore, opts: { type: string; checkpointType?: string }) { if (!opts.type) throw new Error('RecordJournal: a record type is required'); @@ -156,7 +170,9 @@ export class RecordJournal { return { snapshot, entries }; } - /** True after an ambiguous write, until load() has replayed it. */ + /** True after a failure that load() can resolve, until it has replayed the + * tail. An append that threw inside the store is never resolved here: the + * journal stays unwritable until a journal on the reopened store loads it. */ get needsReconcile(): boolean { return this.unreconciled !== null; } @@ -186,19 +202,32 @@ export class RecordJournal { /** Hand one record to the store; any failure from here on is ambiguous. */ private write(type: string, payload: unknown, durable: boolean, onWritten: (ref: JournalRecordRef) => void): JournalRecordRef { + let record: { id: string; sequence: number }; + try { + record = this.store.appendJson(type, payload); + } catch (err) { + this.unreadable = asError(err); + throw err; + } try { - const record = this.store.appendJson(type, payload); const ref = { id: record.id, sequence: record.sequence }; onWritten(ref); if (durable) this.store.sync(); return ref; } catch (err) { - this.unreconciled = err instanceof Error ? err : new Error(String(err)); + this.unreconciled = asError(err); throw err; } } private assertReconciled(): void { + if (this.unreadable) { + throw new Error( + `RecordJournal(${this.type}): an earlier append failed inside the store ` + + `(${this.unreadable.message}), and its record may be in the store's log where this process can't ` + + `read it back; nothing more is written until the store is reopened, as at the next start`, + ); + } if (this.unreconciled) { throw new Error( `RecordJournal(${this.type}): an earlier write failed after reaching the store ` + diff --git a/test/record-journal.test.ts b/test/record-journal.test.ts index 3618ae077..300a5f0f6 100644 --- a/test/record-journal.test.ts +++ b/test/record-journal.test.ts @@ -195,6 +195,58 @@ describe('RecordJournal', () => { }); }); + it('an append that throws inside the store blocks writes for the life of the journal; a journal on the reopened store loads its record', () => { + withStore((store) => { + // As Chronicle's append can fail: the record is written to the log, the + // periodic sync then fails, and the record never reaches the index this + // process reads. Reopening the store indexes it from the log. `hidden` + // is this process's view of the store. + let failAppend: 'after-writing' | 'before-writing' | null = null; + const hidden = new Set(); + const live = { + getRecordIdsByType: (type: string) => store.getRecordIdsByType(type).filter((id) => !hidden.has(id)), + getRecord: (id: string) => (hidden.has(id) ? null : store.getRecord(id)), + appendJson: (type: string, data: unknown) => { + if (failAppend === 'before-writing') { failAppend = null; throw new Error('EIO: write failed'); } + const record = store.appendJson(type, data); + if (failAppend === 'after-writing') { failAppend = null; hidden.add(record.id); throw new Error('EIO: fsync failed'); } + return record; + }, + sync: () => store.sync(), + } as unknown as JsStore; + const journal = new RecordJournal(live, { type: 'journal-test/entry' }); + journal.append({ n: 'A' }); + + failAppend = 'after-writing'; + assert.throws(() => journal.append({ n: 'B' }), /EIO: fsync failed/); + assert.strictEqual(journal.needsReconcile, false, 'no load() on this store can read B back'); + assert.deepStrictEqual(journal.load().entries.map((e) => e.entry.n), ['A'], 'B is outside this view of the store'); + // A checkpoint now would cover B, unreduced, so nothing more is written. + const blocked = /append failed inside the store \(EIO: fsync failed\).*until the store is reopened/; + assert.throws(() => journal.append({ n: 'C' }), blocked); + assert.throws(() => journal.checkpoint({ seen: ['A'] }), blocked); + assert.throws(() => journal.append({ n: 'C' }), blocked, 'load() does not lift it'); + + // A journal on the reopened store loads B with its tail, and writes again. + const reopened = new RecordJournal(store, { type: 'journal-test/entry' }); + assert.deepStrictEqual(reopened.load().entries.map((e) => e.entry.n), ['A', 'B']); + reopened.append({ n: 'C' }); + reopened.checkpoint({ seen: ['A', 'B', 'C'] }); + assert.deepStrictEqual(new RecordJournal(store, { type: 'journal-test/entry' }).load(), + { snapshot: { seen: ['A', 'B', 'C'] }, entries: [] }); + + // An append that fails before writing anything looks the same from + // here, so it blocks the same way; the reopened store has nothing new. + const second = new RecordJournal(live, { type: 'journal-test/entry' }); + second.load(); + failAppend = 'before-writing'; + assert.throws(() => second.append({ n: 'X' }), /EIO: write failed/); + assert.throws(() => second.append({ n: 'D' }), /until the store is reopened/); + assert.deepStrictEqual(new RecordJournal(store, { type: 'journal-test/entry' }).load(), + { snapshot: { seen: ['A', 'B', 'C'] }, entries: [] }); + }); + }); + it('refuses a checkpoint type equal to the entry type', () => { withStore((store) => { assert.throws(() => new RecordJournal(store, { type: 't', checkpointType: 't' }), /distinct record types/); From 38df4fd44b01b6597790665321ef83a1017f936f Mon Sep 17 00:00:00 2001 From: Petra-1012 Date: Mon, 12 Oct 2026 07:26:21 -0400 Subject: [PATCH 0126/1112] docs(drafts): say what a journal that refuses writes means for the drafts An append that fails inside the store now leaves the record journal refusing every write until the store is reopened (5082128b). The drafts' write comment said only what a failure after the append returned does. It now also says that until the next start each write throws, no new draft is kept (hold reports the segments it couldn't keep, and the turn's notice names the failure), and the projection lacks the entry the next start replays. --- src/prose-drafts.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/prose-drafts.ts b/src/prose-drafts.ts index c290a749b..c7002903a 100644 --- a/src/prose-drafts.ts +++ b/src/prose-drafts.ts @@ -307,9 +307,13 @@ export class ProseDraftStore { } /** - * Journal and apply one entry. A write that fails after reaching the store - * leaves the journal unreconciled: rebuild from it before rethrowing, so - * the projection never disagrees with what the journal holds. + * Journal and apply one entry. A write that fails after the store's append + * returned leaves the journal unreconciled: rebuild from it before + * rethrowing, so the projection never disagrees with what the journal + * holds. An append that fails inside the store can't be read back until + * the store is reopened: the journal then refuses every write until the + * next start, which replays that entry, so until then each write throws, + * no new draft is kept, and the projection lacks that entry. */ private write(entry: DraftEntry, durable: boolean, afterCommittedState = false): void { try { From 538791622a27cde918749725a69598644148e8af Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Mon, 12 Oct 2026 06:31:26 -0400 Subject: [PATCH 0127/1112] fix(chronicle): an append that throws inside the store blocks the journal's writes until the store is reopened Chronicle's append (v0.4.0, src/records/log.rs and src/store.rs) assigns the record's id, writes the record to its log and advances the file size, syncs the log every sync_interval writes, and only then adds the record to the index that getRecordIdsByType reads. When that periodic sync fails, the append throws with the record in the log but not in this process's view of the store. RecordJournal treated an append that threw like a failed barrier: load() cleared the unreconciled state, but the record wasn't in what load() read, so a later checkpoint covered it, and the reopened store replayed it as covered, never reduced. Nothing in the store's JS API rereads the log short of reopening the store, so an append that throws now leaves the journal unwritable for the rest of its life: every append and checkpoint throws, saying nothing more is written until the store is reopened, as at the next start. A journal on the reopened store loads that record with its tail, for the caller to reduce. needsReconcile stays false for it, since no load() resolves it. A failure after the append returned (a durable barrier, or reading the id it returned) is resolved by load() as before, and a failure before the record reaches the store (the afterCommittedState sync) still leaves the journal usable. Tests (record-journal, 10): - an append that throws after writing its record, modelled as Chronicle's failed periodic sync (the record hidden from this view of the store until it's reopened), blocks append and checkpoint, before and after load(); a journal on the reopened store loads it and writes again; an append that throws before writing anything blocks the same way; - a durable checkpoint whose barrier throws blocks writes until load() replays it (split out of the old combined test). Resolving an append that throws by load(), as before, fails the first. A native probe outside the suite injected one EIO at the log's periodic fsync (an LD_PRELOAD shim) on Linux, Node 24.21.0, Chronicle 0.4.0. The 100th append threw with its record in the log. The journal then refused the next append and checkpoint, and after a reopen the tail held all 100 entries, the failed one included. With the old rule, the next checkpoint covered it. --- src/record-journal.ts | 57 +++++++++++++++++++++------- test/record-journal.test.ts | 76 +++++++++++++++++++++++++++++++++++++ 2 files changed, 119 insertions(+), 14 deletions(-) diff --git a/src/record-journal.ts b/src/record-journal.ts index 0565e5422..ee6544ee0 100644 --- a/src/record-journal.ts +++ b/src/record-journal.ts @@ -32,16 +32,25 @@ * one THIS journal loaded or appended, so entries another instance appended * to the same type in between would be covered without being reduced. * - * An ambiguous write must be reconciled before the journal writes again. Once - * `append` or `checkpoint` has handed a record to the store, any failure from - * there on (the append itself, or a barrier after it) leaves it unknown - * whether that record exists, and the caller didn't reduce it. A later - * checkpoint would then cover it without its ever entering the reducer. So - * such a failure marks the journal unreconciled: every further `append` and - * `checkpoint` throws until `load()` has replayed the tail — the possibly - * written record included — for the caller to reduce. A failure BEFORE the - * record reaches the store (the `afterCommittedState` sync) wrote nothing and - * leaves the journal usable. + * An ambiguous write must be resolved before the journal writes again. Once + * `append` or `checkpoint` has handed a record to the store, a failure leaves + * a record the caller didn't reduce, or may leave one, and a later checkpoint + * would cover it without its ever entering the reducer. There are two kinds: + * - The store's append itself throws. Its record may still be in the store's + * log, outside this process's view of the store: the store writes a + * record to its log, syncs the log every so often, and only then indexes + * the record, which is what `getRecordIdsByType` reads. So a failed sync + * leaves a record that reopening the store will find and no `load()` on + * this store can. Nothing but a reopen rereads the log, so from then on + * every `append` and `checkpoint` throws, for the life of the journal. A + * journal on the store reopened, as at the next start, loads that record + * with its tail for the caller to reduce. + * - A failure after the append returned: a `durable` barrier, or reading + * the id it returned. The record is in this process's view, so every + * further `append` and `checkpoint` throws until `load()` has replayed the + * tail, that record included, for the caller to reduce. + * A failure BEFORE the record reaches the store (the `afterCommittedState` + * sync) wrote nothing and leaves the journal usable. */ import type { JsStore } from '@animalabs/chronicle'; @@ -78,6 +87,8 @@ interface CheckpointPayload { snapshot: Snapshot; } +const asError = (err: unknown): Error => (err instanceof Error ? err : new Error(String(err))); + /** Record ids are decimal strings from one store-wide counter. */ function idValue(id: string): bigint { if (!/^\d+$/.test(id)) throw new Error(`RecordJournal: unexpected chronicle record id "${id}"`); @@ -99,8 +110,11 @@ export class RecordJournal { /** Highest entry id this journal has loaded or appended (0 when none). */ private lastEntryId = 0n; private sinceCheckpoint = 0; - /** Set by a failure after a record reached the store; cleared by load(). */ + /** Set by a failure after the store's append returned; cleared by load(). */ private unreconciled: Error | null = null; + /** Set when the store's append itself threw: its record may be in the log + * but not in this process's view of the store. Never cleared. */ + private unreadable: Error | null = null; constructor(store: JsStore, opts: { type: string; checkpointType?: string }) { if (!opts.type) throw new Error('RecordJournal: a record type is required'); @@ -156,7 +170,9 @@ export class RecordJournal { return { snapshot, entries }; } - /** True after an ambiguous write, until load() has replayed it. */ + /** True after a failure that load() can resolve, until it has replayed the + * tail. An append that threw inside the store is never resolved here: the + * journal stays unwritable until a journal on the reopened store loads it. */ get needsReconcile(): boolean { return this.unreconciled !== null; } @@ -186,19 +202,32 @@ export class RecordJournal { /** Hand one record to the store; any failure from here on is ambiguous. */ private write(type: string, payload: unknown, durable: boolean, onWritten: (ref: JournalRecordRef) => void): JournalRecordRef { + let record: { id: string; sequence: number }; + try { + record = this.store.appendJson(type, payload); + } catch (err) { + this.unreadable = asError(err); + throw err; + } try { - const record = this.store.appendJson(type, payload); const ref = { id: record.id, sequence: record.sequence }; onWritten(ref); if (durable) this.store.sync(); return ref; } catch (err) { - this.unreconciled = err instanceof Error ? err : new Error(String(err)); + this.unreconciled = asError(err); throw err; } } private assertReconciled(): void { + if (this.unreadable) { + throw new Error( + `RecordJournal(${this.type}): an earlier append failed inside the store ` + + `(${this.unreadable.message}), and its record may be in the store's log where this process can't ` + + `read it back; nothing more is written until the store is reopened, as at the next start`, + ); + } if (this.unreconciled) { throw new Error( `RecordJournal(${this.type}): an earlier write failed after reaching the store ` + diff --git a/test/record-journal.test.ts b/test/record-journal.test.ts index 3618ae077..9760ef792 100644 --- a/test/record-journal.test.ts +++ b/test/record-journal.test.ts @@ -195,6 +195,82 @@ describe('RecordJournal', () => { }); }); + it('an append that throws inside the store blocks writes for the life of the journal; a journal on the reopened store loads its record', () => { + withStore((store) => { + // As Chronicle's append can fail: the record is written to the log, the + // periodic sync then fails, and the record never reaches the index this + // process reads. Reopening the store indexes it from the log. `hidden` + // is this process's view of the store. + let failAppend: 'after-writing' | 'before-writing' | null = null; + const hidden = new Set(); + const live = { + getRecordIdsByType: (type: string) => store.getRecordIdsByType(type).filter((id) => !hidden.has(id)), + getRecord: (id: string) => (hidden.has(id) ? null : store.getRecord(id)), + appendJson: (type: string, data: unknown) => { + if (failAppend === 'before-writing') { failAppend = null; throw new Error('EIO: write failed'); } + const record = store.appendJson(type, data); + if (failAppend === 'after-writing') { failAppend = null; hidden.add(record.id); throw new Error('EIO: fsync failed'); } + return record; + }, + sync: () => store.sync(), + } as unknown as JsStore; + const journal = new RecordJournal(live, { type: 'journal-test/entry' }); + journal.append({ n: 'A' }); + + failAppend = 'after-writing'; + assert.throws(() => journal.append({ n: 'B' }), /EIO: fsync failed/); + assert.strictEqual(journal.needsReconcile, false, 'no load() on this store can read B back'); + assert.deepStrictEqual(journal.load().entries.map((e) => e.entry.n), ['A'], 'B is outside this view of the store'); + // A checkpoint now would cover B, unreduced, so nothing more is written. + const blocked = /append failed inside the store \(EIO: fsync failed\).*until the store is reopened/; + assert.throws(() => journal.append({ n: 'C' }), blocked); + assert.throws(() => journal.checkpoint({ seen: ['A'] }), blocked); + assert.throws(() => journal.append({ n: 'C' }), blocked, 'load() does not lift it'); + + // A journal on the reopened store loads B with its tail, and writes again. + const reopened = new RecordJournal(store, { type: 'journal-test/entry' }); + assert.deepStrictEqual(reopened.load().entries.map((e) => e.entry.n), ['A', 'B']); + reopened.append({ n: 'C' }); + reopened.checkpoint({ seen: ['A', 'B', 'C'] }); + assert.deepStrictEqual(new RecordJournal(store, { type: 'journal-test/entry' }).load(), + { snapshot: { seen: ['A', 'B', 'C'] }, entries: [] }); + + // An append that fails before writing anything looks the same from + // here, so it blocks the same way; the reopened store has nothing new. + const second = new RecordJournal(live, { type: 'journal-test/entry' }); + second.load(); + failAppend = 'before-writing'; + assert.throws(() => second.append({ n: 'X' }), /EIO: write failed/); + assert.throws(() => second.append({ n: 'D' }), /until the store is reopened/); + assert.deepStrictEqual(new RecordJournal(store, { type: 'journal-test/entry' }).load(), + { snapshot: { seen: ['A', 'B', 'C'] }, entries: [] }); + }); + }); + + it('a durable checkpoint whose barrier throws blocks writes until load() replays it', () => { + withStore((store) => { + let failNextSync = false; + const flaky = { + getRecordIdsByType: (type: string) => store.getRecordIdsByType(type), + getRecord: (id: string) => store.getRecord(id), + appendJson: (type: string, data: unknown) => store.appendJson(type, data), + sync: () => { + if (failNextSync) { failNextSync = false; throw new Error('disk full'); } + store.sync(); + }, + } as unknown as JsStore; + const journal = new RecordJournal(flaky, { type: 'journal-test/entry' }); + journal.append({ n: 'A' }); + + failNextSync = true; + assert.throws(() => journal.checkpoint({ seen: ['A'] }, { durable: true }), /disk full/); + assert.strictEqual(journal.needsReconcile, true); + assert.throws(() => journal.append({ n: 'C' }), /load\(\) and reduce its tail/); + assert.deepStrictEqual(journal.load(), { snapshot: { seen: ['A'] }, entries: [] }, 'the checkpoint landed'); + journal.append({ n: 'C' }); + }); + }); + it('refuses a checkpoint type equal to the entry type', () => { withStore((store) => { assert.throws(() => new RecordJournal(store, { type: 't', checkpointType: 't' }), /distinct record types/); From 40197c56569e984c90693065c5be59490d321fef Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Mon, 12 Oct 2026 11:38:04 -0400 Subject: [PATCH 0128/1112] feat(chronicle): RecordJournal.needsReopen, true once an append threw inside the store Since the previous commit, an append that throws inside the store leaves the journal unwritable for the rest of its life, with needsReconcile false, since no load() on that store can read the append's record back. A caller that decides from what load() returns needs to know that its view may lack that record: needsReopen is true from then on, until a journal on the reopened store has loaded it. needsReconcile keeps its meaning, a failure load() resolves. The getter and its two assertions are the same lines the workspace's disk-agreement change adds to RecordJournal, so the two branches carry identical text and merge cleanly in either order. Tests (record-journal, 10): needsReopen is true after an append that threw inside the store, and false after a durable checkpoint's barrier failed. --- src/record-journal.ts | 8 ++++++++ test/record-journal.test.ts | 2 ++ 2 files changed, 10 insertions(+) diff --git a/src/record-journal.ts b/src/record-journal.ts index ee6544ee0..179c4f4b6 100644 --- a/src/record-journal.ts +++ b/src/record-journal.ts @@ -177,6 +177,14 @@ export class RecordJournal { return this.unreconciled !== null; } + /** True once an append threw inside the store, for the rest of the + * journal's life. What `load()` returns may then lack that append's record, + * so a caller that decides from the journal's contents can't trust them + * until a journal on the reopened store has loaded it. */ + get needsReopen(): boolean { + return this.unreadable !== null; + } + /** Append one entry. The returned ref says where it landed, not that it's durable. */ append(entry: Entry, opts: AppendOptions = {}): JournalRecordRef { this.assertReconciled(); diff --git a/test/record-journal.test.ts b/test/record-journal.test.ts index 9760ef792..977751457 100644 --- a/test/record-journal.test.ts +++ b/test/record-journal.test.ts @@ -220,6 +220,7 @@ describe('RecordJournal', () => { failAppend = 'after-writing'; assert.throws(() => journal.append({ n: 'B' }), /EIO: fsync failed/); assert.strictEqual(journal.needsReconcile, false, 'no load() on this store can read B back'); + assert.strictEqual(journal.needsReopen, true, 'what this view holds may lack B until a reopen'); assert.deepStrictEqual(journal.load().entries.map((e) => e.entry.n), ['A'], 'B is outside this view of the store'); // A checkpoint now would cover B, unreduced, so nothing more is written. const blocked = /append failed inside the store \(EIO: fsync failed\).*until the store is reopened/; @@ -265,6 +266,7 @@ describe('RecordJournal', () => { failNextSync = true; assert.throws(() => journal.checkpoint({ seen: ['A'] }, { durable: true }), /disk full/); assert.strictEqual(journal.needsReconcile, true); + assert.strictEqual(journal.needsReopen, false, 'the checkpoint is in this view of the store'); assert.throws(() => journal.append({ n: 'C' }), /load\(\) and reduce its tail/); assert.deepStrictEqual(journal.load(), { snapshot: { seen: ['A'] }, entries: [] }, 'the checkpoint landed'); journal.append({ n: 'C' }); From 930cefa1b6268b7bd7324dfb484f11354482cb59 Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Mon, 12 Oct 2026 11:55:30 -0400 Subject: [PATCH 0129/1112] fix(awareness): the journal decides nothing after a write it can't read back; statuses, retries and refusals say what can be known From a review of the whole change. A write that failed inside the store. Chronicle 0.4.0's append writes its record to the log and then syncs the log every so often; when that sync throws, the record is on disk but outside this process's index, so no load() here can see it, and reopening the store indexes it. The outbox read such a failure back as "not landed": settleApplied reported not-scheduled ("none of its marks will be sent") for a hide or turn undo whose batch the next start then completed and delivered; settleActivation retired a batch whose activation was on disk, and the reducer, which retires only a prepared batch, kept the activation, so its marks went out at the next start; a cancel, retract or outcome written that way was missing from every later decision in the process (a cancelled batch kept sending, a confirmed add was sent again, an activation outranked a retract it couldn't see); and a legacy import was written a second time. With the RecordJournal rule from the commits before this one (an append that throws inside the store leaves the journal unwritable until the store is reopened, needsReopen), the outbox's load() now refuses while the journal needs a reopen, naming the reopen, so every read and write refuses and both settle paths fall to `unresolved` (a later start may complete the batch), which is what can be known. Statuses. An operation's status read `cancelled` before its last attempt, so an add a retract superseded while its request was on the wire, or after it went unanswered, hid that it may land. Status now says first what Discord may have (confirmed, then a request on the wire or never answered), then whether it will be sent again. An operation's status is its latest answer, so a retry Discord confirmed reads confirmed while the first, unanswered request stays counted as unresolved; the header now says so instead of reading as if the operation stayed unknown. Retries. A request refused with an error the permanent pattern didn't know, or never answered, stayed due, and every drain trigger sent it again and journaled two more records, with nothing telling the operator. An operation is no longer sent after three attempts that reached the server without a confirmation (refused or unanswered); an attempt refused before it left the host doesn't count, so a route that is down keeps its work. recordOutcome returns how many operations it ended, and a drain pass that ends any raises one discord-awareness-delivery alert naming the first. Discord's "Unknown Emoji", "Maximum number of reactions reached" and "Reaction was blocked" are permanent. A JSON-RPC error answer, which the connection documents as possibly coming after the server acted, and an error the connection didn't classify, are recorded unknown rather than failed; Discord's own refusals arrive as isError results and stay failed. Surgeries and controls. - A prepare whose write fails says what it leaves: nothing; the batch, retired at once when it is in this process's view; or, after a write that failed inside the store, a batch the next start may hold, to be cancelled, not released, since its surgery didn't run. rollbackToMessage and suppressMessages refuse with OperatorActionError('failed'), "not started, nothing changed". - rollbackToMessage and suppressMessages check `marks` at runtime and refuse anything but 'none' or { scope: 'addressed' | 'all', refs? } with Discord addresses: a bare 'addressed' (the host/command vocabulary) skipped the scope filter and marked every addressable message. - The outbox's operator controls throw DiscordAwarenessRefusal (exported) for a request they won't carry out; the framework maps it to 'invalid' and any other error, a failure to read or write the journal, to 'failed' (it mapped everything to 'invalid'). - A failed surgery whose batch can't be retired says the batch stays prepared, that a start on the source holds it, and to cancel it rather than release it, since its messages were never removed. - A suppression-only batch (no refs) whose activation can't be recorded is logged, with an unresolved alert when it stays prepared. - The outbox is constructed unconditionally in create(), its only caller, so the field is non-null and its dead guards and "no storePath" refusal are gone. syncDiscordAwarenessMarkers honours its server id. Texts: the header says the one thing a restart does with branch state (complete a surgery that crashed after its switch); activationPolicy matters only with suppression intervals; the receipt docs no longer promise a "later reconciliation" (only a start reconciles); addsSuperseded covers adds on the wire or unknown; a repeated cancel's receipt counts what that call stopped; the import doc and the v1 mapping comment; the import record names v1 for a version 1 file; the config doc says discordAwarenessOutboxPath only names a pre-journal ledger to import; the ledger-path doc sits on its function; a stale gate comment is gone. restoreState gives an act from a checkpoint written before authorization order authorization 0, older than anything after it (its batch's adds had outranked a later retract). Tests: discord-awareness-outbox (journal seams gain an append that writes and then hides its record until the store is reopened): a hide's batch, an activation, a cancel and a legacy import written that way (unresolved; the outbox refuses; the next start reads them); a prepare whose write failed, both ways; status precedence; the bound, with not-sent not counted; the permanent patterns; an old checkpoint's batch against a later retract; refusals; a v1 import's source; a one-server sync; a confirmed retry's status and count; error answers and unclassified errors recorded unknown, and the pass's alert. surgery-marks-receipt: invalid marks refused before anything changes; a batch that can't be recorded stops the surgery; control errors mapped. --- src/framework.ts | 242 ++++++++++++-------- src/index.ts | 1 + src/operator-log.ts | 10 +- src/recovery/discord-awareness-outbox.ts | 233 +++++++++++++++----- src/types/framework.ts | 9 +- test/discord-awareness-outbox.test.ts | 268 +++++++++++++++++++++-- test/surgery-marks-receipt.test.ts | 60 ++++- 7 files changed, 663 insertions(+), 160 deletions(-) diff --git a/src/framework.ts b/src/framework.ts index 887afe156..90c79c1e4 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -404,6 +404,7 @@ import { formatZonedDateTime, resolveTimeZone } from './timezone.js'; import { DEFAULT_DISCORD_AWARENESS_EMOJI, DiscordAwarenessOutbox, + DiscordAwarenessRefusal, defaultDiscordAwarenessOutboxPath, extractDiscordAwarenessRefs, selectDiscordAwarenessRefs, @@ -413,6 +414,7 @@ import { type DiscordAwarenessView, type DiscordAwarenessCancelReceipt, type DiscordAwarenessMarks, + type DiscordAwarenessPrepareInput, type DiscordAwarenessReleaseReceipt, type DiscordAwarenessRetractReceipt, type DiscordAwarenessScope, @@ -738,6 +740,31 @@ function marksScope(marks: DiscordAwarenessMarks | undefined): 'none' | DiscordA return !marks || marks === 'none' ? 'none' : marks.scope; } +/** + * A surgery's marks choice as a caller passes it at runtime: `none` (or left + * out), or `{ scope: 'addressed' | 'all', refs? }` with each ref a Discord + * address. Anything else is refused, never guessed: a bare scope string (the + * host/command vocabulary) or a misspelled scope would otherwise skip the + * scope filter and mark every addressable message. + */ +function checkedMarks(marks: unknown): DiscordAwarenessMarks { + if (marks === undefined || marks === 'none') return 'none'; + if (marks && typeof marks === 'object' && !Array.isArray(marks)) { + const { scope, refs } = marks as { scope?: unknown; refs?: unknown }; + const isRef = (ref: unknown) => !!ref && typeof ref === 'object' + && typeof (ref as Record).serverId === 'string' + && typeof (ref as Record).channelId === 'string' + && typeof (ref as Record).messageId === 'string'; + if ((scope === 'addressed' || scope === 'all') && (refs === undefined || (Array.isArray(refs) && refs.every(isRef)))) { + return marks as DiscordAwarenessMarks; + } + } + throw new OperatorActionError( + 'invalid', + `marks must be 'none' or { scope: 'addressed' | 'all', refs? } (got ${boundDiscordAwarenessText(JSON.stringify(marks) ?? String(marks), 200)})`, + ); +} + /** A publication choice, summarized for the operator log (refs counted, not listed). */ function describeMarksChoice(marks: DiscordAwarenessMarks | undefined): Record | 'none' { if (!marks || marks === 'none') return 'none'; @@ -1399,7 +1426,8 @@ export class AgentFramework { * framework started with zero configured servers. */ private mcplInferenceRoutingConfig: import('./mcpl/types.js').InferenceRoutingPolicy | null = null; /** Durable, non-Chronicle projection queue for messages removed by a branch. */ - private discordAwarenessOutbox: DiscordAwarenessOutbox | null = null; + /** The awareness journal, in the store: every framework has one. */ + private readonly discordAwarenessOutbox: DiscordAwarenessOutbox; private discordAwarenessEmoji = DEFAULT_DISCORD_AWARENESS_EMOJI; private discordAwarenessDeadlineMs = DEFAULT_DISCORD_AWARENESS_DEADLINE_MS; /** Durable JSONL record of operator-initiated mutations (see operator-log.ts). */ @@ -1414,7 +1442,6 @@ export class AgentFramework { private surgeryHold: { verb: string; agentName: string; since: number } | null = null; /** Serialize per-server drains so reconnect and an online undo cannot race. */ private discordAwarenessDrains: Map> = new Map(); - /** Framework-global inference gate; older generations cannot release it. */ // EventGate (null when FrameworkConfig.gate is omitted) private eventGate: EventGate | null = null; @@ -1437,7 +1464,7 @@ export class AgentFramework { processLoggingPersist: boolean, processLoggingBroadcast: boolean, timeZone: string, - discordAwarenessOutbox: DiscordAwarenessOutbox | null, + discordAwarenessOutbox: DiscordAwarenessOutbox, discordAwarenessEmoji: string, discordAwarenessDeadlineMs: number, operatorLog: OperatorLog = new OperatorLog(undefined), @@ -1599,21 +1626,19 @@ export class AgentFramework { // switching to exactly the active branch completes its activation; any // other prepared batch is held for an operator; an attempt the previous // process left on the wire is recorded unknown. No content is read. - if (discordAwarenessOutbox) { - try { - const recovered = discordAwarenessOutbox.recoverAtStartup(store.currentBranch().name); - if (recovered.activated.length > 0 || recovered.held.length > 0 || recovered.unknownAttempts > 0) { - console.error( - `[discord-awareness] startup: activated ${recovered.activated.length} interrupted batch(es) ` + - `for ${store.currentBranch().name}, held ${recovered.held.length}, ` + - `recorded ${recovered.unknownAttempts} unanswered attempt(s) as unknown`, - ); - } - } catch (error) { - // The branch may be safe, but reporting the framework ready while its - // durable awareness journal is unreadable creates a half-ready host. - throw new DiscordAwarenessAccountingError('startup reconciliation', error); + try { + const recovered = discordAwarenessOutbox.recoverAtStartup(store.currentBranch().name); + if (recovered.activated.length > 0 || recovered.held.length > 0 || recovered.unknownAttempts > 0) { + console.error( + `[discord-awareness] startup: activated ${recovered.activated.length} interrupted batch(es) ` + + `for ${store.currentBranch().name}, held ${recovered.held.length}, ` + + `recorded ${recovered.unknownAttempts} unanswered attempt(s) as unknown`, + ); } + } catch (error) { + // The branch may be safe, but reporting the framework ready while its + // durable awareness journal is unreadable creates a half-ready host. + throw new DiscordAwarenessAccountingError('startup reconciliation', error); } // Restore persisted usage data (if any) from prior session @@ -6086,6 +6111,23 @@ export class AgentFramework { }; } + /** + * A surgery's marker batch, recorded before its body change. If it can't + * be recorded, the surgery doesn't start, and the error says what the + * journal may hold (DiscordAwarenessOutbox.prepare). + */ + private prepareMarkerBatch(what: string, input: DiscordAwarenessPrepareInput): DiscordAwarenessBatch | null { + try { + return this.discordAwarenessOutbox.prepare(input); + } catch (error) { + throw new OperatorActionError( + 'failed', + `${what} not started, nothing changed: ${error instanceof Error ? error.message : String(error)}`, + { cause: error }, + ); + } + } + /** * After a failed surgery has put the source back: retire its prepared * awareness batch, but only when the store is confirmed on the source. If @@ -6098,7 +6140,7 @@ export class AgentFramework { sourceBranch: string, agentName: string, ): string { - if (!batch || !this.discordAwarenessOutbox) return ''; + if (!batch) return ''; if (this.store.currentBranch().name !== sourceBranch) { return `; awareness batch ${batch.id} kept, since the source branch could not be confirmed`; } @@ -6107,13 +6149,12 @@ export class AgentFramework { return ''; } catch (error) { const detail = boundDiscordAwarenessText(error instanceof Error ? error.message : String(error)); - this.opsAlert( - 'discord-awareness-unresolved', - agentName, - `A failed surgery's awareness batch ${batch.id} could not be retired; startup will hold it: ${detail}`, - { data: { batchId: batch.id } }, - ); - return `; awareness batch ${batch.id} could not be retired (${detail})`; + // Its surgery didn't happen, so its messages are still in context: a + // release would mark them. The operator's act is a cancel. + const what = `awareness batch ${batch.id} could not be retired (${detail}); it stays prepared, and a start on ` + + `${sourceBranch} holds it: cancel it then, not release it, since its messages were never removed`; + this.opsAlert('discord-awareness-unresolved', agentName, `A failed surgery's ${what}`, { data: { batchId: batch.id } }); + return `; ${what}`; } } @@ -6193,9 +6234,10 @@ export class AgentFramework { params: { messageId: opts.messageId, marks: describeMarksChoice(opts.marks) }, }; try { + const marks = checkedMarks(opts.marks); const { cm, tailMessageId, messagesRemoved, discarded } = this.planRollback(agentName, opts.messageId); const removedRefs = extractDiscordAwarenessRefs(discarded); - const selection = selectDiscordAwarenessRefs(discarded, opts.marks ?? 'none'); + const selection = selectDiscordAwarenessRefs(discarded, marks); const sourceBranch = this.store.currentBranch().name; const targetBranch = opts.branchName ?? `rollback/${agentName}/${Date.now()}`; if (targetBranch === sourceBranch) { @@ -6220,16 +6262,16 @@ export class AgentFramework { // promotes this batch by matching targetBranch to the active branch. // If the fork/switch itself throws, the batch is retired so it cannot // arm a later branch that happens to reuse the name. - const markerBatch = this.discordAwarenessOutbox?.prepare({ + const markerBatch = this.prepareMarkerBatch('Rollback', { agentName, sourceBranch, targetBranch, refs: selection.refs, - scope: marksScope(opts.marks), + scope: marksScope(marks), unmarked: selection.unmarked, notRemoved: selection.notRemoved, emoji: this.discordAwarenessEmoji, - }) ?? null; + }); try { branchName = cm.branchAt(tailMessageId as MessageId, targetBranch); @@ -6251,7 +6293,7 @@ export class AgentFramework { // rollback into a failure: marks are an outbox outcome, reported // apart from the body (`markers`). markers = this.activateSurgeryMarkers(markerBatch, 'rollback', agentName, { - scope: marksScope(opts.marks) as SurgeryMarkerReceipt['scope'], + scope: marksScope(marks) as SurgeryMarkerReceipt['scope'], unmarked: selection.unmarked, notRemoved: selection.notRemoved, }); @@ -6342,10 +6384,11 @@ export class AgentFramework { params: { messageIds: requestedIds, marks: describeMarksChoice(opts.marks) }, }; try { + const marks = checkedMarks(opts.marks); const { cm, ordered, messagesRemoved, targeted } = this.planSuppression(agentName, requestedIds); const removedIds = targeted.map((m) => String(m.id)); const removedRefs = extractDiscordAwarenessRefs(targeted); - const selection = selectDiscordAwarenessRefs(targeted, opts.marks ?? 'none'); + const selection = selectDiscordAwarenessRefs(targeted, marks); const sourceBranch = this.store.currentBranch().name; const targetBranch = opts.branchName ?? `suppress/${agentName}/${Date.now()}`; @@ -6363,12 +6406,12 @@ export class AgentFramework { let createdBranch: string; let markers: SurgeryMarkerReceipt; try { - const markerBatch = this.discordAwarenessOutbox?.prepare({ + const markerBatch = this.prepareMarkerBatch('Suppression', { agentName, sourceBranch, targetBranch, refs: selection.refs, - scope: marksScope(opts.marks), + scope: marksScope(marks), unmarked: selection.unmarked, notRemoved: selection.notRemoved, emoji: this.discordAwarenessEmoji, @@ -6376,7 +6419,7 @@ export class AgentFramework { // only this operation activates the batch, after the last redaction. activationPolicy: 'explicit', suppressionIntervals: ordered.map((iv) => ({ fromId: iv.fromId, toId: iv.toId })), - }) ?? null; + }); // A failure short of activation puts the source back first, and only // then retires the batch: a prepared explicit batch is resumed at // startup whenever its exact target branch is active (e.g. after an @@ -6422,7 +6465,7 @@ export class AgentFramework { // interval needs resuming any more, so retiring the batch there // loses nothing.) markers = this.activateSurgeryMarkers(markerBatch, 'suppress', agentName, { - scope: marksScope(opts.marks) as SurgeryMarkerReceipt['scope'], + scope: marksScope(marks) as SurgeryMarkerReceipt['scope'], unmarked: selection.unmarked, notRemoved: selection.notRemoved, }); @@ -6563,7 +6606,7 @@ export class AgentFramework { /** The awareness journal's batches and retract requests, for operator surfaces. */ listDiscordAwareness(): DiscordAwarenessView[] { - return this.discordAwarenessOutbox?.view() ?? []; + return this.discordAwarenessOutbox.view(); } /** @@ -6577,7 +6620,7 @@ export class AgentFramework { opts: { requester?: OperatorRequester; note?: string } = {}, ): DiscordAwarenessCancelReceipt { return this.awarenessOperatorAction('awareness-cancel', { target }, opts, () => - this.requireDiscordAwarenessOutbox().cancel(target, describeRequester(opts.requester))); + this.discordAwarenessOutbox.cancel(target, describeRequester(opts.requester))); } /** @@ -6591,7 +6634,7 @@ export class AgentFramework { opts: { requester?: OperatorRequester; note?: string } = {}, ): DiscordAwarenessRetractReceipt { const receipt = this.awarenessOperatorAction('awareness-retract', { target }, opts, () => - this.requireDiscordAwarenessOutbox().retract(target, describeRequester(opts.requester))); + this.discordAwarenessOutbox.retract(target, describeRequester(opts.requester))); this.deliverDiscordAwarenessInBackground('retract', 'operator'); return receipt; } @@ -6602,18 +6645,11 @@ export class AgentFramework { opts: { requester?: OperatorRequester; note?: string } = {}, ): DiscordAwarenessReleaseReceipt { const receipt = this.awarenessOperatorAction('awareness-release', { batchId }, opts, () => - this.requireDiscordAwarenessOutbox().release(batchId, describeRequester(opts.requester))); + this.discordAwarenessOutbox.release(batchId, describeRequester(opts.requester))); this.deliverDiscordAwarenessInBackground('release', 'operator'); return receipt; } - private requireDiscordAwarenessOutbox(): DiscordAwarenessOutbox { - if (!this.discordAwarenessOutbox) { - throw new OperatorActionError('invalid', 'This framework has no Discord awareness journal (no storePath)'); - } - return this.discordAwarenessOutbox; - } - private awarenessOperatorAction( kind: string, params: Record, @@ -6635,7 +6671,13 @@ export class AgentFramework { } catch (error) { this.recordOperatorAction({ ...logBase, error: error instanceof Error ? error.message : String(error) }); if (error instanceof OperatorActionError) throw error; - throw new OperatorActionError('invalid', error instanceof Error ? error.message : String(error), { cause: error }); + // A request the journal won't carry out is the caller's to change; any + // other error is a failure to read or write the journal. + throw new OperatorActionError( + error instanceof DiscordAwarenessRefusal ? 'invalid' : 'failed', + error instanceof Error ? error.message : String(error), + { cause: error }, + ); } } @@ -6644,20 +6686,19 @@ export class AgentFramework { * has landed, and say what happened. Never throws: the body is already * applied, so a ledger failure here is reported in the receipt rather than * as a failed surgery an operator might retry. A batch that could not be - * activated is retired, so no later reconciliation can deliver it behind a - * receipt saying it was not scheduled (`not-scheduled`). If retiring fails - * too, the receipt says only that the outcome is `unresolved`: the batch is - * still prepared, and it may still be delivered. A rollback's batch - * (activated by target branch) may be promoted by the very next - * reconciliation, the one the delivery it starts runs; a suppression's - * (activated explicitly) waits for the next start. + * activated is retired, so no later start can deliver it behind a receipt + * saying it was not scheduled (`not-scheduled`). If retiring fails too, + * the receipt says only that the outcome is `unresolved`: the batch is + * still prepared, and it may still be delivered. The next start completes + * it if its target branch is active then (a suppression's after resuming + * its body), and holds it otherwise; nothing before a restart does. * * A batch with no refs (a suppression journal with nothing addressable) is * still activated, which closes its resume journal. It schedules no marks, - * so it reports `none` whatever the ledger does. If it can be neither - * activated nor retired, its journal is left prepared and closes at the - * next start: every interval it names has already committed, so the resume - * skips them all and activates it. + * so it reports `none` whatever the ledger does, and a failure is logged. + * If it can be neither activated nor retired, its journal is left prepared + * and closes at the next start on its branch: every interval it names has + * already committed, so the resume skips them all and activates it. */ private activateSurgeryMarkers( batch: DiscordAwarenessBatch | null, @@ -6665,11 +6706,25 @@ export class AgentFramework { agentName: string, facts: SurgeryMarkerFacts, ): SurgeryMarkerReceipt { - if (!batch || !this.discordAwarenessOutbox) return { ...facts, status: 'none', queued: 0 }; + if (!batch) return { ...facts, status: 'none', queued: 0 }; const settled = this.discordAwarenessOutbox.settleActivation(batch.id); - // A suppression-only batch carries no marks: its activation records the - // body complete, and the receipt has nothing to report. - if (batch.refs.length === 0) return { ...facts, status: 'none', queued: 0 }; + if (batch.refs.length === 0) { + // A suppression-only batch carries no marks, so the receipt has + // nothing to report. Its activation closes the body's resume journal: + // if that failed, the journal did, and the operator hears what's left. + if (settled.status !== 'queued') { + const left = settled.status === 'not-scheduled' + ? 'it was retired instead, so nothing is left to resume' + : 'it stays prepared: a start on this branch finds its removals applied and closes it, a start on another holds it'; + const message = `${verb} applied, but its resume journal (awareness batch ${batch.id}, no marks) ` + + `could not be closed (${settled.error}); ${left}`; + console.error(`[discord-awareness] agent=${agentName}: ${message}`); + if (settled.status === 'unresolved') { + this.opsAlert('discord-awareness-unresolved', agentName, message, { data: { batchId: batch.id } }); + } + } + return { ...facts, status: 'none', queued: 0 }; + } return this.settledMarkerReceipt(settled, batch.id, batch.refs.length, verb, agentName, facts); } @@ -6733,7 +6788,7 @@ export class AgentFramework { unmarked: selection.unmarked, notRemoved: selection.notRemoved, }; - if (!this.discordAwarenessOutbox || selection.refs.length === 0) return { ...facts, status: 'none', queued: 0 }; + if (selection.refs.length === 0) return { ...facts, status: 'none', queued: 0 }; let branch: string; try { branch = this.store.currentBranch().name; @@ -6773,7 +6828,6 @@ export class AgentFramework { occasion: 'rollback' | 'suppress' | 'hide' | 'undo' | 'retract' | 'release' | 'startup' | 'connect' | 'list-change' | 'reconnect', agentName: string, ): void { - if (!this.discordAwarenessOutbox) return; this.syncDiscordAwarenessMarkers().catch((error) => { const detail = boundDiscordAwarenessText(error instanceof Error ? error.message : String(error)); console.error(`[discord-awareness] delivery after ${occasion} failed: ${detail}`); @@ -13695,21 +13749,20 @@ export class AgentFramework { } /** - * Deliver every server's queued awareness operations now and wait for this - * pass. Delivery never gates MCPL traffic or turns: callers that do not - * need to wait use the background form. A server that is not connected is - * skipped (its work stays queued until it connects). + * Deliver every server's queued awareness operations now (or only + * `onlyServerId`'s) and wait for this pass. Delivery never gates MCPL + * traffic or turns: callers that do not need to wait use the background + * form. A server that is not connected is skipped (its work stays queued + * until it connects). */ - async syncDiscordAwarenessMarkers(_onlyServerId?: string): Promise { - if (!this.discordAwarenessOutbox) return; + async syncDiscordAwarenessMarkers(onlyServerId?: string): Promise { const serverIds = [...new Set( - this.readDiscordAwarenessDispatches().map((dispatch) => dispatch.key.serverId), + this.readDiscordAwarenessDispatches(onlyServerId).map((dispatch) => dispatch.key.serverId), )]; await Promise.all(serverIds.map((serverId) => this.drainDiscordAwarenessOutbox(serverId))); } private async resumePreparedDiscordSuppressions(): Promise { - if (!this.discordAwarenessOutbox) return; const batches = this.discordAwarenessOutbox.preparedSuppressionsForBranch( this.store.currentBranch().name, ); @@ -13761,9 +13814,6 @@ export class AgentFramework { private drainDiscordAwarenessOutbox( serverId: string, ): Promise { - if (!this.discordAwarenessOutbox) { - return Promise.resolve({ status: 'delivered', delivered: 0, failed: 0 }); - } const existing = this.discordAwarenessDrains.get(serverId); if (existing) return existing; @@ -13791,7 +13841,7 @@ export class AgentFramework { pass: DiscordAwarenessDrainPass, ): Promise { try { - const outbox = this.discordAwarenessOutbox!; + const outbox = this.discordAwarenessOutbox; const connection = this.mcplServerRegistry?.getServer(serverId); if (!connection?.isConnected) { const waiting = this.readDiscordAwarenessDispatches(serverId).length; @@ -13805,6 +13855,8 @@ export class AgentFramework { let delivered = 0; let failed = 0; + let givenUp = 0; + let firstGivenUp: string | undefined; const attempted = new Set(); const signature = (dispatch: { opIds: string[]; action: string }) => `${dispatch.opIds.join(',')}\0${dispatch.action}`; @@ -13855,20 +13907,19 @@ export class AgentFramework { } } catch (error) { detail = error instanceof Error ? error.message : String(error); - if (error instanceof McplRequestError) { - outcome = error.outcome === 'not-sent' - ? 'not-sent' - : error.outcome === 'error-response' ? 'failed' : 'unknown'; - } else { - // Not classified by the connection: it may have been written. - outcome = 'unknown'; - } + // Only a request refused before it was written is known not to + // have reached the server. A JSON-RPC error answer may come after + // the server acted (McplRequestError's contract), and an error the + // connection didn't classify may follow a write: both are unknown. + // Discord's own refusals arrive as an isError result, above. + outcome = error instanceof McplRequestError && error.outcome === 'not-sent' ? 'not-sent' : 'unknown'; } // Classify on the full text, then keep only a bounded copy. const permanent = outcome === 'failed' && isPermanentDiscordReactionFailure(detail ?? ''); if (detail !== undefined) detail = boundDiscordAwarenessText(detail); + let ended: number; try { - outbox.recordOutcome(attempts, outcome, { + ended = outbox.recordOutcome(attempts, outcome, { ...(permanent ? { permanent: true } : {}), ...(detail !== undefined ? { error: detail } : {}), }); @@ -13879,9 +13930,14 @@ export class AgentFramework { delivered++; } else { failed++; + if (ended > 0) { + givenUp += ended; + firstGivenUp ??= `${dispatch.action} ${ref.channelId}/${ref.messageId}: ${outcome}: ${detail ?? ''}`; + } console.error( `[discord-awareness] ${dispatch.action} ${ref.channelId}/${ref.messageId}: ${outcome}` + - `${outcome === 'failed' ? (permanent ? ' (permanent)' : ' (retryable)') : ''}: ${detail ?? ''}`, + `${ended > 0 ? ' (not sent again)' : outcome === 'failed' ? ' (retryable)' : ''}` + + `: ${detail ?? ''}`, ); } } @@ -13891,6 +13947,18 @@ export class AgentFramework { `[discord-awareness] ${serverId}: confirmed=${delivered} not-confirmed=${failed}`, ); } + if (givenUp > 0) { + // One alert per pass for the operations it ended without a + // confirmation: refused for good, or at their third unconfirmed + // attempt. Their history stays in the journal (marks list). + this.opsAlert( + 'discord-awareness-delivery', + 'host', + `${givenUp} awareness mark operation(s) on ${serverId} won't be sent again: Discord refused them for good, ` + + `or three attempts went unconfirmed (first: ${firstGivenUp})`, + { data: { serverId, givenUp } }, + ); + } return { status: 'delivered' as const, delivered, failed }; } finally { pass.ended = true; @@ -13902,7 +13970,7 @@ export class AgentFramework { private readDiscordAwarenessDispatches(serverId?: string) { try { - return this.discordAwarenessOutbox!.pendingDispatches(serverId); + return this.discordAwarenessOutbox.pendingDispatches(serverId); } catch (error) { throw new DiscordAwarenessAccountingError( `journal read${serverId ? ` for ${serverId}` : ''}`, diff --git a/src/index.ts b/src/index.ts index 2e1c021c4..e51f514de 100644 --- a/src/index.ts +++ b/src/index.ts @@ -129,6 +129,7 @@ export type { Membrane, NormalizedMessage, NormalizedRequest, ContentBlock } fro // Offline outage recovery and the Discord awareness-mark journal export { DiscordAwarenessOutbox, + DiscordAwarenessRefusal, DEFAULT_DISCORD_AWARENESS_EMOJI, defaultDiscordAwarenessOutboxPath, extractDiscordAwarenessRefs, diff --git a/src/operator-log.ts b/src/operator-log.ts index a2191b755..2030b8e69 100644 --- a/src/operator-log.ts +++ b/src/operator-log.ts @@ -142,9 +142,11 @@ export interface SurgeryMarkerFacts { * the addressable ones left unmarked, and the surgery's `removedRefs` * (`discordRefs` in the operator log) list every one. * - `queued` — one add request per mark is durably recorded and delivery has - * started. If delivery fails, the requests stay recorded, the + * started. If a delivery pass fails, the requests stay recorded, the * `discord-awareness-delivery` ops alert says so, and a later pass (a - * reconnect, a channel-list change or the next start) delivers them. It is + * reconnect, a channel-list change or the next start) delivers them. The + * same alert names the requests a pass gave up on: refused by Discord for + * good, or after three attempts that reached the server unconfirmed. It is * not a delivery claim: per-request outcomes stay in the awareness journal. * - `not-scheduled` — the body change landed, but the batch was never * recorded, or recording it as active failed and it was retired from the @@ -152,8 +154,8 @@ export interface SurgeryMarkerFacts { * - `unresolved` — the journal could record neither the activation nor the * retirement, or could not be read back to say whether the batch was * recorded at all. `batchId` names the batch, whose scheduling outcome is - * unresolved: any later reconciliation or restart that can read it may - * activate and deliver it, and nothing here promises otherwise. + * unresolved: the next start, if it can read the batch, may activate and + * deliver it, and nothing here promises otherwise. */ export type SurgeryMarkerReceipt = SurgeryMarkerFacts & ( | { status: 'none'; queued: 0 } diff --git a/src/recovery/discord-awareness-outbox.ts b/src/recovery/discord-awareness-outbox.ts index e94ba37f9..3c8d2e55b 100644 --- a/src/recovery/discord-awareness-outbox.ts +++ b/src/recovery/discord-awareness-outbox.ts @@ -13,14 +13,23 @@ import { RecordJournal, type AppendOptions } from '../record-journal.js'; * Marks are an explicit publication choice, separate from the surgery itself * (`DiscordAwarenessMarks`, default `none`). Once requested, they are * one-shot: branch moves and restarts never derive new operations from branch - * state; an operator retracts, cancels or releases explicitly. + * state. The one thing a restart does with branch state is complete a + * surgery that crashed after its switch: a prepared batch whose target is + * the active branch carries out that surgery's own recorded choice (see + * recoverAtStartup). An operator retracts, cancels or releases explicitly. * * The ledger is an append-only journal of what was asked and what was * answered, never a model of what is on Discord. Each operation is recorded - * as `requested`, then `dispatching` (written before the request can leave), - * then an outcome. A request that was dispatched and never answered stays - * `unknown` for good: a later retry's confirmation does not resolve it, and a - * later opposite operation discloses it. + * as `requested`, then each attempt as `dispatching` (written before its + * request can leave), then that attempt's outcome. A request that was + * dispatched and never answered stays `unknown` for good: a later attempt's + * answer doesn't change it, receipts count it as unresolved, and a later + * opposite operation discloses it. An operation's status is its latest + * answer, so an add whose retry Discord confirmed is confirmed, while its + * first request is still counted as unresolved. An operation is not sent + * again after three attempts that reached the server without a confirmation + * (refused or unanswered); an attempt refused before it left the host + * doesn't count, so a route that is down keeps its work queued. * * Operations are keyed by (MCPL server id, channel, message, emoji). The * server id is the configured route to a bot, not the bot's identity: if the @@ -33,7 +42,9 @@ import { RecordJournal, type AppendOptions } from '../record-journal.js'; * holding a group of journal records that apply together. A record that * asserts a branch change (an activation after a switch or redaction, or a * batch recorded after an in-place change) is appended only after that - * state is synced, so it can never outlive it. + * state is synced, so it can never outlive it. A write that fails inside the + * store leaves the journal refusing every read and write until the store is + * reopened (see load). */ export const DEFAULT_DISCORD_AWARENESS_EMOJI = '💤'; @@ -55,29 +66,42 @@ export function boundDiscordAwarenessText(text: string, max = 500): string { return `${head} … [${text.length - head.length - tail.length} chars omitted] … ${tail}`; } -/** - * Where a pre-journal awareness ledger (the JSON file earlier releases kept - * under a store) is found, to import once. - */ /** * A Discord answer that the reaction can never succeed: the message or - * channel is gone or inaccessible. Retrying cannot help. + * channel is gone or inaccessible, the emoji can't be used, or the message + * takes no more reactions (or blocks this one). Retrying cannot help. */ export function isPermanentDiscordReactionFailure(message: string): boolean { - return /unknown message|unknown channel|missing access|missing permissions|missing permission|cannot access|channel .* not found|message .* not found/i + return /unknown message|unknown channel|unknown emoji|missing access|missing permissions|missing permission|cannot access|maximum number of reactions|reaction was blocked|channel .* not found|message .* not found/i .test(message); } +/** + * An operator's request the journal won't carry out: an unknown batch or + * retract request, or one in the wrong state. Nothing was written. Any other + * error from an operator control is a failure to read or write the journal. + */ +export class DiscordAwarenessRefusal extends Error { + constructor(message: string) { + super(message); + this.name = 'DiscordAwarenessRefusal'; + } +} + /** AF's own refusal to write a request on a closed connection. */ const PRE_WRITE_REFUSAL = /^Cannot send request:/; +/** The pre-journal (version 1 and 2) ledger file name, imported once. */ +const LEGACY_LEDGER_NAME = 'discord-awareness-outbox.json'; + +/** + * Where a pre-journal awareness ledger (the JSON file earlier releases kept + * under a store) is found, to import once. + */ export function defaultDiscordAwarenessOutboxPath(storePath: string): string { return join(storePath, 'recovery', LEGACY_LEDGER_NAME); } -/** The pre-journal (version 1 and 2) ledger file name, imported once. */ -const LEGACY_LEDGER_NAME = 'discord-awareness-outbox.json'; - export interface DiscordAwarenessRef { serverId: string; channelId: string; @@ -131,9 +155,13 @@ export interface DiscordAwarenessBatchRecord { /** Authorized refs the surgery did not remove. */ notRemoved?: number; /** - * `target-branch`: startup activates the batch only if its target is - * exactly the active branch (the surgery crashed after switching). - * `explicit`: only the surgery (or startup's resume of its intervals) does. + * Who completes a batch a crash left prepared. With suppression intervals, + * `explicit` means only its surgery, or startup's resume of its intervals + * on its target branch, activates it: that branch being active doesn't + * prove the redactions finished. Without intervals the policy changes + * nothing: startup activates a prepared batch whose target is exactly the + * active branch (its surgery crashed after its switch, or after the + * in-place change it was recorded for) and holds any other. */ activationPolicy?: 'target-branch' | 'explicit'; /** Idempotent interval operations used to resume an interrupted suppression. */ @@ -244,8 +272,9 @@ export interface DiscordAwarenessCancelReceipt { /** The batch id or retract request id that was cancelled. */ target: string; kind: 'batch' | 'retract'; - /** Requests that will now never be sent (for a batch not yet activated, - * every mark it would have requested). */ + /** Requests this call stopped before they were ever sent (for a batch not + * yet activated, every mark it would have requested). A repeated cancel + * stops none. */ cancelled: number; /** Held release actions dropped (a held batch was cancelled). */ heldDropped: number; @@ -269,7 +298,10 @@ export interface DiscordAwarenessRetractReceipt { requestId: string; /** Removals queued: one per selected reaction key, whatever history says. */ removalsQueued: number; - /** Add requests for those keys that had not been sent, now superseded. */ + /** Add requests for those keys that had not ended (due, on the wire, or + * unknown), whose later sends and retries are now retired. One on the + * wire still settles before the removal is sent, and any unresolved one + * may land or have landed (keysWithUnresolvedAdds). */ addsSuperseded: number; /** Keys whose earlier add attempts are unresolved (on the wire or unknown): * such an add may land after this removal. */ @@ -486,7 +518,7 @@ type JournalRecord = | { t: 'request-cancelled'; at: number; requestId: string; by?: string } | { t: 'suppression-complete'; at: number; batchId: string } | { t: 'legacy-evidence'; at: number; evidence: DiscordAwarenessLegacyEvidence } - | { t: 'import-complete'; at: number; source: 'v2'; sha256: string }; + | { t: 'import-complete'; at: number; source: 'v1' | 'v2'; sha256: string }; /** One journal entry (one Chronicle record): records that apply together. */ interface JournalEntryGroup { @@ -566,16 +598,46 @@ export class DiscordAwarenessOutbox { /** * Record a surgery's request before its branch switch. Returns null when it * carries neither marks nor a suppression journal. A prepared batch never - * delivers: activate() (or startup's crash completion) does. + * delivers: activate() (or startup's crash completion) does. If its write + * fails it throws, saying what the journal may hold (failedPrepare), and the + * surgery doesn't run. */ prepare(input: DiscordAwarenessPrepareInput): DiscordAwarenessBatch | null { const batch = batchRecord(input); if (!batch) return null; - // Durable before the surgery's switch, so startup can crash-complete it. - this.append([{ t: 'batch', at: batch.createdAt, batch }], { durable: true }); + try { + // Durable before the surgery's switch, so startup can crash-complete it. + this.append([{ t: 'batch', at: batch.createdAt, batch }], { durable: true }); + } catch (error) { + throw new Error(this.failedPrepare(batch.id, error), { cause: error }); + } return { ...structuredClone(batch), status: 'prepared' }; } + /** + * What a prepare whose write failed leaves, for the surgery's error (the + * surgery then doesn't run): nothing; the batch, retired at once when it is + * in this process's view; or, after a write that failed inside the store, a + * batch this process can't see, which the next start holds for an operator. + * A held batch of a surgery that never ran is to be cancelled, not + * released: its messages were never removed. + */ + private failedPrepare(batchId: string, error: unknown): string { + const detail = `recording awareness batch ${batchId} failed: ${errorText(error)}`; + const cancelIt = 'cancel it then, not release it, since its surgery didn\'t run'; + if (this.journal.needsReopen) { + return `${detail}; the batch may be in the store, unseen until the store is reopened, ` + + `and if so the next start holds it: ${cancelIt}`; + } + if (!this.journal.needsReconcile) return detail; + try { + return this.discard(batchId) ? `${detail}; it reached the journal and is retired` : detail; + } catch (retireError) { + return `${detail}; it reached the journal, and retiring it also failed (${errorText(retireError)}), ` + + `so the next start holds it: ${cancelIt}`; + } + } + /** * Marks for a body change already applied on `branch` (an in-place hide, a * turn undo): a batch for exactly that branch, recorded and activated in @@ -588,8 +650,10 @@ export class DiscordAwarenessOutbox { * that is not there is `not-scheduled` (nothing was recorded); one that is * there is settled like any prepared batch (activated, else retired, else * `unresolved`); and if the journal can't be read back, `unresolved`, since - * the batch may be there. Returns null when there is nothing to mark. - * Never throws. + * the batch may be there. That includes a write that failed inside the + * store: its record may be in the store's log, which this process can't + * read until the store is reopened (see load). Returns null when there is + * nothing to mark. Never throws. */ settleApplied( input: Omit @@ -816,7 +880,7 @@ export class DiscordAwarenessOutbox { if (state.retracts.has(target)) return this.cancelRetract(state, target, by); const batchId = target; const batch = state.batches.get(batchId); - if (!batch) throw new Error(`Discord awareness batch or retract request not found: ${target}`); + if (!batch) throw new DiscordAwarenessRefusal(`Discord awareness batch or retract request not found: ${target}`); const receipt: DiscordAwarenessCancelReceipt = { target: batchId, kind: 'batch', @@ -932,7 +996,7 @@ export class DiscordAwarenessOutbox { ]); } else { const batch = state.batches.get(target); - if (!batch) throw new Error(`Discord awareness batch not found: ${target}`); + if (!batch) throw new DiscordAwarenessRefusal(`Discord awareness batch not found: ${target}`); candidateKeys = uniqueKeys(batch.refs.map((ref) => ({ ...ref, emoji: batch.emoji }))); } const requestId = randomUUID(); @@ -971,9 +1035,9 @@ export class DiscordAwarenessOutbox { release(batchId: string, by?: string): DiscordAwarenessReleaseReceipt { const state = this.load(); const batch = state.batches.get(batchId); - if (!batch) throw new Error(`Discord awareness batch not found: ${batchId}`); + if (!batch) throw new DiscordAwarenessRefusal(`Discord awareness batch not found: ${batchId}`); if (batch.status !== 'held' || !batch.held) { - throw new Error(`Discord awareness batch ${batchId} is ${batch.status}, not held`); + throw new DiscordAwarenessRefusal(`Discord awareness batch ${batchId} is ${batch.status}, not held`); } const at = Date.now(); const actions = batch.held.releaseActions; @@ -997,7 +1061,8 @@ export class DiscordAwarenessOutbox { * wire. A key's operations go out in request order; consecutive requested * operations with the same action share one dispatch. An `unknown` * operation is retried only while it is still the newest live operation - * for its key. + * for its key. No operation is sent again once three of its attempts + * reached the server without a confirmation. */ pendingDispatches(serverId?: string): DiscordAwarenessDispatch[] { const state = this.load(); @@ -1074,7 +1139,10 @@ export class DiscordAwarenessOutbox { const op = live[index]; const status = this.opStatus(op); if (status === 'requested') { head = op; break; } - if (status === 'unknown' && index === live.length - 1) { head = op; break; } + // An unanswered request is sent again only while its operation is the + // newest live one for its key, and within the bound. + if (status === 'unknown' && index === live.length - 1 + && unconfirmedAttempts(op) < MAX_UNCONFIRMED_ATTEMPTS) { head = op; break; } } if (!head) return null; const opIds = [head.opId]; @@ -1086,12 +1154,17 @@ export class DiscordAwarenessOutbox { return { key: { ...head.key }, action: head.action, opIds, order: head.requestedAt }; } - /** Record the outcome of a dispatch's attempts. */ + /** + * Record the outcome of a dispatch's attempts. Returns how many of its + * operations this outcome ended without a confirmation: refused for good, + * or at their third attempt that reached the server unconfirmed, so they + * will not be sent again. + */ recordOutcome( attempts: Array<{ opId: string; attempt: number }>, outcome: DiscordAwarenessOutcome, detail: { permanent?: boolean; error?: string } = {}, - ): void { + ): number { const at = Date.now(); const error = detail.error !== undefined ? boundDiscordAwarenessText(detail.error) : undefined; try { @@ -1107,6 +1180,15 @@ export class DiscordAwarenessOutbox { } finally { for (const { opId } of attempts) this.inFlight.delete(opId); } + if (outcome !== 'failed' && outcome !== 'unknown') return 0; + const state = this.load(); + let ended = 0; + for (const { opId } of attempts) { + const op = state.ops.get(opId); + if (!op || op.cancelled) continue; + if ((outcome === 'failed' && detail.permanent) || unconfirmedAttempts(op) >= MAX_UNCONFIRMED_ATTEMPTS) ended++; + } + return ended; } // -- views ------------------------------------------------------------------ @@ -1195,17 +1277,23 @@ export class DiscordAwarenessOutbox { // -- internals -------------------------------------------------------------- + /** + * What became of an operation, latest answer first. What Discord may have + * comes before whether it will be sent again: a confirmation is a fact + * about Discord, and a request on the wire or never answered may land or + * have landed whatever was decided after it was sent (a cancel, or a later + * opposite act, stops what would be sent next, not a request already + * sent). Then: cancelled, refused for good (permanently, or after the + * bound on unconfirmed attempts), or due. + */ private opStatus(op: DiscordAwarenessOp): DiscordAwarenessOpStatus { const last = op.attempts.at(-1); - // A confirmation is a fact about Discord, even for a request cancelled - // while it was on the wire. if (last?.outcome === 'confirmed') return 'confirmed'; + if (last && last.outcome === undefined) return this.inFlight.has(op.opId) ? 'dispatching' : 'unknown'; + if (last?.outcome === 'unknown') return 'unknown'; if (op.cancelled) return 'cancelled'; - if (!last) return 'requested'; - if (last.outcome === undefined) return this.inFlight.has(op.opId) ? 'dispatching' : 'unknown'; - if (last.outcome === 'unknown') return 'unknown'; - if (last.outcome === 'not-sent') return 'requested'; - return last.permanent ? 'failed' : 'requested'; + if (!last || last.outcome === 'not-sent') return 'requested'; + return last.permanent || unconfirmedAttempts(op) >= MAX_UNCONFIRMED_ATTEMPTS ? 'failed' : 'requested'; } /** @@ -1278,8 +1366,20 @@ export class DiscordAwarenessOutbox { * a reload before writing again): every entry is replayed, after an * earlier build's checkpoint if there is one (see JournalSnapshot). The * first load also imports a pre-journal ledger, once. + * + * After an append that threw inside the store, its record may be in the + * store's log where no load in this process can see it (RecordJournal's + * needsReopen), so nothing decided here could be trusted: every read and + * write refuses until the store is reopened, as at the next start, which + * reads that record with the rest. */ private load(): JournalState { + if (this.journal.needsReopen) { + throw new Error( + 'The Discord awareness journal can\'t be read in this process: an earlier write failed inside the store, ' + + 'and its record may be there unseen. Nothing is read or written until the store is reopened, as at the next start', + ); + } if (this.state && !this.journal.needsReconcile) return this.state; const { snapshot, entries } = this.journal.load(); const state = snapshot ? restoreState(snapshot) : emptyState(); @@ -1336,8 +1436,10 @@ export class DiscordAwarenessOutbox { /** * Append one group of records as one journal entry and apply it. A failed - * append leaves the state to be rebuilt from the journal: whether it was - * written is unknown, and the next operation reloads before writing. + * append leaves the state to be rebuilt from the journal before the next + * operation: one whose record reached this process's view (a barrier + * failed) is read back with it, and one that failed inside the store + * leaves the journal refusing until the store is reopened (see load). */ private append(records: JournalRecord[], opts: AppendOptions = {}): void { if (records.length === 0) return; @@ -1366,13 +1468,24 @@ function liveRequests(records: JournalRecord[]): number { return records.filter((record) => record.t === 'requested' && !cancelled.has(record.opId)).length; } -/** The state an earlier build's checkpoint recorded (see JournalSnapshot). */ +/** + * The state an earlier build's checkpoint recorded (see JournalSnapshot). + * Checkpoints written before authorization order existed carry no + * authorizations and no position: a restored act takes 0, older than + * anything recorded after the restore, which is when it was decided. + */ function restoreState(snapshot: JournalSnapshot): JournalState { const state: JournalState = { - batches: new Map(snapshot.batches.map((batch) => [batch.id, structuredClone(batch)])), - ops: new Map(snapshot.ops.map((op) => [op.opId, structuredClone(op)])), + batches: new Map(snapshot.batches.map((batch) => [ + batch.id, + { ...structuredClone(batch), authorization: batch.authorization ?? 0 }, + ])), + ops: new Map(snapshot.ops.map((op) => [op.opId, { ...structuredClone(op), authorization: op.authorization ?? 0 }])), legacy: new Map(), - retracts: new Map((snapshot.retracts ?? []).map((retract) => [retract.requestId, structuredClone(retract)])), + retracts: new Map((snapshot.retracts ?? []).map((retract) => [ + retract.requestId, + { ...structuredClone(retract), authorization: retract.authorization ?? 0 }, + ])), importedSources: new Set(snapshot.importedSources), position: snapshot.position ?? 0, }; @@ -1560,8 +1673,10 @@ interface LegacyBatch { * What the old ledger recorded about each ref is kept as legacy evidence: * facts (attempt count, last action, status and error), never synthesized * attempts and never a judgment about what is on Discord. Pending work, and - * batches still prepared, are held for an operator: they were queued before - * marks were an explicit choice. Nothing imported is ever dispatched. + * batches still prepared, are held for an operator when they have marks to + * release: they were queued before marks were an explicit choice. A prepared + * suppression with nothing to mark only has its body resumed on its branch + * (see preparedSuppressionsForBranch). Nothing imported is ever dispatched. * * Each batch is imported once. A batch the journal already holds, or a * second listing of one in this file, is skipped whole: its evidence would @@ -1618,7 +1733,9 @@ function importLegacy( const releaseActions: DiscordAwarenessReleaseAction[] = []; for (const entry of raw.refs) { const key = { serverId: entry.serverId, channelId: entry.channelId, messageId: entry.messageId, emoji: raw.emoji }; - // v1 entries carry no delivery state: an active v1 batch was all pending. + // v1 entries carry no delivery state. A v1 batch is `prepared` or + // `pending`, and `pending` was the active one, every mark still to + // deliver: its entries are pending, a prepared one's are not. const status = entry.deliveryStatus ?? (raw.status === 'pending' ? 'pending' : 'applied'); const desired = entry.desired ?? raw.status === 'pending'; if (!legacyActive || status === 'pending') { @@ -1647,7 +1764,7 @@ function importLegacy( }); } } - records.push({ t: 'import-complete', at, source: 'v2', sha256 }); + records.push({ t: 'import-complete', at, source: document.version === 1 ? 'v1' : 'v2', sha256 }); return records; } @@ -1741,6 +1858,18 @@ function markKey(key: DiscordAwarenessRef & { emoji: string }): string { return `${key.emoji}\0${refKey(key)}`; } +/** + * Attempts that reached the server without a confirmation (refused, or never + * answered) after which an operation is not sent again. Without a bound, a + * refusal Discord repeats, or a server that never answers, would be sent + * again at every drain, two journal records each time. + */ +const MAX_UNCONFIRMED_ATTEMPTS = 3; + +function unconfirmedAttempts(op: DiscordAwarenessOp): number { + return op.attempts.filter((attempt) => attempt.outcome === 'failed' || attempt.outcome === 'unknown').length; +} + /** Unresolved requests among these operations' attempts, one per dispatch. */ function unresolvedDispatches( ops: DiscordAwarenessOp[], diff --git a/src/types/framework.ts b/src/types/framework.ts index 92d70fe99..ac492dc89 100644 --- a/src/types/framework.ts +++ b/src/types/framework.ts @@ -110,9 +110,12 @@ export interface FrameworkConfig { storePath?: string; /** - * Branch-independent Discord awareness-marker outbox. Defaults to - * `/recovery/discord-awareness-outbox.json`. Set explicitly when - * providing an app-owned `store` without `storePath`; omit to disable. + * A pre-journal Discord awareness ledger to import once. The awareness + * journal itself lives in the store (typed records), so every framework + * has one; this only names the JSON file earlier releases kept, by default + * `/recovery/discord-awareness-outbox.json`. Set it when + * providing an app-owned `store` without `storePath` whose earlier + * releases kept such a file. */ discordAwarenessOutboxPath?: string; diff --git a/test/discord-awareness-outbox.test.ts b/test/discord-awareness-outbox.test.ts index b891f48ba..1445c5654 100644 --- a/test/discord-awareness-outbox.test.ts +++ b/test/discord-awareness-outbox.test.ts @@ -19,7 +19,9 @@ import { JsStore } from '@animalabs/chronicle'; import { DISCORD_AWARENESS_RECORD_TYPE, DiscordAwarenessOutbox, + DiscordAwarenessRefusal, boundDiscordAwarenessText, + isPermanentDiscordReactionFailure, extractDiscordAwarenessRefs, selectDiscordAwarenessRefs, type DiscordAwarenessBatchView, @@ -79,8 +81,8 @@ function answer( ) { const claimed = outbox.claimDispatch('discord', (d) => d.key.messageId !== messageId); assert.ok(claimed, `a dispatch for ${messageId} is due`); - outbox.recordOutcome(claimed.attempts, outcome, detail); - return claimed; + const ended = outbox.recordOutcome(claimed.attempts, outcome, detail); + return { ...claimed, ended }; } const batches = (outbox: DiscordAwarenessOutbox) => @@ -309,7 +311,8 @@ test('a newer opposite request stops the retries of a retryably failed one, in b answer(outbox, 'm1', 'failed', { error: 'Discord 503' }); // retryable const receipt = outbox.retract(batch.id); assert.equal(receipt.addsSuperseded, 1); - for (let i = 0; i < 3; i++) { + // Retried while refused, within the bound of three unconfirmed attempts. + for (let i = 0; i < 2; i++) { const claimed = outbox.claimDispatch('discord')!; assert.equal(claimed.dispatch.action, 'remove'); outbox.recordOutcome(claimed.attempts, 'failed', { error: 'Discord 503' }); @@ -468,6 +471,13 @@ test('a retry confirming an unknown add never erases the unresolved first attemp assert.deepEqual(retry.dispatch.opIds, first.dispatch.opIds); assert.equal(retry.attempts[0].attempt, 2); outbox.recordOutcome(retry.attempts, 'confirmed'); + // The operation's status is its latest answer; the first request stays + // unknown in its history and is counted as unresolved. + const [op] = outbox.operations(); + assert.equal(outbox.operationStatus(op), 'confirmed'); + assert.deepEqual(op.attempts.map((attempt) => attempt.outcome), ['unknown', 'confirmed']); + assert.equal(batches(outbox)[0].adds.confirmed, 1); + assert.equal(batches(outbox)[0].adds.unknown, 0); assert.equal(batches(outbox)[0].unresolvedAttempts, 1); const receipt = outbox.retract(batch.id); assert.equal(receipt.unresolvedAddAttempts, 1); @@ -756,7 +766,7 @@ async function muted(fn: () => Promise): Promise { test('framework drain classifies outcomes from what reached the server', async () => { const h = storeHarness(); try { - activeBatch(h.outbox, [ref('m1'), ref('m2'), ref('m3'), ref('m4'), ref('m5')]); + activeBatch(h.outbox, [ref('m1'), ref('m2'), ref('m3'), ref('m4'), ref('m5'), ref('m6'), ref('m7')]); const calls: string[] = []; let connected = true; const connection = { @@ -766,6 +776,8 @@ test('framework drain classifies outcomes from what reached the server', async ( if (args.messageId === 'm2') return { isError: true, content: [{ type: 'text', text: 'Unknown Message' }] }; if (args.messageId === 'm3') throw new McplRequestError('Cannot send request: connection to "discord" is closed', 'not-sent'); if (args.messageId === 'm4') throw new McplRequestError('did not respond', 'no-response'); + if (args.messageId === 'm6') throw new McplRequestError('Internal error', 'error-response', -32603); + if (args.messageId === 'm7') throw new TypeError('transport went away'); return { content: [{ type: 'text', text: 'Reaction applied' }] }; }, }; @@ -774,13 +786,20 @@ test('framework drain classifies outcomes from what reached the server', async ( framework.discordAwarenessDrains = new Map(); framework.discordAwarenessDeadlineMs = 1000; framework.mcplServerRegistry = { getServer: () => connection }; + const alerts: Array<{ kind: string; message: string }> = []; + framework.opsAlert = (kind: string, _agent: string, message: string) => { alerts.push({ kind, message }); }; await muted(() => framework.drainDiscordAwarenessOutbox('discord')); - assert.deepEqual(calls.sort(), ['add_reaction:m1', 'add_reaction:m2', 'add_reaction:m3', 'add_reaction:m4', 'add_reaction:m5']); + assert.deepEqual(calls.sort(), ['m1', 'm2', 'm3', 'm4', 'm5', 'm6', 'm7'].map((m) => `add_reaction:${m}`)); const status = new Map(h.outbox.operations().map((op) => [op.key.messageId, h.outbox.operationStatus(op)])); assert.equal(status.get('m1'), 'confirmed'); assert.equal(status.get('m2'), 'failed', 'Discord said the message is gone: final'); assert.equal(status.get('m3'), 'requested', 'never written: still queued'); assert.equal(status.get('m4'), 'unknown', 'written, no answer: unknown'); + assert.equal(status.get('m6'), 'unknown', 'a JSON-RPC error may come after the server acted'); + assert.equal(status.get('m7'), 'unknown', 'an error the connection did not classify may follow a write'); + assert.equal(alerts.length, 1, 'one alert for the pass'); + assert.equal(alerts[0].kind, 'discord-awareness-delivery'); + assert.match(alerts[0].message, /^1 awareness mark operation\(s\) on discord won't be sent again.*add discord:g1:c1\/m2: failed: Unknown Message/); // A disconnected route leaves its work queued without writing anything. connected = false; @@ -792,7 +811,7 @@ test('framework drain classifies outcomes from what reached the server', async ( connected = true; calls.length = 0; await muted(() => framework.drainDiscordAwarenessOutbox('discord')); - assert.deepEqual(calls.sort(), ['add_reaction:m3', 'add_reaction:m4']); + assert.deepEqual(calls.sort(), ['add_reaction:m3', 'add_reaction:m4', 'add_reaction:m6', 'add_reaction:m7']); } finally { h.cleanup(); } @@ -1174,18 +1193,28 @@ const afterAnAppend = (calls: string[]) => calls.includes(JOURNAL_APPEND); * The harness store with its journal seams observed and faultable. `fail` * makes calls at a seam throw while `matches` holds for the calls before * them (up to `times` of them): `before` the real call, so nothing reaches - * the store, or `after` it, so it did. + * the store, or `after` it, so it did. `unindexed` is an append as + * Chronicle 0.4.0's can fail: the record is written to the log, the periodic + * sync then throws, and the record stays out of this process's view of the + * store (its lists and reads) until the store is reopened. A view of the + * underlying store, as a restarted process has, sees it. */ function journalSeams(store: JsStore) { type Seam = 'sync' | 'append' | 'list'; + type When = 'before' | 'after' | 'unindexed'; const calls: string[] = []; - const faults: Array<{ seam: Seam; when: 'before' | 'after'; matches: (calls: string[]) => boolean; times: number }> = []; - const through = (seam: Seam, label: string, call: () => T): T => { + const hidden = new Set(); + const faults: Array<{ seam: Seam; when: When; matches: (calls: string[]) => boolean; times: number }> = []; + const through = (seam: Seam, label: string, call: () => T, hide?: (result: T) => void): T => { const fault = faults.find((candidate) => candidate.seam === seam && candidate.times > 0 && candidate.matches(calls)); calls.push(label); if (fault) fault.times--; if (fault?.when === 'before') throw new Error(`injected ${seam} failure`); const result = call(); + if (fault?.when === 'unindexed') { + hide?.(result); + throw new Error(`injected ${seam} failure after writing`); + } if (fault?.when === 'after') throw new Error(`injected ${seam} failure`); return result; }; @@ -1193,11 +1222,14 @@ function journalSeams(store: JsStore) { get(target, prop) { if (prop === 'sync') return () => through('sync', 'sync', () => target.sync()); if (prop === 'appendJson') { - return (type: string, payload: unknown) => through('append', `append ${type}`, () => target.appendJson(type, payload)); + return (type: string, payload: unknown) => through( + 'append', `append ${type}`, () => target.appendJson(type, payload), (record) => hidden.add(record.id), + ); } if (prop === 'getRecordIdsByType') { - return (type: string) => through('list', `list ${type}`, () => target.getRecordIdsByType(type)); + return (type: string) => through('list', `list ${type}`, () => target.getRecordIdsByType(type).filter((id) => !hidden.has(id))); } + if (prop === 'getRecord') return (id: string) => (hidden.has(id) ? null : target.getRecord(id)); const value = Reflect.get(target, prop); return typeof value === 'function' ? value.bind(target) : value; }, @@ -1205,7 +1237,7 @@ function journalSeams(store: JsStore) { return { store: proxy as JsStore, calls, - fail: (seam: Seam, when: 'before' | 'after', matches: (calls: string[]) => boolean, times = 1) => { + fail: (seam: Seam, when: When, matches: (calls: string[]) => boolean, times = 1) => { faults.push({ seam, when, matches, times }); }, }; @@ -1235,7 +1267,7 @@ async function hideOverFaultyJournal(arrange: (seams: ReturnType framework.handleHostCommand('discord', { command: 'hide', agentName: 'cairn', fromMessageId: 'm1', marks: 'all', })); - return { h, seams, result, alerts }; + return { h, seams, outbox, result, alerts }; } test('a hide records its batch only after the removal is synced: the batch can never outlive the change', async () => { @@ -1334,3 +1366,213 @@ test('an applied change\'s batch whose journal cannot be read back after a faile h.cleanup(); } }); + +test('an applied change\'s batch written but out of this view of the store is unresolved, and the journal refuses until reopened', async () => { + // The batch's append writes its record, then Chronicle's periodic sync + // fails: the record is in the store's log, unindexed until a reopen. Read + // back in this process, it would look like nothing was recorded. + const { h, outbox, result, alerts } = await hideOverFaultyJournal((seams) => { + seams.fail('append', 'unindexed', (calls) => calls.includes('remove')); + }); + try { + assert.equal(result.ok, true, 'the hide stands'); + assert.equal(result.markers.status, 'unresolved', 'not a not-scheduled receipt over a batch the next start may deliver'); + assert.match(result.markers.error, /recording the batch failed: injected append failure after writing.*until the store is reopened/); + assert.deepEqual(alerts, ['discord-awareness-unresolved']); + assert.throws(() => outbox.batches(), /can't be read in this process.*until the store is reopened/); + assert.throws(() => outbox.claimDispatch('discord'), /until the store is reopened/); + // The next start reads the batch and, as the receipt says it may, completes it. + const fresh = new DiscordAwarenessOutbox(h.store); + const [batch] = fresh.batches(); + assert.equal(batch.id, result.markers.batchId); + assert.equal(batch.status, 'prepared'); + assert.deepEqual(fresh.recoverAtStartup('main').activated, [batch.id]); + } finally { + h.cleanup(); + } +}); + +test('an activation written but out of this view is unresolved: nothing retires it, and the next start delivers it', withJournal((_outbox, h) => { + const seams = journalSeams(h.store); + const outbox = new DiscordAwarenessOutbox(seams.store); + const batch = outbox.prepare({ agentName: 'cairn', sourceBranch: 'main', targetBranch: 'rollback/cairn/1', refs: [ref('m1')], scope: 'all' })!; + seams.fail('append', 'unindexed', () => true); + const settled = outbox.settleActivation(batch.id); + assert.equal(settled.status, 'unresolved'); + assert.match((settled as { error: string }).error, /injected append failure after writing; retiring it also failed: .*until the store is reopened/); + assert.throws(() => outbox.pendingDispatches('discord'), /until the store is reopened/); + const fresh = new DiscordAwarenessOutbox(h.store); + assert.equal(fresh.batches()[0].status, 'active', 'the activation reached the store, and no retirement follows it'); + assert.deepEqual(fresh.pendingDispatches('discord').map((d) => d.key.messageId), ['m1']); +})); + +test('a cancel written but out of this view: this process sends nothing more, and the next start reads the cancel', withJournal((_outbox, h) => { + const seams = journalSeams(h.store); + const outbox = new DiscordAwarenessOutbox(seams.store); + const batch = activeBatch(outbox, [ref('m1'), ref('m2')]); + seams.fail('append', 'unindexed', () => true); + assert.throws(() => outbox.cancel(batch.id), /injected append failure after writing/); + assert.throws(() => outbox.claimDispatch('discord'), /until the store is reopened/, 'nothing is claimed from a view that may lack the cancel'); + assert.throws(() => outbox.view(), /until the store is reopened/); + const fresh = new DiscordAwarenessOutbox(h.store); + assert.ok(fresh.batches()[0].cancelled); + assert.equal(fresh.pendingDispatches('discord').length, 0); +})); + +test('a legacy import written but out of this view is not imported again in this process', withJournal((_outbox, h) => { + mkdirSync(join(h.dir, 'recovery'), { recursive: true }); + writeFileSync(h.legacyPath, JSON.stringify({ + version: 2, + batches: [{ + id: 'old', status: 'active', agentName: 'resident', sourceBranch: 'main', targetBranch: 'rollback/resident/1', + emoji: '💤', createdAt: 1, refs: [{ ...ref('a1'), lastAction: 'add', desired: true, deliveryStatus: 'pending', attempts: 2, lastError: 'did not respond' }], + }], + })); + const seams = journalSeams(h.store); + const outbox = new DiscordAwarenessOutbox(seams.store, { legacyPath: h.legacyPath }); + seams.fail('append', 'unindexed', () => true); + assert.throws(() => outbox.batches(), /injected append failure after writing/); + assert.throws(() => outbox.batches(), /until the store is reopened/, 'no second import is written'); + assert.equal(h.store.getRecordIdsByType(DISCORD_AWARENESS_RECORD_TYPE).length, 1); + // The next start reads the one import, and the file it names isn't imported again. + const fresh = h.reopen(); + const [view] = batches(fresh); + assert.equal(view.id, 'old'); + assert.equal(view.legacy?.entries, 1); + assert.equal(h.store.getRecordIdsByType(DISCORD_AWARENESS_RECORD_TYPE).length, 1); +})); + +test('a prepare whose write failed says what the journal may hold, and retires a batch it can see', withJournal((_outbox, h) => { + const seams = journalSeams(h.store); + const outbox = new DiscordAwarenessOutbox(seams.store); + outbox.batches(); + const input = (n: number) => ({ agentName: 'cairn', sourceBranch: 'main', targetBranch: `rollback/cairn/${n}`, refs: [ref('m1')], scope: 'all' as const }); + // The record reached this view and its durability barrier failed: retired at once. + seams.fail('sync', 'before', rightAfterAppend); + assert.throws(() => outbox.prepare(input(1)), /recording awareness batch \S+ failed: injected sync failure; it reached the journal and is retired/); + assert.deepEqual(new DiscordAwarenessOutbox(h.store).batches().map((b) => b.status), ['discarded']); + // Written but out of this view: the next start holds it, and the error says to cancel it. + seams.fail('append', 'unindexed', () => true); + assert.throws(() => outbox.prepare(input(2)), /may be in the store, unseen until the store is reopened, and if so the next start holds it: cancel it then, not release it/); + const fresh = new DiscordAwarenessOutbox(h.store); + assert.equal(fresh.recoverAtStartup('main').held.length, 1); +})); + +test('an operation\'s status says first what Discord may have: an add on the wire that a retract superseded is dispatching, then unknown', withJournal((outbox, h) => { + const batch = activeBatch(outbox, [ref('m1')]); + const claimed = outbox.claimDispatch('discord')!; + outbox.retract(batch.id); + const add = (journal: DiscordAwarenessOutbox) => journal.operations().find((op) => op.action === 'add')!; + assert.ok(add(outbox).cancelled, 'the retract retired its later sends'); + assert.equal(outbox.operationStatus(add(outbox)), 'dispatching', 'its request is on the wire'); + outbox.recordOutcome(claimed.attempts, 'unknown'); + assert.equal(outbox.operationStatus(add(outbox)), 'unknown', 'never answered: it may have landed'); + assert.equal(batches(outbox)[0].adds.unknown, 1); + assert.equal(batches(outbox)[0].adds.cancelled, 0); + const restarted = h.reopen(); + assert.equal(restarted.operationStatus(add(restarted)), 'unknown'); + // An add retired before it was ever sent is just cancelled. + const second = activeBatch(restarted, [ref('m2')]); + restarted.cancel(second.id); + const unsent = restarted.operations().find((op) => op.key.messageId === 'm2')!; + assert.equal(restarted.operationStatus(unsent), 'cancelled'); +})); + +test('an operation is not sent again after three attempts that reached the server unconfirmed; one refused before it left does not count', withJournal((outbox) => { + activeBatch(outbox, [ref('m1'), ref('m2'), ref('m3')]); + const op = (messageId: string) => outbox.operations().find((o) => o.key.messageId === messageId)!; + const due = () => outbox.pendingDispatches('discord').map((d) => d.key.messageId).sort(); + // m1: a refusal the pattern doesn't know, three times. + assert.equal(answer(outbox, 'm1', 'failed', { error: 'Internal Server Error' }).ended, 0); + assert.equal(outbox.operationStatus(op('m1')), 'requested'); + assert.equal(answer(outbox, 'm1', 'failed', { error: 'Internal Server Error' }).ended, 0); + assert.equal(answer(outbox, 'm1', 'failed', { error: 'Internal Server Error' }).ended, 1, 'the third ends it'); + assert.equal(outbox.operationStatus(op('m1')), 'failed'); + // m2: twice refused before it left (the route was down), then three unanswered requests. + answer(outbox, 'm2', 'not-sent'); + answer(outbox, 'm2', 'not-sent'); + assert.equal(answer(outbox, 'm2', 'unknown').ended, 0); + assert.equal(answer(outbox, 'm2', 'unknown').ended, 0); + assert.equal(answer(outbox, 'm2', 'unknown').ended, 1); + assert.equal(outbox.operationStatus(op('m2')), 'unknown', 'it may have landed'); + // m3: refused for good at once. + assert.equal(answer(outbox, 'm3', 'failed', { permanent: true, error: 'Unknown Message' }).ended, 1); + assert.deepEqual(due(), [], 'none of them is sent again'); + assert.equal(op('m2').attempts.length, 5); +})); + +test('Discord refusals that no retry can change are permanent', () => { + for (const text of ['Unknown Message', 'Unknown Channel', 'Unknown Emoji', 'Missing Access', 'Missing Permissions', + 'Maximum number of reactions reached (20)', 'Reaction was blocked']) { + assert.equal(isPermanentDiscordReactionFailure(text), true, text); + } + for (const text of ['Internal Server Error', 'You are being rate limited.', 'did not respond within 10000ms']) { + assert.equal(isPermanentDiscordReactionFailure(text), false, text); + } +}); + +test('a checkpoint from before authorization order restores its acts as older than anything after it', withJournal((outbox, h) => { + const batch = outbox.prepare({ agentName: 'cairn', sourceBranch: 'main', targetBranch: 'rollback/cairn/1', refs: [ref('m1')], scope: 'all' })!; + // The earliest checkpointing build wrote batches and operations with no + // authorization, and no position. + const earlier = new RecordJournal<{ records: unknown[] }, unknown>(h.store, { type: DISCORD_AWARENESS_RECORD_TYPE }); + earlier.load(); + const { authorization: _authorization, ...unauthorized } = outbox.batches()[0]; + earlier.checkpoint({ batches: [unauthorized], ops: [], legacy: [], importedSources: [] }); + const restored = h.reopen(); + // A retract after the restore is the later act: the batch's activation + // (its surgery's earlier choice) is recorded superseded and never sent. + restored.retract('all'); + assert.equal(restored.activate(batch.id), 0); + assert.ok(restored.operations().filter((op) => op.action === 'add').every((op) => op.cancelled?.reason === 'superseded')); + assert.deepEqual(restored.pendingDispatches('discord').map((d) => d.action), ['remove']); +})); + +test('an operator\'s request the journal won\'t carry out is a refusal; a journal failure is not', withJournal((_outbox, h) => { + const seams = journalSeams(h.store); + const outbox = new DiscordAwarenessOutbox(seams.store); + const batch = activeBatch(outbox, [ref('m1')]); + assert.throws(() => outbox.cancel('no-such-batch'), DiscordAwarenessRefusal); + assert.throws(() => outbox.retract('no-such-batch'), DiscordAwarenessRefusal); + assert.throws(() => outbox.release(batch.id), (error) => error instanceof DiscordAwarenessRefusal && /is active, not held/.test(error.message)); + seams.fail('sync', 'before', () => true); + assert.throws(() => outbox.cancel(batch.id), (error) => !(error instanceof DiscordAwarenessRefusal) && /injected sync failure/.test((error as Error).message)); +})); + +test('a version 1 ledger\'s import is recorded as v1', withJournal((_outbox, h) => { + mkdirSync(join(h.dir, 'recovery'), { recursive: true }); + writeFileSync(h.legacyPath, JSON.stringify({ + version: 1, + batches: [{ id: 'v1-batch', status: 'pending', agentName: 'resident', sourceBranch: 'main', targetBranch: 'rollback/resident/1', emoji: '💤', createdAt: 1, refs: [ref('a1')] }], + })); + h.reopen().batches(); + const records = h.store.getRecordIdsByType(DISCORD_AWARENESS_RECORD_TYPE) + .flatMap((id) => (JSON.parse(Buffer.from(h.store.getRecord(id)!.payload).toString('utf-8')) as { records: Array<{ t: string; source?: string }> }).records); + assert.deepEqual(records.filter((record) => record.t === 'import-complete').map((record) => record.source), ['v1']); +})); + +test('a sync for one server drains only that server', async () => { + const h = storeHarness(); + try { + activeBatch(h.outbox, [ref('m1'), { serverId: 'discord-b', channelId: 'discord:g2:c1', messageId: 'n1' }]); + const calls: string[] = []; + const connection = (id: string) => ({ + isConnected: true, + sendToolsCallWithDeadline: async (_name: string, args: Record) => { + calls.push(`${id}:${args.messageId}`); + return { content: [{ type: 'text', text: 'Reaction added' }] }; + }, + }); + const framework = Object.create(AgentFramework.prototype) as any; + framework.discordAwarenessOutbox = h.outbox; + framework.discordAwarenessDrains = new Map(); + framework.discordAwarenessDeadlineMs = 1000; + framework.mcplServerRegistry = { getServer: (id: string) => connection(id) }; + await muted(() => framework.syncDiscordAwarenessMarkers('discord-b')); + assert.deepEqual(calls, ['discord-b:n1']); + await muted(() => framework.syncDiscordAwarenessMarkers()); + assert.deepEqual(calls, ['discord-b:n1', 'discord:m1']); + } finally { + h.cleanup(); + } +}); diff --git a/test/surgery-marks-receipt.test.ts b/test/surgery-marks-receipt.test.ts index d7b058f24..8eae9c1d6 100644 --- a/test/surgery-marks-receipt.test.ts +++ b/test/surgery-marks-receipt.test.ts @@ -25,7 +25,7 @@ import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { JsStore } from '@animalabs/chronicle'; -import { AgentFramework, DiscordAwarenessOutbox } from '../src/index.js'; +import { AgentFramework, DiscordAwarenessOutbox, OperatorActionError } from '../src/index.js'; import type { OperatorLogEntry } from '../src/index.js'; import { MockMembrane } from './helpers/mock-membrane.js'; @@ -133,6 +133,64 @@ describe('surgery and awareness marks', () => { rmSync(dir, { recursive: true, force: true }); }); + it('a marks value that is not a choice is refused before anything changes', async () => { + await start(false); + const { tail, removed } = seed(3); + const branch = cm().currentBranch().name; + const invalid = (error: unknown) => error instanceof OperatorActionError && error.code === 'invalid' + && /marks must be 'none' or \{ scope: 'addressed' \| 'all', refs\? \}/.test(error.message); + // A bare scope (host/command's vocabulary), a misspelled scope, refs that + // aren't Discord addresses: each would otherwise mark every message. + for (const marks of ['addressed', { scope: 'adressed' }, { scope: 'all', refs: ['amb-0'] }]) { + await assert.rejects(framework.rollbackToMessage('resident', { messageId: tail, marks: marks as never }), invalid); + await assert.rejects(framework.suppressMessages('resident', { messageIds: [removed[0]], marks: marks as never }), invalid); + } + assert.equal(cm().currentBranch().name, branch); + assert.equal(cm().getAllMessages().length, 5); + assert.equal(outbox().batches().length, 0); + }); + + it('a surgery whose marker batch cannot be recorded does not start, and says so', async () => { + await start(false); + const { tail } = seed(2); + const branch = cm().currentBranch().name; + const original = DiscordAwarenessOutbox.prototype.prepare; + DiscordAwarenessOutbox.prototype.prepare = function () { + throw new Error('recording awareness batch b1 failed: disk full'); + }; + try { + await assert.rejects( + framework.rollbackToMessage('resident', { messageId: tail, marks: { scope: 'all' } }), + (error: unknown) => error instanceof OperatorActionError && error.code === 'failed' + && /^Rollback not started, nothing changed: recording awareness batch b1 failed: disk full/.test(error.message), + ); + } finally { + DiscordAwarenessOutbox.prototype.prepare = original; + } + assert.equal(cm().currentBranch().name, branch); + assert.equal(cm().getAllMessages().length, 4); + }); + + it('an operator control the journal refuses is invalid; one it fails to record is failed', async () => { + await start(false); + assert.throws( + () => framework.cancelDiscordAwareness('no-such-batch'), + (error: unknown) => error instanceof OperatorActionError && error.code === 'invalid', + ); + const original = DiscordAwarenessOutbox.prototype.retract; + DiscordAwarenessOutbox.prototype.retract = function () { + throw new Error('disk full'); + }; + try { + assert.throws( + () => framework.retractDiscordAwareness('all'), + (error: unknown) => error instanceof OperatorActionError && error.code === 'failed' && /disk full/.test(error.message), + ); + } finally { + DiscordAwarenessOutbox.prototype.retract = original; + } + }); + it('a rollback is local by default: no reaction, and the receipt counts what stayed unmarked', async () => { await start(true); const { tail } = seed(5, [3]); From d225da31b4dc5ced88720f40912b5acdbd6a8166 Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Mon, 12 Oct 2026 12:01:21 -0400 Subject: [PATCH 0130/1112] fix(mcpl): a policy exchange opens only its own transport's planes; held requests and answers never cross a transport boundary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit From a review of the whole change. - A transport that drops during a reconnect's §5.3 exchange. The close bumps transportEpoch, so the in-flight marker stopped holding the connection against other openers (a resume, startup's open of every connection), and the exchange's own completion, its request rejected and swallowed, opened both planes on the closed connection and flushed its buffer. A policy answer that arrived just before the drop could also establish its grant after the boundary reset it. Now readyMcplPlanes leaves a connection that isn't connected as it is; the exchange (settlePolicyExchange) opens the planes only if the transport it ran on is still the connection's and connected; and registerMcplServerFeatures applies an answer only on the transport it was asked on. An unanswered or refused exchange on a live transport still opens the planes with the grant empty (MCP-only), as at initial connect. - A runtime connect awaited its exchange unmarked, so a resume during it opened the new connection's planes before its grant existed, and its channels/register was refused against the empty grant, the case the reconnect fix was for. Connect and reconnect now run the exchange through one helper, settlePolicyExchange, which marks it in flight synchronously for the transport it runs on. - Requests across a boundary. Items buffered while a plane was closed survived the transport that sent them: a held server request was admitted against the next transport's grant, and its responder wrote the old id to the next transport, where a new server process may match it to a request of its own. The boundary now drops buffered server requests (their sender is gone, and logs how many); responder-less events stay, to be admitted against the next grant. A responder answers only on the transport its request came on, so a handler that answers after a reconnect writes nothing. - A close() during a reconnect's handshake was undone: attemptReconnect checked reconnectEnabled before awaiting the handshake and adopted the new transport after it, setting closed back to false and emitting 'reconnect'. It rechecks after the await and closes the new transport instead. - reconnectAfterFailure had had no caller since the awareness recycle was removed, and its doc described that recycle: removed. readyMcplPlanes' doc says a connection whose exchange is in flight is held whole, not only its data plane. Tests: - mcpl-reconnect-events: a reconnected transport's data traffic is held until the grant exists and the planes open (removing the boundary's data close drops it against the empty grant); a server request held across a boundary is dropped, and an answer after the reconnect isn't written; a close during the handshake closes the new transport and emits nothing. - mcpl-reconnect-registration: the staged-startup test waits for the policy-held event instead of 100 ms and checks that startup's open of every connection ran while the retry's exchange was in flight; a transport that drops during its exchange stays closed and its late answer grants nothing; a reconnect on a quiesced host opens only the control plane until resume; a runtime connect's exchange holds its planes against a resume. - mcpl-reconnect-reregistration: its fake connection is a live transport. --- src/framework.ts | 85 +++++++++---- src/mcpl/server-connection.ts | 80 +++++++----- test/mcpl-reconnect-events.test.ts | 90 ++++++++++++++ test/mcpl-reconnect-registration.test.ts | 137 ++++++++++++++++++--- test/mcpl-reconnect-reregistration.test.ts | 5 + 5 files changed, 323 insertions(+), 74 deletions(-) diff --git a/src/framework.ts b/src/framework.ts index 90c79c1e4..87f78fa34 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -13980,24 +13980,28 @@ export class AgentFramework { } /** - * Per connection, the transport epoch whose §5.3 policy exchange a - * reconnect has in flight. Until it settles, that transport's traffic must - * wait for its grant, so its planes stay closed whoever else opens planes - * meanwhile (startup's staged open, resume()); the reconnect opens them. + * Per connection, the transport epoch whose §5.3 policy exchange is in + * flight (a connect's or a reconnect's). Until it settles, that transport's + * traffic must wait for its grant, so its planes stay closed whoever else + * opens planes meanwhile (startup's staged open, resume()); the exchange's + * own caller opens them (settlePolicyExchange). */ private readonly mcplPolicyExchanges = new WeakMap(); /** - * Open MCPL connections' planes (every connection when none is given), - * except a connection whose reconnect's policy exchange is still in flight - * (see mcplPolicyExchanges). A quiesced host (issue #122) opens only - * control planes, so data planes stay held for the whole maintenance - * window; resume() calls this again once the flag is cleared. Control - * planes always come up, since host/command (and with it resume) rides the - * control plane. + * Open MCPL connections' planes (every connection when none is given). A + * connection that isn't connected is left as it is: its buffered traffic + * belongs to a transport that is gone, and the next transport's exchange + * opens it. So is one whose policy exchange is in flight on its current + * transport (see mcplPolicyExchanges): both its planes stay held. A + * quiesced host (issue #122) opens only control planes, so data planes + * stay held for the whole maintenance window; resume() calls this again + * once the flag is cleared. Otherwise a control plane comes up even while + * quiesced, since host/command (and with it resume) rides it. */ private readyMcplPlanes(connections?: McplServerConnection[]): void { for (const connection of connections ?? this.mcplServerRegistry?.getAllServers() ?? []) { + if (!connection.isConnected) continue; const exchange = this.mcplPolicyExchanges.get(connection); if (exchange !== undefined && exchange === connection.transportEpoch) continue; if (this.quiesced) connection.readyControlPlane(); @@ -14005,6 +14009,30 @@ export class AgentFramework { } } + /** + * Run a connection's §5.3 policy exchange with its planes held: it is + * marked in flight for the transport it runs on, so no other opener + * releases that transport's traffic before its grant exists. Returns + * whether the caller may open the planes: the exchange settled (answered or + * not) on the transport that is still the connection's, and it is + * connected. A transport that dropped meanwhile is left closed; the next + * one's exchange opens the planes. + */ + private async settlePolicyExchange( + config: import('./mcpl/types.js').McplServerConfig, + connection: McplServerConnection, + ): Promise { + const epoch = connection.transportEpoch; + this.mcplPolicyExchanges.set(connection, epoch); + try { + await this.registerMcplServerFeatures(config, connection); + } finally { + // A newer transport's exchange may have taken the marker over. + if (this.mcplPolicyExchanges.get(connection) === epoch) this.mcplPolicyExchanges.delete(connection); + } + return connection.transportEpoch === epoch && connection.isConnected; + } + /** * Connect a single MCPL server: register routing entries, open the * connection, wire events, and initialize feature sets / scopes / @@ -14042,13 +14070,14 @@ export class AgentFramework { // §5.3 initial policy handshake with both planes still closed (PR #79 // review blocker 3: an early control flush released channels-register/ // changed before any grant existed). The grant is established (or - // knowingly left empty) before ANY buffered traffic flows. - await this.registerMcplServerFeatures(config, connection); + // knowingly left empty) before ANY buffered traffic flows, whoever else + // opens planes meanwhile (a resume during a runtime connect). + const current = await this.settlePolicyExchange(config, connection); this.emitTrace({ type: 'module:added', moduleName: `mcpl:${config.id}` }); // Staged startup opens every connection together (initializeMcpl). if (staged) return; - this.readyMcplPlanes([connection]); + if (current) this.readyMcplPlanes([connection]); // A connection can make queued awareness work deliverable; it never // waits on it. this.deliverDiscordAwarenessInBackground('connect', 'host'); @@ -14255,6 +14284,8 @@ export class AgentFramework { connection: McplServerConnection, ): Promise { if (!connection.capabilities) return; + // A grant belongs to the transport it was negotiated on. + const epoch = connection.transportEpoch; // §5.4: compute the effective grant from the (config-masked) // advertisement. Not yet active — expansion activates only after the @@ -14284,6 +14315,12 @@ export class AgentFramework { try { const receipt = await connection.sendFeatureSetsUpdateRequest(updateParams); + if (connection.transportEpoch !== epoch) { + // The transport this exchange was for is gone: its answer grants + // nothing to the next one, which negotiates its own. + console.error(`[mcpl] ${config.id}: policy answer arrived after its transport closed; not applied`); + return; + } if (receipt && receipt.accepted === false) { const fallback = receipt.fallback ?? 'mcp-only'; console.error( @@ -14689,20 +14726,18 @@ export class AgentFramework { connection.on('reconnect', (info?: { attempts?: number }) => { // The connection closed both planes before wiring the fresh transport, // as at initial connect. Re-establish the grant first: the §5.3 policy - // Request (whose response is never buffered) must be answered before - // any plane opens, otherwise the first post-reconnect traffic, control + // Request (whose response is never buffered) must settle before any + // plane opens, otherwise the first post-reconnect traffic, control // included (the server's channels/register), lands while the grant is // still empty and is rejected fail-closed instead of delivered. The - // exchange is marked in flight for this transport, so no other opener - // (startup's staged open, resume) releases the traffic early. + // exchange is marked in flight for this transport synchronously, here, + // so no other opener (startup's staged open, resume) releases the + // traffic early (settlePolicyExchange). const epoch = connection.transportEpoch; - this.mcplPolicyExchanges.set(connection, epoch); void (async () => { try { const config = this.mcplServerConfigs.get(connection.id); - if (config) { - await this.registerMcplServerFeatures(config, connection); - } + if (config) await this.settlePolicyExchange(config, connection); } catch (error) { console.error( `MCPL server "${connection.id}" re-registration after reconnect failed:`, @@ -14711,9 +14746,9 @@ export class AgentFramework { } // Settled, granted or not (an unanswered or refused policy leaves the // grant empty and the connection MCP-only, as at initial connect): - // open the planes, unless a newer transport's exchange has taken over. - if (this.mcplPolicyExchanges.get(connection) === epoch) { - this.mcplPolicyExchanges.delete(connection); + // open the planes if this transport is still the connection's. One + // that dropped meanwhile stays closed; the next reconnect opens it. + if (connection.transportEpoch === epoch && connection.isConnected) { this.readyMcplPlanes([connection]); } this.handleToolsListChanged(connection.id); diff --git a/src/mcpl/server-connection.ts b/src/mcpl/server-connection.ts index 8dab9ac99..6e186055f 100644 --- a/src/mcpl/server-connection.ts +++ b/src/mcpl/server-connection.ts @@ -211,6 +211,18 @@ export class McplServerConnection extends EventEmitter { this.grant = CapabilityGrant.empty(); this.policyEstablished = false; this.transportEpoch++; + // A server request held in the buffer came on the transport that is now + // gone: its sender can't be answered, and admitting it against the next + // transport's grant, or answering its id there, would act for a request + // the next server never made. Events without a responder stay, to be + // admitted against the next grant. + const held = this.bufferedEvents.length; + this.bufferedEvents = this.bufferedEvents.filter((item) => !McplServerConnection.hasResponder(item)); + if (this.bufferedEvents.length < held) { + console.error( + `[mcpl] ${this.id}: dropped ${held - this.bufferedEvents.length} held request(s) from a transport that closed`, + ); + } // RFC-007 §6.5: a filter is per connection epoch — after a reconnect the // server sends it again, and until then receives metadata only. this.toolObserveFilter = null; @@ -416,10 +428,7 @@ export class McplServerConnection extends EventEmitter { // strand that call forever (the server timing out and reconnecting is the // accepted bound there, and reconnect re-runs the barrier funnel). if (this.flushDepth === 0 && this.bufferedEvents.length >= McplServerConnection.MAX_BUFFERED_EVENTS) { - const hasResponder = (item: { args: unknown[] }): boolean => { - const responder = item.args[1] as { respond?: unknown } | undefined; - return typeof responder?.respond === 'function'; - }; + const hasResponder = McplServerConnection.hasResponder; if (!hasResponder({ args })) { this.droppedBufferedEvents++; if (this.droppedBufferedEvents % 100 === 1) { @@ -441,6 +450,12 @@ export class McplServerConnection extends EventEmitter { return true; } + /** A buffered server→host request: it carries a responder its server waits on. */ + private static hasResponder(item: { args: unknown[] }): boolean { + const responder = item.args[1] as { respond?: unknown } | undefined; + return typeof responder?.respond === 'function'; + } + /** Event-name form of METHOD_TO_REQUIRED_CAPABILITY, for admission-time * enforcement in emit(). */ private static readonly EVENT_TO_REQUIRED_CAPABILITY: Record = { @@ -889,27 +904,6 @@ export class McplServerConnection extends EventEmitter { this.emit('close'); } - /** - * Tear down a live transport after an awareness-accounting failure without - * disabling configured auto-reconnect. The normal transport close handler - * rejects requests, emits lifecycle state, and schedules the next bounded - * reconnect attempt. With reconnect disabled this leaves the server closed. - */ - async reconnectAfterFailure(): Promise { - this.pauseDataPlane(); - if (this.closed) { - this.scheduleReconnect(); - return; - } - if (this.transport) { - await this.transport.close(); - return; - } - this.closed = true; - this.emit('close'); - this.scheduleReconnect(); - } - /** * Schedule a background reconnection attempt. * @@ -952,6 +946,12 @@ export class McplServerConnection extends EventEmitter { this.config, this.hostCapabilities, ); + if (!this.reconnectEnabled) { + // close() ran while the handshake was in flight: the host is done + // with this server, so the new transport is closed, not adopted. + await transport.close().catch(() => {}); + return; + } // Close both planes before the new transport can emit, as at initial // connect: its traffic is admitted against the grant, which this @@ -1188,12 +1188,15 @@ export class McplServerConnection extends EventEmitter { if (eventName) { // Emit the typed event with params and (for requests) a respond callback if (request.id != null) { - // Server expects a response — provide a respond helper + // Server expects a response — provide a respond helper, bound to + // the transport the request came on: an answer after a reconnect + // would reach a server that never asked it. + const epoch = this.transportEpoch; this.emit(eventName, request.params, { id: request.id, - respond: (result: unknown) => this.sendResponse(request.id!, result), + respond: (result: unknown) => this.sendResponse(request.id!, result, epoch), respondError: (code: number, message: string, data?: unknown) => - this.sendErrorResponse(request.id!, code, message, data), + this.sendErrorResponse(request.id!, code, message, data, epoch), }); } else { // Notification — no response expected @@ -1263,17 +1266,30 @@ export class McplServerConnection extends EventEmitter { /** * Send a successful JSON-RPC response back to the server. */ - private sendResponse(id: string | number, result: unknown): void { - if (this.closed) return; + private sendResponse(id: string | number, result: unknown, epoch = this.transportEpoch): void { + if (this.closed || !this.sameTransport(id, epoch)) return; const response: JsonRpcResponse = { jsonrpc: '2.0', id, result }; this.transport?.writeLine(JSON.stringify(response)); } + /** Whether a response to a request received on `epoch` may go out now. */ + private sameTransport(id: string | number, epoch: number): boolean { + if (epoch === this.transportEpoch) return true; + console.error(`[mcpl] ${this.id}: not answering request ${String(id)}: the transport it came on has closed`); + return false; + } + /** * Send a JSON-RPC error response back to the server. */ - private sendErrorResponse(id: string | number, code: number, message: string, data?: unknown): void { - if (this.closed) return; + private sendErrorResponse( + id: string | number, + code: number, + message: string, + data?: unknown, + epoch = this.transportEpoch, + ): void { + if (this.closed || !this.sameTransport(id, epoch)) return; const response: JsonRpcResponse = { jsonrpc: '2.0', id, diff --git a/test/mcpl-reconnect-events.test.ts b/test/mcpl-reconnect-events.test.ts index 5a58b2abc..07e268dcc 100644 --- a/test/mcpl-reconnect-events.test.ts +++ b/test/mcpl-reconnect-events.test.ts @@ -191,4 +191,94 @@ describe('McplServerConnection — reconnect lifecycle events', () => { connection.ready(); assert.deepStrictEqual(controlEvents, ['tools-list-changed', 'tools-list-changed'], 'and released when the planes open'); }); + + it('holds a reconnected transport\'s data traffic until the host has its grant and opens the planes', async () => { + writeFileSync(FLAG_FILE, ''); + connection = await McplServerConnection.connect(makeConfig(), HOST_CAPS); + const pushes: unknown[] = []; + connection.on('push-event', (params: unknown) => pushes.push(params)); + let reconnected = false; + connection.on('reconnect', () => { reconnected = true; }); + connection.ready(); + connection.sendToolsList().catch(() => {}); + await waitFor(() => reconnected, 5000, 'reconnect after crash'); + // Admitted now, it would meet the empty grant and be refused; held, it + // meets the grant the host establishes. + connection.emit('push-event', { n: 1 }); + assert.deepStrictEqual(pushes, [], 'data traffic is held after the boundary'); + connection.establishGrant(new CapabilityGrant(new Set(['pushEvents']), [])); + connection.ready(); + assert.deepStrictEqual(pushes, [{ n: 1 }], 'and delivered against the new grant'); + }); + + it('drops a server request held across a transport boundary, and never answers one on the next transport', async () => { + writeFileSync(FLAG_FILE, ''); + connection = await McplServerConnection.connect(makeConfig(), HOST_CAPS); + connection.establishGrant(new CapabilityGrant(new Set(['inferenceRequest']), [])); + const requests: Array<{ id: unknown; respond: (result: unknown) => void }> = []; + connection.on('inference-request', (_params: unknown, responder: { id: unknown; respond: (result: unknown) => void }) => { + requests.push(responder); + }); + let reconnected = 0; + connection.on('reconnect', () => { reconnected++; }); + connection.ready(); + const serverSends = (message: Record) => + (connection as unknown as { transport: { emit(event: string, line: string): void } }).transport + .emit('line', JSON.stringify({ jsonrpc: '2.0', ...message })); + // One request is admitted and its handler still holds the responder; a + // second is held while the data plane is paused (a quiesce). + serverSends({ id: 5, method: 'inference/request', params: { n: 5 } }); + assert.equal(requests.length, 1); + connection.pauseDataPlane(); + serverSends({ id: 7, method: 'inference/request', params: { n: 7 } }); + assert.equal(requests.length, 1, 'held while paused'); + + connection.sendToolsList().catch(() => {}); + await waitFor(() => reconnected === 1, 5000, 'reconnect after crash'); + const written: string[] = []; + const transport = (connection as unknown as { transport: { writeLine(line: string): void } }).transport; + const writeLine = transport.writeLine.bind(transport); + transport.writeLine = (line: string) => { written.push(line); writeLine(line); }; + + // The handler answers its request after the reconnect: not written. + requests[0]!.respond({ ok: true }); + connection.establishGrant(new CapabilityGrant(new Set(['inferenceRequest']), [])); + connection.ready(); + assert.equal(requests.length, 1, 'the held request from the closed transport is not admitted on the new one'); + assert.deepStrictEqual(written.filter((line) => /"id":(5|7)\b/.test(line)), [], 'nothing answers the closed transport\'s requests'); + }); + + it('a close during a reconnect\'s handshake closes the new transport instead of adopting it', async () => { + // Flag file absent: the initial connect fails and we get a stub that + // reconnects in the background. + connection = await McplServerConnection.connectWithReconnect(makeConfig({ reconnectIntervalMs: 20 }), HOST_CAPS); + const statics = McplServerConnection as unknown as { handshake: (...args: unknown[]) => Promise }; + const handshake = statics.handshake; + let release!: (value: unknown) => void; + let started = false; + let closedNew = false; + statics.handshake = () => { + started = true; + return new Promise((resolve) => { release = resolve; }); + }; + try { + await waitFor(() => started, 5000, 'a reconnect handshake in flight'); + const reconnected: unknown[] = []; + connection.on('reconnect', (p: unknown) => reconnected.push(p)); + await connection.close(); + release({ + transport: { close: async () => { closedNew = true; }, on: () => {}, writeLine: () => {} }, + capabilities: { version: '0.5' }, + droppedCapabilities: [], + mcpToolsAdvertised: false, + }); + await waitFor(() => closedNew, 2000, 'the new transport closed'); + assert.deepStrictEqual(reconnected, [], 'nothing is emitted for it'); + assert.strictEqual(connection.isConnected, false); + assert.strictEqual(connection.willReconnect, false); + } finally { + statics.handshake = handshake; + connection = null; + } + }); }); diff --git a/test/mcpl-reconnect-registration.test.ts b/test/mcpl-reconnect-registration.test.ts index 5bc7d40cb..4563848b4 100644 --- a/test/mcpl-reconnect-registration.test.ts +++ b/test/mcpl-reconnect-registration.test.ts @@ -114,21 +114,36 @@ describe('MCPL reconnect and channel registration', () => { enabledFeatureSets: ['chat'], ...(id === 'retry' ? { reconnect: true, reconnectIntervalMs: 20 } : {}), }); - framework = await AgentFramework.create({ - storePath: join(dir, 'store'), - membrane: new MockMembrane().asMembrane(), - agents: [{ name: 'resident', model: 'test-model', systemPrompt: 'You are a resident.' }], - modules: [], - maintenanceIntervalMs: 0, - mcplServers: [ - server('retry', { FAIL_FIRST_PATH: join(dir, 'failed-once'), POLICY_PENDING_PATH: pending, POLICY_RELEASE_PATH: release }), - server('stager', { INITIALIZE_AFTER_PATH: pending }), - ], - }); + // Record each open of every connection's planes (startup's staged open, + // or a resume), and whether the retry's exchange was in flight then. + const opensOfEvery: boolean[] = []; + const proto = AgentFramework.prototype as unknown as { readyMcplPlanes: (connections?: unknown[]) => void }; + const readyMcplPlanes = proto.readyMcplPlanes; + proto.readyMcplPlanes = function (this: any, connections?: unknown[]) { + if (!connections) { + const retryConnection = this.mcplServerRegistry?.getServer('retry'); + opensOfEvery.push(!!retryConnection && this.mcplPolicyExchanges.get(retryConnection) === retryConnection.transportEpoch); + } + return readyMcplPlanes.call(this, connections); + }; + try { + framework = await AgentFramework.create({ + storePath: join(dir, 'store'), + membrane: new MockMembrane().asMembrane(), + agents: [{ name: 'resident', model: 'test-model', systemPrompt: 'You are a resident.' }], + modules: [], + maintenanceIntervalMs: 0, + mcplServers: [ + server('retry', { FAIL_FIRST_PATH: join(dir, 'failed-once'), POLICY_PENDING_PATH: pending, POLICY_RELEASE_PATH: release }), + server('stager', { INITIALIZE_AFTER_PATH: pending }), + ], + }); + } finally { + proto.readyMcplPlanes = readyMcplPlanes; + } const retry = () => serverEvents(eventsPath).filter((e) => e.role === 'retry'); - // Startup has opened every plane it may; the retry's policy is still held. - await new Promise((resolve) => setTimeout(resolve, 100)); - assert.ok(retry().some((e) => e.event === 'policy-held'), `events: ${JSON.stringify(retry())}`); + await waitFor('the retry\'s policy answer held', () => retry().some((e) => e.event === 'policy-held')); + assert.ok(opensOfEvery.includes(true), `startup opened every connection's planes while the retry's exchange was in flight: ${JSON.stringify(opensOfEvery)}`); assert.equal( retry().find((e) => e.event === 'register-answered'), undefined, @@ -149,15 +164,18 @@ describe('MCPL reconnect: a pending policy exchange holds its planes', () => { function harness() { const answers: Array<(receipt: { accepted: true }) => void> = []; let opened = 0; + let controlOnly = 0; + let granted = 0; const connection = Object.assign(new EventEmitter(), { id: 'srv', transportEpoch: 0, + isConnected: true, capabilities: { version: '0.5', pushEvents: true, featureSets: { chat: { description: 'chat', uses: ['pushEvents'] } } }, mcpToolsAdvertised: false, - establishGrant: () => {}, + establishGrant: () => { granted++; }, sendFeatureSetsUpdateRequest: () => new Promise<{ accepted: true }>((resolve) => { answers.push(resolve); }), ready: () => { opened++; }, - readyControlPlane: () => { opened++; }, + readyControlPlane: () => { controlOnly++; }, }); const fw = Object.create(AgentFramework.prototype) as any; fw.traceListeners = []; @@ -173,7 +191,11 @@ describe('MCPL reconnect: a pending policy exchange holds its planes', () => { connection, answers, opened: () => opened, - reconnect: () => { connection.transportEpoch++; connection.emit('reconnect', { attempts: 1 }); }, + controlOnly: () => controlOnly, + granted: () => granted, + reconnect: () => { connection.transportEpoch++; connection.isConnected = true; connection.emit('reconnect', { attempts: 1 }); }, + /** The transport closes, as its close handler leaves the connection. */ + drop: () => { connection.transportEpoch++; connection.isConnected = false; }, }; } @@ -202,3 +224,84 @@ describe('MCPL reconnect: a pending policy exchange holds its planes', () => { assert.equal(h.opened(), 1); }); }); + +describe('MCPL policy exchanges: a transport that drops, a quiesced host, a runtime connect', () => { + function harness() { + const answers: Array<(receipt: { accepted: true }) => void> = []; + const counts = { opened: 0, controlOnly: 0, granted: 0 }; + const connection = Object.assign(new EventEmitter(), { + id: 'srv', + transportEpoch: 0, + isConnected: true, + capabilities: { version: '0.5', pushEvents: true, featureSets: { chat: { description: 'chat', uses: ['pushEvents'] } } }, + mcpToolsAdvertised: false, + establishGrant: () => { counts.granted++; }, + sendFeatureSetsUpdateRequest: () => new Promise<{ accepted: true }>((resolve) => { answers.push(resolve); }), + ready: () => { counts.opened++; }, + readyControlPlane: () => { counts.controlOnly++; }, + }); + const fw = Object.create(AgentFramework.prototype) as any; + fw.traceListeners = []; + fw.mcplPolicyExchanges = new WeakMap(); + fw.mcplServerConfigs = new Map([['srv', { id: 'srv', command: 'unused', enabledFeatureSets: ['chat'] }]]); + fw.mcplPrefixMap = new Map(); + fw.featureSetManager = new FeatureSetManager(); + fw.checkpointManager = null; + fw.mcplHostCapabilities = { version: '0.5' }; + fw.mcplServerRegistry = { getAllServers: () => [connection], getServer: () => connection, addServer: async () => connection }; + fw.handleToolsListChanged = () => {}; + fw.deliverDiscordAwarenessInBackground = () => {}; + const settle = () => new Promise((resolve) => setImmediate(resolve)); + return { + fw, + connection, + answers, + counts, + settle, + reconnect: () => { connection.transportEpoch++; connection.isConnected = true; connection.emit('reconnect', { attempts: 1 }); }, + drop: () => { connection.transportEpoch++; connection.isConnected = false; }, + }; + } + + it('a transport that drops during its exchange leaves the planes closed, and its late answer grants nothing', async () => { + const h = harness(); + h.fw.wireMcplEvents(h.connection); + h.reconnect(); + h.drop(); + h.fw.readyMcplPlanes(); // a resume while the connection is down + assert.deepEqual(h.counts, { opened: 0, controlOnly: 0, granted: 0 }, 'a closed connection is not opened'); + h.answers[0]({ accepted: true }); // answered just as the transport went + await h.settle(); + assert.deepEqual(h.counts, { opened: 0, controlOnly: 0, granted: 0 }, 'the dead transport\'s exchange opens and grants nothing'); + // The next transport's exchange grants and opens. + h.reconnect(); + h.answers[1]({ accepted: true }); + await h.settle(); + assert.deepEqual(h.counts, { opened: 1, controlOnly: 0, granted: 1 }); + }); + + it('a reconnect on a quiesced host opens only the control plane; resume opens the data plane', async () => { + const h = harness(); + h.fw.wireMcplEvents(h.connection); + h.fw.quiesced = true; + h.reconnect(); + h.answers[0]({ accepted: true }); + await h.settle(); + assert.deepEqual(h.counts, { opened: 0, controlOnly: 1, granted: 1 }, 'data stays held for the maintenance window'); + h.fw.quiesced = false; + h.fw.readyMcplPlanes(); // resume() + assert.equal(h.counts.opened, 1); + }); + + it('a runtime connect\'s exchange holds its planes against a resume until it settles', async () => { + const h = harness(); + const connecting = h.fw.connectMcplServerInternal({ id: 'srv', command: 'unused', enabledFeatureSets: ['chat'] }); + await h.settle(); + assert.equal(h.answers.length, 1, 'its policy Request is in flight'); + h.fw.readyMcplPlanes(); // a resume during the connect + assert.deepEqual(h.counts, { opened: 0, controlOnly: 0, granted: 0 }, 'held while the exchange is in flight'); + h.answers[0]({ accepted: true }); + await connecting; + assert.deepEqual(h.counts, { opened: 1, controlOnly: 0, granted: 1 }, 'opened by the connect once the grant exists'); + }); +}); diff --git a/test/mcpl-reconnect-reregistration.test.ts b/test/mcpl-reconnect-reregistration.test.ts index eb3d20630..0d48867ee 100644 --- a/test/mcpl-reconnect-reregistration.test.ts +++ b/test/mcpl-reconnect-reregistration.test.ts @@ -68,6 +68,11 @@ class FakeConnection extends EventEmitter { // models a server whose initialize carried capabilities.tools. mcpToolsAdvertised = true; policyEstablished = false; + // A live transport (the real connection's view after a reconnect); the + // framework opens planes only on a connected transport, the one its + // exchange ran on. + isConnected = true; + transportEpoch = 0; establishGrant(grant: CapabilityGrant): void { this.grant = grant; this.policyEstablished = true; From a3cf4d24677d6ed1a627e43b9dbe9cd69d44f004 Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Mon, 12 Oct 2026 12:04:27 -0400 Subject: [PATCH 0131/1112] fix(recover): the CLI opens only a store that exists, refuses options its verb ignores, and its failures name what they leave From a review of the whole change. - A store that exists, or nothing. The awareness verbs and the recovery opened the store with openOrCreate, so a mistyped --store created an empty store (and its operator log) there, and a list then reported no batches while a cancel reported "not found". The CLI and createOfflineRecoveryBranch now open an existing store only, refusing a path that holds none with "No Chronicle store could be opened at ", and a recovery on such a path writes no log entry, since the log lives in the store. - --awareness with a recovery's options. The awareness verbs returned before the recovery options were read, so --marks, --branch or an anchor were silently ignored, and an operator could believe a choice applied. The awareness verbs take only --store and --outbox; anything else is refused, named. - A failed recovery's error names the branch it kept for diagnosis, and whether it may be partly suppressed (don't start on it): a generated name appeared nowhere. - A batch the failure kept, because the source branch couldn't be confirmed, is completed by a start on its target branch, where its body is in effect, and held by a start on any other; the comment said startup holds its marks, and the error (live and offline) now says which start does what. - A batch whose retirement failed after the source was restored: the error said only "startup will hold it", while the usage presented release as the next step, which would mark messages that were never removed. The offline and live errors now say the batch stays prepared, that a start on the source holds it, and to cancel it, not release it. The usage says the same under release. - The usage says the operator log is best-effort (a log that can't be written says so on stderr, and the act stands), that a list writes no entry but imports a pre-journal ledger if one is present, as the host's next start would, and that the store must exist. Tests: - recover-cli-operator-log: a list, a control and a recovery against a missing store are refused and create nothing; recovery options with --awareness are refused, with nothing retracted or recorded; the cancel and retract the CLI reports are read back from the journal. - offline-recovery-branch: a restored failure names its kept branch; a failed retirement after a restore says to cancel the batch, and a start on the source holds it; the unconfirmed-source case names the branch and what each start does. --- src/framework.ts | 3 +- src/recovery/offline-branch.ts | 36 ++++++++++++----- src/recovery/recover-cli.ts | 48 ++++++++++++++++++---- test/offline-recovery-branch.test.ts | 33 +++++++++++++-- test/recover-cli-operator-log.test.ts | 58 +++++++++++++++++++++++++++ 5 files changed, 155 insertions(+), 23 deletions(-) diff --git a/src/framework.ts b/src/framework.ts index 87f78fa34..7f7152279 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -6142,7 +6142,8 @@ export class AgentFramework { ): string { if (!batch) return ''; if (this.store.currentBranch().name !== sourceBranch) { - return `; awareness batch ${batch.id} kept, since the source branch could not be confirmed`; + return `; awareness batch ${batch.id} kept, since the source branch could not be confirmed: a start on ` + + `${batch.targetBranch} completes it, and a start on another branch holds it`; } try { this.discordAwarenessOutbox.discard(batch.id); diff --git a/src/recovery/offline-branch.ts b/src/recovery/offline-branch.ts index ef0541e42..158be8af0 100644 --- a/src/recovery/offline-branch.ts +++ b/src/recovery/offline-branch.ts @@ -86,7 +86,13 @@ export async function createOfflineRecoveryBranch( throw new Error('messages must be a positive integer'); } - const store = JsStore.openOrCreate({ path: options.storePath }); + // The store must exist: a mistyped path is refused, never created. + let store: JsStore; + try { + store = JsStore.open({ path: options.storePath }); + } catch (error) { + throw new Error(`No Chronicle store could be opened at ${options.storePath}: ${error instanceof Error ? error.message : String(error)}`); + } let contextManager: ContextManager | null = null; try { contextManager = await ContextManager.open({ @@ -226,12 +232,13 @@ export async function createOfflineRecoveryBranch( })), }); + let createdBranch: string | undefined; try { // branchAt uses the target message's origin sequence. No message // content is compiled or submitted to Membrane during this operation. // Creating the branch is part of the body: a failure here settles the // batch like a failed switch or redaction. - const createdBranch = contextManager.branchAt(target.id, targetBranch); + createdBranch = contextManager.branchAt(target.id, targetBranch); await contextManager.switchBranch(createdBranch); // Work from newest to oldest so earlier redactions never perturb the // live positions used by later range endpoints. @@ -242,11 +249,12 @@ export async function createOfflineRecoveryBranch( } catch (error) { // A partially suppressed branch is not safe to boot into. Preserve it // for diagnosis, but put Chronicle back on the untouched source branch. - // Once the source is confirmed, the batch is retired (after the - // restored state is synced): its surgery never happened. If it can't - // be confirmed, the batch stays prepared as the unfinished body's - // record (startup holds its marks, and resumes the body only on its - // branch). The error says which. + // Once the source is confirmed, the batch is retired (discard syncs the + // restored state before it records that): its surgery never happened. + // If it can't be confirmed, the batch stays prepared as the body's + // record: a start on its target branch completes it (a suppression's + // body first), since its body is in effect there, and a start on any + // other branch holds it. The error says which, and names the branch. const failure = error instanceof Error ? error.message : String(error); let restore = ''; try { @@ -259,19 +267,25 @@ export async function createOfflineRecoveryBranch( let retire = ''; if (batch) { if (store.currentBranch().name !== sourceBranch) { - retire = `; awareness batch ${batch.id} kept, since the source branch could not be confirmed`; + retire = `; awareness batch ${batch.id} kept, since the source branch could not be confirmed: a start on ` + + `${targetBranch} completes it, and a start on another branch holds it`; } else { try { outbox.discard(batch.id); } catch (discardError) { retire = `; awareness batch ${batch.id} could not be retired (${boundDiscordAwarenessText( discardError instanceof Error ? discardError.message : String(discardError), - )}); startup will hold it`; + )}); it stays prepared, and a start on ${sourceBranch} holds it: cancel it then ` + + `(--awareness cancel ${batch.id}), not release it, since its messages were never removed`; } } } - if (!restore && !retire) throw error; - throw new Error(`${failure}${restore}${retire}`, { cause: error }); + const kept = createdBranch === undefined ? '' + : suppression.intervals.length > 0 + ? `; branch ${createdBranch} is kept for diagnosis: it may be partly suppressed, so don't start on it` + : `; branch ${createdBranch} is kept for diagnosis`; + if (!restore && !retire && !kept) throw error; + throw new Error(`${failure}${restore}${kept}${retire}`, { cause: error }); } // The body change landed. Marker bookkeeping is reported apart from it diff --git a/src/recovery/recover-cli.ts b/src/recovery/recover-cli.ts index 4940165f8..4f5464c61 100644 --- a/src/recovery/recover-cli.ts +++ b/src/recovery/recover-cli.ts @@ -31,8 +31,14 @@ interface CliOptions { marks: 'none' | 'addressed' | 'all'; awareness?: { verb: 'list' | 'cancel' | 'retract' | 'release'; target?: string }; dryRun: boolean; + /** Every option given, as typed, so a combination can be refused rather + * than some of it ignored. */ + given: string[]; } +/** What an awareness verb takes besides itself; every other option belongs to a recovery. */ +const AWARENESS_OPTIONS = new Set(['--store', '--outbox', '--awareness']); + function usage(): string { return `Usage: agent-framework-recover --store --agent [options] @@ -73,21 +79,28 @@ Awareness journal (host stopped): Queue removal of this bot's marks (through each ref's configured MCPL route) on a batch's refs, or on every ref an add was ever sent for - --awareness release Queue a held batch's recorded operations - Each acts at once: inspect first with --awareness list. + --awareness release Queue a held batch's recorded operations. A held + batch whose surgery never ran (its error said + so) is to be cancelled, not released: its + messages were never removed + Each acts at once: inspect first with --awareness list. The awareness verbs + take only --store and --outbox; anything else is refused. Queued work is delivered when the host next connects to the Discord server. -Each recovery, and each cancel, retract or release, is recorded in -/operator-actions.jsonl, the host's operator log, with the OS account -that ran it, whether it was carried out or refused. A dry run and a list -record nothing. +Each recovery, and each cancel, retract or release, is recorded, best-effort, +in /operator-actions.jsonl, the host's operator log, with the OS +account that ran it, whether it was carried out or refused: a log that can't +be written says so on stderr, and the act stands. A dry run and a list write +no entry. A list imports a pre-journal ledger if one is present, as the +host's next start would. The store must exist: neither creates one. The agent host must be stopped before running this command.`; } function parseArgs(args: string[]): CliOptions { - const options: CliOptions = { dryRun: false, suppressMessageIds: [], suppressRanges: [], marks: 'none' }; + const options: CliOptions = { dryRun: false, suppressMessageIds: [], suppressRanges: [], marks: 'none', given: [] }; for (let i = 0; i < args.length; i++) { const arg = args[i]; + if (arg.startsWith('--')) options.given.push(arg); const value = () => { const next = args[++i]; if (!next) throw new Error(`${arg} requires a value`); @@ -176,7 +189,7 @@ function errorMessage(error: unknown): string { function awareness(options: CliOptions): void { // The journal lives in the store; with the host stopped, this process is // its only writer. - const store = JsStore.openOrCreate({ path: options.storePath! }); + const store = openExistingStore(options.storePath!); try { const outbox = new DiscordAwarenessOutbox(store, { legacyPath: options.outboxPath ?? defaultDiscordAwarenessOutboxPath(options.storePath!), @@ -210,12 +223,27 @@ function awareness(options: CliOptions): void { } } +/** The store at `path`, which must exist: a mistyped path is refused, never created. */ +function openExistingStore(path: string): JsStore { + try { + return JsStore.open({ path }); + } catch (error) { + throw new Error(`No Chronicle store could be opened at ${path}: ${errorMessage(error)}`); + } +} + async function main(): Promise { const options = parseArgs(process.argv.slice(2)); if (options.awareness) { if (!options.storePath) { throw new Error(`--awareness needs --store\n\n${usage()}`); } + // A recovery's options with an awareness verb are refused, never + // ignored: the operator would believe a choice applied that didn't. + const foreign = [...new Set(options.given.filter((arg) => !AWARENESS_OPTIONS.has(arg) && arg !== '--dry-run'))]; + if (foreign.length > 0) { + throw new Error(`--awareness takes only --store and --outbox; ${foreign.join(', ')} belong to a recovery`); + } // There is no preview of a control: refuse rather than act on a command // that asked not to change anything. if (options.dryRun) { @@ -232,6 +260,10 @@ async function main(): Promise { throw new Error(`--store, --agent, and one anchor (--message-id, --context-id, or --messages) are required\n\n${usage()}`); } + // A path that holds no store is refused before anything else, and gets no + // log entry: the log lives in the store. + openExistingStore(options.storePath).close(); + // A CLI choice and its application are the same moment (the host is // stopped), so the scope is evaluated now rather than frozen to refs. const marks: DiscordAwarenessMarks = options.marks === 'none' ? 'none' : { scope: options.marks }; diff --git a/test/offline-recovery-branch.test.ts b/test/offline-recovery-branch.test.ts index ee9b6e9fb..5ba0b47dc 100644 --- a/test/offline-recovery-branch.test.ts +++ b/test/offline-recovery-branch.test.ts @@ -400,17 +400,44 @@ test('a failed offline suppression retires its batch once the source is confirme try { ContextManager.prototype.removeMessage = function () { throw new Error('injected redaction failure'); }; - // The source comes back: the batch is retired; nothing will ever resume or mark. + // The source comes back: the batch is retired; nothing will ever resume + // or mark. The partly suppressed branch is named. await seed(); await assert.rejects( createOfflineRecoveryBranch({ storePath, agentName: 'cairn', messageId: 'm-current', suppressMessageIds: ['m-a'], marks: { scope: 'all' }, branchName: 'recovery/cairn/fail', }), - (error: Error) => /injected redaction failure/.test(error.message) && !/kept/.test(error.message), + (error: Error) => /^injected redaction failure; branch recovery\/cairn\/fail is kept for diagnosis: it may be partly suppressed, so don't start on it$/.test(error.message), ); assert.deepEqual(inspect(), { branch: 'main', statuses: ['discarded'], due: 0 }); + // The source comes back but retiring the batch fails: the error says the + // surgery didn't happen, and to cancel the held batch rather than release it. + await seed(); + const discard = DiscordAwarenessOutbox.prototype.discard; + DiscordAwarenessOutbox.prototype.discard = function () { throw new Error('injected retire failure'); }; + try { + await assert.rejects( + createOfflineRecoveryBranch({ + storePath, agentName: 'cairn', messageId: 'm-current', suppressMessageIds: ['m-a'], + marks: { scope: 'all' }, branchName: 'recovery/cairn/fail1', + }), + /awareness batch (\S+) could not be retired \(injected retire failure\); it stays prepared, and a start on main holds it: cancel it then \(--awareness cancel \1\), not release it, since its messages were never removed/, + ); + } finally { + DiscordAwarenessOutbox.prototype.discard = discard; + } + assert.deepEqual(inspect(), { branch: 'main', statuses: ['prepared'], due: 0 }); + { + const store = JsStore.open({ path: storePath }); + try { + assert.equal(new DiscordAwarenessOutbox(store).recoverAtStartup('main').held.length, 1, 'a start on the source holds it'); + } finally { + store.close(); + } + } + // The source can't be confirmed: the batch stays as the unfinished body's record. await seed(); ContextManager.prototype.switchBranch = async function (this: ContextManager, name: string) { @@ -422,7 +449,7 @@ test('a failed offline suppression retires its batch once the source is confirme storePath, agentName: 'cairn', messageId: 'm-current', suppressMessageIds: ['m-a'], marks: { scope: 'all' }, branchName: 'recovery/cairn/fail2', }), - /injected redaction failure; restoring main failed: injected restore failure; awareness batch .* kept, since the source branch could not be confirmed/, + /injected redaction failure; restoring main failed: injected restore failure; branch recovery\/cairn\/fail2 is kept for diagnosis: it may be partly suppressed, so don't start on it; awareness batch .* kept, since the source branch could not be confirmed: a start on recovery\/cairn\/fail2 completes it, and a start on another branch holds it/, ); assert.deepEqual(inspect(), { branch: 'recovery/cairn/fail2', statuses: ['prepared'], due: 0 }); } finally { diff --git a/test/recover-cli-operator-log.test.ts b/test/recover-cli-operator-log.test.ts index 91c470490..2755aac5f 100644 --- a/test/recover-cli-operator-log.test.ts +++ b/test/recover-cli-operator-log.test.ts @@ -56,9 +56,26 @@ test('the offline awareness controls record each act in the operator log, done o assert.equal(cancelled.status, 0, cancelled.stderr); const receipt = JSON.parse(cancelled.stdout) as Record; assert.equal(receipt.cancelled, 2); + // What the CLI says it did is what the journal now holds. + const journal = () => { + const reopened = JsStore.open({ path: storePath }); + try { + const reader = new DiscordAwarenessOutbox(reopened); + return { batches: reader.batches(), operations: reader.operations() }; + } finally { + reopened.close(); + } + }; + assert.ok(journal().batches[0].cancelled, 'the cancel is in the journal'); + assert.ok(journal().operations.every((op) => op.cancelled?.reason === 'batch-cancelled')); const retracted = recover(storePath, '--awareness', 'retract', 'all'); assert.equal(retracted.status, 0, retracted.stderr); + assert.deepEqual( + journal().operations.filter((op) => op.action === 'remove' && !op.cancelled).map((op) => op.key.messageId).sort(), + ['m1', 'm2'], + 'the retract\'s removals are in the journal', + ); const refused = recover(storePath, '--awareness', 'release', batch.id); assert.equal(refused.status, 1, 'releasing a batch that is not held is refused'); @@ -173,3 +190,44 @@ test('an offline recovery records its marks choice and receipt in the operator l rmSync(dir, { recursive: true, force: true }); } }); + +test('a path that holds no store is refused, never created, for a list, a control or a recovery', () => { + const dir = mkdtempSync(join(tmpdir(), 'recover-cli-log-')); + try { + const missing = join(dir, 'no-such-store'); + for (const args of [['--awareness', 'list'], ['--awareness', 'cancel', 'b1'], ['--agent', 'cairn', '--message-id', 'm1']]) { + const run = recover(missing, ...args); + assert.equal(run.status, 1, args.join(' ')); + assert.match(run.stderr, /No Chronicle store could be opened at/); + assert.equal(existsSync(missing), false, `${args.join(' ')} created nothing there`); + } + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test('a recovery\'s options with an awareness verb are refused, not ignored', () => { + const dir = mkdtempSync(join(tmpdir(), 'recover-cli-log-')); + const storePath = join(dir, 'store'); + try { + const store = JsStore.openOrCreate({ path: storePath }); + const outbox = new DiscordAwarenessOutbox(store); + const batch = outbox.prepare({ + agentName: 'cairn', sourceBranch: 'main', targetBranch: 'rollback/cairn/1', refs: [ref('m1')], scope: 'all', + })!; + outbox.activate(batch.id); + store.close(); + const run = recover(storePath, '--awareness', 'retract', 'all', '--marks', 'all', '--branch', 'x'); + assert.equal(run.status, 1); + assert.match(run.stderr, /--awareness takes only --store and --outbox; --marks, --branch belong to a recovery/); + const reopened = JsStore.open({ path: storePath }); + try { + assert.equal(new DiscordAwarenessOutbox(reopened).operations().filter((op) => op.action === 'remove').length, 0, 'nothing was retracted'); + } finally { + reopened.close(); + } + assert.deepEqual(operatorLog(storePath), [], 'and nothing was recorded'); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); From 43de3d716d9cce0f9a594b3d82671adb1ffb058a Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Mon, 12 Oct 2026 12:09:10 -0400 Subject: [PATCH 0132/1112] test(awareness): checks that can fail when their contract breaks, and the two tests the delivery-handoff change had From a review of the whole change. surgery-marks-receipt: - The never-sent check after a cancel waited a fixed 100 ms after a sync that returns at once when nothing is due, without joining a drain in flight. It now asserts nothing is due and awaits every drain pass, after which nothing can send without a new trigger. - The retract over a three-ref batch accepted removalsQueued >= 1; it asserts three, and three removals on the wire. - The probe wait accepted any answer; it asserts the answer carries no error. - The 3 s bounds on calls that never wait on Discord judged latency; they are 20 s hang detectors, and answered() === 0 carries the property. - A list that may be empty under every(), and a rejects with no matcher, now assert the batch and its status, and match the error. - A choice bound to refs that includes one the surgery doesn't remove reports notRemoved 1, for a rollback and a suppression, and the operator log records { scope, authorizedRefs }; only a zero notRemoved was ever asserted. - The caller-supplied store test checks that the journal is records in that store and that no ledger file is written, as its name says. - The failed-retire and kept-batch errors are matched in their new words. - Ported from the delivery-handoff change, to this journal: a delivery that fails to start raises an ops alert and leaves no unhandled rejection, and a suppression with nothing addressable reports none whether or not its journal can be closed (now with its unresolved alert). discord-awareness-delivery: - Marks queued while a configured route is down go out when it reconnects, without a restart: the reconnect kick had no test (the route-down test configures no route and recovers through a restart). The fixture takes a CRASH_PATH (exit once) and a DOWN_PATH (refuse to start) for it. - Startup is awaited with a hang detector, so a startup that waited on the withheld reactions fails instead of hanging the file. --- test/discord-awareness-delivery.test.ts | 67 ++++++++++- test/fixtures/surgery-marks-mcpl-server.mjs | 11 +- test/surgery-marks-receipt.test.ts | 124 ++++++++++++++++++-- 3 files changed, 190 insertions(+), 12 deletions(-) diff --git a/test/discord-awareness-delivery.test.ts b/test/discord-awareness-delivery.test.ts index cdf39b48d..4c6da6ffd 100644 --- a/test/discord-awareness-delivery.test.ts +++ b/test/discord-awareness-delivery.test.ts @@ -29,6 +29,21 @@ function jsonl(path: string): Array> { return readFileSync(path, 'utf8').split('\n').filter(Boolean).map((line) => JSON.parse(line)); } +/** Resolve with the promise's value, or fail if it is still pending after ms: a hang detector. */ +async function settlesWithin(promise: Promise, ms: number, what: string): Promise { + let timer: ReturnType | undefined; + try { + return await Promise.race([ + promise, + new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error(`${what} still pending after ${ms} ms`)), ms); + }), + ]); + } finally { + if (timer) clearTimeout(timer); + } +} + async function waitFor(description: string, predicate: () => boolean, timeoutMs = 5_000): Promise { const deadline = Date.now() + timeoutMs; while (Date.now() < deadline) { @@ -46,6 +61,8 @@ function setup() { events: join(dir, 'events.jsonl'), hold: join(dir, 'release'), probe: join(dir, 'probe'), + crash: join(dir, 'crash'), + down: join(dir, 'down'), }; /** Queue marks in the store's journal while no host has it open. */ const queueMarks = (messageIds: string[]) => { @@ -68,7 +85,7 @@ function setup() { /** The running framework's own journal instance. */ const journal = (framework: AgentFramework): DiscordAwarenessOutbox => (framework as unknown as { discordAwarenessOutbox: DiscordAwarenessOutbox }).discordAwarenessOutbox; - const config = (opts: { discord?: boolean; requestTimeoutMs?: number; awarenessDeadlineMs?: number } = {}) => ({ + const config = (opts: { discord?: boolean; requestTimeoutMs?: number; awarenessDeadlineMs?: number; reconnect?: boolean } = {}) => ({ storePath, membrane: new MockMembrane().asMembrane(), agents: [{ name: 'resident', model: 'test', systemPrompt: 'test' }], @@ -81,8 +98,12 @@ function setup() { command: process.execPath, args: [FIXTURE], ...(opts.requestTimeoutMs !== undefined ? { requestTimeoutMs: opts.requestTimeoutMs } : {}), - env: { CALLS_PATH: paths.calls, EVENTS_PATH: paths.events, HOLD_PATH: paths.hold, PROBE_PATH: paths.probe }, + env: { + CALLS_PATH: paths.calls, EVENTS_PATH: paths.events, HOLD_PATH: paths.hold, PROBE_PATH: paths.probe, + CRASH_PATH: paths.crash, DOWN_PATH: paths.down, + }, enabledFeatureSets: ['chat'], + ...(opts.reconnect ? { reconnect: true, reconnectIntervalMs: 50, reconnectMaxIntervalMs: 200 } : {}), }], }), }); @@ -95,7 +116,8 @@ test('startup does not wait for queued marks, and channel traffic flows while th try { queueMarks(['m1', 'm2']); writeFileSync(paths.probe, '1'); // the server sends channel traffic as soon as it can - framework = await AgentFramework.create(config()); + // A hang detector: startup that waited on the withheld reactions would never return. + framework = await settlesWithin(AgentFramework.create(config()), 30_000, 'AgentFramework.create with reactions withheld'); await waitFor('the queued add to reach the server', () => jsonl(paths.calls).length >= 1); await waitFor('the channel traffic to be answered', () => jsonl(paths.events).some((event) => event.event === 'probe-answered')); @@ -208,3 +230,42 @@ test('a branch switch neither adds nor removes marks', async () => { rmSync(dir, { recursive: true, force: true }); } }); + +test('marks queued while a configured route is down go out when it reconnects, without a restart', async () => { + const { dir, journal, paths, config } = setup(); + let framework: AgentFramework | undefined; + try { + writeFileSync(paths.hold, '1'); + framework = await AgentFramework.create(config({ reconnect: true })); + const cm = framework.getAgent('resident')!.getContextManager(); + const tail = String(cm.addMessage('Operator', [{ type: 'text', text: 'tail' }], { + serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'tail', + })); + cm.addMessage('Member', [{ type: 'text', text: 'hi' }], { + serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'm1', tags: ['chat:addressed'], + }); + const connection = (framework as unknown as { mcplServerRegistry: { getServer(id: string): { isConnected: boolean } } }) + .mcplServerRegistry.getServer('discord'); + + // The route goes down and stays down while the marks are chosen. + writeFileSync(paths.down, '1'); + writeFileSync(paths.crash, '1'); + await waitFor('the route to go down', () => !connection.isConnected); + const result = await framework.rollbackToMessage('resident', { messageId: tail, marks: { scope: 'addressed' } }); + assert.equal(result.markers.status, 'queued'); + assert.deepEqual(jsonl(paths.calls), [], 'nothing reached a route that is down'); + + // It comes back: the reconnect itself starts delivery. + rmSync(paths.down, { force: true }); + await waitFor('the queued add confirmed after the reconnect', () => { + const outbox = journal(framework!); + return outbox.operations().length === 1 && outbox.operations().every((op) => outbox.operationStatus(op) === 'confirmed'); + }, 10_000); + assert.deepEqual(jsonl(paths.calls).map((call) => call.name), ['add_reaction']); + } finally { + writeFileSync(paths.hold, '1'); + rmSync(paths.down, { force: true }); + await framework?.stop(); + rmSync(dir, { recursive: true, force: true }); + } +}); diff --git a/test/fixtures/surgery-marks-mcpl-server.mjs b/test/fixtures/surgery-marks-mcpl-server.mjs index 9c5314901..0f1c852ff 100644 --- a/test/fixtures/surgery-marks-mcpl-server.mjs +++ b/test/fixtures/surgery-marks-mcpl-server.mjs @@ -5,7 +5,12 @@ // long reaction drain. When PROBE_PATH appears, the server sends one // inference-bearing channels/incoming request and appends to EVENTS_PATH when // the host answers it, so a test can see whether the MCPL data plane was held. -import { appendFileSync, existsSync } from 'node:fs'; +// When CRASH_PATH appears, the server consumes it and exits, and a server that +// starts while DOWN_PATH exists exits at once, so a test can take the route +// down and bring it back while the host reconnects. +import { appendFileSync, existsSync, rmSync } from 'node:fs'; + +if (process.env.DOWN_PATH && existsSync(process.env.DOWN_PATH)) process.exit(1); const callsPath = process.env.CALLS_PATH; const eventsPath = process.env.EVENTS_PATH; @@ -93,6 +98,10 @@ process.stdin.on('data', (chunk) => { }); const timer = setInterval(() => { + if (process.env.CRASH_PATH && existsSync(process.env.CRASH_PATH)) { + rmSync(process.env.CRASH_PATH, { force: true }); + process.exit(1); + } if (held.length > 0 && holdPath && existsSync(holdPath)) { for (const message of held.splice(0)) answer(message); } diff --git a/test/surgery-marks-receipt.test.ts b/test/surgery-marks-receipt.test.ts index 8eae9c1d6..e72f8d4e4 100644 --- a/test/surgery-marks-receipt.test.ts +++ b/test/surgery-marks-receipt.test.ts @@ -26,6 +26,7 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { JsStore } from '@animalabs/chronicle'; import { AgentFramework, DiscordAwarenessOutbox, OperatorActionError } from '../src/index.js'; +import { DISCORD_AWARENESS_RECORD_TYPE } from '../src/recovery/discord-awareness-outbox.js'; import type { OperatorLogEntry } from '../src/index.js'; import { MockMembrane } from './helpers/mock-membrane.js'; @@ -229,6 +230,101 @@ describe('surgery and awareness marks', () => { ]); }); + it('a choice bound to refs counts the ones its surgery did not remove, and the log records how many were authorized', async () => { + await start(false); + const { tail, removed } = seed(3, [0, 1, 2]); + const elsewhere = { serverId: 'discord', channelId: 'discord:g1:c2', messageId: 'not-removed' }; + const refs = [...framework.previewSurgeryMarks('resident', { rollbackTo: tail }).scopes.all.refs, elsewhere]; + const rolled = await framework.rollbackToMessage('resident', { messageId: tail, marks: { scope: 'all', refs } }); + assert.equal(rolled.markers.notRemoved, 1); + assert.equal(rolled.markers.status === 'queued' && rolled.markers.queued, 3); + const logged = framework.getOperatorLog({ limit: 5 }).find((entry) => entry.kind === 'rollback'); + assert.deepEqual(logged?.params?.marks, { scope: 'all', authorizedRefs: 4 }); + void removed; + + // A suppression counts the same way. + const kept = String(cm().addMessage('Member', [{ type: 'text', text: 'later' }], { + serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'later', tags: ['chat:addressed'], + })); + const later = { serverId: 'discord', channelId: 'discord:g1:c1', messageId: 'later' }; + const suppressed = await framework.suppressMessages('resident', { messageIds: [kept], marks: { scope: 'all', refs: [later, elsewhere] } }); + assert.equal(suppressed.markers.notRemoved, 1); + }); + + it('a delivery that fails to start raises an ops alert and leaves no unhandled rejection', async () => { + await start(true); + const { tail } = seed(2); + const alerts: Array<{ kind: string; message: string }> = []; + framework.onTrace((event) => { + if (event.type === 'ops:alert') alerts.push(event as unknown as { kind: string; message: string }); + }); + const unhandled: unknown[] = []; + const onUnhandled = (reason: unknown) => { unhandled.push(reason); }; + process.on('unhandledRejection', onUnhandled); + const pendingDispatches = DiscordAwarenessOutbox.prototype.pendingDispatches; + DiscordAwarenessOutbox.prototype.pendingDispatches = function () { + throw new Error('injected journal read failure'); + }; + let result: Awaited>; + try { + result = await settlesWithin( + framework.rollbackToMessage('resident', { messageId: tail, marks: { scope: 'all' } }), + 20_000, + 'rollbackToMessage whose delivery fails to start', + ); + await waitFor('the delivery alert', () => alerts.some((alert) => alert.kind === 'discord-awareness-delivery')); + await new Promise((resolve) => setImmediate(resolve)); + } finally { + DiscordAwarenessOutbox.prototype.pendingDispatches = pendingDispatches; + process.off('unhandledRejection', onUnhandled); + } + assert.deepEqual(unhandled, [], 'the background failure is handled'); + assert.match( + alerts.find((alert) => alert.kind === 'discord-awareness-delivery')!.message, + /after rollback failed: .*injected journal read failure/, + ); + // Scheduling succeeded: the batch stays durably active for the next pass. + assert.equal(result.markers.status, 'queued'); + assert.equal(outbox().batches()[0].status, 'active'); + }); + + it('a suppression with nothing addressable reports none, whether or not its journal can be closed', async () => { + await start(false); + const local = String(cm().addMessage('resident', [{ type: 'text', text: 'local only' }], {})); + const other = String(cm().addMessage('resident', [{ type: 'text', text: 'also local' }], {})); + cm().addMessage('resident', [{ type: 'text', text: 'after' }], {}); + const none = { scope: 'none', unmarked: 0, notRemoved: 0, status: 'none', queued: 0 }; + + const closed = await framework.suppressMessages('resident', { messageIds: [local] }); + assert.deepEqual(closed.markers, none); + const [journal] = outbox().batches(); + assert.equal(journal?.status, 'active', 'activated, which closes the resume journal'); + assert.deepEqual(journal.refs, []); + + // When the journal can be neither activated nor retired, it still + // schedules nothing, and the receipt says none, not unresolved; the + // operator hears that the journal failed. + const alerts: Array<{ kind: string; message: string }> = []; + framework.onTrace((event) => { + if (event.type === 'ops:alert') alerts.push(event as unknown as { kind: string; message: string }); + }); + const activate = DiscordAwarenessOutbox.prototype.activate; + const discard = DiscordAwarenessOutbox.prototype.discard; + DiscordAwarenessOutbox.prototype.activate = function () { throw new Error('injected ledger failure'); }; + DiscordAwarenessOutbox.prototype.discard = function () { throw new Error('injected retire failure'); }; + let stuck: Awaited>; + try { + stuck = await framework.suppressMessages('resident', { messageIds: [other] }); + } finally { + DiscordAwarenessOutbox.prototype.activate = activate; + DiscordAwarenessOutbox.prototype.discard = discard; + } + assert.deepEqual(stuck.markers, none); + assert.equal(outbox().batches().filter((batch) => batch.status === 'prepared').length, 1, 'its journal is left prepared'); + const alert = alerts.find((candidate) => candidate.kind === 'discord-awareness-unresolved'); + assert.match(alert?.message ?? '', /suppress applied, but its resume journal \(awareness batch \S+, no marks\) could not be closed.*it stays prepared/); + }); + it('a choice bound to previewed refs never widens to messages that arrive before it applies', async () => { await start(true); const { tail } = seed(3, [0, 2]); @@ -255,9 +351,11 @@ describe('surgery and awareness marks', () => { await start(true); const { tail, removed } = seed(4); + // A hang detector, not a latency verdict: answered() === 0 below is what + // shows the rollback didn't wait on Discord. const result = await settlesWithin( framework.rollbackToMessage('resident', { messageId: tail, marks: { scope: 'all' } }), - 3_000, + 20_000, 'rollbackToMessage with reactions held', ); assert.equal(result.markers.status, 'queued'); @@ -272,7 +370,7 @@ describe('surgery and awareness marks', () => { const fresh = cm().getAllMessages().at(-1)!; const second = await settlesWithin( framework.suppressMessages('resident', { messageIds: [String(fresh.id)] }), - 3_000, + 20_000, 'a second surgery during delivery', ); assert.equal(second.markers.status, 'none'); @@ -281,6 +379,7 @@ describe('surgery and awareness marks', () => { writeFileSync(probePath, '1'); await waitFor('the probe to be answered while reactions are still held', () => jsonl(eventsPath).some((e) => e.event === 'probe-answered')); + assert.equal(jsonl(eventsPath).find((e) => e.event === 'probe-answered')?.error, null, 'accepted, not refused'); assert.equal(answered(), 0, 'the probe was not waiting behind the marks'); writeFileSync(holdPath, '1'); @@ -372,7 +471,7 @@ describe('surgery and awareness marks', () => { assert.equal(cm().currentBranch().name, result.targetBranch, 'the suppression is not undone'); assert.ok(!cm().getAllMessages().some((m) => String(m.id) === removed[1])); assert.equal(result.markers.status, 'not-scheduled'); - assert.equal(outbox().batches().every((batch) => batch.status === 'discarded' || batch.status === 'active'), true); + assert.deepEqual(outbox().batches().map((batch) => batch.status), ['discarded'], 'its one batch is retired'); }); it('cancel stops unsent marks without removing any; retract removes this bot\'s marks and discloses an unknown add', async () => { @@ -398,7 +497,7 @@ describe('surgery and awareness marks', () => { assert.ok(calls().every((c) => c.name === 'add_reaction')); const retracted = framework.retractDiscordAwareness(batchId, { requester: { via: 'test', name: 'operator' } }); - assert.ok(retracted.removalsQueued >= 1); + assert.equal(retracted.removalsQueued, 3, 'one removal per selected message, whatever history says'); assert.ok(retracted.keysWithUnresolvedAdds >= 1, 'the unknown add is disclosed, not assumed absent'); await waitFor('the removals to be sent', () => calls().slice(sentBeforeRetract).filter((c) => c.name === 'remove_reaction').length === retracted.removalsQueued); @@ -420,6 +519,9 @@ describe('surgery and awareness marks', () => { assert.equal(result.markers.status, 'queued'); assert.equal(result.markers.queued, 1); assert.equal(result.markers.unmarked, 1); + // The journal is records in the caller's store, and no ledger file is written. + assert.ok(framework.getStore().getRecordIdsByType(DISCORD_AWARENESS_RECORD_TYPE).length > 0); + assert.equal(existsSync(join(storePath, 'recovery', 'discord-awareness-outbox.json')), false); }); it('cancel during delivery stops the marks not yet sent: exactly one request reaches the server', async () => { @@ -433,8 +535,11 @@ describe('surgery and awareness marks', () => { writeFileSync(holdPath, '1'); await waitFor('the held add answered and recorded', () => outbox().operations().some((op) => op.attempts[0]?.outcome === 'confirmed')); + // Nothing is due, and once every drain pass has ended nothing can send + // without a new trigger. + assert.equal(outbox().pendingDispatches('discord').length, 0); await framework.syncDiscordAwarenessMarkers(); - await new Promise((resolve) => setTimeout(resolve, 100)); + await Promise.all([...(framework as unknown as { discordAwarenessDrains: Map> }).discordAwarenessDrains.values()]); assert.equal(calls().length, 1, 'the cancelled marks were never sent'); }); @@ -451,7 +556,7 @@ describe('surgery and awareness marks', () => { try { await assert.rejects( framework.rollbackToMessage('resident', { messageId: tail, marks: { scope: 'all' } }), - /awareness batch .* kept, since the source branch could not be confirmed/, + /awareness batch .* kept, since the source branch could not be confirmed: a start on rollback\/resident\/\d+ completes it, and a start on another branch holds it/, ); } finally { live.switchBranch = realSwitch; @@ -472,7 +577,7 @@ describe('surgery and awareness marks', () => { try { await assert.rejects( framework.suppressMessages('resident', { messageIds: [removed[0]], marks: { scope: 'all' } }), - /could not be retired \(injected retire failure\)/, + /could not be retired \(injected retire failure\); it stays prepared, and a start on main holds it: cancel it then, not release it, since its messages were never removed/, ); } finally { live.fork = realFork; @@ -482,7 +587,10 @@ describe('surgery and awareness marks', () => { // With retirement working, the same failure retires it (source confirmed). live.fork = async () => { throw new Error('injected fork failure'); }; try { - await assert.rejects(framework.suppressMessages('resident', { messageIds: [removed[1]], marks: { scope: 'all' } })); + await assert.rejects( + framework.suppressMessages('resident', { messageIds: [removed[1]], marks: { scope: 'all' } }), + (error: unknown) => error instanceof OperatorActionError && /^Suppression failed creating .*injected fork failure$/.test(error.message), + ); } finally { live.fork = realFork; } From 0762b0b0b55e7cb17fb13f28e57162149de736b9 Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Mon, 12 Oct 2026 12:10:46 -0400 Subject: [PATCH 0133/1112] docs(awareness): the README and changelog say what the round changed, and what an upgrade or downgrade meets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit From a review of the whole change. - Breaking: "no sibling version is required" held for the framework but not for an older discord-mcpl, whose /hide reacts 💤 to every message in hiddenRefs itself until its companion release; the entry now says so. It names the exports that went with the JSON ledger (DiscordAwarenessEntry, DiscordAwarenessOperation) and their replacements, the offline result's changes (no outboxPath; discordMarkersQueued with markers beside it; new counts), that the CLI and createOfflineRecoveryBranch need a store that exists, that a release before this one doesn't read the journal (a downgrade starts without the imported or queued work), DiscordAwarenessRefusal, and that invalid marks are refused. - Changed: an unanswered request stays unknown in its operation's history and every unresolved count while the operation's status is its latest answer; the bound of three unconfirmed attempts, the permanent refusals and the pass's alert; an error answer recorded unknown; a journal write the store took but this process can't read back. - Fixed (MCPL): a runtime connect holds its planes too; a transport that drops during its exchange stays closed; held requests and answers never cross a transport boundary; a close during the handshake. - Added: the controls' invalid and failed refusals; --awareness takes only --store and --outbox. - README: the same, where the live surgery and the offline CLI are described, and that a held batch whose surgery never ran is cancelled, not released. --- README.md | 55 +++++++++++++------ .../mcpl-reconnect-registration.fixed.md | 11 +++- changelog.d/surgery-marks-controls.added.md | 12 ++-- changelog.d/surgery-marks-one-shot.changed.md | 18 +++++- changelog.d/surgery-marks-opt-in.breaking.md | 29 ++++++++-- 5 files changed, 97 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 9ba8b06f8..8d61f7cb8 100644 --- a/README.md +++ b/README.md @@ -176,7 +176,9 @@ If the branch is made but the marks can't be recorded, the output's `markers` says so and the branch stands. Like a live surgery, the recovery (with its marks choice and receipt) is recorded in the store's operator log, `operator-actions.jsonl`, with the OS account that ran it; so is each -`--awareness` cancel, retract and release. A dry run records nothing. +`--awareness` cancel, retract and release. A dry run and a list record +nothing. The store must exist: a path that holds none is refused, never +created. Portal and other non-Discord records are ignored. When the safe point is an assistant/tool-side record rather than a Discord @@ -223,7 +225,8 @@ open. With the host stopped, the same command inspects and controls the awareness journal: `--awareness list`, `--awareness cancel `, `--awareness retract ` and `--awareness release ` behave as -the live controls described below. +the live controls described below. They take only `--store` and `--outbox`, +and refuse a recovery's options given with them. ### MCPL (MCP Live) @@ -347,23 +350,31 @@ const result = await framework.rollbackToMessage('cairn', { `addressed` covers messages tagged `chat:addressed` (mentions, replies to the bot, DMs). Passing a preview's `refs` binds the choice to exactly that set: messages that arrive before the surgery applies are removed locally but - never marked. The choice is recorded in the operator log. + never marked. Any other value is refused (`invalid`), never guessed. The + choice is recorded in the operator log. - **The surgery returns once marks are scheduled**, never after Discord has accepted them. `result.markers` says whether they were scheduled (`queued` with a count and batch id), not chosen or not in scope (`none`), not to be delivered because recording the batch failed after the body change and it was retired or never recorded (`not-scheduled`), or `unresolved` (neither activation nor retirement could be recorded, or the journal could not be - read back to say whether the batch was; it names a batch that may still be - delivered). Every receipt also counts the removed messages left unmarked. - Marker bookkeeping never fails or undoes an applied rollback or - suppression. + read back to say whether the batch was; it names a batch the next start + may still deliver). Every receipt also counts the removed messages left + unmarked. Marker bookkeeping never fails or undoes an applied rollback or + suppression; a batch that can't be recorded before the body change stops + the surgery before anything changes (`failed`). - **Delivery runs in the background** and never holds MCPL traffic or turns. A route that is not connected keeps its work queued until it connects. Each reaction call has a mandatory deadline independent of `requestTimeoutMs` (including `0`): `discordAwarenessDeadlineMs` (default 10000ms, clamped to 50..60000ms). The emoji is `discordAwarenessEmoji` - (offline: `--emoji`). + (offline: `--emoji`). An operation is not sent again after three attempts + that reached the server without a confirmation (refused, or never + answered), or after a refusal no retry can change (the message, channel + or emoji is gone, access is missing, or the message takes no more + reactions); a pass that gives up on operations raises one + `discord-awareness-delivery` ops alert. A JSON-RPC error answer, which + may come after the server acted, counts as unanswered. - **Marks are one-shot.** Switching branches, undo/redo and restarts never add or remove a mark. The journal is an append-only history of requests and their attempts, kept as typed records in the Chronicle store: it survives @@ -371,18 +382,24 @@ const result = await framework.rollbackToMessage('cairn', { Each request is written ahead of its dispatch, and a dispatch is admitted from the journal as it is at that moment, so a cancel takes effect even while an earlier request is on the wire. A request written and never - answered is recorded as `unknown`, and a later confirmation of a different - attempt never resolves it. Between opposite requests for one reaction, + answered stays `unknown` in its operation's history and in every receipt's + unresolved count, even after a later attempt is confirmed; an operation's + status is its latest answer. Between opposite requests for one reaction, the later *authorization* wins, whenever each request was created: a surgery's adds carry the moment of its marks choice, even when activation comes later, so a retract made in between prevails; a release is a new act. A record that certifies a body change (an activation, a completed suppression, a retirement, or the batch a hide or turn undo records after - its change) is written only after that change is synced. - A batch whose surgery was interrupted before its branch switch was - recorded is held at startup until an operator releases it; an interrupted - suppression's redactions are resumed whenever its branch is active at - startup, whatever its marks' state. + its change) is written only after that change is synced. A journal write + the store took but this process can't read back (Chronicle's periodic sync + failing inside an append) leaves the journal refusing every read and write + until the store is reopened; the next start reads that record with the + rest. A batch whose surgery crashed after its switch (its target branch + active at startup) is completed then; one whose surgery was interrupted + before its branch switch was recorded is held at startup until an operator + releases or cancels it; an interrupted suppression's redactions are + resumed whenever its branch is active at startup, whatever its marks' + state. - **Operator controls,** each recorded in the operator log: - `listDiscordAwareness()` lists batches and retract requests; - `cancelDiscordAwareness(batch | retractRequest)` stops all further sends @@ -398,7 +415,13 @@ const result = await framework.rollbackToMessage('cairn', { any an earlier choice would request later; cancelling it never revives them. Its receipt discloses earlier requests whose outcome is unknown and imported history that leaves outcomes unrecorded; - - `releaseDiscordAwareness(batch)` queues a held batch. + - `releaseDiscordAwareness(batch)` queues a held batch. A held batch whose + surgery never ran (its error said so) is cancelled instead: its messages + were never removed. + + A request the journal won't carry out (an unknown batch, or one in the + wrong state) is refused as `invalid`; a failure to read or write the + journal is `failed`. - **Over `host/command`** (servers granted `allowHostCommands`): `undo` (by messages or by turns) and `hide` take `marks: 'none' | 'addressed' | 'all'` (default `none`; any other value is refused) and return `markers`. Each diff --git a/changelog.d/mcpl-reconnect-registration.fixed.md b/changelog.d/mcpl-reconnect-registration.fixed.md index daaff2abc..4544f1282 100644 --- a/changelog.d/mcpl-reconnect-registration.fixed.md +++ b/changelog.d/mcpl-reconnect-registration.fixed.md @@ -7,4 +7,13 @@ now holds control traffic as well as data until the new grant is established, as initial connect already did, both for a connection that was open and for one whose first connect had failed, and even when - startup or a quiesce resume opens every connection meanwhile. + startup or a quiesce resume opens every connection meanwhile. A runtime + connect holds its planes the same way. +- A transport that drops during that policy exchange no longer has its + planes opened: they stay closed until the next transport's exchange + settles, and an answer that arrives after its transport closed grants + nothing. A server request held while a plane was closed is dropped when + its transport closes, rather than admitted under the next transport's + grant, and an answer to a request from a closed transport is never written + to the next one. A `close()` during a reconnect's handshake now closes the + new transport instead of adopting it. diff --git a/changelog.d/surgery-marks-controls.added.md b/changelog.d/surgery-marks-controls.added.md index b562b6481..3553c4d1a 100644 --- a/changelog.d/surgery-marks-controls.added.md +++ b/changelog.d/surgery-marks-controls.added.md @@ -18,10 +18,14 @@ unrecorded. - `releaseDiscordAwareness(batch)` queues a held batch. - Each control is recorded in the operator log. The same controls are - available as the `host/command` `marks` verb (`action: list | cancel | - retract | release`, `target`) and, with the host stopped, as - `agent-framework-recover --awareness list|cancel|retract|release`. + Each control is recorded in the operator log. A request the journal won't + carry out (an unknown batch, or one in the wrong state) is refused as + `invalid`, and a failure to read or write the journal is `failed`. The same + controls are available as the `host/command` `marks` verb (`action: list | + cancel | retract | release`, `target`) and, with the host stopped, as + `agent-framework-recover --awareness list|cancel|retract|release`, which + takes only `--store` and `--outbox` and refuses a recovery's options given + with it. - A turn-based `host/command` `undo` that undoes a turn records one `undo-turns` entry in the operator log: the requested turns, the marks choice (`none` included) and the receipt, beside the per-turn `undo-turn` diff --git a/changelog.d/surgery-marks-one-shot.changed.md b/changelog.d/surgery-marks-one-shot.changed.md index 44b4a2f37..d4c20bd45 100644 --- a/changelog.d/surgery-marks-one-shot.changed.md +++ b/changelog.d/surgery-marks-one-shot.changed.md @@ -3,8 +3,16 @@ when the source branch became active and re-added when the recovery branch did. Each request is written ahead of its dispatch, and a dispatch is admitted from the journal as it is at that moment. A request that went out - and got no answer is recorded as `unknown`, and a later confirmation of a - different attempt never resolves it. Between opposite requests for one + and got no answer stays `unknown` in its operation's history and in every + receipt's unresolved count, even after a later attempt is confirmed; an + operation's status is its latest answer. An operation is not sent again + after three attempts that reached the server without a confirmation + (refused, or never answered), or after a refusal no retry can change (a + message, channel or emoji that is gone, missing access, a message that + takes no more reactions), and a delivery pass that gives up on operations + raises one `discord-awareness-delivery` ops alert. A JSON-RPC error answer, + which may come after the server acted, is recorded `unknown`. Between + opposite requests for one reaction, the later authorization wins, whenever each was created. A batch whose surgery was interrupted before its branch switch was recorded is held at startup until an operator releases it. An interrupted suppression's @@ -17,7 +25,11 @@ rollback the store had lost. If writing a `hide` or turn `undo` batch fails, its receipt follows what the journal then holds: a batch that reached the store is activated, retired or reported `unresolved`, never - reported `not-scheduled` while it could still be delivered. + reported `not-scheduled` while it could still be delivered. A write the + store took but this process can't read back (Chronicle's periodic sync + failing inside an append) leaves the journal refusing every read and write + until the store is reopened, and such a receipt says `unresolved`: the next + start reads that record with the rest. - Awareness delivery no longer holds MCPL traffic. Previously every MCPL data plane waited at startup, at reconnect, at a tools list change and after each surgery until every queued mark had been attempted, which at Discord's pace diff --git a/changelog.d/surgery-marks-opt-in.breaking.md b/changelog.d/surgery-marks-opt-in.breaking.md index 34d954a6a..964969adf 100644 --- a/changelog.d/surgery-marks-opt-in.breaking.md +++ b/changelog.d/surgery-marks-opt-in.breaking.md @@ -24,9 +24,14 @@ - `host/command` `hide` and turn-based `undo` now hold the store like every surgery, so they are refused (`agent-busy`) while any agent sharing the store is mid-turn. - - No sibling version is required: a host that ignores the new option simply - places no marks. The connectome-host and discord-mcpl companions add the - choice to their surfaces. + - No sibling version is required for the framework to place no marks + unless asked: a host that ignores the new option places none. An older + discord-mcpl's `/hide` is the exception: it reacts 💤 to every message in + `hiddenRefs` itself, whatever the framework does, until its companion + release, which stops that and passes the operator's choice instead. The + connectome-host companion adds the choice to its surface. + - `rollbackToMessage` and `suppressMessages` refuse a `marks` value that + isn't `'none'` or `{ scope: 'addressed' | 'all', refs? }` (`invalid`). - **Code using `DiscordAwarenessOutbox` directly:** the awareness ledger is now an append-only journal of typed records in the Chronicle store (`af:discord-awareness`, on the shared `RecordJournal`), so the constructor @@ -40,4 +45,20 @@ there later imports only batches the journal doesn't already hold. What it recorded is kept as evidence, never as synthesized attempts or as proof that a suppression's body completed, and its undelivered work is held for an - explicit release. + explicit release. A release before this one doesn't read the journal, so a + host downgraded after the import starts without that work, or anything + queued since: those marks aren't delivered. + - The exported `DiscordAwarenessEntry` and `DiscordAwarenessOperation` + types went with the JSON ledger: an operation is `DiscordAwarenessOp` + (one request and its attempts), a batch `DiscordAwarenessBatch` + (`DiscordAwarenessBatchRecord` as recorded), and `view()` returns + `DiscordAwarenessView`. + - An operator control the outbox won't carry out throws + `DiscordAwarenessRefusal`; any other error is a failure to read or write + the journal. +- **`createOfflineRecoveryBranch` and `agent-framework-recover`:** the result + (and the CLI's JSON output) no longer has `outboxPath`, since the journal is + in the store. `discordMarkersQueued` is the marks queued (planned, for a dry + run), with `markers`, the receipt, beside it, and `discordAddressable`, + `marksScope`, `unmarked` and `notRemoved` are new. Both need a store that + exists: a path that holds none is refused, where it used to be created. From 6470d89337319ba6d282cf13eee7b96bc7513db0 Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Mon, 12 Oct 2026 12:24:16 -0400 Subject: [PATCH 0134/1112] fix(mcpl): a transport that closed leaves nothing it sent to be admitted later, and one that dropped during its exchange isn't announced From an independent read of the round before this one. - The boundary dropped held server requests but kept responder-less events "to be admitted against the next grant". Those are the server's notifications, and they describe a process that is gone: a tools/observe filter from the old process was admitted after the next exchange and set the connection's filter (the boundary had cleared it because a filter is per epoch), winning over the new process's if it sent none and racing it if it did; a channel registration from the old process would admit a channel the new one may not have. The boundary now drops everything the server sent that is still held (every event routed from its transport, METHOD_TO_EVENT's), and keeps the host's own reports about the transport: its lifecycle (connect-failed, reconnect-failed, which a stub buffers until the host calls ready()), the server's stderr, and orphaned-response audits. - The reconnect listener gated its open of the planes on its transport still being current, but then announced the reconnect anyway: it emitted mcpl:server-reconnected and module:added after the close's module:removed, so a module-lifecycle consumer believed a server was back that had dropped again, and it ran the tools-list refresh and a drain for it. It now returns before all of that when its transport isn't the connection's or isn't connected. A runtime connect emits module:added only when its exchange's transport is current. With this, the listener's own check refuses something readyMcplPlanes doesn't, so it is no longer redundant. - A policy Request rejected because its transport closed landed in the "did not answer ... unanswered expansion" log; registerMcplServerFeatures' catch now says the transport closed and that the next one negotiates its own. Tests: mcpl-reconnect-events drops a tools/observe filter a transport sent during its exchange before dying, so the next grant never admits it; mcpl-reconnect-registration checks a dropped transport is announced by nothing and the next one by both traces, and that a runtime connect whose transport drops during its exchange is neither opened nor announced. Each fails with its fix undone (tmp mutants: boundary filter, listener return, runtime connect's trace). --- .../mcpl-reconnect-registration.fixed.md | 17 +++++++------ src/framework.ts | 16 +++++++++---- src/mcpl/server-connection.ts | 22 +++++++++++------ test/mcpl-reconnect-events.test.ts | 24 +++++++++++++++++++ test/mcpl-reconnect-registration.test.ts | 19 ++++++++++++++- 5 files changed, 78 insertions(+), 20 deletions(-) diff --git a/changelog.d/mcpl-reconnect-registration.fixed.md b/changelog.d/mcpl-reconnect-registration.fixed.md index 4544f1282..18460823e 100644 --- a/changelog.d/mcpl-reconnect-registration.fixed.md +++ b/changelog.d/mcpl-reconnect-registration.fixed.md @@ -10,10 +10,13 @@ startup or a quiesce resume opens every connection meanwhile. A runtime connect holds its planes the same way. - A transport that drops during that policy exchange no longer has its - planes opened: they stay closed until the next transport's exchange - settles, and an answer that arrives after its transport closed grants - nothing. A server request held while a plane was closed is dropped when - its transport closes, rather than admitted under the next transport's - grant, and an answer to a request from a closed transport is never written - to the next one. A `close()` during a reconnect's handshake now closes the - new transport instead of adopting it. + planes opened or is announced as reconnected (`mcpl:server-reconnected`, + `module:added`): its planes stay closed until the next transport's + exchange settles, and an answer that arrives after its transport closed + grants nothing. What a server sent that is still held when its transport + closes, requests and notifications alike (a `tools/observe` filter, a + channel registration), is dropped rather than admitted under the next + transport's grant: the next process announces its own. An answer to a + request from a closed transport is never written to the next one. A + `close()` during a reconnect's handshake now closes the new transport + instead of adopting it. diff --git a/src/framework.ts b/src/framework.ts index 7f7152279..b48ed519f 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -14074,7 +14074,9 @@ export class AgentFramework { // knowingly left empty) before ANY buffered traffic flows, whoever else // opens planes meanwhile (a resume during a runtime connect). const current = await this.settlePolicyExchange(config, connection); - this.emitTrace({ type: 'module:added', moduleName: `mcpl:${config.id}` }); + // A transport that dropped during its exchange isn't announced: its close + // reported it gone, and its reconnect announces it. + if (current) this.emitTrace({ type: 'module:added', moduleName: `mcpl:${config.id}` }); // Staged startup opens every connection together (initializeMcpl). if (staged) return; @@ -14346,6 +14348,10 @@ export class AgentFramework { } } } catch (err) { + if (connection.transportEpoch !== epoch) { + console.error(`[mcpl] ${config.id}: its transport closed during the initial policy Request; the next one negotiates its own`); + return; + } // A pre-0.5 server treats the Request as a notification and never // answers. §6.7: an unanswered expansion simply does not activate — // grant stays empty, connection is effectively MCP-only. This WILL @@ -14748,10 +14754,10 @@ export class AgentFramework { // Settled, granted or not (an unanswered or refused policy leaves the // grant empty and the connection MCP-only, as at initial connect): // open the planes if this transport is still the connection's. One - // that dropped meanwhile stays closed; the next reconnect opens it. - if (connection.transportEpoch === epoch && connection.isConnected) { - this.readyMcplPlanes([connection]); - } + // that dropped meanwhile stays closed and isn't announced: the close + // already reported it gone, and the next reconnect announces it. + if (connection.transportEpoch !== epoch || !connection.isConnected) return; + this.readyMcplPlanes([connection]); this.handleToolsListChanged(connection.id); this.emitTrace({ type: 'mcpl:server-reconnected', diff --git a/src/mcpl/server-connection.ts b/src/mcpl/server-connection.ts index 6e186055f..2d14a3dbc 100644 --- a/src/mcpl/server-connection.ts +++ b/src/mcpl/server-connection.ts @@ -211,16 +211,19 @@ export class McplServerConnection extends EventEmitter { this.grant = CapabilityGrant.empty(); this.policyEstablished = false; this.transportEpoch++; - // A server request held in the buffer came on the transport that is now - // gone: its sender can't be answered, and admitting it against the next - // transport's grant, or answering its id there, would act for a request - // the next server never made. Events without a responder stay, to be - // admitted against the next grant. + // Everything the server sent that is still held in the buffer came on the + // transport that is now gone. Its requests can't be answered, and + // admitting one against the next transport's grant, or answering its id + // there, would act for a request the next server never made. Its + // notifications describe a process that is gone (a tools/observe filter, + // a channel registration): the next one announces its own. The host's + // own reports about the transport stay: its lifecycle (connect-failed, + // reconnect-failed), the server's stderr, and orphaned-response audits. const held = this.bufferedEvents.length; - this.bufferedEvents = this.bufferedEvents.filter((item) => !McplServerConnection.hasResponder(item)); + this.bufferedEvents = this.bufferedEvents.filter((item) => !McplServerConnection.SERVER_EVENTS.has(item.event)); if (this.bufferedEvents.length < held) { console.error( - `[mcpl] ${this.id}: dropped ${held - this.bufferedEvents.length} held request(s) from a transport that closed`, + `[mcpl] ${this.id}: dropped ${held - this.bufferedEvents.length} held message(s) from a transport that closed`, ); } // RFC-007 §6.5: a filter is per connection epoch — after a reconnect the @@ -456,6 +459,11 @@ export class McplServerConnection extends EventEmitter { return typeof responder?.respond === 'function'; } + /** Events that carry a server's own messages, as routed from its transport. */ + private static get SERVER_EVENTS(): ReadonlySet { + return new Set(Object.values(McplServerConnection.METHOD_TO_EVENT)); + } + /** Event-name form of METHOD_TO_REQUIRED_CAPABILITY, for admission-time * enforcement in emit(). */ private static readonly EVENT_TO_REQUIRED_CAPABILITY: Record = { diff --git a/test/mcpl-reconnect-events.test.ts b/test/mcpl-reconnect-events.test.ts index 07e268dcc..fd108e4fc 100644 --- a/test/mcpl-reconnect-events.test.ts +++ b/test/mcpl-reconnect-events.test.ts @@ -248,6 +248,30 @@ describe('McplServerConnection — reconnect lifecycle events', () => { assert.deepStrictEqual(written.filter((line) => /"id":(5|7)\b/.test(line)), [], 'nothing answers the closed transport\'s requests'); }); + it('drops what a transport that closed during its exchange sent, so the next transport\'s grant never admits it', async () => { + writeFileSync(FLAG_FILE, ''); + connection = await McplServerConnection.connect(makeConfig(), HOST_CAPS); + const filters: unknown[] = []; + connection.on('tools-observe', (params: unknown) => filters.push(params)); + let reconnected = 0; + connection.on('reconnect', () => { reconnected++; }); + connection.ready(); + const serverSends = (message: Record) => + (connection as unknown as { transport: { emit(event: string, line: string): void } }).transport + .emit('line', JSON.stringify({ jsonrpc: '2.0', ...message })); + connection.sendToolsList().catch(() => {}); + await waitFor(() => reconnected === 1, 5000, 'first reconnect'); + // The fresh transport sends its filter right after initialize, while + // its exchange holds both planes, and then dies before the exchange + // settles. + serverSends({ method: 'tools/observe', params: { rules: [{ tool: 'old-process' }] } }); + connection.sendToolsList().catch(() => {}); + await waitFor(() => reconnected === 2, 5000, 'second reconnect'); + connection.establishGrant(new CapabilityGrant(new Set(['toolLifecycle.observe']), [])); + connection.ready(); + assert.deepStrictEqual(filters, [], 'the closed transport\'s filter is not admitted under the next grant'); + }); + it('a close during a reconnect\'s handshake closes the new transport instead of adopting it', async () => { // Flag file absent: the initial connect fails and we get a stub that // reconnects in the background. diff --git a/test/mcpl-reconnect-registration.test.ts b/test/mcpl-reconnect-registration.test.ts index 4563848b4..93f994eed 100644 --- a/test/mcpl-reconnect-registration.test.ts +++ b/test/mcpl-reconnect-registration.test.ts @@ -265,6 +265,8 @@ describe('MCPL policy exchanges: a transport that drops, a quiesced host, a runt it('a transport that drops during its exchange leaves the planes closed, and its late answer grants nothing', async () => { const h = harness(); + const traces: string[] = []; + h.fw.traceListeners = [(event: { type: string }) => traces.push(event.type)]; h.fw.wireMcplEvents(h.connection); h.reconnect(); h.drop(); @@ -273,11 +275,13 @@ describe('MCPL policy exchanges: a transport that drops, a quiesced host, a runt h.answers[0]({ accepted: true }); // answered just as the transport went await h.settle(); assert.deepEqual(h.counts, { opened: 0, controlOnly: 0, granted: 0 }, 'the dead transport\'s exchange opens and grants nothing'); - // The next transport's exchange grants and opens. + assert.deepEqual(traces.filter((type) => type === 'mcpl:server-reconnected' || type === 'module:added'), [], 'and announces nothing'); + // The next transport's exchange grants, opens and announces. h.reconnect(); h.answers[1]({ accepted: true }); await h.settle(); assert.deepEqual(h.counts, { opened: 1, controlOnly: 0, granted: 1 }); + assert.deepEqual(traces.filter((type) => type === 'mcpl:server-reconnected' || type === 'module:added'), ['mcpl:server-reconnected', 'module:added']); }); it('a reconnect on a quiesced host opens only the control plane; resume opens the data plane', async () => { @@ -293,6 +297,19 @@ describe('MCPL policy exchanges: a transport that drops, a quiesced host, a runt assert.equal(h.counts.opened, 1); }); + it('a runtime connect whose transport drops during its exchange is neither opened nor announced', async () => { + const h = harness(); + const traces: string[] = []; + h.fw.traceListeners = [(event: { type: string }) => traces.push(event.type)]; + const connecting = h.fw.connectMcplServerInternal({ id: 'srv', command: 'unused', enabledFeatureSets: ['chat'] }); + await h.settle(); + h.drop(); + h.answers[0]({ accepted: true }); + await connecting; + assert.deepEqual(h.counts, { opened: 0, controlOnly: 0, granted: 0 }); + assert.equal(traces.includes('module:added'), false, 'its close reported it gone; its reconnect announces it'); + }); + it('a runtime connect\'s exchange holds its planes against a resume until it settles', async () => { const h = harness(); const connecting = h.fw.connectMcplServerInternal({ id: 'srv', command: 'unused', enabledFeatureSets: ['chat'] }); From 58ea8e4fc3cef0fe6c11039a7a042fb2d4a605cb Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Mon, 12 Oct 2026 20:07:26 -0400 Subject: [PATCH 0135/1112] fix(code-execution): a script is told what its tool functions return, and a result JSON can't hold is an error naming the tool A review of this change against #181 found three things to carry over. - The code_execution description said each function returns "the same text the tool would have returned to you directly". For a result with structured content it returns a JSON object instead. The description now gives that object, { content, structuredContent, isError } with error when isError is true, a reference in content as { type, refId, savedTo, ... } (savedTo null when the payload wasn't saved; refId is fetch_reference's ref_id), and the oversized form. handleScriptToolCall's comment, which said results are text-only by contract, now says how each kind of result is serialized. - A result that can't be converted to JSON, a cyclic object or a BigInt that an in-process module returned, made scriptStructuredResult's or the history serializer's JSON.stringify throw. The runner caught it and gave the script its bare error, such as "Error: Converting circular structure ...", naming no tool, and a script parsing a structured result failed in json.loads. Each path now returns its own error form. A structured result gets the documented object, empty: { content: [], structuredContent: null, isError: true, error }, so a script that reads content first still works. Anything else gets "Error: ...". The error names the tool, gives JSON.stringify's reason and says the call itself completed, so a script that reports it doesn't invite a blind retry. The host's log gets the same line, since the tool needs fixing whether or not the script reports it. - test/code-execution-structured-results.test.ts, adapted from #181's test of the same name, runs the whole boundary: a real MCPL peer over WebSocket, the framework's dispatch, and a real Python script. The script reads each kind of result: structured with text, with an empty object, structured only, plain text, empty, a structured error, and one over the cap with no workspace to save it in. It also calls an in-process module's cyclic, BigInt and cyclic-data tools. The model's own calls read the text, or the structured JSON when the content has none, and the stored history holds the script's stdout and none of its intermediate results. With the JSON handling reverted, the script fails at json.loads of the runner's bare error. - changelog: says both, and credits #181's author. --- .../mcp-standard-tool-results.changed.md | 12 ++ src/code-execution/tool-definition.ts | 11 +- src/framework.ts | 51 +++++- .../code-execution-structured-results.test.ts | 173 ++++++++++++++++++ 4 files changed, 236 insertions(+), 11 deletions(-) create mode 100644 test/code-execution-structured-results.test.ts diff --git a/changelog.d/mcp-standard-tool-results.changed.md b/changelog.d/mcp-standard-tool-results.changed.md index 4d809347f..3565ec180 100644 --- a/changelog.d/mcp-standard-tool-results.changed.md +++ b/changelog.d/mcp-standard-tool-results.changed.md @@ -49,6 +49,18 @@ `{ isError, oversized: { chars, savedTo } }`. If it can't be saved, the script gets a failure that says the tool may have completed. Results without structured content keep the existing script contract. +- A result that can't be converted to JSON, such as a cyclic object or a + `BigInt` from an in-process module, reaches the script as an error that + names the tool and says the call itself completed: for a structured + result, the object above with `content: []`, `structuredContent: null`, + `isError: true` and the `error`, and `Error: …` otherwise. Before, the + script got the runner's bare error, such as `Converting circular + structure…`. +- The `code_execution` tool's description now says what a script's tool + function returns: the text, or for a structured result the JSON object + above, with references and the oversized form. These two changes, and an + end-to-end test with real Python adapted from #181's, come from + @Meganeuridae's review of this change against #181. - The direct tool path (`executeToolCall`, `ModuleContext.callTool`) now treats an MCP tool error (`isError: true`) as a failure, with `success: false` and the error text; before, every answer counted as diff --git a/src/code-execution/tool-definition.ts b/src/code-execution/tool-definition.ts index cab62de22..e2cdbee9a 100644 --- a/src/code-execution/tool-definition.ts +++ b/src/code-execution/tool-definition.ts @@ -32,8 +32,15 @@ export function buildCodeExecutionToolDefinition(opts?: { "non-identifier character replaced by '_' " + "(e.g. tool 'mcpl--discord--fetch_history' is the function mcpl__discord__fetch_history, " + "and tool 'mcpl--dog-events--status' is mcpl__dog_events__status). " + - 'Each function takes a single dict of arguments and returns a string — the same text ' + - 'the tool would have returned to you directly; parse structured results with json.loads. ' + + 'Each function takes a single dict of arguments and returns a string: the text the tool ' + + 'would have returned to you directly, or "Error: ..." if the call failed. For a result ' + + 'with structured content, the string is a JSON object to parse with json.loads: ' + + '{"content": [...], "structuredContent": ..., "isError": false}, with "error" added when ' + + `"isError" is true. "content" holds the server's content blocks. A reference to a payload ` + + 'appears there as {"type": ..., "refId": ..., "savedTo": ...}, where "savedTo" is the ' + + `workspace path the payload was saved to (null if it wasn't) and "refId" is fetch_reference's ` + + 'ref_id. A structured result over 5 MB is saved to the workspace instead, and the string is ' + + '{"isError": ..., "oversized": {"chars": ..., "savedTo": ...}}. ' + "An exact-name lookup dict is also available: tools['mcpl--discord--fetch_history']({...}). " + 'Use top-level await; run independent calls in parallel with asyncio.gather. ' + 'Only what you print() (plus stderr and the exit code) comes back to you — intermediate ' + diff --git a/src/framework.ts b/src/framework.ts index 5c3eaf2e2..3acdd3838 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -958,6 +958,22 @@ interface EphemeralRun { toolCallsCount: number; } +/** + * What a script gets for a tool result that can't be converted to JSON, + * such as a cyclic object or a BigInt that an in-process module returned. + * It names the tool, and says the call itself completed, so a script that + * reports it doesn't invite a blind retry. The host's log gets the same + * line, since the tool that returned it needs fixing whether or not the + * script reports it. + */ +function unconvertibleScriptResult(toolName: string, err: unknown): string { + const reason = err instanceof Error ? err.message : String(err); + const text = `tool '${toolName}' returned a result that can't be converted to JSON, so the script can't receive it ` + + `(${reason}). The call itself completed.`; + console.error(`[code_execution] ${text}`); + return text; +} + /** Cap an API error message for inline use in a rewind marker: keep enough to * identify the failure class without pasting a wall of provider JSON into the * agent's context. */ @@ -11863,12 +11879,17 @@ export class AgentFramework { /** * Resolve one script-inner tool call to the string the script's tool - * function returns. Never rejects — errors become "Error: ..." strings, + * function returns. Never rejects: errors become "Error: ..." strings, * matching the managed PTC runtime ("Claude's code receives this error"). * - * Results are serialized with the HISTORY serializer (images become - * placeholders): programmatic tool results are text-only by contract, and - * a script variable holding megabytes of base64 helps nobody. + * A result without structured content is serialized with the HISTORY + * serializer: images become placeholders, since a script variable holding + * megabytes of base64 helps nobody. A result with structured content is + * one JSON object holding both of the server's views + * (scriptStructuredResult), so its content blocks reach the script as the + * dispatch gave them, images and payloads included. A result that can't be + * converted to JSON, such as a cyclic object or a BigInt from an in-process + * module, becomes an error that names the tool, in each path's error form. */ private async handleScriptToolCall( agentName: string, @@ -11902,8 +11923,12 @@ export class AgentFramework { // Scripts get (nearly) full data — the script environment IS the spill: // filtering big results in code is the whole point. 5MB protocol safety // cap only; no context-cap truncation, no file spill. - if (structured) return this.scriptStructuredResult(result); - return toolResultDataToHistoryString(result.data, 5_000_000); + if (structured) return this.scriptStructuredResult(result, toolName); + try { + return toolResultDataToHistoryString(result.data, AgentFramework.SCRIPT_RESULT_MAX_CHARS); + } catch (err) { + return `Error: ${unconvertibleScriptResult(toolName, err)}`; + } } /** The 5MB protocol safety cap on what a script receives. */ @@ -11921,16 +11946,24 @@ export class AgentFramework { * saved to the workspace, and the script gets * `{ isError, oversized: { chars, savedTo } }`. If it can't be saved, the * script gets a failure to materialize the result, saying the tool may - * already have completed, never a success missing its result. + * already have completed, never a success missing its result. A result + * that can't be converted to JSON (a cyclic object, a BigInt) gets the + * same object empty, `{ content: [], structuredContent: null, isError: + * true, error }`, the error naming the tool. */ - private async scriptStructuredResult(result: ToolResult): Promise { + private async scriptStructuredResult(result: ToolResult, toolName: string): Promise { const object = { content: Array.isArray(result.data) ? result.data : [], structuredContent: result.structured, isError: result.isError === true, ...(result.isError ? { error: result.error ?? '' } : {}), }; - const json = JSON.stringify(object); + let json: string; + try { + json = JSON.stringify(object); + } catch (err) { + return JSON.stringify({ content: [], structuredContent: null, isError: true, error: unconvertibleScriptResult(toolName, err) }); + } if (json.length <= AgentFramework.SCRIPT_RESULT_MAX_CHARS) return json; const workspace = this.getWorkspaceModule(); const mount = workspace ? this.firstWritableMountName(workspace) : null; diff --git a/test/code-execution-structured-results.test.ts b/test/code-execution-structured-results.test.ts new file mode 100644 index 000000000..21343077d --- /dev/null +++ b/test/code-execution-structured-results.test.ts @@ -0,0 +1,173 @@ +/** + * A tool's structured result end to end: a real MCPL connection, the + * framework's dispatch, and a real Python script. Adapted from + * anima-research/agent-framework#181's test of the same name, which first + * found scripts losing structuredContent. Beside what a script reads, it + * checks that the model's own calls and the stored history keep the readable + * text, and that a result an in-process module returns but JSON can't hold + * reaches the script as an error naming the tool. + */ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import type { AddressInfo } from 'node:net'; +import { WebSocketServer } from 'ws'; + +import { AgentFramework } from '../src/index.js'; +import type { EventResponse, Module, ModuleContext, ToolCall, ToolDefinition, ToolResult } from '../src/index.js'; +import { MockMembrane, createMockResponse } from './helpers/mock-membrane.js'; + +const envelope = { + runId: 'r41', state: 'running', exit: null, nextCursor: 17, truncated: false, + output: 'café 🌱', details: { retries: 0, lines: ['one', 'two'] }, +}; +const transcript = 'Readable transcript: work continues.'; +const text = (t: string) => [{ type: 'text', text: t }]; + +/** tools/call results by variant, as an MCPL peer sends them. */ +const fixtures: Record = { + dual: { content: text(transcript), structuredContent: envelope }, + empty: { content: text('not the empty object'), structuredContent: {} }, + only: { content: [], structuredContent: envelope }, + plain: { content: text('plain text') }, + absent: { content: [] }, + error: { content: text('run refused'), structuredContent: { state: 'rejected' }, isError: true }, + large: { content: text('short rendering'), structuredContent: { output: 'x'.repeat(5_000_000) } }, +}; + +/** An in-process module whose results JSON can't hold. */ +class UnconvertibleModule implements Module { + readonly name = 'loop'; + async start(_ctx: ModuleContext): Promise {} + async stop(): Promise {} + async onProcess(): Promise { return {}; } + getTools(): ToolDefinition[] { + return ['cyclic', 'bigint', 'cyclic_data'].map((name) => ({ name, description: name, inputSchema: { type: 'object' } })); + } + async handleToolCall(call: ToolCall): Promise { + const name = call.name.includes('--') ? call.name.slice(call.name.lastIndexOf('--') + 2) : call.name; + const cyclic: Record = { kind: 'loop' }; + cyclic.self = cyclic; + if (name === 'cyclic') return { success: true, data: text('a loop'), structured: cyclic }; + if (name === 'bigint') return { success: true, data: text('a big number'), structured: { n: 1n } }; + return { success: true, data: cyclic }; + } +} + +test("a script reads a structured result whole, the model's calls and history keep the readable text, and a result JSON can't hold is an error naming the tool", async () => { + const calls: string[] = []; + const wss = new WebSocketServer({ port: 0, host: '127.0.0.1' }); + wss.on('connection', (ws) => ws.on('message', (data) => { + const msg = JSON.parse(String(data)); + const reply = (result: unknown) => ws.send(JSON.stringify({ jsonrpc: '2.0', id: msg.id, result })); + if (msg.method === 'initialize') { + reply({ + protocolVersion: '2024-11-05', serverInfo: { name: 'fixture', version: '1.0.0' }, + capabilities: { tools: {}, experimental: { mcpl: { version: '0.4', pushEvents: true } } }, + }); + } else if (msg.method === 'featureSets/update') { + reply({}); + } else if (msg.method === 'tools/list') { + reply({ tools: [{ name: 'inspect', description: 'Read work status', inputSchema: { type: 'object' } }] }); + } else if (msg.method === 'tools/call') { + calls.push(msg.params.arguments.variant); + reply(fixtures[msg.params.arguments.variant]); + } + })); + if (!wss.address()) await new Promise((resolve) => wss.once('listening', resolve)); + const dir = mkdtempSync(join(tmpdir(), 'pytc-structured-')); + const membrane = new MockMembrane(); + let framework: AgentFramework | undefined; + try { + framework = await AgentFramework.create({ + storePath: join(dir, 'store.chronicle'), membrane: membrane.asMembrane(), + agents: [], modules: [new UnconvertibleModule()], syncIntervalMs: 0, codeExecution: { enabled: true }, + mcplServers: [{ id: 'fixture', url: `ws://127.0.0.1:${(wss.address() as AddressInfo).port}`, toolPrefix: 'fixture' }], + }); + await framework.start(); + for (let i = 0; i < 100 && !framework.getAllTools().some((t) => t.name === 'fixture--inspect'); i++) { + await new Promise((resolve) => setTimeout(resolve, 50)); + } + + // The promise-based path, which modules and outside callers use, carries + // both views. + const direct = await framework.executeToolCall({ id: 'direct-api', name: 'fixture--inspect', input: { variant: 'dual' } }); + assert.deepEqual(direct.data, text(transcript)); + assert.deepEqual(direct.structured, envelope); + + const code = [ + 'import json', + `ENVELOPE = json.loads(${JSON.stringify(JSON.stringify(envelope))})`, + `TRANSCRIPT = ${JSON.stringify(transcript)}`, + 'r = json.loads(await fixture__inspect({"variant": "dual"}))', + 'assert r == {"content": [{"type": "text", "text": TRANSCRIPT}], "structuredContent": ENVELOPE, "isError": False}, r', + 'empty = json.loads(await fixture__inspect({"variant": "empty"}))', + 'assert empty["structuredContent"] == {} and empty["content"] == [{"type": "text", "text": "not the empty object"}], empty', + 'only = json.loads(await fixture__inspect({"variant": "only"}))', + 'assert only == {"content": [], "structuredContent": ENVELOPE, "isError": False}, only', + // Without structured content, a result is the text it always was. + 'plain = await fixture__inspect({"variant": "plain"})', + 'assert plain == json.dumps("plain text"), plain', + 'absent = await fixture__inspect({"variant": "absent"})', + 'assert absent == "", absent', + 'error = json.loads(await fixture__inspect({"variant": "error"}))', + 'assert error == {"content": [{"type": "text", "text": "run refused"}], "structuredContent": {"state": "rejected"}, "isError": True, "error": "run refused"}, error', + // Over the cap, with no workspace to save it to: a failure, never a success missing its result. + 'large = json.loads(await fixture__inspect({"variant": "large"}))', + 'assert large["isError"] is True and large["oversized"]["savedTo"] is None and large["oversized"]["chars"] > 5000000, large', + 'assert "could not be delivered" in large["error"], large', + // Results JSON can't hold: an error naming the tool, in each path's form. + // A structured result's error keeps the documented shape, so a script reading "content" first still works. + 'cyclic = json.loads(await loop__cyclic({}))', + 'assert cyclic["content"] == [] and cyclic["structuredContent"] is None and cyclic["isError"] is True, cyclic', + 'assert "loop--cyclic" in cyclic["error"] and "circular" in cyclic["error"], cyclic', + 'bigint = json.loads(await loop__bigint({}))', + 'assert bigint["content"] == [] and bigint["structuredContent"] is None and bigint["isError"] is True, bigint', + 'assert "loop--bigint" in bigint["error"] and "BigInt" in bigint["error"], bigint', + 'cyclic_data = await loop__cyclic_data({})', + 'assert cyclic_data.startswith("Error: tool ") and "loop--cyclic_data" in cyclic_data and "circular" in cyclic_data, cyclic_data', + 'assert all("The call itself completed." in e for e in [cyclic["error"], bigint["error"], cyclic_data]), "completed"', + 'print(r["structuredContent"]["runId"], r["structuredContent"]["state"], r["structuredContent"]["output"])', + ].join('\n'); + membrane.pushResponse(createMockResponse([ + { type: 'tool_use', id: 'model-direct', name: 'fixture--inspect', input: { variant: 'dual' } }, + { type: 'tool_use', id: 'model-only', name: 'fixture--inspect', input: { variant: 'only' } }, + { type: 'tool_use', id: 'model-python', name: 'code_execution', input: { code } }, + ], 'tool_use')); + membrane.pushResponse(createMockResponse([])); + const created = await framework.createEphemeralAgent({ name: 'worker', model: 'test-model', systemPrompt: 'Inspect work.', allowedTools: 'all' }); + created.contextManager.addMessage('user', [{ type: 'text', text: 'Go.' }]); + await framework.runEphemeralToCompletion(created.agent, created.contextManager); + + const results = membrane.lastStream!.receivedToolResults.flat() as Array<{ toolUseId: string; content: string }>; + const resultOf = (id: string) => results.find((r) => r.toolUseId === id)!.content; + // The model reads the content: the text when there is some, and the + // structured JSON only when the content has none. + assert.equal(resultOf('model-direct'), JSON.stringify(transcript)); + assert.match(resultOf('model-only'), /\\"runId\\":\\"r41\\"/); + const python = JSON.parse(resultOf('model-python')); + assert.equal(python.stderr, ''); + assert.equal(python.return_code, 0); + assert.equal(python.stdout.trim(), 'r41 running café 🌱'); + assert.deepEqual(calls, ['dual', 'dual', 'only', 'dual', 'empty', 'only', 'plain', 'absent', 'error', 'large']); + + // The stored history holds what the model read: the readable direct + // result and the script's chosen stdout, never the script's intermediate + // results. + const stored = created.contextManager.getAllMessages().flatMap((m) => m.content) + .filter((b) => b.type === 'tool_result') as unknown as Array<{ toolUseId: string; content: string }>; + const storedOf = (id: string) => JSON.stringify(stored.find((b) => b.toolUseId === id)!.content); + assert.ok(storedOf('model-direct').includes('Readable transcript')); + assert.ok(!storedOf('model-direct').includes('nextCursor')); + assert.ok(!storedOf('model-python').includes('nextCursor')); + assert.ok(!storedOf('model-python').includes('short rendering')); + assert.ok(!storedOf('model-python').includes('structuredContent')); + } finally { + await framework?.stop(); + for (const client of wss.clients) client.terminate(); + await new Promise((resolve) => wss.close(() => resolve())); + rmSync(dir, { recursive: true, force: true }); + } +}); From 0baafd0bb2f9c6ff3babdad70b9aec78285e97b7 Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Mon, 12 Oct 2026 21:43:58 -0400 Subject: [PATCH 0136/1112] fix(mcp): the modern engine's teardown follows the MCPL engine's contract, and a reap failure names its server ModernMcpConnection.close() emitted 'close' as soon as the host closed a live session, even when its child couldn't be confirmed exited. The framework then traced mcpl:server-closed and module:removed for a modern server whose child might still be running, while a legacy server is traced closed only once its teardown is confirmed (main's rule, from #221). Now 'close' is emitted once, by the close that confirms the teardown: a failed close emits none, and the retry that reaps the child does. Its failures take the MCPL engine's shape too: a single failure is the transport's own error, several an AggregateError naming the server. Before, every failure was wrapped in a new Error, so a caller couldn't match the transport's error itself. Concurrent calls still queue behind the close in flight: after a success a queued call finds nothing to close, and after a failure it re-checks what wasn't reaped. The stdio reap error now names its server ("stdio child N of MCP server "id" did not exit within 2000ms of SIGKILL; it could not be reaped"), and so does the SIGTERM log line. Since a single failure is rethrown as itself, stop()'s rejection otherwise named a stuck child only by its pid. Tests: a failed disconnect on each engine traces no close, and the retry that confirms the exit traces one; the reap error names its server (stop(), the transport, the legacy concurrent-close test); a modern close's single failure is the failure itself. Each part reverted on its own fails a test: the close gating (the new trace test), the server name (four tests), and the single failure's identity (the connect-cleanup test). Changelog: the breaking note says stop() names each such child and its server. --- .../legacy-mcp-protocol-boundary.breaking.md | 14 +++++----- src/mcpl/modern-connection.ts | 26 ++++++++++++++----- src/mcpl/transport.ts | 9 ++++--- test/mcp-modern-outcomes.test.ts | 7 ++--- test/mcp-server-ownership.test.ts | 23 ++++++++++++++-- test/mcpl-legacy-lifetime.test.ts | 4 +-- 6 files changed, 59 insertions(+), 24 deletions(-) diff --git a/changelog.d/legacy-mcp-protocol-boundary.breaking.md b/changelog.d/legacy-mcp-protocol-boundary.breaking.md index 40a7c12d9..80ffaa865 100644 --- a/changelog.d/legacy-mcp-protocol-boundary.breaking.md +++ b/changelog.d/legacy-mcp-protocol-boundary.breaking.md @@ -28,10 +28,10 @@ about 1 ms, so every request timed out at once. Every value that worked, `0` included, passes. - **Hosts:** `stop()` now rejects, after its whole teardown, when an MCP - server's child couldn't be confirmed exited, naming each such child. - Before, closing a stdio server returned as soon as a signal was sent, so - `stop()` never failed for this. Code that chains work on `stop()`, such as - exiting a TUI or switching sessions, should catch that rejection: the - framework is stopped either way. As with any failed teardown, an owned - store stays open, and a later `stop()` retries the reap, completing once - the child has exited. + server's child couldn't be confirmed exited, naming each such child and + its server. Before, closing a stdio server returned as soon as a signal + was sent, so `stop()` never failed for this. Code that chains work on + `stop()`, such as exiting a TUI or switching sessions, should catch that + rejection: the framework is stopped either way. As with any failed + teardown, an owned store stays open, and a later `stop()` retries the + reap, completing once the child has exited. diff --git a/src/mcpl/modern-connection.ts b/src/mcpl/modern-connection.ts index cafc4ddf3..ff52ddd94 100644 --- a/src/mcpl/modern-connection.ts +++ b/src/mcpl/modern-connection.ts @@ -786,12 +786,19 @@ export class ModernMcpConnection extends EventEmitter { /** The close in flight, which later calls wait behind. */ private closing: Promise | null = null; + /** Set when the host closes a live session; cleared, with the one + * 'close', by the close that confirms its teardown. */ + private closeIncomplete = false; + /** * Close for good: no reconnect, no reopened subscription, no connect left - * in flight. Awaits the in-flight connect's end, and rejects if a child - * could not be reaped. Calls queue behind the one in flight, so none - * resolves while teardown is still pending. A call after a failed close - * re-checks what wasn't reaped. + * in flight. Awaits the in-flight connect's end, and rejects while a child + * can't be confirmed exited: with that failure itself, the transport's + * own error, or an AggregateError when there are several, as the MCPL + * engine does. Calls queue behind the one in flight, so none resolves + * while teardown is still pending. A call after a failed close re-checks + * what wasn't reaped. 'close' is emitted once, when the teardown of a + * session the host closed while live completes, not on a failed close. */ close(): Promise { const previous = this.closing; @@ -808,6 +815,7 @@ export class ModernMcpConnection extends EventEmitter { const opening = this.opening; const session = this.session; this.session = null; + if (session) this.closeIncomplete = true; // Every generation this connection still owns: a connect in flight, the // live one, and any a failed cleanup couldn't reap. const targets: Session[] = []; @@ -832,9 +840,13 @@ export class ModernMcpConnection extends EventEmitter { await opening.done; for (const held of this.unreaped) if (!failures.includes(held.failure)) failures.push(held.failure); } - if (session) this.emit('close', { reason: 'closed by host' }); - if (failures.length > 0) { - throw new Error(`MCP server "${this.id}" did not close cleanly: ${failures.map((f) => f.message).join('; ')}`); + if (failures.length === 1) throw failures[0]; + if (failures.length > 1) { + throw new AggregateError(failures, `MCP server "${this.id}" did not close cleanly: ${failures.map((f) => f.message).join('; ')}`); + } + if (this.closeIncomplete) { + this.closeIncomplete = false; + this.emit('close', { reason: 'closed by host' }); } } diff --git a/src/mcpl/transport.ts b/src/mcpl/transport.ts index 529b5feb2..18375a2c9 100644 --- a/src/mcpl/transport.ts +++ b/src/mcpl/transport.ts @@ -172,6 +172,8 @@ export class StdioTransport extends McplTransport { readonly kind = 'stdio' as const; private constructor( + /** The server this child serves, named in what its reaping reports. */ + private readonly serverId: string, private readonly child: ChildProcess, private readonly rl: ReadlineInterface, ) { @@ -215,7 +217,7 @@ export class StdioTransport extends McplTransport { env: buildChildEnv(config), }); const rl = createInterface({ input: child.stdout! }); - return new StdioTransport(child, rl); + return new StdioTransport(config.id, child, rl); } writeLine(json: string): void { @@ -261,11 +263,12 @@ export class StdioTransport extends McplTransport { }); child.kill('SIGTERM'); if (await exitWithin(STDIO_EXIT_GRACE_MS)) return; - console.error(`[mcpl] stdio child ${child.pid} ignored SIGTERM for ${STDIO_EXIT_GRACE_MS}ms — sending SIGKILL`); + const named = `stdio child ${child.pid} of MCP server "${this.serverId}"`; + console.error(`[mcpl] ${named} ignored SIGTERM for ${STDIO_EXIT_GRACE_MS}ms — sending SIGKILL`); child.kill('SIGKILL'); if (await exitWithin(STDIO_EXIT_GRACE_MS)) return; throw new Error( - `stdio child ${child.pid} did not exit within ${STDIO_EXIT_GRACE_MS}ms of SIGKILL; it could not be reaped`, + `${named} did not exit within ${STDIO_EXIT_GRACE_MS}ms of SIGKILL; it could not be reaped`, ); } } diff --git a/test/mcp-modern-outcomes.test.ts b/test/mcp-modern-outcomes.test.ts index 62b06cbc1..338e00ee0 100644 --- a/test/mcp-modern-outcomes.test.ts +++ b/test/mcp-modern-outcomes.test.ts @@ -324,7 +324,7 @@ test('a child that cannot be reaped fails close() explicitly, never a claimed ex const realKill = child.kill.bind(child); child.kill = () => true; // signals never arrive: the child outlives both bounds cleanups.push(() => { realKill('SIGKILL'); }); - await assert.rejects(transport.close(), /did not exit within 2000ms of SIGKILL; it could not be reaped/); + await assert.rejects(transport.close(), /^Error: stdio child \d+ of MCP server "stuck" did not exit within 2000ms of SIGKILL; it could not be reaped$/); // Calls share the one attempt and its verdict. await assert.rejects(transport.close(), /could not be reaped/); }); @@ -571,16 +571,17 @@ test('close() during a connect waits for that connect\'s own cleanup, and report const statics = ModernMcpConnection as unknown as { closeSession: (session: unknown) => Promise }; const real = statics.closeSession; let closes = 0; + const unconfirmed = new Error('the child could not be confirmed gone (simulated)'); // close() closes the generation first; the connect's own cleanup closes it // again, and that second close is the one that can't confirm the exit. statics.closeSession = async (session: unknown) => { const verdict = await real.call(ModernMcpConnection, session); - return ++closes === 2 ? new Error('the child could not be confirmed gone (simulated)') : verdict; + return ++closes === 2 ? unconfirmed : verdict; }; cleanups.push(() => { statics.closeSession = real; }); const starting = connection.start(); await until(() => wire(log).some((e) => e.method === 'server/discover'), 'the launch waiting on discover'); - await assert.rejects(connection.close(), /did not close cleanly: the child could not be confirmed gone \(simulated\)/); + await assert.rejects(connection.close(), (error) => error === unconfirmed, 'the failure itself, as the MCPL engine rethrows one'); assert.equal(closes, 2, 'close() resolved only after the connect\'s own cleanup'); await starting; statics.closeSession = real; diff --git a/test/mcp-server-ownership.test.ts b/test/mcp-server-ownership.test.ts index 569ae0f7f..b48f11380 100644 --- a/test/mcp-server-ownership.test.ts +++ b/test/mcp-server-ownership.test.ts @@ -302,8 +302,8 @@ test('stop() closes every connection, keeps the ones it cannot reap registered, await assert.rejects(framework.stop(), (error) => { assert.ok(error instanceof AggregateError, String(error)); assert.equal(error.errors.length, 2, 'one failure from each engine'); - assert.match(error.message, /MCP server "m" did not close cleanly: .*could not be reaped/); - assert.match(error.message, /stdio child \d+ did not exit within \d+ms of SIGKILL; it could not be reaped/); + assert.match(error.message, /stdio child \d+ of MCP server "m" did not exit within \d+ms of SIGKILL; it could not be reaped/); + assert.match(error.message, /stdio child \d+ of MCP server "l" did not exit within \d+ms of SIGKILL; it could not be reaped/); return true; }); const internals = framework as unknown as { @@ -320,6 +320,25 @@ test('stop() closes every connection, keeps the ones it cannot reap registered, assert.equal(internals.mcplServerRegistry.getServer('l'), null); }); +test('a teardown that cannot confirm the exit traces no close; the retry that confirms it traces one, on both engines', async () => { + await withFramework(async ({ framework, dir }) => { + const closed: string[] = []; + framework.onTrace((event) => { if (event.type === 'mcpl:server-closed') closed.push(event.serverId); }); + const reaping = unreapable(); + await framework.connectMcplServer(modern('m', '', join(dir, 'raw.jsonl'))); + await framework.connectMcplServer(legacy('l', 'ok', join(dir, 'legacy.log'))); + await Promise.all(['m', 'l'].map((id) => assert.rejects( + framework.disconnectMcplServer(id), + new RegExp(`stdio child \\d+ of MCP server "${id}" did not exit .*stays registered, closed`), + ))); + assert.deepEqual(closed, [], 'neither is traced closed while its child may still be running'); + for (const stuck of reaping.stuck) await killAndWait(stuck); + await framework.disconnectMcplServer('m'); + await framework.disconnectMcplServer('l'); + assert.deepEqual(closed.sort(), ['l', 'm'], 'each is traced closed once, by the retry that confirms its exit'); + }); +}); + test("stop()'s teardown budget bounds a modern server's close too, and the server stays registered while its close is pending", async () => { const dir = mkdtempSync(join(tmpdir(), 'mcp-ownership-')); cleanups.push(() => rmSync(dir, { recursive: true, force: true })); diff --git a/test/mcpl-legacy-lifetime.test.ts b/test/mcpl-legacy-lifetime.test.ts index 99ef67faa..285cd5db9 100644 --- a/test/mcpl-legacy-lifetime.test.ts +++ b/test/mcpl-legacy-lifetime.test.ts @@ -139,7 +139,7 @@ test('a close that cannot reap fails, and so do concurrent ones; a later close r assert.deepEqual(settled, [], 'neither resolves while the teardown is pending'); await Promise.all([first, second]); assert.equal(settled.length, 2); - assert.match(settled[0]!, /^first:stdio child \d+ did not exit within \d+ms of SIGKILL; it could not be reaped/); + assert.match(settled[0]!, /^first:stdio child \d+ of MCP server "legacy" did not exit within \d+ms of SIGKILL; it could not be reaped/); assert.match(settled[1]!, /^second:.*could not be reaped/, 'the second shares the attempt and its verdict, rather than claiming success'); await killAndWait(stuck[0]!); await connection.close(); @@ -355,7 +355,7 @@ test('a WebSocket dial whose connection is closed meanwhile ends at once, not at test("a write to a stdio child that has just exited is reported, never thrown", (t) => { const child = Object.assign(new EventEmitter(), { stdin: new EventEmitter(), stderr: new EventEmitter(), killed: false, exitCode: null }); const rl = new EventEmitter(); - const transport = new (StdioTransport as unknown as new (c: unknown, r: unknown) => StdioTransport)(child, rl); + const transport = new (StdioTransport as unknown as new (id: string, c: unknown, r: unknown) => StdioTransport)('legacy', child, rl); const logged: unknown[] = []; t.mock.method(console, 'error', (...args: unknown[]) => { logged.push(args); }); const epipe = Object.assign(new Error('write EPIPE'), { code: 'EPIPE' }); From 2f0382a5aa5cdbcd6a767cbf6dc4f78584fd416d Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Mon, 12 Oct 2026 22:49:03 -0400 Subject: [PATCH 0137/1112] fix(mcp): an http(s) url can't carry userinfo, and a script reads a standard peer's malformed result as the model does From slimepriestess's review of 0baafd0: - An http(s):// url with userinfo is refused (serverConfigProblems). fetch refuses such a url, and its TypeError quotes the whole url, password included, into the connect failure's log line, the mcpl:server-connect-failed traces and the ops alert (logs/failures.log and the ops webhook). A setting nothing could send is refused rather than ignored; the credential goes in token or accessProvider. The cleartext warning's userinfo item now applies to ws:// alone, since an http:// url's is refused. A ws:// url still sends its userinfo. - redactUrl replaces a query parameter name that looks like a credential, so a bare token in a valueless parameter (?) no longer survives as the name. - A script's structured result from a standard peer whose content isn't a list of blocks is now the malformed result the model reads ("[malformed result: its content is not a list of blocks; ...]", its structured content kept), where it was an empty success. An MCPL peer's reading is unchanged on both paths: no content, and the structured view. The notice text is shared (MALFORMED_CONTENT_NOTICE). - A structured script result that can't be converted keeps the tool's own error (", with this error: ..."), and a thrown value with no text form is described without String() throwing. Tests: the userinfo refusal for http and https, loopback or not and user-only too, through serverConfigProblems and ModernMcpConnection.create, with no warning beside it, and a ws:// url passing; redactUrl with a credential-like valueless name and a short plain one; a malformed structured result through both peers' model and script readings; a script receiving a cyclic error result keeps its own error. Each fails on 0baafd0's sources, and each piece reverted on its own fails one. --- src/framework.ts | 26 +++++++++++++---- src/mcpl/protocol-family.ts | 29 ++++++++++++++----- src/mcpl/tool-result-normalize.ts | 7 ++++- .../code-execution-structured-results.test.ts | 6 +++- test/mcp-protocol-family.test.ts | 22 ++++++++++++-- test/script-structured-references.test.ts | 24 ++++++++++++--- 6 files changed, 92 insertions(+), 22 deletions(-) diff --git a/src/framework.ts b/src/framework.ts index 38cd232ab..5a4da8c4a 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -395,7 +395,7 @@ import type { McplServerConnection } from './mcpl/server-connection.js'; import { McplProtocolVersionError, McplUnreapedLaunchError } from './mcpl/server-connection.js'; import { ModernMcpConnection } from './mcpl/modern-connection.js'; import { redactUrl, resolveServerBinding, serverConfigProblems } from './mcpl/protocol-family.js'; -import { normalizeStandardToolResult } from './mcpl/tool-result-normalize.js'; +import { MALFORMED_CONTENT_NOTICE, normalizeStandardToolResult } from './mcpl/tool-result-normalize.js'; import type { McplServerConfig, McplHostCapabilities, @@ -967,10 +967,16 @@ interface EphemeralRun { * line, since the tool that returned it needs fixing whether or not the * script reports it. */ -function unconvertibleScriptResult(toolName: string, err: unknown): string { - const reason = err instanceof Error ? err.message : String(err); +function unconvertibleScriptResult(toolName: string, err: unknown, ownError?: string): string { + let reason: string; + try { + reason = err instanceof Error ? err.message : String(err); + } catch { + reason = 'a value with no text form'; // String() throws on an object with no prototype + } + const own = ownError ? `, with this error: ${ownError}` : '.'; const text = `tool '${toolName}' returned a result that can't be converted to JSON, so the script can't receive it ` + - `(${reason}). The call itself completed.`; + `(${reason}). The call itself completed${own}`; console.error(`[code_execution] ${text}`); return text; } @@ -12002,7 +12008,8 @@ export class AgentFramework { try { json = JSON.stringify(object); } catch (err) { - return JSON.stringify({ content: [], structuredContent: null, isError: true, error: unconvertibleScriptResult(toolName, err) }); + const error = unconvertibleScriptResult(toolName, err, result.isError ? result.error : undefined); + return JSON.stringify({ content: [], structuredContent: null, isError: true, error }); } if (json.length <= AgentFramework.SCRIPT_RESULT_MAX_CHARS) return json; const workspace = this.getWorkspaceModule(); @@ -15432,7 +15439,14 @@ export class AgentFramework { return this.mcpToolResult(result, call.id, mcpl); } const direct = AgentFramework.directToolResult(result); - return mcpl && Array.isArray(direct.data) ? { ...direct, data: direct.data.map(referenceBlockForScript) } : direct; + if (mcpl) return Array.isArray(direct.data) ? { ...direct, data: direct.data.map(referenceBlockForScript) } : direct; + // A standard peer's content that isn't a list of blocks is a malformed + // result on the model's reading (normalizeStandardToolResult), so the + // script reads it that way too, its structured content kept. + if (!result.isError && result.content !== undefined && !Array.isArray(result.content)) { + return { success: false, isError: true, error: MALFORMED_CONTENT_NOTICE, data: [], structured: result.structuredContent }; + } + return direct; }; if (modern) { diff --git a/src/mcpl/protocol-family.ts b/src/mcpl/protocol-family.ts index 54682351c..ed33f98f0 100644 --- a/src/mcpl/protocol-family.ts +++ b/src/mcpl/protocol-family.ts @@ -75,8 +75,8 @@ function credentialLikeSegment(segment: string): boolean { * A url as text a person or a log reads it: its scheme, host and port, and * its path with any credential-like segment replaced. The userinfo is * removed and each query parameter's value replaced, since a credential - * rides there too. A url that doesn't parse can't be redacted, so it isn't - * shown at all. + * rides there too, and so is a parameter name that looks like one. A url + * that doesn't parse can't be redacted, so it isn't shown at all. */ export function redactUrl(url: string | undefined): string { let parsed: URL; @@ -89,7 +89,9 @@ export function redactUrl(url: string | undefined): string { parsed.password = ''; parsed.hash = ''; parsed.pathname = parsed.pathname.split('/').map((segment) => (credentialLikeSegment(segment) ? 'redacted' : segment)).join('/'); - parsed.search = new URLSearchParams([...parsed.searchParams].map(([name]): [string, string] => [name, 'redacted'])).toString(); + // A bare token rides as a name with no value (`?`), so a name that + // looks like a credential is replaced too. + parsed.search = new URLSearchParams([...parsed.searchParams].map(([name]): [string, string] => [credentialLikeSegment(name) ? 'redacted' : name, 'redacted'])).toString(); return parsed.toString(); } @@ -183,6 +185,17 @@ export function serverConfigProblems(config: McplServerConfig): string[] { return [(error as Error).message]; } const problems: string[] = []; + if (binding.transport === 'http') { + // fetch refuses a url with userinfo, and its error quotes the whole url, + // password included, into the connect failure's log line, trace and + // ops alert. A setting nothing could send is refused, not ignored. + const url = new URL(config.url!); + if (url.username || url.password) { + problems.push( + `MCP server "${config.id}": an http(s):// url can't carry userinfo (user:password@), since the HTTP client refuses such a url; give the credential as "token" or "accessProvider"`, + ); + } + } if (config.protocol !== undefined) { if (config.protocol !== 'legacy' && config.protocol !== 'modern') { problems.push(`MCP server "${config.id}": protocol must be 'legacy' or 'modern', got ${JSON.stringify(config.protocol)}`); @@ -241,9 +254,9 @@ const CREDENTIAL_PARAM = /token|key|secret|password|auth|signature/i; * as messages; empty when there is nothing. A credential for an `http://` or * `ws://` url whose host isn't loopback crosses the network in cleartext: * `token` or `accessProvider` (in the `Authorization` header, or the - * WebSocket `?token=` query), and a secret the url carries itself (userinfo, - * a query parameter whose name says it's a credential, or a path segment - * that looks like one, as {@link redactUrl} judges it). That can be + * WebSocket `?token=` query), and a secret the url carries itself (a ws:// + * url's userinfo, a query parameter whose name says it's a credential, or a + * path segment that looks like one, as {@link redactUrl} judges it). That can be * deliberate, on a private network or behind a TLS-terminating proxy, so it * is a warning and never a refusal. */ @@ -259,7 +272,9 @@ export function serverConfigWarnings(config: McplServerConfig): string[] { if ((url.protocol !== 'http:' && url.protocol !== 'ws:') || isLoopbackHost(url.hostname)) return []; const carried: string[] = []; if (config.token || config.accessProvider) carried.push('its credential'); - if (url.username || url.password) carried.push("its url's userinfo"); + // An http:// url's userinfo is refused (serverConfigProblems); a ws:// + // url sends it. + if (url.protocol === 'ws:' && (url.username || url.password)) carried.push("its url's userinfo"); if (url.pathname.split('/').some(credentialLikeSegment)) carried.push("a credential-like segment of its url's path"); const params = [...new Set([...url.searchParams.keys()].filter((name) => CREDENTIAL_PARAM.test(name)))]; if (params.length > 0) { diff --git a/src/mcpl/tool-result-normalize.ts b/src/mcpl/tool-result-normalize.ts index b38f20b59..daa5cf29f 100644 --- a/src/mcpl/tool-result-normalize.ts +++ b/src/mcpl/tool-result-normalize.ts @@ -30,6 +30,11 @@ */ import type { ToolResult } from '../types/index.js'; +/** What a result whose content is present but isn't a list of blocks reads + * as, on the model's path and a script's alike. */ +export const MALFORMED_CONTENT_NOTICE = + '[malformed result: its content is not a list of blocks; the tool may already have completed]'; + /** Saves a payload under a host-chosen name; resolves to its workspace path, * or null when there is nowhere writable to put it. */ export type SaveToolPayload = (fileName: string, bytes: Buffer, mimeType: string) => Promise; @@ -258,7 +263,7 @@ export async function normalizeStandardToolResult( if (malformed) { return { success: false, - error: `[malformed result: its content is not a list of blocks; the tool may already have completed]${text ? `\n${text}` : ''}`, + error: `${MALFORMED_CONTENT_NOTICE}${text ? `\n${text}` : ''}`, isError: true, ...structured, }; diff --git a/test/code-execution-structured-results.test.ts b/test/code-execution-structured-results.test.ts index 21343077d..4b75ee439 100644 --- a/test/code-execution-structured-results.test.ts +++ b/test/code-execution-structured-results.test.ts @@ -44,7 +44,7 @@ class UnconvertibleModule implements Module { async stop(): Promise {} async onProcess(): Promise { return {}; } getTools(): ToolDefinition[] { - return ['cyclic', 'bigint', 'cyclic_data'].map((name) => ({ name, description: name, inputSchema: { type: 'object' } })); + return ['cyclic', 'bigint', 'cyclic_data', 'cyclic_error'].map((name) => ({ name, description: name, inputSchema: { type: 'object' } })); } async handleToolCall(call: ToolCall): Promise { const name = call.name.includes('--') ? call.name.slice(call.name.lastIndexOf('--') + 2) : call.name; @@ -52,6 +52,7 @@ class UnconvertibleModule implements Module { cyclic.self = cyclic; if (name === 'cyclic') return { success: true, data: text('a loop'), structured: cyclic }; if (name === 'bigint') return { success: true, data: text('a big number'), structured: { n: 1n } }; + if (name === 'cyclic_error') return { success: false, isError: true, error: 'db timeout', data: text('failed'), structured: cyclic }; return { success: true, data: cyclic }; } } @@ -129,6 +130,9 @@ test("a script reads a structured result whole, the model's calls and history ke 'cyclic_data = await loop__cyclic_data({})', 'assert cyclic_data.startswith("Error: tool ") and "loop--cyclic_data" in cyclic_data and "circular" in cyclic_data, cyclic_data', 'assert all("The call itself completed." in e for e in [cyclic["error"], bigint["error"], cyclic_data]), "completed"', + // An error result that can't be converted keeps the tool's own error. + 'cyclic_error = json.loads(await loop__cyclic_error({}))', + 'assert cyclic_error["isError"] is True and cyclic_error["error"].endswith("The call itself completed, with this error: db timeout"), cyclic_error', 'print(r["structuredContent"]["runId"], r["structuredContent"]["state"], r["structuredContent"]["output"])', ].join('\n'); membrane.pushResponse(createMockResponse([ diff --git a/test/mcp-protocol-family.test.ts b/test/mcp-protocol-family.test.ts index 761d3a971..f95be6a35 100644 --- a/test/mcp-protocol-family.test.ts +++ b/test/mcp-protocol-family.test.ts @@ -84,6 +84,9 @@ test('errors and the status view show a url redacted: no userinfo, no query valu assert.equal(redactUrl('https://mcp.example/server/123e4567-e89b-12d3-a456-426614174000/mcp'), 'https://mcp.example/server/redacted/mcp'); assert.equal(redactUrl(`https://mcp.example/${'a'.repeat(32)}/mcp`), 'https://mcp.example/redacted/mcp'); assert.equal(redactUrl('https://mcp.example/v2/2026-07-28/organization-settings/mcp'), 'https://mcp.example/v2/2026-07-28/organization-settings/mcp'); + // A bare token rides as a valueless name; a short plain name stays. + assert.equal(redactUrl('https://h/mcp?sk-live-abcdef0123456789'), 'https://h/mcp?redacted=redacted'); + assert.equal(redactUrl('https://h/mcp?verbose'), 'https://h/mcp?verbose=redacted'); assert.equal(redactUrl('not a url ?token=abc'), '(an invalid url, not shown)'); assert.equal(redactUrl(undefined), '(an invalid url, not shown)'); // The configuration errors that name a url give it redacted, or not at all. @@ -205,14 +208,14 @@ test('a credential sent in cleartext to a host that is not loopback is warned ab test('a secret the url carries itself is warned about too, and the warning never quotes it', () => { const warned = (url: string, extra: Partial = {}) => serverConfigWarnings(cfg({ url, ...extra })); - assert.match(warned('http://user:pw@10.0.0.5/mcp')[0]!, /its url's userinfo goes to 10\.0\.0\.5 unencrypted \(http:\/\/\).*read it;/); + assert.match(warned('ws://user:pw@10.0.0.5/mcpl')[0]!, /its url's userinfo goes to 10\.0\.0\.5 unencrypted \(ws:\/\/\).*read it;/); assert.match(warned('ws://mcp.lan/mcpl?api_key=k1')[0]!, /its url's query parameter "api_key" goes to mcp\.lan unencrypted.*use wss:\/\//); for (const name of ['token', 'Access_Token', 'apikey', 'client_secret', 'password', 'auth', 'X-Amz-Signature']) { assert.equal(warned(`http://10.0.0.5/mcp?${name}=v`).length, 1, name); } - const both = warned('http://u:p@10.0.0.5/mcp?token=a&sig_key=b®ion=eu', { token: 't' })[0]!; + const both = warned('ws://u:p@10.0.0.5/mcpl?token=a&sig_key=b®ion=eu', { token: 't' })[0]!; assert.match(both, /its credential, its url's userinfo and its url's query parameters "token", "sig_key" go to 10\.0\.0\.5 unencrypted.*read them;/); - for (const text of [both, ...warned('http://user:pw@10.0.0.5/mcp?token=sekret')]) assert.ok(!/sekret|pw|u:p/.test(text), text); + for (const text of [both, ...warned('ws://user:pw@10.0.0.5/mcpl?token=sekret')]) assert.ok(!/sekret|pw|u:p/.test(text), text); const path = warned('http://10.0.0.5/api/mcp/s/Zt4kQ9xW2vLm8pRs7YbN/mcp')[0]!; assert.match(path, /a credential-like segment of its url's path goes to 10\.0\.0\.5 unencrypted.*read it;/); assert.ok(!path.includes('Zt4kQ9xW2vLm8pRs7YbN'), path); @@ -249,6 +252,19 @@ test('an access provider that throws is logged by its error class and length, no assert.equal(describeCredentialError(undefined), 'a thrown undefined'); }); +test("an http(s) url with userinfo is refused, since the HTTP client can't send it and its error would quote it", () => { + for (const url of ['http://user:S3CRET@127.0.0.1:9/mcp', 'http://user:S3CRET@10.0.0.5/mcp', 'https://user:S3CRET@mcp.example/mcp', 'http://justauser@127.0.0.1:9/mcp']) { + const problems = serverConfigProblems(cfg({ url })); + assert.deepEqual(problems, [ + 'MCP server "s": an http(s):// url can\'t carry userinfo (user:password@), since the HTTP client refuses such a url; give the credential as "token" or "accessProvider"', + ], url); + // The modern engine checks at create(), so nothing is dialed with it. + assert.throws(() => ModernMcpConnection.create(cfg({ id: 's', url })), (error: Error) => error.message === problems[0]); + assert.deepEqual(serverConfigWarnings(cfg({ url })), [], 'a refusal, not also a warning'); + } + assert.deepEqual(serverConfigProblems(cfg({ url: 'ws://user:pw@127.0.0.1:9/mcpl' })), [], 'a WebSocket url sends its userinfo'); +}); + test('both engines admit a configuration through checkServerConfig, which logs its warnings', () => { const lines: string[] = []; const realWarn = console.warn; diff --git a/test/script-structured-references.test.ts b/test/script-structured-references.test.ts index 55a6b8f30..232e133a3 100644 --- a/test/script-structured-references.test.ts +++ b/test/script-structured-references.test.ts @@ -65,8 +65,8 @@ interface Harness { answer: (result: Answer) => void; /** A script's call to a tool of the MCPL peer `files`, or of the MCP-only peer `plain`. */ script: (tool: string) => Promise; - /** A call the model makes to a tool of `files`, through the same dispatch. */ - modelCall: (tool: string) => Promise; + /** A call the model makes to a tool of `files` (or of `plain`), through the same dispatch. */ + modelCall: (tool: string, server?: 'files' | 'plain') => Promise; /** The URIs the fetcher was asked for, saved records aside. */ fetched: string[]; workspace: WorkspaceModule; @@ -111,7 +111,7 @@ async function withPeers(fn: (h: Harness) => Promise, opts: { eager?: bool return { ok: true, path: SAVED }; }, }); - const modelCall = (tool: string) => new Promise((resolve) => { + const modelCall = (tool: string, server: 'files' | 'plain' = 'files') => new Promise((resolve) => { const id = `model-${randomUUID()}`; const push = internals.pushEvent; internals.pushEvent = (event) => { @@ -119,7 +119,7 @@ async function withPeers(fn: (h: Harness) => Promise, opts: { eager?: bool internals.pushEvent = push; resolve(event.result!); }; - internals.dispatchMcplToolCall('agent', { id, name: `mcpl--files--${tool}`, input: {} }, 'files', 'mcpl--files'); + internals.dispatchMcplToolCall('agent', { id, name: `mcpl--${server}--${tool}`, input: {} }, server, `mcpl--${server}`); }); framework.start(); try { @@ -176,6 +176,22 @@ test("an MCP-only peer's structured result reaches a script as the server sent i }); }); +test("content that isn't a list of blocks reaches a script as the model reads it, for each kind of peer", async () => { + await withPeers(async ({ answer, script, modelCall }) => { + answer({ content: 'not a list' as unknown as unknown[], structuredContent: { a: 1 } }); + const notice = '[malformed result: its content is not a list of blocks; the tool may already have completed]'; + // A standard peer's reading calls it a malformed result, on both paths. + const plainModel = await modelCall('attach', 'plain'); + assert.equal(plainModel.isError, true); + assert.ok(plainModel.error?.startsWith(notice), plainModel.error); + assert.deepEqual(JSON.parse(await script('mcpl--plain--attach')), { content: [], structuredContent: { a: 1 }, isError: true, error: notice }); + // An MCPL peer's reading has no content and reads the structured view, on both paths. + const filesModel = await modelCall('attach', 'files'); + assert.equal(filesModel.isError, false); + assert.deepEqual(JSON.parse(await script('mcpl--files--attach')), { content: [], structuredContent: { a: 1 }, isError: false }); + }); +}); + test('without structured content, an MCPL peer\'s result reaches a script as stubs naming the same reference id and saved path', async () => { await withPeers(async ({ answer, script }) => { // Only blocks the text lane recognizes: one it doesn't sends the whole From 75391e7256b87927a7fd3d50dfa2e9cf3c7dddf0 Mon Sep 17 00:00:00 2001 From: Basil-1007 Date: Mon, 12 Oct 2026 22:49:02 -0400 Subject: [PATCH 0138/1112] test(mcp): an auth wall on a later round of a call is a no-response, driven over HTTP MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ObservedTransport.observe marks a call continued when an answer isn't a complete result, and requestFailure reads that mark to classify a 401 or 403 on a later round as no-response: the server took the first round and may have acted on it, so a blind retry could run the call twice. The only test of that rule built `{ continued: true }` by hand and called requestFailure, so nothing drove observe: with its `continued` line deleted, every modern-engine suite stayed green, and a 401 on round 2 was reported as error-response, "refused tools/call: HTTP 401". The HTTP test server takes an optional hand-answer hook, for a JSON-RPC request the SDK server can't be made to answer as a case needs. The new test answers tools/call for `cont`, which the SDK server doesn't list, with a state-only input_required for the first round and 401 for every later one. It checks the verdict (no-response, "asked for another round … refused a later one: HTTP 401"), and that the later rounds carried the first round's state, so the refusal really met a later round. Red check: with observe's `continued` line deleted, the test fails on the outcome (error-response). --- test/mcp-modern-http.test.ts | 49 +++++++++++++++++++++++++++++++++--- 1 file changed, 46 insertions(+), 3 deletions(-) diff --git a/test/mcp-modern-http.test.ts b/test/mcp-modern-http.test.ts index 235bccb72..a0a2ea6cb 100644 --- a/test/mcp-modern-http.test.ts +++ b/test/mcp-modern-http.test.ts @@ -6,11 +6,12 @@ * - credentials: a cached bearer from `accessProvider`, refreshed once on 401; * - the required headers reach the server; * - a deadline aborts the request stream, which the server sees as cancellation; - * - an auth wall is an `error-response` carrying the HTTP status. + * - an auth wall is an `error-response` carrying the HTTP status, unless an + * earlier round of the call was answered, when it is a `no-response`. */ import { test, afterEach } from 'node:test'; import assert from 'node:assert/strict'; -import { createServer as createHttpServer, type Server } from 'node:http'; +import { createServer as createHttpServer, type Server, type ServerResponse } from 'node:http'; import { Readable } from 'node:stream'; import { join, dirname } from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; @@ -24,7 +25,11 @@ const here = dirname(fileURLToPath(import.meta.url)); interface Seen { method: string; auth: string | null; version: string | null; mcpMethod: string | null; mcpName: string | null } -async function startServer(accepted: (token: string | null) => boolean): Promise<{ url: string; seen: Seen[]; events: string[]; server: Server }> { +/** A JSON-RPC request a test answers itself, at the HTTP layer, where an SDK + * server can't be made to answer as the case needs. True once answered. */ +type HandAnswer = (message: { method?: unknown; id?: unknown; params?: Record }, res: ServerResponse) => boolean; + +async function startServer(accepted: (token: string | null) => boolean, answer?: HandAnswer): Promise<{ url: string; seen: Seen[]; events: string[]; server: Server }> { const { createServer } = await import(pathToFileURL(join(here, 'fixtures', 'modern-tools.mjs')).href) as { createServer: (opts: { log: (line: string) => void }) => unknown; }; @@ -46,6 +51,11 @@ async function startServer(accepted: (token: string | null) => boolean): Promise } const chunks: Buffer[] = []; for await (const chunk of req) chunks.push(chunk as Buffer); + if (answer && req.method === 'POST') { + let message: unknown; + try { message = JSON.parse(Buffer.concat(chunks).toString('utf8')); } catch { message = undefined; } + if (message !== null && typeof message === 'object' && !Array.isArray(message) && answer(message as never, res)) return; + } const headers = new Headers(); for (const [key, value] of Object.entries(req.headers)) { if (typeof value === 'string') headers.set(key, value); @@ -138,3 +148,36 @@ test('HTTP: an auth wall on a call is an error-response with the status in data' return true; }); }); + +test('HTTP: an auth wall on a later round of a call is a no-response, since the first round may have acted', async () => { + // The SDK server can't be made to ask for another round, so `cont`, which + // it doesn't list, is answered here: a state-only input_required for the + // first round, then 401 for every later one. + const states: unknown[] = []; + const { url, server } = await startServer(() => true, (message, res) => { + if (message.method !== 'tools/call' || message.params?.name !== 'cont') return false; + const state = message.params.requestState; + states.push(state); + if (state === undefined) { + res.writeHead(200, { 'content-type': 'application/json' }).end(JSON.stringify({ + jsonrpc: '2.0', id: message.id, result: { resultType: 'input_required', requestState: 'state-1' }, + })); + } else { + res.writeHead(401, { 'content-type': 'text/plain' }).end('unauthorized'); + } + return true; + }); + cleanups.push(() => new Promise((resolve) => { server.closeAllConnections?.(); server.close(() => resolve()); })); + const connection = await ModernMcpConnection.connect({ id: 'remote', url, token: 't' }); + cleanups.push(() => connection.close()); + await assert.rejects(connection.callTool('cont', {}), (err: unknown) => { + assert.ok(err instanceof McplRequestError); + // Not the error-response a refused first round gets: the server took + // the first round, and may have acted on it. + assert.equal(err.outcome, 'no-response'); + assert.match(err.message, /asked for another round \(input_required\), and the server refused a later one: HTTP 401/); + return true; + }); + assert.equal(states[0], undefined, 'the first round carried no state'); + assert.ok(states.length >= 2 && states.slice(1).every((s) => s === 'state-1'), `each later round carried it: ${JSON.stringify(states)}`); +}); From 82ebcfd119b78e4fe1bf8ff207415dab75d14c53 Mon Sep 17 00:00:00 2001 From: Basil-1007 Date: Mon, 12 Oct 2026 22:49:02 -0400 Subject: [PATCH 0139/1112] fix(mcp): a restart refuses a colliding tool prefix before tearing the server down restartMcplServer checks the new configuration before it disconnects, so a configuration the connect would refuse leaves the running server as it was. The tool-prefix collision checks weren't part of that: they ran in connectMcplServer, after the disconnect. Restarting `first` with `toolPrefix: 'mcpl--second'` while `second` was live rejected with "collides with server second", and `first` and its tools were gone. The two collision rules (a module of that name, another server holding the prefix) are now one method, mcplPrefixCollision, which connectMcplServer and restartMcplServer both call. A restart calls it before its disconnect. The server's own prefix is free to keep, as before, and so is that of the server a restart is about to disconnect, so a restart under another id isn't refused for the prefix it frees. Test: with `second` connected, restarting `first` onto `second`'s prefix is refused naming the collision, and `first` is still connected with its tools listed. Red check: without the restart's pre-check, `first` is gone after the refusal and the test fails. --- src/framework.ts | 38 +++++++++++++++++------------ test/mcpl-runtime-lifecycle.test.ts | 13 ++++++++++ 2 files changed, 36 insertions(+), 15 deletions(-) diff --git a/src/framework.ts b/src/framework.ts index 5a4da8c4a..64eabb950 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -14465,20 +14465,8 @@ export class AgentFramework { await this.initializeMcpl([], this.mcplInferenceRoutingConfig ?? undefined); } - // Same collision rules create() enforces at startup: the tool prefix must - // not shadow a module name or another server's prefix. - const prefix = config.toolPrefix ?? `mcpl--${config.id}`; - if (this.moduleRegistry.getAllModules().some(m => m.name === prefix)) { - throw new Error( - `MCPL server "${config.id}" toolPrefix "${prefix}" collides with module "${prefix}"`, - ); - } - const prefixOwner = this.mcplPrefixMap.get(prefix); - if (prefixOwner && prefixOwner !== config.id) { - throw new Error( - `MCPL server "${config.id}" toolPrefix "${prefix}" collides with server "${prefixOwner}"`, - ); - } + const collision = this.mcplPrefixCollision(config); + if (collision) throw new Error(collision); const oldToolNames = new Set(this.mcplTools.map(t => t.name)); await this.connectMcplServerInternal(config); @@ -14571,12 +14559,32 @@ export class AgentFramework { throw new Error(`MCPL server "${id}" is not configured`); } // A configuration the connect would refuse is refused here, before the - // running server is torn down for it. + // running server is torn down for it: its binding and settings, and a + // tool prefix another server or a module already holds. AgentFramework.usableConfigFamily(config); + const collision = this.mcplPrefixCollision(config, id); + if (collision) throw new Error(collision); await this.disconnectMcplServer(id); await this.connectMcplServer(config); } + /** The collision rules create() enforces at startup, as a runtime connect + * (or a restart, before its disconnect) meets them: the tool prefix must + * not shadow a module name or another server's prefix. The server's own + * prefix is free to keep, and so is that of the server a restart is + * about to disconnect (`leaving`). */ + private mcplPrefixCollision(config: import('./mcpl/types.js').McplServerConfig, leaving?: string): string | null { + const prefix = config.toolPrefix ?? `mcpl--${config.id}`; + if (this.moduleRegistry.getAllModules().some(m => m.name === prefix)) { + return `MCPL server "${config.id}" toolPrefix "${prefix}" collides with module "${prefix}"`; + } + const prefixOwner = this.mcplPrefixMap.get(prefix); + if (prefixOwner && prefixOwner !== config.id && prefixOwner !== leaving) { + return `MCPL server "${config.id}" toolPrefix "${prefix}" collides with server "${prefixOwner}"`; + } + return null; + } + /** The protocol family a configuration resolves to, or a refusal naming * every problem with it, before anything is spawned, dialed or torn down. */ private static usableConfigFamily(config: import('./mcpl/types.js').McplServerConfig): 'legacy' | 'modern' { diff --git a/test/mcpl-runtime-lifecycle.test.ts b/test/mcpl-runtime-lifecycle.test.ts index 56c10c52b..195c8bbd6 100644 --- a/test/mcpl-runtime-lifecycle.test.ts +++ b/test/mcpl-runtime-lifecycle.test.ts @@ -116,6 +116,19 @@ test('restartMcplServer accepts an updated config (new toolPrefix)', async () => }); }); +test('restartMcplServer refuses a toolPrefix another server holds before tearing anything down', async () => { + await withFramework(true, async (framework) => { + await framework.connectMcplServer(serverConfig('second') as never); + await assert.rejects( + framework.restartMcplServer('first', serverConfig('first', { toolPrefix: 'mcpl--second' }) as never), + /collides with server "second"/, + ); + const first = framework.listMcplServers().find(s => s.id === 'first'); + assert.equal(first?.connected, true, 'the server is still connected'); + assert.ok(framework.getAllTools().some(t => t.name === 'mcpl--first--echo'), 'and its tools still listed'); + }); +}); + test('connectMcplServer rejects a duplicate id and a colliding prefix', async () => { await withFramework(true, async (framework) => { await assert.rejects( From 1b815b92f7208b1d9d6ae7c405fb6873152298c1 Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Mon, 12 Oct 2026 23:04:46 -0400 Subject: [PATCH 0140/1112] fix(mcp): one rule for a query name that may itself be a credential; tests for absent content and a bare throw From the review of the previous round: - A query parameter name that may itself be a credential (one shaped like a token, or one with no value whose name says it carries a credential, since a bare token rides as a name alone) is replaced by redactUrl and counted, never quoted, by the cleartext warning. Before, the two used different rules: redactUrl went by shape alone, so a short bare token with a keyword name survived, and the warning went by keyword alone and quoted the name it flagged, so a bare token shaped like one crossed unencrypted with no warning. A name with a value is a label and stays. - The status view's url doc says such a name is replaced too. - Changelog: the warning's fragment names the ws:// userinfo, the query name rule and the http(s) userinfo refusal; the standard results fragment says a standard peer's non-list content now reaches a script as the malformed result. Tests: redactUrl with a short keyword-named bare token, a labelled token=abc and a plain valueless name; the warning for a bare token alone (its exact text) and for a labelled token beside two hidden names, none of them quoted; a structured result with no content field reaches a script as no content, not malformed, on both peers; a structured result whose toJSON throws a value with no prototype reaches a script as an error naming "a value with no text form". Each fails with its piece reverted. --- .../cleartext-mcp-credential-warning.added.md | 10 +++-- .../mcp-standard-tool-results.changed.md | 5 ++- src/framework.ts | 3 +- src/mcpl/protocol-family.ts | 40 +++++++++++++------ .../code-execution-structured-results.test.ts | 6 ++- test/mcp-protocol-family.test.ts | 13 +++++- test/script-structured-references.test.ts | 6 +++ 7 files changed, 64 insertions(+), 19 deletions(-) diff --git a/changelog.d/cleartext-mcp-credential-warning.added.md b/changelog.d/cleartext-mcp-credential-warning.added.md index dadbe0514..6b801f0c2 100644 --- a/changelog.d/cleartext-mcp-credential-warning.added.md +++ b/changelog.d/cleartext-mcp-credential-warning.added.md @@ -1,9 +1,13 @@ - Connecting to an MCP server warns once when a credential would cross the network unencrypted, to an `http://` or `ws://` url whose host isn't this machine: its `token` or `accessProvider`, or a secret the url carries - itself (userinfo, a query parameter whose name says credential: `token`, - `key`, `secret`, `password`, `auth` or `signature`, or a path segment - that looks like one, as the redacted url judges it). Loopback hosts are + itself (a `ws://` url's userinfo, a query parameter whose name says + credential: `token`, `key`, `secret`, `password`, `auth` or `signature`, + a query parameter name that may itself be one, or a path segment that + looks like one, as the redacted url judges it). A name that may itself be + a credential is counted in the warning, never quoted. An `http(s)://` url + with userinfo is refused instead, since the HTTP client can't send it and + its error would quote it. Loopback hosts are exempt, and so are the unspecified addresses `0.0.0.0` and `[::]`, which reach this host too. It warns rather than refuses, since a private network or a TLS-terminating proxy can make that deliberate. diff --git a/changelog.d/mcp-standard-tool-results.changed.md b/changelog.d/mcp-standard-tool-results.changed.md index 9cd732f2e..e4dfb7fd2 100644 --- a/changelog.d/mcp-standard-tool-results.changed.md +++ b/changelog.d/mcp-standard-tool-results.changed.md @@ -29,7 +29,10 @@ server's result itself, as one JSON object: `{ content, structuredContent, isError, error? }`. From a server that isn't an MCPL peer, `content` is the blocks as sent, so images and other payloads survive. `structuredContent` - is the tool's own JSON, as sent. + is the tool's own JSON, as sent. Content that is present but isn't a + list of blocks, from such a server, is the malformed result the model + reads (`isError`, with the malformed notice as `error`), its structured + content kept, where it was an empty success. - An MCPL peer's blocks in a script's `content` get the treatment every lane the model reads gives them, since what a script prints comes back to the model. An RFC-005 reference arrives without its URI, as the host's diff --git a/src/framework.ts b/src/framework.ts index 64eabb950..e778986d7 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -14660,7 +14660,8 @@ export class AgentFramework { command?: string; /** The url as {@link redactUrl} shows it, since a credential may ride in * it: no userinfo, each query parameter's value replaced, and any path - * segment that looks like a credential replaced. */ + * segment or query parameter name that looks like a credential + * replaced. */ url?: string; }> { const result: Array<{ diff --git a/src/mcpl/protocol-family.ts b/src/mcpl/protocol-family.ts index ed33f98f0..238adc106 100644 --- a/src/mcpl/protocol-family.ts +++ b/src/mcpl/protocol-family.ts @@ -71,12 +71,26 @@ function credentialLikeSegment(segment: string): boolean { return (segment.length >= 16 && /[0-9]/.test(segment)) || segment.length >= 32; } +/** Query parameter names that say they carry a credential. */ +const CREDENTIAL_PARAM = /token|key|secret|password|auth|signature/i; + +/** + * A query parameter name that may itself be a credential, which + * {@link redactUrl} replaces and a warning never quotes: one shaped like a + * token, or one with no value whose name says it carries a credential, since + * a bare token rides as a name alone (`?`). + */ +function credentialLikeQueryName(name: string, value: string): boolean { + return credentialLikeSegment(name) || (value === '' && CREDENTIAL_PARAM.test(name)); +} + /** * A url as text a person or a log reads it: its scheme, host and port, and * its path with any credential-like segment replaced. The userinfo is * removed and each query parameter's value replaced, since a credential - * rides there too, and so is a parameter name that looks like one. A url - * that doesn't parse can't be redacted, so it isn't shown at all. + * rides there too, and so is a parameter name that may itself be one + * ({@link credentialLikeQueryName}). A url that doesn't parse can't be + * redacted, so it isn't shown at all. */ export function redactUrl(url: string | undefined): string { let parsed: URL; @@ -89,9 +103,7 @@ export function redactUrl(url: string | undefined): string { parsed.password = ''; parsed.hash = ''; parsed.pathname = parsed.pathname.split('/').map((segment) => (credentialLikeSegment(segment) ? 'redacted' : segment)).join('/'); - // A bare token rides as a name with no value (`?`), so a name that - // looks like a credential is replaced too. - parsed.search = new URLSearchParams([...parsed.searchParams].map(([name]): [string, string] => [credentialLikeSegment(name) ? 'redacted' : name, 'redacted'])).toString(); + parsed.search = new URLSearchParams([...parsed.searchParams].map(([name, value]): [string, string] => [credentialLikeQueryName(name, value) ? 'redacted' : name, 'redacted'])).toString(); return parsed.toString(); } @@ -246,17 +258,16 @@ function isLoopbackHost(hostname: string): boolean { /^\[::ffff:7f[0-9a-f]{2}:[0-9a-f]{1,4}\]$/.test(host) || host === '0.0.0.0' || host === '[::]'; } -/** Query parameter names that say they carry a credential. */ -const CREDENTIAL_PARAM = /token|key|secret|password|auth|signature/i; - /** * What is usable in a server configuration but worth an operator's attention, * as messages; empty when there is nothing. A credential for an `http://` or * `ws://` url whose host isn't loopback crosses the network in cleartext: * `token` or `accessProvider` (in the `Authorization` header, or the * WebSocket `?token=` query), and a secret the url carries itself (a ws:// - * url's userinfo, a query parameter whose name says it's a credential, or a - * path segment that looks like one, as {@link redactUrl} judges it). That can be + * url's userinfo, a query parameter whose name says it's a credential or may + * itself be one, or a path segment that looks like one, as {@link redactUrl} + * judges it). A name that may itself be a credential is counted, never + * quoted. That can be * deliberate, on a private network or behind a TLS-terminating proxy, so it * is a warning and never a refusal. */ @@ -276,13 +287,18 @@ export function serverConfigWarnings(config: McplServerConfig): string[] { // url sends it. if (url.protocol === 'ws:' && (url.username || url.password)) carried.push("its url's userinfo"); if (url.pathname.split('/').some(credentialLikeSegment)) carried.push("a credential-like segment of its url's path"); - const params = [...new Set([...url.searchParams.keys()].filter((name) => CREDENTIAL_PARAM.test(name)))]; + const entries = [...url.searchParams]; + const hidden = entries.filter(([name, value]) => credentialLikeQueryName(name, value)).length; + const params = [...new Set(entries.filter(([name, value]) => !credentialLikeQueryName(name, value) && CREDENTIAL_PARAM.test(name)).map(([name]) => name))]; if (params.length > 0) { carried.push(`its url's query ${params.length === 1 ? 'parameter' : 'parameters'} ${params.map((name) => JSON.stringify(name)).join(', ')}`); } + if (hidden > 0) { + carried.push(hidden === 1 ? 'a query parameter whose name looks like a credential' : `${hidden} query parameters whose names look like credentials`); + } if (carried.length === 0) return []; const what = carried.length === 1 ? carried[0] : `${carried.slice(0, -1).join(', ')} and ${carried[carried.length - 1]}`; - const one = carried.length === 1 && params.length <= 1; + const one = carried.length === 1 && params.length <= 1 && hidden <= 1; const secure = url.protocol === 'http:' ? 'https' : 'wss'; return [ `MCP server "${config.id}": ${what} ${one ? 'goes' : 'go'} to ${url.host} unencrypted (${url.protocol}//), ` + diff --git a/test/code-execution-structured-results.test.ts b/test/code-execution-structured-results.test.ts index 4b75ee439..2a8e1624f 100644 --- a/test/code-execution-structured-results.test.ts +++ b/test/code-execution-structured-results.test.ts @@ -44,7 +44,7 @@ class UnconvertibleModule implements Module { async stop(): Promise {} async onProcess(): Promise { return {}; } getTools(): ToolDefinition[] { - return ['cyclic', 'bigint', 'cyclic_data', 'cyclic_error'].map((name) => ({ name, description: name, inputSchema: { type: 'object' } })); + return ['cyclic', 'bigint', 'cyclic_data', 'cyclic_error', 'bare_throw'].map((name) => ({ name, description: name, inputSchema: { type: 'object' } })); } async handleToolCall(call: ToolCall): Promise { const name = call.name.includes('--') ? call.name.slice(call.name.lastIndexOf('--') + 2) : call.name; @@ -53,6 +53,8 @@ class UnconvertibleModule implements Module { if (name === 'cyclic') return { success: true, data: text('a loop'), structured: cyclic }; if (name === 'bigint') return { success: true, data: text('a big number'), structured: { n: 1n } }; if (name === 'cyclic_error') return { success: false, isError: true, error: 'db timeout', data: text('failed'), structured: cyclic }; + // A toJSON that throws a value with no prototype, so String() can't describe it either. + if (name === 'bare_throw') return { success: true, data: text('bare'), structured: { toJSON() { throw Object.create(null); } } }; return { success: true, data: cyclic }; } } @@ -133,6 +135,8 @@ test("a script reads a structured result whole, the model's calls and history ke // An error result that can't be converted keeps the tool's own error. 'cyclic_error = json.loads(await loop__cyclic_error({}))', 'assert cyclic_error["isError"] is True and cyclic_error["error"].endswith("The call itself completed, with this error: db timeout"), cyclic_error', + 'bare_throw = json.loads(await loop__bare_throw({}))', + 'assert bare_throw["isError"] is True and "(a value with no text form)" in bare_throw["error"], bare_throw', 'print(r["structuredContent"]["runId"], r["structuredContent"]["state"], r["structuredContent"]["output"])', ].join('\n'); membrane.pushResponse(createMockResponse([ diff --git a/test/mcp-protocol-family.test.ts b/test/mcp-protocol-family.test.ts index f95be6a35..382eb148d 100644 --- a/test/mcp-protocol-family.test.ts +++ b/test/mcp-protocol-family.test.ts @@ -84,8 +84,12 @@ test('errors and the status view show a url redacted: no userinfo, no query valu assert.equal(redactUrl('https://mcp.example/server/123e4567-e89b-12d3-a456-426614174000/mcp'), 'https://mcp.example/server/redacted/mcp'); assert.equal(redactUrl(`https://mcp.example/${'a'.repeat(32)}/mcp`), 'https://mcp.example/redacted/mcp'); assert.equal(redactUrl('https://mcp.example/v2/2026-07-28/organization-settings/mcp'), 'https://mcp.example/v2/2026-07-28/organization-settings/mcp'); - // A bare token rides as a valueless name; a short plain name stays. + // A bare token rides as a valueless name: one shaped like a token, or one + // whose name says it's a credential, goes. A name with a value is a label + // and stays, and so does a short plain name. assert.equal(redactUrl('https://h/mcp?sk-live-abcdef0123456789'), 'https://h/mcp?redacted=redacted'); + assert.equal(redactUrl('https://h/mcp?apikey_x9'), 'https://h/mcp?redacted=redacted'); + assert.equal(redactUrl('https://h/mcp?token=abc'), 'https://h/mcp?token=redacted'); assert.equal(redactUrl('https://h/mcp?verbose'), 'https://h/mcp?verbose=redacted'); assert.equal(redactUrl('not a url ?token=abc'), '(an invalid url, not shown)'); assert.equal(redactUrl(undefined), '(an invalid url, not shown)'); @@ -216,6 +220,13 @@ test('a secret the url carries itself is warned about too, and the warning never const both = warned('ws://u:p@10.0.0.5/mcpl?token=a&sig_key=b®ion=eu', { token: 't' })[0]!; assert.match(both, /its credential, its url's userinfo and its url's query parameters "token", "sig_key" go to 10\.0\.0\.5 unencrypted.*read them;/); for (const text of [both, ...warned('ws://user:pw@10.0.0.5/mcpl?token=sekret')]) assert.ok(!/sekret|pw|u:p/.test(text), text); + // A name that may itself be a credential is warned about and counted, never quoted. + assert.deepEqual(warned('ws://mcp.lan/mcpl?sk-live-abcdef0123456789'), [ + 'MCP server "s": a query parameter whose name looks like a credential goes to mcp.lan unencrypted (ws://), where anything on the network path can read it; use wss:// unless that network is trusted', + ]); + const named = warned('ws://mcp.lan/mcpl?token=a&apikey_x9&x7y8z9w0v1u2t3s4r5')[0]!; + assert.match(named, /its url's query parameter "token" and 2 query parameters whose names look like credentials go to mcp\.lan unencrypted.*read them;/); + assert.ok(!/apikey_x9|x7y8z9/.test(named), named); const path = warned('http://10.0.0.5/api/mcp/s/Zt4kQ9xW2vLm8pRs7YbN/mcp')[0]!; assert.match(path, /a credential-like segment of its url's path goes to 10\.0\.0\.5 unencrypted.*read it;/); assert.ok(!path.includes('Zt4kQ9xW2vLm8pRs7YbN'), path); diff --git a/test/script-structured-references.test.ts b/test/script-structured-references.test.ts index 232e133a3..8942a3854 100644 --- a/test/script-structured-references.test.ts +++ b/test/script-structured-references.test.ts @@ -189,6 +189,12 @@ test("content that isn't a list of blocks reaches a script as the model reads it const filesModel = await modelCall('attach', 'files'); assert.equal(filesModel.isError, false); assert.deepEqual(JSON.parse(await script('mcpl--files--attach')), { content: [], structuredContent: { a: 1 }, isError: false }); + // Content left out isn't malformed: no content, on either peer. + answer({ structuredContent: { a: 1 } } as unknown as Answer); + assert.equal((await modelCall('attach', 'plain')).isError, false); + for (const tool of ['mcpl--plain--attach', 'mcpl--files--attach']) { + assert.deepEqual(JSON.parse(await script(tool)), { content: [], structuredContent: { a: 1 }, isError: false }, tool); + } }); }); From d69add31acc11d012a6917d777d4fb66c66f6fba Mon Sep 17 00:00:00 2001 From: Basil-1007 Date: Mon, 12 Oct 2026 23:05:48 -0400 Subject: [PATCH 0141/1112] fix(mcp): a restart refuses a taken id or a stopping framework before tearing the server down restartMcplServer's pre-check covered the configuration's binding and settings and the tool-prefix collisions, but not the two refusals the connect makes from the framework's own state. So two restarts the connect refuses still tore the running server down first: - a new configuration under another live server's id (restartMcplServer('first', )): the prefix belongs to `second`, which is the config's own id, so the collision check passed, `first` was disconnected and its configuration deleted, and the connect then refused "already registered"; - a restart once stop() has begun: `first` was disconnected, and the connect refused "the framework is stopping or stopped". The connect's id refusal (a connect or disconnect in flight, or a registered connection) is now one method, mcpServerIdTaken, which connectMcplServerInternal and restartMcplServer both call. A restart asks it for a new id only: the id it restarts is meant to be registered, and the disconnect frees it. A restart once stop() has begun is refused as "was not restarted: the framework is stopping or stopped", before it looks anything up. Not covered: a connect that takes the prefix or the new id while the restart's disconnect awaits its teardown is still refused only after it. Holding them across the disconnect would need a hold on prefixes that ids don't have. Tests (mcpl-runtime-lifecycle): - one test with `second` connected and a module named `notes` added: restarting `first` onto `second`'s prefix, onto `notes`, and under `second`'s id are each refused naming why, and after each both servers are still connected with their tools listed; - a restart of `first` under a new id that keeps `mcpl--first` succeeds: `first` is replaced, and the prefix it freed serves the new id; - a restart called just after stop() is refused with the new text, and disconnectMcplServer is never called. Red checks, each undone on its own: without the id check, the first test finds `first` gone after the third refusal; without the module branch of the collision rule, the restart onto `notes` succeeds; without the exemption for the prefix the restart frees, the new-id restart is refused; without the admission check, the stopping restart disconnects `first` and fails with the connect's text. --- src/framework.ts | 45 ++++++++++++++-------- test/mcpl-runtime-lifecycle.test.ts | 58 +++++++++++++++++++++++++++-- 2 files changed, 84 insertions(+), 19 deletions(-) diff --git a/src/framework.ts b/src/framework.ts index e778986d7..27aa2c037 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -13967,18 +13967,8 @@ export class AgentFramework { // protocol on a URL, a modern deadline of 0, MCPL policy on a modern // server. Every valid legacy shape resolves exactly as before. const family = AgentFramework.usableConfigFamily(config); - // One owner per id, across both engines (holdMcpServerId): a connect or - // disconnect in flight holds it, and so does a registered connection, - // including one kept because its launch couldn't be confirmed exited. - const op = this.mcpServerOps?.get(config.id); - if (op?.kind === 'disconnect') { - throw new Error(`MCP server "${config.id}" is still being disconnected; connect it once that has finished`); - } - const registered = this.modernServers().get(config.id) ?? this.mcplServerRegistry.getServer(config.id); - if (op || registered) { - const idle = !op && registered && !registered.isConnected && !registered.willReconnect; - throw new Error(`MCPL server "${config.id}" is already registered${idle ? ' (not connected and not reconnecting: disconnect it first)' : ''}`); - } + const taken = this.mcpServerIdTaken(config.id); + if (taken) throw new Error(taken); const hold = this.holdMcpServerId(config.id, 'connect'); try { await this.connectHeldMcplServer(config, family, deferAwareness, hold.displaced); @@ -14554,20 +14544,45 @@ export class AgentFramework { id: string, newConfig?: import('./mcpl/types.js').McplServerConfig, ): Promise { + // What the connect would refuse is refused here, before the running + // server is torn down for it: a framework whose stop() has begun, the + // configuration's binding and settings, a new id another server holds, + // and a tool prefix another server or a module already holds. + if (this.mcpServerAdmissionClosed) { + throw new Error(`MCP server "${id}" was not restarted: the framework is stopping or stopped`); + } const config = newConfig ?? this.mcplServerConfigs.get(id); if (!config) { throw new Error(`MCPL server "${id}" is not configured`); } - // A configuration the connect would refuse is refused here, before the - // running server is torn down for it: its binding and settings, and a - // tool prefix another server or a module already holds. AgentFramework.usableConfigFamily(config); + if (config.id !== id) { + const taken = this.mcpServerIdTaken(config.id); + if (taken) throw new Error(taken); + } const collision = this.mcplPrefixCollision(config, id); if (collision) throw new Error(collision); await this.disconnectMcplServer(id); await this.connectMcplServer(config); } + /** Why a connect for this id is refused, or null when the id is free. One + * owner per id, across both engines (holdMcpServerId): a connect or + * disconnect in flight holds it, and so does a registered connection, + * including one kept because its launch couldn't be confirmed exited. */ + private mcpServerIdTaken(id: string): string | null { + const op = this.mcpServerOps?.get(id); + if (op?.kind === 'disconnect') { + return `MCP server "${id}" is still being disconnected; connect it once that has finished`; + } + const registered = this.modernServers().get(id) ?? this.mcplServerRegistry?.getServer(id); + if (op || registered) { + const idle = !op && registered && !registered.isConnected && !registered.willReconnect; + return `MCPL server "${id}" is already registered${idle ? ' (not connected and not reconnecting: disconnect it first)' : ''}`; + } + return null; + } + /** The collision rules create() enforces at startup, as a runtime connect * (or a restart, before its disconnect) meets them: the tool prefix must * not shadow a module name or another server's prefix. The server's own diff --git a/test/mcpl-runtime-lifecycle.test.ts b/test/mcpl-runtime-lifecycle.test.ts index 195c8bbd6..4e5befdb5 100644 --- a/test/mcpl-runtime-lifecycle.test.ts +++ b/test/mcpl-runtime-lifecycle.test.ts @@ -14,8 +14,19 @@ import { WebSocketServer } from 'ws'; import type { AddressInfo } from 'node:net'; import { AgentFramework } from '../src/index.js'; +import type { EventResponse, Module, ModuleContext, ToolDefinition, ToolResult } from '../src/index.js'; import { MockMembrane } from './helpers/mock-membrane.js'; +/** A module with no tools: only a name, for a server's tool prefix to collide with. */ +class NamedModule implements Module { + constructor(readonly name: string) {} + async start(_ctx: ModuleContext): Promise {} + async stop(): Promise {} + async onProcess(): Promise { return {}; } + getTools(): ToolDefinition[] { return []; } + async handleToolCall(): Promise { return { success: true }; } +} + // A tiny stdio MCPL server (same shape as mcpl-ws-transport.test.ts): reads // NDJSON on stdin, answers initialize / tools/list / tools/call on stdout. const STDIO_SERVER = ` @@ -116,16 +127,55 @@ test('restartMcplServer accepts an updated config (new toolPrefix)', async () => }); }); -test('restartMcplServer refuses a toolPrefix another server holds before tearing anything down', async () => { +test("restartMcplServer refuses what the connect would refuse before tearing anything down: another server's prefix or id, or a module's name", async () => { await withFramework(true, async (framework) => { await framework.connectMcplServer(serverConfig('second') as never); + await framework.addModule(new NamedModule('notes')); + const untouched = () => { + const live = framework.listMcplServers().filter(s => s.connected).map(s => s.id).sort(); + assert.deepEqual(live, ['first', 'second'], 'both servers still connected'); + const tools = framework.getAllTools().map(t => t.name); + assert.ok(tools.includes('mcpl--first--echo') && tools.includes('mcpl--second--echo'), 'and their tools still listed'); + }; await assert.rejects( framework.restartMcplServer('first', serverConfig('first', { toolPrefix: 'mcpl--second' }) as never), /collides with server "second"/, ); - const first = framework.listMcplServers().find(s => s.id === 'first'); - assert.equal(first?.connected, true, 'the server is still connected'); - assert.ok(framework.getAllTools().some(t => t.name === 'mcpl--first--echo'), 'and its tools still listed'); + untouched(); + await assert.rejects( + framework.restartMcplServer('first', serverConfig('first', { toolPrefix: 'notes' }) as never), + /collides with module "notes"/, + ); + untouched(); + // Under another live server's id, whose prefix is that server's own, so + // only the id is taken. + await assert.rejects( + framework.restartMcplServer('first', serverConfig('second') as never), + /MCPL server "second" is already registered/, + ); + untouched(); + }); +}); + +test('restartMcplServer under a new id may keep the tool prefix the restarted server frees', async () => { + await withFramework(true, async (framework) => { + await framework.restartMcplServer('first', serverConfig('renamed', { toolPrefix: 'mcpl--first' }) as never); + const servers = framework.listMcplServers(); + assert.deepEqual(servers.map(s => [s.id, s.connected]), [['renamed', true]], 'first was replaced'); + assert.ok(framework.getAllTools().some(t => t.name === 'mcpl--first--echo'), 'and the prefix it freed serves the new id'); + }); +}); + +test('restartMcplServer refuses once stop() has begun, before it disconnects anything', async (t) => { + await withFramework(true, async (framework) => { + const disconnect = t.mock.method(framework, 'disconnectMcplServer'); + const stopping = framework.stop(); + await assert.rejects( + framework.restartMcplServer('first'), + /MCP server "first" was not restarted: the framework is stopping or stopped/, + ); + await stopping; + assert.equal(disconnect.mock.callCount(), 0, 'the restart disconnected nothing'); }); }); From 3ef9f6b2d4f249fbeeb14454540fcb684da70d44 Mon Sep 17 00:00:00 2001 From: Petra-1012 Date: Tue, 13 Oct 2026 00:37:21 -0400 Subject: [PATCH 0142/1112] test(drafts): drive the explicit gateway through its admitting entry After the merge of main, deliverProseEnvelope takes an envelope admitted with its target resolved (admitProse), not raw text. Five calls in the drafts tests still passed raw text, so the envelope's text read as undefined and bounceProse threw ("Cannot read properties of undefined (reading 'includes')"). They now call deliverProse, which admits the text's envelopes and delivers each. Admission is synchronous, so the in-flight checks see the same timing as before. --- test/prose-drafts.test.ts | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/test/prose-drafts.test.ts b/test/prose-drafts.test.ts index 1068ff601..05c7f13f6 100644 --- a/test/prose-drafts.test.ts +++ b/test/prose-drafts.test.ts @@ -1105,7 +1105,8 @@ describe('drafts resend ownership', () => { const internals = framework as unknown as { proseDrafts: ProseDraftStore; handleDraftsTool(agent: string, input: unknown): Promise<{ success: boolean; data?: Array<{ text: string }>; error?: string }>; - deliverProseEnvelope(agent: unknown, text: string, hold: { notice: 'later' }): Promise; + // The explicit gateway's entry: it admits the text's envelopes, then delivers each. + deliverProse(agent: unknown, text: string, hold: { notice: 'later' }): Promise; }; const say = (r: { success: boolean; data?: Array<{ text: string }>; error?: string }) => r.success ? r.data!.map((b) => b.text).join('') : r.error!; @@ -1286,7 +1287,7 @@ describe('drafts resend ownership', () => { const batch = t.internals.handleDraftsTool('scout', { action: 'resend', draftIds: [t.a.id, bounce!.id], destination: 'chan' }); await t.untilPublishing(); const agent = t.framework.getAgent('scout')!; - await t.internals.deliverProseEnvelope(agent, '>>chan {{unsent}} and a postscript', { notice: 'later' }); + await t.internals.deliverProse(agent, '>>chan {{unsent}} and a postscript', { notice: 'later' }); assert.deepEqual(t.pending.map((p) => p.text), ['A words'], 'nothing more was sent'); const fresh = t.internals.proseDrafts.latestBounce('scout')!; assert.notEqual(fresh.id, bounce!.id); @@ -1319,12 +1320,12 @@ describe('drafts resend ownership', () => { // The journal refuses the dismissal: the bounce stays, and the resident is told. const dismiss = t.internals.proseDrafts.dismiss.bind(t.internals.proseDrafts); t.internals.proseDrafts.dismiss = () => { throw new Error('EIO'); }; - await t.internals.deliverProseEnvelope(agent, '>>skip_reply {{unsent}}', { notice: 'later' }); + await t.internals.deliverProse(agent, '>>skip_reply {{unsent}}', { notice: 'later' }); assert.ok(words().includes(`[prose-routing] {{unsent}} is draft ${bounce!.id}, which could not be set aside (EIO): ` + 'it is still held, and {{unsent}} still names it.'), words().join('\n')); assert.equal(t.internals.proseDrafts.latestBounce('scout')?.id, bounce!.id); t.internals.proseDrafts.dismiss = dismiss; - await t.internals.deliverProseEnvelope(agent, '>>skip_reply {{unsent}}', { notice: 'later' }); + await t.internals.deliverProse(agent, '>>skip_reply {{unsent}}', { notice: 'later' }); assert.equal(draftState(t.internals.proseDrafts.get('scout', bounce!.id)!), 'dismissed'); assert.ok(words().some((w) => w.startsWith(`[prose-routing] Draft ${bounce!.id} is set aside. Note that ${bounce!.id}'s attempt to (chan) at `) && w.endsWith('may already have been posted: no receipt.')), words().join('\n')); @@ -1340,7 +1341,7 @@ describe('drafts resend ownership', () => { const sending = t.internals.handleDraftsTool('scout', { action: 'resend', draftIds: [bounce!.id], destination: 'chan' }); await t.untilPublishing(); const agent = t.framework.getAgent('scout')!; - const envelope = (text: string) => t.internals.deliverProseEnvelope(agent, text, { notice: 'later' }); + const envelope = (text: string) => t.internals.deliverProse(agent, text, { notice: 'later' }); await envelope('>>skip_reply {{unsent}}'); assert.equal(t.internals.proseDrafts.get('scout', bounce!.id)!.dismissedAt, undefined); const words = agent.getContextManager().getAllMessages().flatMap((m) => m.content).map((b) => (b as { text?: string }).text ?? ''); @@ -1360,7 +1361,7 @@ describe('drafts resend ownership', () => { try { const [bounce] = t.internals.proseDrafts.hold('scout', [{ text: 'bounced words', source }], 'bounced').held; const agent = t.framework.getAgent('scout')!; - const delivering = t.internals.deliverProseEnvelope(agent, '>>chan {{unsent}}', { notice: 'later' }); + const delivering = t.internals.deliverProse(agent, '>>chan {{unsent}}', { notice: 'later' }); await t.untilPublishing(); assert.deepEqual(t.pending.map((p) => p.text), ['bounced words'], 'the bounce is out through {{unsent}}'); const refused = t.say(await t.internals.handleDraftsTool('scout', { action: 'dismiss', draftIds: [bounce!.id] })); From b0a2dafc0fce3439116f9e5eda345cd83725d715 Mon Sep 17 00:00:00 2001 From: Rosalind-2359 Date: Tue, 13 Oct 2026 00:40:37 -0400 Subject: [PATCH 0143/1112] fix(mcp): redact configured credentials from recorded server text A transport error can quote the server URL through an echoed HTTP request path or redirect target. Recording that error verbatim copied known URL credentials into connection traces, console errors and ops alerts. Tool transport failures could also copy the same text into a resident's history. redactServerText derives exact strings from the configured URL and static token using the URL redaction predicates. It covers credential-like path segments and query names, credential-shaped or credential-named query values, other nonempty query pairs, and WebSocket usernames and passwords. Encoded and decoded forms are included. Standalone strings need eight characters; shorter query values are removed with their name=value pair. Escaped alternatives match longest first in one replacement pass, so the replacement text is never rescanned. Cached access-provider tokens and secrets absent from the configuration remain outside this contract. Apply the helper where both framework engines record connection failures, errors and stderr, in startup and cleanup diagnostics, and in the engines' own error console lines. Modern requestFailure sanitizes transport text before returning it to tool dispatch. A JSON-RPC tool error remains the server's answer, and connection errors thrown directly to host callers remain unchanged. Document those limits and add a changelog fragment. Tests cover encoded/decoded forms, static tokens, WebSocket userinfo, short-value pairs, unrelated words, regex characters, longest-first matching and idempotence. Boundary tests cover both engines' recordings, legacy console failures, startup failures, HTTP 404 bodies, and a real tool 500 reaching framework dispatch. The connection 404 test replaces SDK negotiation with a tools/list send over the real HTTP transport: pin-mode discovery otherwise rewrites a 404 as a generic version-negotiation error. JSON-RPC errors and direct host connect errors retain their original text. Co-Authored-By: GPT-6.1 Sol --- README.md | 2 + .../redact-recorded-mcp-credentials.fixed.md | 1 + src/framework.ts | 37 ++-- src/mcpl/modern-connection.ts | 8 +- src/mcpl/protocol-family.ts | 59 ++++++ src/mcpl/server-connection.ts | 7 +- test/mcp-modern-http.test.ts | 55 ++++++ test/mcp-protocol-family.test.ts | 34 ++++ test/mcp-recorded-text.test.ts | 183 ++++++++++++++++++ 9 files changed, 363 insertions(+), 23 deletions(-) create mode 100644 changelog.d/redact-recorded-mcp-credentials.fixed.md create mode 100644 test/mcp-recorded-text.test.ts diff --git a/README.md b/README.md index 77ef73f74..27cff15bb 100644 --- a/README.md +++ b/README.md @@ -314,6 +314,8 @@ supports, or its error text when it lists none, and when the list includes `2026-07-28` it says to configure the server with `protocol: 'modern'` (stdio) or reach it over http(s). +Server and transport diagnostics recorded by the framework remove known credentials from the configured URL and static `token`. This covers connection traces, console errors, stderr traces and ops alerts, plus transport failures returned by modern tool calls. Credential-like URL path segments and query names, credential-shaped or credential-named query values, and WebSocket userinfo are replaced in their encoded and decoded forms. Other non-empty query parameters are replaced as whole `name=value` strings. Standalone strings shorter than eight characters stay unchanged because they may be ordinary words; short query values are still removed with their parameter names. The cached token from an `accessProvider` and secrets the framework does not know remain outside this filtering. JSON-RPC tool answers and direct host connection errors pass through as sent. + #### Feature sets `enabledFeatureSets` and `disabledFeatureSets` select which of a server's diff --git a/changelog.d/redact-recorded-mcp-credentials.fixed.md b/changelog.d/redact-recorded-mcp-credentials.fixed.md new file mode 100644 index 000000000..85d447944 --- /dev/null +++ b/changelog.d/redact-recorded-mcp-credentials.fixed.md @@ -0,0 +1 @@ +- Server and transport diagnostics recorded by the framework redact known URL credentials and the configured static token, including encoded and decoded forms. An HTTP error that echoes a credential-bearing request path no longer copies the credential into connection traces, console errors, ops alerts or modern tool-call failures. Stderr traces use the same filtering. Short standalone strings remain unchanged; short query values are removed as whole `name=value` strings. Access-provider cached tokens and unknown secrets are outside this filtering, and JSON-RPC tool answers and direct host connection errors retain their pass-through contract. diff --git a/src/framework.ts b/src/framework.ts index 27aa2c037..3dbfbff0b 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -394,7 +394,7 @@ import type { SessionUsageSnapshot, UsageUpdatedEvent } from './usage/types.js'; import type { McplServerConnection } from './mcpl/server-connection.js'; import { McplProtocolVersionError, McplUnreapedLaunchError } from './mcpl/server-connection.js'; import { ModernMcpConnection } from './mcpl/modern-connection.js'; -import { redactUrl, resolveServerBinding, serverConfigProblems } from './mcpl/protocol-family.js'; +import { redactUrl, redactServerText, resolveServerBinding, serverConfigProblems } from './mcpl/protocol-family.js'; import { MALFORMED_CONTENT_NOTICE, normalizeStandardToolResult } from './mcpl/tool-result-normalize.js'; import type { McplServerConfig, @@ -13479,7 +13479,8 @@ export class AgentFramework { } // Fail-open: log and continue with remaining servers const err = error instanceof Error ? error : new Error(String(error)); - console.error(`Failed to connect MCPL server "${config.id}":`, err.message); + const message = redactServerText(err.message, config); + console.error(`Failed to connect MCPL server "${config.id}":`, message); // Reaching this catch means no stub was created, so nothing will // retry — except in the edge case where addServer succeeded (with a // reconnecting stub) and a later setup step threw; config.reconnect @@ -13487,7 +13488,7 @@ export class AgentFramework { this.emitTrace({ type: 'mcpl:server-connect-failed', serverId: config.id, - error: err.message, + error: message, attempt: 0, // A protocol-version verdict is never retried (no common revision; // only configuration can fix it), and neither is a configuration @@ -13498,7 +13499,7 @@ export class AgentFramework { // So the server doesn't vanish quietly: alert at once, as the // reconnect path does for the same verdict, since no attempt follows. if (error instanceof McplProtocolVersionError) { - this.opsAlert('mcpl-down', config.id, `MCPL server unreachable at startup: ${err.message}`); + this.opsAlert('mcpl-down', config.id, `MCPL server unreachable at startup: ${message}`); } } } @@ -14026,7 +14027,7 @@ export class AgentFramework { // and holds the id: nothing launches beside it, and a disconnect // retries the reap. const message = cleanup instanceof Error ? cleanup.message : String(cleanup); - this.emitTrace({ type: 'mcpl:server-error', serverId: config.id, error: message }); + this.emitTrace({ type: 'mcpl:server-error', serverId: config.id, error: redactServerText(message, config) }); if (this.modernServers().get(config.id) === modern) { throw AgentFramework.unreapedConnectFailure(config.id, new Error(`${reason.message}; its cleanup also failed: ${message}`, { cause: reason })); } @@ -14750,10 +14751,12 @@ export class AgentFramework { * Push events and inference requests are deferred to Steps 6/7. */ private wireMcplEvents(connection: McplServerConnection): void { + const config = this.mcplServerConfigs.get(connection.id) ?? {}; + const recorded = (text: string): string => redactServerText(text, config); // Forward subprocess stderr lines as trace events so consumers (conhost, // log sinks, TUI badges) can persist and surface them. connection.on('stderr', (params: { line: string }) => { - this.emitTrace({ type: 'mcpl:server-stderr', serverId: connection.id, line: params.line }); + this.emitTrace({ type: 'mcpl:server-stderr', serverId: connection.id, line: recorded(params.line) }); }); // featureSets/changed is REMOVED in 0.5.0 — the connection answers it @@ -14971,21 +14974,21 @@ export class AgentFramework { this.emitTrace({ type: 'mcpl:server-connect-failed', serverId: connection.id, - error: params.error, + error: recorded(params.error), attempt: params.attempt, willRetry: connection.willReconnect, }); // `permanent`: the failed launch couldn't be reaped, so no retry will // start beside it. Alert now, as for a permanent reconnect failure. if (params.permanent) { - this.opsAlert('mcpl-down', connection.id, `MCPL server unreachable (attempt ${params.attempt}): ${params.error}`); + this.opsAlert('mcpl-down', connection.id, `MCPL server unreachable (attempt ${params.attempt}): ${recorded(params.error)}`); } }); connection.on('reconnect-failed', (params: { error: string; attempt: number; permanent?: boolean }) => { this.emitTrace({ type: 'mcpl:server-connect-failed', serverId: connection.id, - error: params.error, + error: recorded(params.error), attempt: params.attempt, willRetry: connection.willReconnect, }); @@ -15000,7 +15003,7 @@ export class AgentFramework { this.opsAlert( 'mcpl-down', connection.id, - `MCPL server unreachable (attempt ${params.attempt}): ${params.error}`, + `MCPL server unreachable (attempt ${params.attempt}): ${recorded(params.error)}`, ); } }); @@ -15032,7 +15035,7 @@ export class AgentFramework { this.emitTrace({ type: 'mcpl:server-error', serverId: connection.id, - error: err.message, + error: recorded(err.message), }); }); @@ -15077,8 +15080,10 @@ export class AgentFramework { * MCPL data-plane traffic, which this family doesn't carry. */ private wireModernEvents(connection: ModernMcpConnection): void { + const config = this.mcplServerConfigs.get(connection.id) ?? {}; + const recorded = (text: string): string => redactServerText(text, config); connection.on('stderr', (params: { line: string }) => { - this.emitTrace({ type: 'mcpl:server-stderr', serverId: connection.id, line: params.line }); + this.emitTrace({ type: 'mcpl:server-stderr', serverId: connection.id, line: recorded(params.line) }); }); connection.on('tools-list-changed', () => { this.handleToolsListChanged(connection.id); @@ -15092,20 +15097,20 @@ export class AgentFramework { this.emitTrace({ type: 'mcpl:server-connect-failed', serverId: connection.id, - error: params.error, + error: recorded(params.error), attempt: params.attempt, willRetry: connection.willReconnect, }); // `permanent`: reconnecting halted on a launch that couldn't be reaped. // No later attempt will come, so alert now. if (params.permanent || params.attempt >= 5) { - this.opsAlert('mcpl-down', connection.id, `MCP server unreachable (attempt ${params.attempt}): ${params.error}`); + this.opsAlert('mcpl-down', connection.id, `MCP server unreachable (attempt ${params.attempt}): ${recorded(params.error)}`); } }; connection.on('connect-failed', failed); connection.on('reconnect-failed', failed); connection.on('error', (err: Error) => { - this.emitTrace({ type: 'mcpl:server-error', serverId: connection.id, error: err.message }); + this.emitTrace({ type: 'mcpl:server-error', serverId: connection.id, error: recorded(err.message) }); }); connection.on('close', () => { // A closed provider's class hints are stale (RFC-008), as for MCPL. @@ -15224,7 +15229,7 @@ export class AgentFramework { this.emitTrace({ type: 'mcpl:server-error', serverId: server.id, - error: `tools/list failed: ${error instanceof Error ? error.message : String(error)}`, + error: `tools/list failed: ${redactServerText(error instanceof Error ? error.message : String(error), config ?? {})}`, }); } } diff --git a/src/mcpl/modern-connection.ts b/src/mcpl/modern-connection.ts index ff52ddd94..6515b143f 100644 --- a/src/mcpl/modern-connection.ts +++ b/src/mcpl/modern-connection.ts @@ -60,7 +60,7 @@ import { import { StdioTransport, type McplTransport } from './transport.js'; import { McplProtocolVersionError, McplRequestError } from './server-connection.js'; -import { LEGACY_MCP_PROTOCOL_VERSION, MAX_TIMER_MS, MODERN_MCP_PROTOCOL_VERSION, checkServerConfig, describeCredentialError } from './protocol-family.js'; +import { LEGACY_MCP_PROTOCOL_VERSION, MAX_TIMER_MS, MODERN_MCP_PROTOCOL_VERSION, checkServerConfig, describeCredentialError, redactServerText } from './protocol-family.js'; import type { McplServerConfig, McpToolDefinition } from './types.js'; const DEFAULT_REQUEST_TIMEOUT_MS = 60_000; @@ -365,7 +365,7 @@ export class ModernMcpConnection extends EventEmitter { // An 'error' with no listener would throw out of an SDK callback and take // the host down; out-of-band errors here are diagnostics, never fatal. this.on('error', (error: Error) => { - if (this.listenerCount('error') === 1) console.error(`[mcp] ${this.id}: ${error.message}`); + if (this.listenerCount('error') === 1) console.error(`[mcp] ${this.id}: ${redactServerText(error.message, this.config)}`); }); this.on('newListener', (event: string | symbol) => { if (event !== 'stderr' || this.stderrBacklog.length === 0) return; @@ -463,7 +463,7 @@ export class ModernMcpConnection extends EventEmitter { this.emit('connect-failed', { error: `${(error as Error).message}; ${halted}`, attempt: 0, permanent: true }); return; } - console.error(`MCP server "${this.id}" initial connect failed, will retry:`, (error as Error).message); + console.error(`MCP server "${this.id}" initial connect failed, will retry:`, redactServerText((error as Error).message, this.config)); this.reconnectAttempts = 1; this.emit('connect-failed', { error: (error as Error).message, attempt: 0 }); this.scheduleReconnect(); @@ -884,7 +884,7 @@ export class ModernMcpConnection extends EventEmitter { */ private requestFailure(what: string, record: WireRecord, error: unknown, timeoutMs?: number): Error { const prefix = `MCP server "${this.id}"`; - const message = error instanceof Error ? error.message : String(error); + const message = redactServerText(error instanceof Error ? error.message : String(error), this.config); const advice = record.method === 'tools/list' ? 'Listing changes nothing on the server, so it is safe to retry.' : 'Verify state before retrying; a blind retry of a side-effecting tool may duplicate it.'; diff --git a/src/mcpl/protocol-family.ts b/src/mcpl/protocol-family.ts index 238adc106..a6483bd7a 100644 --- a/src/mcpl/protocol-family.ts +++ b/src/mcpl/protocol-family.ts @@ -107,6 +107,65 @@ export function redactUrl(url: string | undefined): string { return parsed.toString(); } +/** + * Server/transport diagnostics as the framework records them. Replace only + * known URL credential strings and the configured static token, in their + * encoded and decoded forms. Short standalone strings (under 8 characters) + * are indistinguishable from ordinary words; short query values are removed + * as name=value instead. A single longest-first replacement never scans its + * own replacement text. Unknown secrets, including accessProvider's cached + * bearer token, are outside this contract. JSON-RPC tool answers and errors + * thrown directly to a host caller keep their separate pass-through contract. + */ +export function redactServerText(text: string, config: Pick): string { + const secrets = new Set(); + const standalone = (value: string): void => { if (value.length >= 8) secrets.add(value); }; + const decode = (value: string, query = false): string => { + try { return decodeURIComponent(query ? value.replace(/\+/g, ' ') : value); } + catch { return value; } + }; + standalone(config.token ?? ''); + let url: URL | undefined; + try { url = new URL(config.url ?? ''); } catch { /* Still remove the static token. */ } + if (url) { + for (const segment of url.pathname.split('/')) { + if (credentialLikeSegment(segment)) { + standalone(segment); + standalone(decode(segment)); + } + } + for (const parameter of url.search.slice(1).split('&')) { + const equals = parameter.indexOf('='); + const encodedName = equals < 0 ? parameter : parameter.slice(0, equals); + const encodedValue = equals < 0 ? '' : parameter.slice(equals + 1); + const name = decode(encodedName, true); + const value = decode(encodedValue, true); + if (credentialLikeQueryName(name, value)) { + standalone(encodedName); + standalone(name); + } + if (CREDENTIAL_PARAM.test(name) || credentialLikeSegment(value)) { + standalone(encodedValue); + standalone(value); + } + if (value !== '') { + secrets.add(`${encodedName}=${encodedValue}`); + secrets.add(`${name}=${value}`); + } + } + if (url.protocol === 'ws:' || url.protocol === 'wss:') { + for (const value of [url.username, url.password]) { + standalone(value); + standalone(decode(value)); + } + } + } + if (secrets.size === 0) return text; + const alternatives = [...secrets].sort((a, b) => b.length - a.length) + .map(value => value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')); + return text.replace(new RegExp(alternatives.join('|'), 'g'), 'redacted'); +} + /** * A credential provider's failure as a log line may show it: the error's * class and its message's length, never the text, which may quote the diff --git a/src/mcpl/server-connection.ts b/src/mcpl/server-connection.ts index b95ababf5..f3aa696ed 100644 --- a/src/mcpl/server-connection.ts +++ b/src/mcpl/server-connection.ts @@ -19,6 +19,7 @@ import { LEGACY_MCP_PROTOCOL_VERSION, MODERN_MCP_PROTOCOL_VERSION, checkServerConfig, + redactServerText, } from './protocol-family.js'; import type { @@ -870,12 +871,12 @@ export class McplServerConnection extends EventEmitter { // not lost in the window before wireMcplEvents runs. if (error instanceof McplUnreapedLaunchError) { const halted = error.connection; - console.error(`MCPL server "${config.id}" initial connect failed, not retrying:`, error.message); + console.error(`MCPL server "${config.id}" initial connect failed, not retrying:`, redactServerText(error.message, config)); halted.emit('connect-failed', { error: `${(error.cause as Error).message}; ${halted.unreapedFailure()}`, attempt: 0, permanent: true }); return halted; } // Non-blocking start: create a disconnected stub that will reconnect in background - console.error(`MCPL server "${config.id}" initial connect failed, will retry:`, (error as Error).message); + console.error(`MCPL server "${config.id}" initial connect failed, will retry:`, redactServerText((error as Error).message, config)); const stub = McplServerConnection.createDisconnectedStub(config, hostCapabilities); stub.emit('connect-failed', { error: (error as Error).message, attempt: 0 }); stub.scheduleReconnect(); @@ -1331,7 +1332,7 @@ export class McplServerConnection extends EventEmitter { // Closed by the host meanwhile: not a failed attempt. if (signal.aborted) return; const failure = unreaped ? error.failure : errorOf(error); - console.error(`MCPL server "${this.id}" reconnect failed:`, failure.message); + console.error(`MCPL server "${this.id}" reconnect failed:`, redactServerText(failure.message, this.config ?? {})); this.reconnectAttempts = attempt + 1; // No common protocol revision: every further attempt would restart the // server to hear the same answer. Stop retrying; the configuration is diff --git a/test/mcp-modern-http.test.ts b/test/mcp-modern-http.test.ts index a0a2ea6cb..30775d702 100644 --- a/test/mcp-modern-http.test.ts +++ b/test/mcp-modern-http.test.ts @@ -14,6 +14,10 @@ import assert from 'node:assert/strict'; import { createServer as createHttpServer, type Server, type ServerResponse } from 'node:http'; import { Readable } from 'node:stream'; import { join, dirname } from 'node:path'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { AgentFramework } from '../src/framework.js'; +import { MockMembrane } from './helpers/mock-membrane.js'; import { fileURLToPath, pathToFileURL } from 'node:url'; import type { AddressInfo } from 'node:net'; @@ -181,3 +185,54 @@ test('HTTP: an auth wall on a later round of a call is a no-response, since the assert.equal(states[0], undefined, 'the first round carried no state'); assert.ok(states.length >= 2 && states.slice(1).every((s) => s === 'state-1'), `each later round carried it: ${JSON.stringify(states)}`); }); + +test('HTTP: an echoed path in a tool transport failure is redacted, but a JSON-RPC error is the server answer', async (t) => { + const consoleLines: string[] = []; + t.mock.method(console, 'error', (...args: unknown[]) => consoleLines.push(args.map(String).join(' '))); + const secret = 'pathSecret0123456789'; + let rpc = true; + const { url, server } = await startServer(() => true, (message, res) => { + if (message.method !== 'tools/call') return false; + if (rpc) { + res.writeHead(200, { 'content-type': 'application/json' }).end(JSON.stringify({ jsonrpc: '2.0', id: message.id, error: { code: -32603, message: secret } })); + } else { + res.writeHead(500, { 'content-type': 'text/plain' }).end(`Cannot POST /api/mcp/s/${secret}/mcp`); + } + return true; + }); + cleanups.push(() => new Promise((resolve) => { server.closeAllConnections?.(); server.close(() => resolve()); })); + const connection = await ModernMcpConnection.connect({ id: 'remote', url: url.replace('/mcp', `/api/mcp/s/${secret}/mcp`) }); + cleanups.push(() => connection.close()); + await assert.rejects(connection.callTool('echo', {}), (error: unknown) => { + assert.ok(error instanceof McplRequestError); + assert.match(error.message, new RegExp(secret)); + assert.equal(error.outcome, 'error-response'); + return true; + }); + rpc = false; + await assert.rejects(connection.callTool('echo', {}), (error: unknown) => { + assert.ok(error instanceof McplRequestError); + assert.match(error.message, /Cannot POST \/api\/mcp\/s\/redacted\/mcp/); + assert.ok(!error.message.includes(secret)); + assert.equal(error.outcome, 'no-response'); + return true; + }); + assert.ok(consoleLines.some(line => line.includes('Cannot POST /api/mcp/s/redacted/mcp'))); + assert.ok(!consoleLines.join('\n').includes(secret)); + const dir = mkdtempSync(join(tmpdir(), 'mcp-tool-transport-')); + const framework = await AgentFramework.create({ + storePath: join(dir, 'store'), membrane: new MockMembrane().asMembrane(), + agents: [{ name: 'agent', model: 'test', systemPrompt: 'test' }], + modules: [], + mcplServers: [{ id: 'remote', url: url.replace('/mcp', `/api/mcp/s/${secret}/mcp`) }], + }); + try { + const result = await framework.executeToolCall({ id: 'echo-failed', name: 'mcpl--remote--echo', input: {} }); + assert.equal(result.success, false); + assert.match(result.error!, /Cannot POST \/api\/mcp\/s\/redacted\/mcp/); + assert.ok(!JSON.stringify(result).includes(secret), 'the model-facing tool result contains no path credential'); + } finally { + await framework.stop(); + rmSync(dir, { recursive: true, force: true }); + } +}); diff --git a/test/mcp-protocol-family.test.ts b/test/mcp-protocol-family.test.ts index 382eb148d..d6c490c95 100644 --- a/test/mcp-protocol-family.test.ts +++ b/test/mcp-protocol-family.test.ts @@ -13,6 +13,7 @@ import { serverConfigWarnings, checkServerConfig, redactUrl, + redactServerText, describeCredentialError, MCPL_ONLY_POLICY_FIELDS, MAX_TIMER_MS, @@ -294,3 +295,36 @@ test('both engines admit a configuration through checkServerConfig, which logs i console.warn = realWarn; } }); + +test('recorded server text removes encoded and decoded URL credentials as exact strings', () => { + const config = cfg({ url: 'https://h/api/s/secret%2B0123456789/mcp?password=p%2Bassw0rd%21&bare%2Bsecret0123456789&plain=value0123456789abcdef®ion=eu&label=a%20b' }); + const secrets = ['secret%2B0123456789', 'secret+0123456789', 'p%2Bassw0rd%21', 'p+assw0rd!', 'bare%2Bsecret0123456789', 'bare+secret0123456789', 'value0123456789abcdef', 'region=eu', 'label=a%20b', 'label=a b']; + for (const secret of secrets) assert.equal(redactServerText(`queue full: ${secret}; eu`, config), 'queue full: redacted; eu', secret); +}); + +test('recorded server text removes WebSocket userinfo and the static token, escaping regex characters', () => { + const config = cfg({ url: 'wss://user%2B12345678:pass%2B12345678@h/mcp', token: 'token.[1234]' }); + for (const secret of ['user%2B12345678', 'user+12345678', 'pass%2B12345678', 'pass+12345678', config.token!]) { + assert.equal(redactServerText(`error: ${secret}`, config), 'error: redacted'); + } + assert.equal(redactServerText('tokenX11234]', config), 'tokenX11234]'); + assert.equal(redactServerText('static-token123', cfg({ token: 'static-token123' })), 'redacted'); + assert.equal(redactServerText('static-token123', cfg({ url: 'invalid', token: 'static-token123' })), 'redacted'); +}); + +test('recorded server text keeps short standalone values and unrelated words, but removes whole query pairs', () => { + const config = cfg({ url: 'https://h/mcp?auth=x&token=auth®ion=eu&debug', token: 'x' }); + assert.equal(redactServerText('auth x eu queue full debug', config), 'auth x eu queue full debug'); + assert.equal(redactServerText('auth=x token=auth region=eu', config), 'redacted redacted redacted'); + assert.equal(redactServerText('1234567 12345678', cfg({token:'1234567'})), '1234567 12345678'); + assert.equal(redactServerText('12345678', cfg({token:'12345678'})), 'redacted'); +}); + +test('recorded server text uses longest-first, one-pass replacements and is idempotent', () => { + const config = cfg({ url: 'https://h/mcp?secret=token0123456789-tail&redacted=eu', token: 'token0123456789' }); + const original = 'secret=token0123456789-tail token0123456789 redacted=eu queue full'; + const clean = redactServerText(original, config); + assert.equal(clean, 'redacted redacted redacted queue full'); + assert.equal(redactServerText('token0123456789-tail', config), 'redacted'); + assert.equal(redactServerText(clean, config), clean); +}); diff --git a/test/mcp-recorded-text.test.ts b/test/mcp-recorded-text.test.ts new file mode 100644 index 000000000..8350fac45 --- /dev/null +++ b/test/mcp-recorded-text.test.ts @@ -0,0 +1,183 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { createServer } from 'node:http'; +import type { AddressInfo } from 'node:net'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { Client, type Transport } from '@modelcontextprotocol/client'; +import { EventEmitter } from 'node:events'; +import type { ModuleContext } from '../src/index.js'; +import { AgentFramework } from '../src/framework.js'; +import { McplServerConnection, McplUnreapedLaunchError } from '../src/mcpl/server-connection.js'; +import { ModernMcpConnection } from '../src/mcpl/modern-connection.js'; +import { MockMembrane } from './helpers/mock-membrane.js'; + +/** Discovery succeeds; the inventory request gets a normal echoed-path 404. */ +function echo404Server() { + return createServer(async (req, res) => { + const chunks: Buffer[] = []; + for await (const chunk of req) chunks.push(chunk as Buffer); + const message = JSON.parse(Buffer.concat(chunks).toString('utf8')); + if (message.method === 'server/discover') { + res.writeHead(200, { 'content-type': 'application/json', 'mcp-protocol-version': '2026-07-28' }).end(JSON.stringify({ + jsonrpc: '2.0', id: message.id, result: { resultType: 'complete', supportedVersions: ['2026-07-28'], capabilities: { tools: {} }, ttlMs: 60000, cacheScope: 'private', _meta: { 'io.modelcontextprotocol/serverInfo': { name: 'echo', version: '1' } } }, + })); + } else { + res.writeHead(404, { 'content-type': 'text/plain' }).end(`Cannot POST ${req.url}`); + } + }); +} + +/** The SDK's pin-mode discovery rewrites an HTTP 404 into a version error. + * Send tools/list through the real transport at the connect seam so this test + * isolates how connection recordings handle an SDK error with the real body. */ +function failConnectWithHttpBody(t: import('node:test').TestContext): void { + t.mock.method(Client.prototype, 'connect', async (transport: Transport) => { + await transport.start(); + try { await transport.send({ jsonrpc: '2.0', id: 1, method: 'tools/list' }); } + finally { await transport.close(); } + }); +} + +const secret = 'pathSecret0123456789'; +const config = { id: 'remote', url: `https://h/api/mcp/s/${secret}/mcp`, token: 'staticToken012345' }; + +test('both framework engines scrub failure, error and stderr records before tracing and alerting', () => { + for (const method of ['wireMcplEvents', 'wireModernEvents']) { + const traces: unknown[] = []; + const alerts: string[] = []; + const framework = Object.create(AgentFramework.prototype) as any; + framework.mcplServerConfigs = new Map([[config.id, config]]); + framework.emitTrace = (event: unknown) => traces.push(event); + framework.opsAlert = (_kind: string, _id: string, text: string) => alerts.push(text); + const connection = Object.assign(new EventEmitter(), { id: config.id, willReconnect: true }); + framework[method](connection); + connection.emit('connect-failed', { error: `Cannot POST ${secret}`, attempt: 0, permanent: true }); + connection.emit('reconnect-failed', { error: `Cannot POST ${secret}`, attempt: 5 }); + connection.emit('error', new Error(`transport ${secret}`)); + connection.emit('stderr', { line: `server ${config.token}` }); + assert.equal(traces.length, 4); + assert.equal(alerts.length, 2); + assert.ok(!JSON.stringify(traces).includes(secret), method); + assert.ok(!JSON.stringify(traces).includes(config.token), method); + assert.ok(alerts.every(text => text.includes('Cannot POST redacted')), method); + } +}); + +test('an SDK HTTP 404 error keeps its path but hides the secret in connection console, traces and ops alerts', async (t) => { + failConnectWithHttpBody(t); + const dir = mkdtempSync(join(tmpdir(), 'mcp-recorded-text-')); + const consoleLines: string[] = []; + t.mock.method(console, 'error', (...args: unknown[]) => consoleLines.push(args.map(String).join(' '))); + const traces: Array<{ type: string; error?: string; message?: string; attempt?: number }> = []; + const server = echo404Server(); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + const port = (server.address() as AddressInfo).port; + let framework: AgentFramework | undefined; + try { + framework = await AgentFramework.create({ + storePath: join(dir, 'store'), membrane: new MockMembrane().asMembrane(), + agents: [{ name: 'agent', model: 'test', systemPrompt: 'test' }], + modules: [{ name: 'watcher', async start(ctx: ModuleContext) { ctx.onTrace(event => traces.push(event as never)); }, async stop() {}, getTools: () => [], async handleToolCall() { return { success: false }; }, async onProcess() { return {}; } }], + mcplServers: [{ id: 'echo', url: `http://127.0.0.1:${port}/api/mcp/s/${secret}/mcp`, reconnect: true, reconnectIntervalMs: 10, reconnectMaxIntervalMs: 20 }], + }); + const deadline = Date.now() + 5000; + while (!traces.some(e => e.type === 'ops:alert') && Date.now() < deadline) await new Promise(resolve => setTimeout(resolve, 10)); + const failed = traces.filter(e => e.type === 'mcpl:server-connect-failed'); + assert.ok(failed.some(e => e.attempt === 0)); + assert.ok(failed.some(e => (e.attempt ?? 0) >= 5)); + assert.ok(failed.every(e => e.error?.includes('Cannot POST /api/mcp/s/redacted/mcp')), JSON.stringify(failed)); + assert.ok(traces.some(e => e.type === 'ops:alert' && e.message?.includes('Cannot POST /api/mcp/s/redacted/mcp'))); + assert.ok(consoleLines.some(line => line.includes('Cannot POST /api/mcp/s/redacted/mcp'))); + assert.ok(!JSON.stringify(traces).includes(secret)); + assert.ok(!consoleLines.join('\n').includes(secret)); + } finally { + await framework?.stop(); + server.closeAllConnections(); + await new Promise(resolve => server.close(() => resolve())); + rmSync(dir, { recursive: true, force: true }); + } +}); + + +test('startup records redact an echoed path while the direct host connect error stays as raised', async (t) => { + failConnectWithHttpBody(t); + const dir = mkdtempSync(join(tmpdir(), 'mcp-startup-text-')); + const consoleLines: string[] = []; + t.mock.method(console, 'error', (...args: unknown[]) => consoleLines.push(args.map(String).join(' '))); + const traces: Array<{ type: string; error?: string }> = []; + const server = echo404Server(); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + const url = `http://127.0.0.1:${(server.address() as AddressInfo).port}/api/mcp/s/${secret}/mcp`; + let framework: AgentFramework | undefined; + try { + await assert.rejects(ModernMcpConnection.connect({ id: 'raw', url }), error => { + assert.ok(error instanceof Error && error.message.includes(secret)); + return true; + }); + framework = await AgentFramework.create({ + storePath: join(dir, 'store'), membrane: new MockMembrane().asMembrane(), + agents: [{ name: 'agent', model: 'test', systemPrompt: 'test' }], + modules: [{ name: 'watcher', async start(ctx: ModuleContext) { ctx.onTrace(event => traces.push(event as never)); }, async stop() {}, getTools: () => [], async handleToolCall() { return { success: false }; }, async onProcess() { return {}; } }], + mcplServers: [{ id: 'echo', url }], + }); + assert.ok(traces.some(e => e.type === 'mcpl:server-connect-failed' && e.error?.includes('Cannot POST /api/mcp/s/redacted/mcp'))); + assert.ok(consoleLines.some(line => line.includes('Failed to connect MCPL server') && line.includes('Cannot POST /api/mcp/s/redacted/mcp'))); + assert.ok(!JSON.stringify(traces).includes(secret)); + assert.ok(!consoleLines.join('\n').includes(secret)); + } finally { + await framework?.stop(); + server.closeAllConnections(); + await new Promise(resolve => server.close(() => resolve())); + rmSync(dir, { recursive: true, force: true }); + } +}); + + +test('legacy initial and reconnect console records redact known credentials', async (t) => { + const lines: string[] = []; + t.mock.method(console, 'error', (...args: unknown[]) => lines.push(args.map(String).join(' '))); + t.mock.method(McplServerConnection as any, 'handshake', async () => { throw new Error(`transport ${secret}`); }); + const legacyConfig = { id: 'legacy', url: `ws://127.0.0.1/s/${secret}`, reconnect: true, reconnectIntervalMs: 10, reconnectMaxIntervalMs: 20 }; + const capabilities = { version: '0.5', pushEvents: true }; + const connection = await McplServerConnection.connectWithReconnect(legacyConfig, capabilities); + try { + const deadline = Date.now() + 5000; + while (!lines.some(line => line.includes('reconnect failed:')) && Date.now() < deadline) await new Promise(resolve => setTimeout(resolve, 10)); + assert.ok(lines.some(line => line.includes('initial connect failed, will retry: transport redacted'))); + assert.ok(lines.some(line => line.includes('reconnect failed: transport redacted'))); + assert.ok(!lines.join('\n').includes(secret)); + } finally { await connection.close(); } + const halted = Object.assign(new EventEmitter(), { unreapedFailure: () => `cleanup ${secret}` }); + t.mock.method(McplServerConnection as any, 'connectChecked', async () => { + throw new McplUnreapedLaunchError(halted as unknown as McplServerConnection, new Error(`transport ${secret}`), new Error(`cleanup ${secret}`)); + }); + await McplServerConnection.connectWithReconnect(legacyConfig, capabilities); + assert.ok(lines.some(line => line.includes('initial connect failed, not retrying: transport redacted; its cleanup also failed: cleanup redacted'))); + assert.ok(!lines.join('\n').includes(secret)); +}); + +test('a real discover-then-tools/list 404 is redacted in the framework error trace', async (t) => { + t.mock.method(console, 'error', () => {}); + const dir = mkdtempSync(join(tmpdir(), 'mcp-list-text-')); + const traces: Array<{ type: string; error?: string }> = []; + const server = echo404Server(); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + let framework: AgentFramework | undefined; + try { + framework = await AgentFramework.create({ + storePath: join(dir, 'store'), membrane: new MockMembrane().asMembrane(), + agents: [{ name: 'agent', model: 'test', systemPrompt: 'test' }], + modules: [{ name: 'watcher', async start(ctx: ModuleContext) { ctx.onTrace(event => traces.push(event as never)); }, async stop() {}, getTools: () => [], async handleToolCall() { return { success: false }; }, async onProcess() { return {}; } }], + mcplServers: [{ id: 'echo', url: `http://127.0.0.1:${(server.address() as AddressInfo).port}/api/mcp/s/${secret}/mcp` }], + }); + assert.ok(traces.some(e => e.type === 'mcpl:server-error' && e.error?.includes('Cannot POST /api/mcp/s/redacted/mcp')), JSON.stringify(traces)); + assert.ok(!JSON.stringify(traces).includes(secret)); + } finally { + await framework?.stop(); + server.closeAllConnections(); + await new Promise(resolve => server.close(() => resolve())); + rmSync(dir, { recursive: true, force: true }); + } +}); From 11bfe4779e62ef7b225a7941b347970b2e83427a Mon Sep 17 00:00:00 2001 From: Rosalind-2359 Date: Tue, 13 Oct 2026 00:52:40 -0400 Subject: [PATCH 0144/1112] test(mcp): initialize server configuration in the observe wire fixture The observe parser wire tests bypass the framework constructor with Object.create. Recording handlers now look up the configured server to redact known credentials, so this stand-in must initialize that map just as a real framework does. Give its observer a stdio configuration; no URL or static token is configured in these admission tests. The expanded protocol/HTTP/recording/observe-parser focused run passes 41 tests with no failures, after typecheck and build. Co-Authored-By: GPT-6.1 Sol --- test/tools-observe-parser-conformance.test.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/tools-observe-parser-conformance.test.ts b/test/tools-observe-parser-conformance.test.ts index c493ed8f6..1a73a70c1 100644 --- a/test/tools-observe-parser-conformance.test.ts +++ b/test/tools-observe-parser-conformance.test.ts @@ -5,6 +5,7 @@ import { test } from 'node:test'; import assert from 'node:assert/strict'; import { AgentFramework } from '../src/framework.js'; +import type { McplServerConfig } from '../src/mcpl/types.js'; import { McplServerConnection } from '../src/mcpl/server-connection.js'; import { McplTransport } from '../src/mcpl/transport.js'; import { CapabilityGrant } from '../src/mcpl/capability-grant.js'; @@ -79,8 +80,10 @@ function wireHarness() { ) => McplServerConnection; const connection = new Connection('observer', null, transport); const framework = Object.create(AgentFramework.prototype) as { + mcplServerConfigs: Map; wireMcplEvents(connection: McplServerConnection): void; }; + framework.mcplServerConfigs = new Map([['observer', { id: 'observer', command: 'unused' }]]); framework.wireMcplEvents(connection); connection.establishGrant(new CapabilityGrant(new Set(['toolLifecycle.observe']), [])); connection.ready(); From b5f0adb310a9b3838c881d035a3576e4b8a6e759 Mon Sep 17 00:00:00 2001 From: Nathan-2367 Date: Tue, 13 Oct 2026 01:26:03 -0400 Subject: [PATCH 0145/1112] fix(prose): retain failed envelopes and settle queued delivery outcomes Explicit envelopes now retain a failed or unexpectedly thrown delivery as a bounce draft with its private notice. They follow the hybrid failure path, while structured unknown outcomes remain marker-only rather than becoming fresh words offered for an ordinary resend. Re-bounces use the authored body before unsent-token expansion so an unresolved source attempt keeps its duplication risk. Await the speech chain once, unconditionally, before final prose dispatch and the completed-turn receipt. The prior awaits were inside final-text branches; an empty completion could receipt and settle a turn while its earlier post and held prose were still queued. Hybrid skip-reply envelopes can dismiss the latest bounce inside a private round, while same-round private-think text remains uninterpreted, matching explicit mode. Forward a bounded display of connector error data through the speech failure log, trace and private failure marker. The returned publish outcome retains the original structured data. Bound both string and malformed message ids in contradictory-receipt diagnostics without rewriting a receipt id. Wrap a synchronous transport write rejection as not-sent and clear its pending entry and timer. Correct the resident help and draft-state/provenance comments. Strengthen the existing dismissal test to observe a real held draft first and its dismissed state afterward. A silent turn intentionally gets no delivery receipt, so its privacy test now asserts that observable contract rather than claiming a resident-visible count. Tests cover explicit failed and thrown envelopes, inherited risk after an unresolved unsent attempt, uncertain delivery guards and real-connection markers, private hybrid dismissal with a private-think control, empty completion behind a controlled slow post and hold, structured connector data, bounded contradictory identifiers and recovery after an unwritten request. The four touched files pass 126 tests after a clean VPS typecheck/build; against the preceding source, ten tests fail on the relevant behavior and the other cases are controls. The immutable committed tree is the input to the full per-file check and individual piece removals. Co-authored-by: GPT-6.1 Sol --- .../prose-drafts-delivery-outcomes.fixed.md | 4 + src/framework.ts | 53 +++++--- src/mcpl/channel-registry.ts | 26 +++- src/mcpl/server-connection.ts | 13 +- src/prose-drafts.ts | 17 +-- test/explicit-prose-routing.test.ts | 113 +++++++++++++++--- test/fixtures/speech-route-mcpl-server.mjs | 6 +- test/mcpl-request-outcome.test.ts | 16 +++ test/prose-drafts.test.ts | 76 +++++++++++- test/publish-outcome.test.ts | 48 +++++++- 10 files changed, 320 insertions(+), 52 deletions(-) create mode 100644 changelog.d/prose-drafts-delivery-outcomes.fixed.md diff --git a/changelog.d/prose-drafts-delivery-outcomes.fixed.md b/changelog.d/prose-drafts-delivery-outcomes.fixed.md new file mode 100644 index 000000000..982bffac9 --- /dev/null +++ b/changelog.d/prose-drafts-delivery-outcomes.fixed.md @@ -0,0 +1,4 @@ +- Explicit prose whose delivery definitely fails is held as a bounce draft with a private notice, matching hybrid envelopes. Unknown delivery outcomes remain failure-marker-only, so their words are not offered for an ordinary resend that might duplicate a post. +- A completing turn waits for its queued speech and hybrid holds even when its final response is empty, so its receipt includes that work and its draft notices settle with the same turn. +- Plain-speech failure notices and traces include a bounded display of the connector's structured error data, such as parts already posted. Contradictory delivery receipts bound the message id in their diagnostic without changing the receipt's id. A synchronous MCPL write rejection reports that the request was not sent and removes its pending state. +- A hybrid `>>>skip_reply {{unsent}}` in a skip-reply round sets the latest bounce aside while keeping publication private; same-round private-think text stays uninterpreted, matching explicit mode. diff --git a/src/framework.ts b/src/framework.ts index 4df304853..9753c5e60 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -351,9 +351,9 @@ const PROSE_ROUTING_HELP = ' immediately when this one ends, instead of pausing until the next event.\n' + ' >>skip_reply — text stays in your context only, like the skip_reply tool.\n' + 'Text without a destination is not delivered: it is held as a draft, and a notice\n' + - 'names it — reply e.g. ">>#channel {{unsent}}" to deliver your latest held text\n' + + 'names it — reply e.g. ">>#channel {{unsent}}" to deliver your latest bounce draft\n' + 'unchanged, or use the drafts tool to list, resend or dismiss any draft by id.\n' + - '">>skip_reply {{unsent}}" sets the latest one aside. Send tools (send_message,\n' + + '">>skip_reply {{unsent}}" sets the latest bounce aside. Send tools (send_message,\n' + 'send_dm, …) are unaffected.'; const PROSE_HELP_TOOL: import('./types/index.js').ToolDefinition = { @@ -9059,6 +9059,16 @@ export class AgentFramework { ): void { if (segments.length === 0) return; if (cause === 'private' || privateThink) { + // Match explicit mode: skip_reply keeps publication private but can + // still set a bounce aside. A private-think round interprets no text. + if (cause === 'private' && !privateThink && agent.proseRouting === 'hybrid') { + for (const segment of segments) { + for (const envelope of splitHybridEnvelopes(segment)) { + const parsed = parseHybridProsePrefix(envelope); + if (parsed.kind === 'private' && parsed.body.includes('{{unsent}}')) this.dismissLatestBounce(agent); + } + } + } this.turnProsePrivate.set(agent.name, (this.turnProsePrivate.get(agent.name) ?? 0) + segments.length); return; } @@ -10297,6 +10307,7 @@ export class AgentFramework { body = rawText; } + const envelopeBody = body; const usedClipboard = body.includes('{{unsent}}'); const unsent = usedClipboard ? this.takeUnsent(agent, body, hold) : {}; if (unsent.refused) return; @@ -10310,9 +10321,15 @@ export class AgentFramework { try { const outcome = await this.deliverWithUnsent(agent, body, targetChannel, unsent.draft); - if (outcome.status === 'delivered') this.proseBounceStreaks.delete(name); + if (outcome.status === 'delivered') { + this.proseBounceStreaks.delete(name); + } else if (outcome.status === 'failed') { + // Unknown deliveries are marker-only: offering a fresh draft would + // invite a duplicate. Keep the authored body for the re-bounce rule. + this.bounceProse(agent, envelopeBody, `delivery to ${targetChannel} failed: ${outcome.reason ?? 'refused'}`, undefined, '>>', hold); + } } catch (err) { - console.error(`[prose] ${name}: delivery to ${targetChannel} failed:`, err); + this.bounceProse(agent, envelopeBody, `delivery to ${targetChannel} failed: ${err instanceof Error ? err.message : String(err)}`, undefined, '>>', hold); } } @@ -11947,6 +11964,11 @@ export class AgentFramework { ); } + // Queued deliveries and hybrid holds belong to this turn even + // when its final response is empty. Settle them before final + // dispatch, the receipt and the turn-end draft settlement. + await turnSpeechChain; + // Host-owned output routing (see LOCUS-ROUTING-DESIGN). On a // text-only turn (no tool calls => speechContent populated), // publish the agent's speech to the conversational locus — the @@ -11960,9 +11982,6 @@ export class AgentFramework { const speechSegments = finalSpeechSegments; const textOnlyHold = { round: draftRound + 1, notice: 'later' as const }; if (speechSegments.length > 0) { - // A turn resumed after its earlier rounds delivers after - // them: their queued deliveries are this chain's. - await turnSpeechChain; if (turnProseRouting === 'disabled') { console.error(`[routing] ${agent.name}: text-only prose NOT routed (proseRouting=disabled)`); this.recordProseSuppression(agent.name, speechSegments.length); @@ -12044,11 +12063,6 @@ export class AgentFramework { const runs = finalRuns; const segments = finalSegments; - // Preserve in-channel ordering: everything enqueued live must - // land before the trailing prose. Awaited even when silenced — - // the chain may still be flushing earlier rounds' posts. - await turnSpeechChain; - if (turnProseRouting === 'disabled') { if (segments.length > 0) { console.error( @@ -12118,9 +12132,9 @@ export class AgentFramework { // doesn't depend on the status field, so it correctly runs after. // Logical turn end (normal completion): drop the `[delivered]` - // receipt. All deliveries are settled — the live chain was - // awaited before trailing dispatch, and trailing/text-only - // segments were awaited in-loop. Locus mode only; explicit-mode + // receipt. Queued deliveries and holds settled before final + // dispatch; trailing/text-only segments were awaited in-loop. + // Locus and hybrid modes only; explicit-mode // envelopes acknowledge themselves through the prose gateway. if (!trigger?.suppressProse && turnProseRouting !== 'explicit') { this.appendProseDeliveryReceipt(agent); @@ -14915,7 +14929,7 @@ export class AgentFramework { // addMessage() alone does not request inference, so this never wakes // her (matching the `discord-send-failed-skip` gate intent: context // yes, wake no). - onRouteFailure: ({ channelId, reason, textLen, outcome }) => { + onRouteFailure: ({ channelId, reason, textLen, outcome, detail }) => { try { // Render a human-readable channel name when we can — a bare // snowflake in the marker is unresolvable for the agent (the @@ -14934,13 +14948,14 @@ export class AgentFramework { 'user', [{ type: 'text', - text: outcome === 'unknown' + text: (outcome === 'unknown' // A request went out and no valid receipt came back: it may // have been posted (in part or whole). Never claim it wasn't. ? `[discord-send-failed] Delivery of your previous reply (${textLen} chars) to ${where} was not confirmed (${reason}). It may or may not have been posted; check the channel before sending it again. It is saved in your archive.` - : `[discord-send-failed] Your previous reply (${textLen} chars) could not be delivered to ${where} (${reason}). Nothing was posted; it is saved in your archive.`, + : `[discord-send-failed] Your previous reply (${textLen} chars) could not be delivered to ${where} (${reason}). Nothing was posted; it is saved in your archive.`) + + (detail !== undefined ? ` Connector detail: ${detail}` : ''), }], - { system: true, kind: 'discord-send-failed', channelId: channelId ?? '', reason, ...(outcome ? { outcome } : {}) }, + { system: true, kind: 'discord-send-failed', channelId: channelId ?? '', reason, ...(outcome ? { outcome } : {}), ...(detail !== undefined ? { detail } : {}) }, ); } catch (err) { console.error('onRouteFailure: failed to record send-failure marker:', err); diff --git a/src/mcpl/channel-registry.ts b/src/mcpl/channel-registry.ts index 222fb2d03..a14ebb70a 100644 --- a/src/mcpl/channel-registry.ts +++ b/src/mcpl/channel-registry.ts @@ -460,6 +460,19 @@ const CHANNEL_TOOL_DEFINITIONS: ToolDefinition[] = [ // Constructor Options // ============================================================================ +/** Bound connector values for notices/logs; the PublishOutcome keeps the original. */ +function connectorText(value: unknown): string { + let text: string; + try { + text = typeof value === 'string' ? value : JSON.stringify(value) ?? String(value); + } catch { + return '[unserializable connector value]'; + } + const limit = 2000; + const suffix = '… [truncated]'; + return text.length > limit ? text.slice(0, limit - suffix.length) + suffix : text; +} + /** Where a publish was resolved to go — named by the registry, never by * the caller's spelling. */ export interface PublishDestination { @@ -534,6 +547,8 @@ interface ChannelRegistryOptions { /** `failed`: nothing was posted; `unknown`: the request was dispatched * and no valid receipt came back, so it may or may not have been. */ outcome?: 'failed' | 'unknown'; + /** Bounded display of the connector's structured error data, when present. */ + detail?: string; }) => void; /** * Called when channels were opened WITHOUT the agent asking (subscription @@ -3208,9 +3223,9 @@ export class ChannelRegistry { destination, ...(messageId ? { messageId } : {}), reason: messageId - ? `server "${entry.serverId}" reported delivered:false but named posted message ${messageId} (delivery uncertain)` + ? `server "${entry.serverId}" reported delivered:false but named posted message ${connectorText(messageId)} (delivery uncertain)` : `server "${entry.serverId}" reported delivered:false beside a malformed message id ` + - `${JSON.stringify(receipt.messageId)} (contradictory receipt; delivery uncertain)`, + `${connectorText(receipt.messageId)} (contradictory receipt; delivery uncertain)`, at: at(), }; } @@ -3246,7 +3261,9 @@ export class ChannelRegistry { ): Promise { const fail = (channelId: string | null, reason: string, outcome: PublishOutcome): PublishOutcome => { const status = outcome.status === 'unknown' ? 'unknown' : 'failed'; - console.error(`[routeSpeech] ${conversationId}: ${reason} — speech ${status === 'unknown' ? 'delivery NOT confirmed' : 'NOT routed'} (${text.length} chars stay in chronicle)`); + const detail = outcome.detail !== undefined ? connectorText(outcome.detail) : undefined; + const detailText = detail !== undefined ? `; connector detail: ${detail}` : ''; + console.error(`[routeSpeech] ${conversationId}: ${reason}${detailText} — speech ${status === 'unknown' ? 'delivery NOT confirmed' : 'NOT routed'} (${text.length} chars stay in chronicle)`); this.emitTraceFn({ type: 'mcpl:speech-route-failed', conversationId, @@ -3254,8 +3271,9 @@ export class ChannelRegistry { reason, textLen: text.length, outcome: status, + ...(detail !== undefined ? { detail } : {}), }); - this.onRouteFailure?.({ conversationId, channelId, reason, textLen: text.length, outcome: status }); + this.onRouteFailure?.({ conversationId, channelId, reason, textLen: text.length, outcome: status, ...(detail !== undefined ? { detail } : {}) }); return outcome; }; diff --git a/src/mcpl/server-connection.ts b/src/mcpl/server-connection.ts index 3e7658b10..9de3b48b2 100644 --- a/src/mcpl/server-connection.ts +++ b/src/mcpl/server-connection.ts @@ -1077,7 +1077,18 @@ export class McplServerConnection extends EventEmitter { pending.timer.unref?.(); } this.pendingRequests.set(id, pending); - this.transport!.writeLine(JSON.stringify(request)); + try { + this.transport!.writeLine(JSON.stringify(request)); + } catch (err) { + // A synchronous write rejection sent no request. Remove its pending + // state and watchdog before reporting that it is safe to retry. + this.pendingRequests.delete(id); + if (pending.timer) clearTimeout(pending.timer); + reject(new McplRequestError( + `Cannot send ${method} to MCPL server "${this.id}": ${err instanceof Error ? err.message : String(err)}`, + 'not-sent', + )); + } }); } diff --git a/src/prose-drafts.ts b/src/prose-drafts.ts index c7002903a..33bfcb3e8 100644 --- a/src/prose-drafts.ts +++ b/src/prose-drafts.ts @@ -38,9 +38,9 @@ * host, and its outcome after: an attempt with no recorded outcome (a crash * mid-send) reads as unknown, never as unsent. A delivered draft carries a * historical receipt — confirmed at a time, to a destination, with the - * connector's message id — not a claim that the message still exists. - * Nothing here publishes anything: every send is the resident's explicit - * resend, and nothing replays at boot or on a branch move. + * connector's message id when it names one — not a claim that the message + * still exists. A draft is sent only by its resident's explicit resend or + * an envelope's {{unsent}}; nothing replays at boot or on a branch move. */ import { randomInt } from 'node:crypto'; @@ -66,8 +66,9 @@ export type DraftReason = export interface DraftSource { /** Chronicle branch current when the words were held. */ branch: string; - /** The resident's logical turn (process-local counter) and the round in it. */ + /** The resident's logical turn (process-local counter). */ turn: number; + /** Round within this stream; numbering restarts at a budget restart. */ round: number; /** Position of the segment within its round's speech. */ segment: number; @@ -137,9 +138,11 @@ export interface Draft { } /** - * - `held`: never attempted, or every attempt definitely failed — resend freely; - * - `unconfirmed`: some attempt's outcome is unknown (or was never - * recorded), and none was confirmed: it may have been posted, so a resend + * - `held`: no inherited risk, and never attempted or every attempt + * definitely failed — resend freely; + * - `unconfirmed`: no attempt was confirmed and the draft is not dismissed; + * either an attempt's outcome is unknown (or was never recorded), or the + * words carry inherited risk regardless of their own attempts. A resend * needs explicit confirmation. A later attempt that fails does not resolve * an earlier one that may have landed — the uncertainty stays; * - `delivered`: an attempt was confirmed — its receipt is historical; diff --git a/test/explicit-prose-routing.test.ts b/test/explicit-prose-routing.test.ts index 2d6ed09e6..f338f5286 100644 --- a/test/explicit-prose-routing.test.ts +++ b/test/explicit-prose-routing.test.ts @@ -27,6 +27,7 @@ import type { import { AgentFramework } from '../src/index.js'; import { MockMembrane, createMockResponse } from './helpers/mock-membrane.js'; import { ProseStreamRouter } from '../src/mcpl/prose-stream-router.js'; +import { draftState, type ProseDraftStore } from '../src/prose-drafts.js'; import type { ContentBlock } from '@animalabs/membrane'; class ToolboxModule implements Module { @@ -67,7 +68,7 @@ class ToolboxModule implements Module { } /** Registry stub: two known channels (#alpha guild, laria's DM) + capture. */ -function stubRegistry(framework: AgentFramework, plan: Array<'delivered' | 'false' | 'throw'> = []) { +function stubRegistry(framework: AgentFramework, plan: Array<'delivered' | 'false' | 'unknown' | 'throw'> = []) { const routed: Array<{ text: string; locus: string | null }> = []; const known: Record = { '#alpha': { channelId: 'chan-alpha', label: '#alpha' }, @@ -78,21 +79,27 @@ function stubRegistry(framework: AgentFramework, plan: Array<'delivered' | 'fals '@laria': { channelId: 'discord:dm:99', label: 'DM: laria' }, 'discord:dm:99': { channelId: 'discord:dm:99', label: 'DM: laria' }, }; + const deliverSpeech = async (_agent: string, text: string, locus?: string | null) => { + routed.push({ text, locus: locus ?? null }); + const behavior = plan.shift() ?? 'delivered'; + if (behavior === 'throw') throw new Error('synthetic publication failure'); + const destination = { serverId: 'stub', channelId: locus ?? '' }; + if (behavior === 'unknown') { + return { status: 'unknown' as const, destination, reason: 'stub delivery uncertain', at: Date.now() }; + } + return behavior === 'delivered' + ? { status: 'delivered' as const, destination, at: Date.now() } + : { status: 'failed' as const, destination, reason: 'stub reported delivered:false', at: Date.now() }; + }; const explicit: Record = { resolveProseTarget: (spec: string) => - known[spec] ?? { error: `no channel matches "${spec}"` }, - routeSpeech: async (_agent: string, text: string, locus?: string | null) => { - routed.push({ text, locus: locus ?? null }); - const behavior = plan.shift() ?? 'delivered'; - if (behavior === 'throw') throw new Error('synthetic publication failure'); - return { delivered: behavior === 'delivered', channelId: locus ?? '' }; - }, - // The registry's outcome-returning form of routeSpeech, over the same plan. - deliverSpeech: async (agent: string, text: string, locus?: string | null) => { - const result = await (explicit.routeSpeech as (a: string, t: string, l?: string | null) => Promise<{ delivered: boolean; channelId: string }>)(agent, text, locus); - return result.delivered - ? { status: 'delivered', destination: { serverId: 'stub', channelId: result.channelId }, at: Date.now() } - : { status: 'failed', destination: { serverId: 'stub', channelId: result.channelId }, reason: 'stub reported delivered:false', at: Date.now() }; + known[spec] ?? { error: 'no channel matches "' + spec + '"' }, + deliverSpeech, + routeSpeech: async (agent: string, text: string, locus?: string | null) => { + const outcome = await deliverSpeech(agent, text, locus); + return outcome.status === 'delivered' + ? { delivered: true, channelId: outcome.destination.channelId } + : null; }, resolveDestination: ({ channelId }: { channelId: string }) => ({ destination: { serverId: 'stub', channelId } }), resolveLocus: () => 'world:commons', @@ -272,10 +279,75 @@ describe('explicit prose routing', () => { }; } - type DraftsView = { open(agent: string): unknown[]; latestBounce(agent: string): unknown }; + type DraftsView = ProseDraftStore; const draftsOf = (framework: AgentFramework): DraftsView => (framework as unknown as { proseDrafts: DraftsView }).proseDrafts; + for (const failure of ['false', 'throw'] as const) { + it('an explicit ' + failure + ' delivery holds its body as a bounce and names it privately', async () => { + const framework = await createFramework(); + try { + const routed = stubRegistry(framework, [failure]); + const agent = framework.getAgent('assistant')!; + const gateway = framework as unknown as { + deliverProse(a: unknown, text: string, hold: { notice: 'later' }): Promise; + }; + await gateway.deliverProse(agent, '>>#alpha keep these words', { notice: 'later' }); + assert.deepEqual(routed, [{ text: 'keep these words', locus: 'chan-alpha' }]); + const held = draftsOf(framework).open('assistant'); + assert.equal(held.length, 1, 'the failed envelope became one draft'); + assert.equal(held[0]!.reason, 'bounced'); + assert.equal(held[0]!.text, 'keep these words'); + assert.ok(context(framework).texts.some((t) => t.includes('[prose-routing]') && t.includes(held[0]!.id))); + } finally { + await framework.stop(); + } + }); + } + + for (const mode of ['explicit', 'hybrid'] as const) { + it('an unknown ' + mode + ' delivery is not offered as a fresh bounce draft', async () => { + const framework = await createFramework(mode); + try { + const routed = stubRegistry(framework, ['unknown']); + const agent = framework.getAgent('assistant')!; + const gateway = framework as unknown as { + deliverProse(a: unknown, text: string, hold: { notice: 'later' }): Promise; + deliverHybridProse(a: unknown, text: string, locus: null, allowLocus: boolean, hold: { notice: 'later' }): Promise; + }; + if (mode === 'explicit') await gateway.deliverProse(agent, '>>#alpha uncertain words', { notice: 'later' }); + else await gateway.deliverHybridProse(agent, '>>>#alpha uncertain words', null, true, { notice: 'later' }); + assert.deepEqual(routed, [{ text: 'uncertain words', locus: 'chan-alpha' }], 'the uncertain attempt was made'); + assert.deepEqual(draftsOf(framework).open('assistant'), []); + assert.equal(context(framework).bounces, 0); + } finally { + await framework.stop(); + } + }); + } + + it('a thrown explicit unsent delivery re-bounces the authored body with its unresolved attempt risk', async () => { + const framework = await createFramework(); + try { + stubRegistry(framework, ['throw']); + const store = draftsOf(framework); + const original = store.hold('assistant', [{ + text: 'earlier words', source: { branch: 'main', turn: 1, round: 1, segment: 0 }, + }], 'bounced').held[0]!; + const gateway = framework as unknown as { + deliverProse(a: unknown, text: string, hold: { notice: 'later' }): Promise; + }; + await gateway.deliverProse(framework.getAgent('assistant')!, '>>#alpha {{unsent}} plus a line', { notice: 'later' }); + const copy = store.open('assistant').find((d) => d.id !== original.id)!; + assert.ok(copy, 'the failed envelope is retained'); + assert.equal(copy.text, 'earlier words plus a line'); + assert.equal(copy.inheritedRisk?.draftId, original.id); + assert.equal(draftState(copy), 'unconfirmed', 'a fresh draft cannot erase its source attempt risk'); + } finally { + await framework.stop(); + } + }); + it('a round that calls skip_reply keeps its unprefixed prose private: nothing delivered, bounced, retained or woken for', async () => { membrane.pushResponse(createMockResponse([ { type: 'text', text: 'Weighing whether this needs an answer from me.' }, @@ -392,6 +464,14 @@ describe('explicit prose routing', () => { const framework = await createFramework(); const routed = stubRegistry(framework); const skip = answerSkipReply(framework); + const retained: Array>> = []; + let turns = 0; + framework.onTrace((event) => { + if (event.type === 'inference:started' && ++turns === 2) { + const draft = draftsOf(framework).latestBounce('assistant'); + if (draft) retained.push(draft); + } + }); feedTurn(framework, 2, [ { type: 'text', text: '>>skip_reply {{unsent}}' }, { type: 'tool_use', id: 'sr4', name: 'skip_reply', input: {} }, @@ -402,6 +482,9 @@ describe('explicit prose routing', () => { assert.equal(skip.calls, 1, 'the bounce wake ran the skip_reply round'); assert.equal(routed.length, 0); + assert.equal(retained.length, 1, 'turn one really held a bounce before the dismissal'); + assert.equal(retained[0]!.text, 'An answer with nowhere to go.'); + assert.equal(draftState(draftsOf(framework).get('assistant', retained[0]!.id)!), 'dismissed'); assert.equal(draftsOf(framework).latestBounce('assistant'), undefined, 'the bounced draft is set aside, as >>skip_reply {{unsent}} says'); await framework.stop(); diff --git a/test/fixtures/speech-route-mcpl-server.mjs b/test/fixtures/speech-route-mcpl-server.mjs index 8cebbf70c..39d2c4e4d 100644 --- a/test/fixtures/speech-route-mcpl-server.mjs +++ b/test/fixtures/speech-route-mcpl-server.mjs @@ -30,11 +30,12 @@ const log = (event, extra = {}) => { }; const send = (message) => process.stdout.write(JSON.stringify(message) + '\n'); const reply = (id, result) => send({ jsonrpc: '2.0', id, result }); -const replyError = (id, message) => send({ jsonrpc: '2.0', id, error: { code: -32000, message } }); +const replyError = (id, message, data) => send({ jsonrpc: '2.0', id, error: { code: -32000, message, ...(data !== undefined ? { data } : {}) } }); let nextId = 500; let processedCommands = 0; let publishMode = 'delivered'; +let publishErrorData; let posted = 0; const histories = new Map(); @@ -122,6 +123,7 @@ function pollCommands() { log('renamed', { channelId: c.channelId, label: c.label }); } else if (c.op === 'publish-mode') { publishMode = c.mode; + publishErrorData = c.detail; log('publish-mode', { mode: c.mode }); } else if (c.op === 'history') { histories.set(c.channelId, c.history); @@ -202,7 +204,7 @@ rl.on('line', (line) => { } if (publishMode === 'error') { log('publish-error', { channelId, text }); - if (msg.id !== undefined && msg.id !== null) replyError(msg.id, 'connector exploded'); + if (msg.id !== undefined && msg.id !== null) replyError(msg.id, 'connector exploded', publishErrorData); return; } if (publishMode === 'not-delivered') { diff --git a/test/mcpl-request-outcome.test.ts b/test/mcpl-request-outcome.test.ts index 1b0f45647..6a681f56e 100644 --- a/test/mcpl-request-outcome.test.ts +++ b/test/mcpl-request-outcome.test.ts @@ -90,3 +90,19 @@ test('McplRequestError is an Error that keeps its message and name', () => { assert.equal(err.name, 'McplRequestError'); assert.equal(err.message, 'boom'); }); + +test('a synchronous write rejection is not-sent, leaves no pending request, and does not poison the next request', async () => { + connection = await McplServerConnection.connect(config(), HOST_CAPS); + const state = connection as unknown as { + transport: { writeLine(json: string): void }; + pendingRequests: Map; + }; + const write = state.transport.writeLine.bind(state.transport); + state.transport.writeLine = () => { throw new Error('synthetic write rejection'); }; + const err = await rejectsWith(connection.sendToolsCall('fail', {}), 'not-sent'); + assert.match(err.message, /synthetic write rejection/); + assert.equal(state.pendingRequests.size, 0, 'nothing remains awaiting an unwritten request'); + state.transport.writeLine = write; + const next = await rejectsWith(connection.sendToolsCall('fail', {}), 'error-response'); + assert.deepEqual(next.data, { posted: ['111'] }, 'a subsequent written request still reaches the real server'); +}); diff --git a/test/prose-drafts.test.ts b/test/prose-drafts.test.ts index 05c7f13f6..1ab212b5f 100644 --- a/test/prose-drafts.test.ts +++ b/test/prose-drafts.test.ts @@ -583,6 +583,79 @@ describe('held prose drafts, end to end', () => { } }); + for (const mode of ['explicit', 'hybrid'] as const) { + it('through the real connection, unknown ' + mode + ' speech is marked but not held for an ordinary resend', async () => { + const h = await harness({ agents: [{ name: 'scout', proseRouting: mode }] }); + try { + h.command({ op: 'publish-mode', mode: 'no-receipt' }); + await h.turn([createMockResponse([text((mode === 'explicit' ? '>>' : '>>>') + ROOM + ' uncertain words')])]); + assert.deepEqual(h.publishes().map((p) => p.text), ['uncertain words'], 'one request may already have posted'); + assert.deepEqual(h.drafts(), [], 'uncertainty is not a fresh resendable bounce'); + const markers = h.texts().filter((t) => t.startsWith('[discord-send-failed]')); + assert.equal(markers.length, 1); + assert.match(markers[0]!, /was not confirmed.*may or may not have been posted/); + assert.ok(!h.messages().some((m) => (m.metadata as { kind?: string } | undefined)?.kind === 'prose-bounce'), 'no bounce notice offers a retry'); + } finally { + await h.close(); + } + }); + } + + for (const privateThink of [false, true]) { + it('hybrid skip_reply unsent ' + (privateThink ? 'stays uninterpreted beside a private think' : 'dismisses the bounce while keeping the round private'), async () => { + const h = await harness({ agents: [{ name: 'scout', proseRouting: 'hybrid', sameRoundThinkTextPolicy: 'private' }] }); + try { + const original = h.store().hold('scout', [{ text: 'held bounce', source }], 'bounced').held[0]!; + assert.equal(draftState(original), 'held'); + const blocks = [text('>>>skip_reply {{unsent}}'), call('skip1', 'skip_reply', {})]; + if (privateThink) blocks.push(call('think1', 'think', { thought: 'private reasoning' })); + await h.turn([createMockResponse(blocks, 'tool_use'), createMockResponse([])]); + assert.equal(draftState(h.store().get('scout', original.id)!), privateThink ? 'held' : 'dismissed'); + assert.deepEqual(h.publishes(), [], 'deliberate privacy posts nothing'); + assert.equal(h.store().open('scout').length, privateThink ? 1 : 0, 'no fresh draft was made from private words'); + } finally { + await h.close(); + } + }); + } + + it('an empty completion waits for earlier queued speech and holds before writing their receipt', async () => { + const h = await harness({ agents: [{ name: 'scout', proseRouting: 'hybrid' }] }); + let release: (() => void) | undefined; + try { + const registry = (h.framework as unknown as { channelRegistry: { deliverSpeech: (...args: unknown[]) => Promise } }).channelRegistry; + const deliver = registry.deliverSpeech.bind(registry); + let entered = false; + registry.deliverSpeech = async (...args: unknown[]) => { + if (args[1] === 'slow queued words') { + entered = true; + await new Promise((resolve) => { release = resolve; }); + } + return deliver(...args); + }; + // Deliberately not rounds(): the last response has no prior content. + const running = h.turn([ + createMockResponse([text('>>>' + ROOM + ' slow queued words'), call('list1', 'channel_list', {})], 'tool_use'), + createMockResponse([text('held behind the slow delivery'), explicitSend('send1')], 'tool_use'), + createMockResponse([]), + ]); + await h.until(() => entered && (h.membrane.lastStream?.receivedToolResults.length ?? 0) >= 2, 'both tool rounds behind the pending delivery'); + assert.deepEqual(h.drafts(), [], 'the hold is still queued'); + release!(); + await running; + await h.until(() => h.publishes().some((p) => p.text === 'slow queued words') && h.drafts().length === 1, 'the earlier delivery and queued hold'); + const draft = h.drafts()[0]!; + assert.equal(draft.text, 'held behind the slow delivery'); + const receipt = h.texts().find((t) => t.startsWith('[delivered]')); + assert.ok(receipt, 'the completed turn receipts the queued work'); + assert.ok(receipt.includes(ROOM), 'the delayed delivery is in the receipt'); + assert.ok(receipt.includes(draft.id), 'so is the hold settled behind it'); + } finally { + release?.(); + await h.close(); + } + }); + it('through the real connection, a server that never answers is unconfirmed once the request times out', async () => { const h = await harness({ requestTimeoutMs: 1_000 }); try { @@ -597,7 +670,7 @@ describe('held prose drafts, end to end', () => { } }); - it("a silent turn's prose stays private beside a send: counted, never held or named", async () => { + it("a silent turn's prose stays private beside a send: never held, named or receipted", async () => { const h = await harness(); try { await h.turn([ @@ -607,6 +680,7 @@ describe('held prose drafts, end to end', () => { assert.deepEqual(h.drafts(), [], 'nothing was held as a draft'); assert.ok(!h.texts().some((t) => t.includes('held as draft')), 'and no notice names one'); assert.deepEqual(h.publishes().map((p) => p.text), ['an explicit note elsewhere'], 'only the explicit send went out'); + assert.ok(!h.texts().some((t) => t.startsWith('[delivered]')), 'a silent turn receives no plain-speech receipt'); } finally { await h.close(); } diff --git a/test/publish-outcome.test.ts b/test/publish-outcome.test.ts index 846a9835f..78f85412e 100644 --- a/test/publish-outcome.test.ts +++ b/test/publish-outcome.test.ts @@ -27,6 +27,7 @@ type Publish = (params: { channelId: string }) => Promise; function registryWith( servers: Record Promise }>, failures: unknown[] = [], + traces: Array<{ type: string; [key: string]: unknown }> = [], ) { const published: Array<{ serverId: string; channelId: string }> = []; const opened: Array<{ serverId: string; channelId: string }> = []; @@ -51,7 +52,7 @@ function registryWith( { getServer: (id: string) => mocks.get(id) } as unknown as McplServerRegistry, {} as FeatureSetManager, () => {}, - () => {}, + (event) => { traces.push(event); }, { onRouteFailure: (info) => { failures.push(info); } }, ); const channels = (registry as unknown as { @@ -176,6 +177,37 @@ describe('ChannelRegistry.publish outcomes', () => { assert.equal(outcome.destination?.channelId, 'c'); }); + for (const id of ['message-' + 'x'.repeat(10_000), { malformed: 'x'.repeat(10_000) }]) { + it('bounds a contradictory ' + (typeof id === 'string' ? 'string' : 'object') + ' message id in the reason, preserving the receipt', async () => { + const { registry, seed } = registryWith({ discord: { publish: async () => ({ delivered: false, messageId: id }) } }); + seed('discord', 'c', '#c'); + const outcome = await registry.publish('agent', 'x', { channelId: 'c' }); + assert.equal(outcome.status, 'unknown'); + assert.ok(outcome.reason!.length < 2300, 'the diagnostic is bounded'); + assert.match(outcome.reason!, /\[truncated\]/); + assert.match(outcome.reason!, /delivery uncertain/); + assert.equal(outcome.messageId, typeof id === 'string' ? id : undefined, 'diagnostic clipping does not rewrite a connector id'); + }); + } + + it('speech failure forwards bounded connector data to both notice and trace, preserving the original outcome', async () => { + const data = { posted: ['111'], note: 'x'.repeat(10_000) }; + const failures: unknown[] = []; + const traces: Array<{ type: string; [key: string]: unknown }> = []; + const { registry, seed } = registryWith({ + discord: { publish: async () => { throw new McplRequestError('part 2 failed', 'error-response', -32000, data); } }, + }, failures, traces); + seed('discord', 'c', '#c'); + const outcome = await registry.deliverSpeech('agent', 'words', 'c'); + assert.equal(outcome.status, 'unknown'); + assert.deepEqual(outcome.detail, data, 'the outcome still carries the complete structured data'); + const info = failures[0] as { detail?: string }; + assert.ok(info.detail && info.detail.length <= 2000); + assert.match(info.detail, /"posted":\["111"\]/); + assert.match(info.detail, /\[truncated\]$/); + assert.equal(traces.find((e) => e.type === 'mcpl:speech-route-failed')?.detail, info.detail); + }); + it('treats a timeout or a lost connection as unknown', async () => { const { registry, seed } = registryWith({ discord: { publish: async () => { throw new McplRequestError('MCPL server "discord" did not respond to channels/publish (id=7) within 10ms', 'no-response'); } }, @@ -263,8 +295,10 @@ test('the resident\'s failure marker never claims an uncertain delivery did not const markers = () => framework.getAgent('scout')!.getContextManager().getAllMessages() .filter((m) => (m.metadata as { kind?: string } | undefined)?.kind === 'discord-send-failed') .map((m) => (m.content[0] as { text: string }).text); - const turn = async (mode: string, n: number) => { - appendFileSync(commandPath, JSON.stringify({ op: 'publish-mode', mode }) + '\n'); + const traces: Array<{ type: string; detail?: unknown }> = []; + framework.onTrace((event) => traces.push(event)); + const turn = async (mode: string, n: number, detail?: unknown) => { + appendFileSync(commandPath, JSON.stringify({ op: 'publish-mode', mode, detail }) + '\n'); membrane.pushResponse(createMockResponse([{ type: 'text', text: `reply ${n}` }])); appendFileSync(commandPath, JSON.stringify({ op: 'incoming', channelId: 'discord:g1:room', messageId: `m-${n}`, mode: 'addressed', text: 'hello?' }) + '\n'); await until(() => markers().length === n, `marker ${n}`); @@ -278,6 +312,14 @@ test('the resident\'s failure marker never claims an uncertain delivery did not assert.doesNotMatch(uncertain!, /did not receive|Nothing was posted/); assert.match(failed!, /could not be delivered/); assert.match(failed!, /Nothing was posted/); + await turn('error', 3, { posted: ['111'], note: 'x'.repeat(10_000) }); + const partial = markers()[2]!; + assert.match(partial, /was not confirmed/); + assert.match(partial, /Connector detail:.*"posted":\["111"\]/); + assert.match(partial, /\[truncated\]$/); + assert.ok(partial.length < 2500, 'the resident receives a bounded display, not the whole error object'); + const trace = traces.find((event) => event.type === 'mcpl:speech-route-failed' && event.detail !== undefined); + assert.ok(trace?.detail, 'the same connector account reaches the failure trace'); } finally { await framework.stop(); rmSync(dir, { recursive: true, force: true }); From d4f849353fe99dfc0987907674b0a7ad2d0a2c02 Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Tue, 13 Oct 2026 01:28:45 -0400 Subject: [PATCH 0146/1112] perf(mcp): a connection's records build their redaction once redactServerText found the configuration's strings and built its pattern on every call, and a server's stderr is recorded a line at a time, so a busy server rebuilt them for every line. serverTextRedactor(config) finds them and builds the pattern once and returns the function that applies it; both engines' event wiring builds one per connection, and redactServerText is that function applied once, for the occasional error. The contract and its tests are unchanged. --- src/framework.ts | 8 +++----- src/mcpl/protocol-family.ts | 13 +++++++++++-- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/src/framework.ts b/src/framework.ts index 3dbfbff0b..67b8157ce 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -394,7 +394,7 @@ import type { SessionUsageSnapshot, UsageUpdatedEvent } from './usage/types.js'; import type { McplServerConnection } from './mcpl/server-connection.js'; import { McplProtocolVersionError, McplUnreapedLaunchError } from './mcpl/server-connection.js'; import { ModernMcpConnection } from './mcpl/modern-connection.js'; -import { redactUrl, redactServerText, resolveServerBinding, serverConfigProblems } from './mcpl/protocol-family.js'; +import { redactUrl, redactServerText, resolveServerBinding, serverConfigProblems, serverTextRedactor } from './mcpl/protocol-family.js'; import { MALFORMED_CONTENT_NOTICE, normalizeStandardToolResult } from './mcpl/tool-result-normalize.js'; import type { McplServerConfig, @@ -14751,8 +14751,7 @@ export class AgentFramework { * Push events and inference requests are deferred to Steps 6/7. */ private wireMcplEvents(connection: McplServerConnection): void { - const config = this.mcplServerConfigs.get(connection.id) ?? {}; - const recorded = (text: string): string => redactServerText(text, config); + const recorded = serverTextRedactor(this.mcplServerConfigs.get(connection.id) ?? {}); // Forward subprocess stderr lines as trace events so consumers (conhost, // log sinks, TUI badges) can persist and surface them. connection.on('stderr', (params: { line: string }) => { @@ -15080,8 +15079,7 @@ export class AgentFramework { * MCPL data-plane traffic, which this family doesn't carry. */ private wireModernEvents(connection: ModernMcpConnection): void { - const config = this.mcplServerConfigs.get(connection.id) ?? {}; - const recorded = (text: string): string => redactServerText(text, config); + const recorded = serverTextRedactor(this.mcplServerConfigs.get(connection.id) ?? {}); connection.on('stderr', (params: { line: string }) => { this.emitTrace({ type: 'mcpl:server-stderr', serverId: connection.id, line: recorded(params.line) }); }); diff --git a/src/mcpl/protocol-family.ts b/src/mcpl/protocol-family.ts index a6483bd7a..0f021057a 100644 --- a/src/mcpl/protocol-family.ts +++ b/src/mcpl/protocol-family.ts @@ -118,6 +118,14 @@ export function redactUrl(url: string | undefined): string { * thrown directly to a host caller keep their separate pass-through contract. */ export function redactServerText(text: string, config: Pick): string { + return serverTextRedactor(config)(text); +} + +/** + * {@link redactServerText} for one configuration, its strings found and its + * pattern built once: for a connection's records, such as each stderr line. + */ +export function serverTextRedactor(config: Pick): (text: string) => string { const secrets = new Set(); const standalone = (value: string): void => { if (value.length >= 8) secrets.add(value); }; const decode = (value: string, query = false): string => { @@ -160,10 +168,11 @@ export function redactServerText(text: string, config: Pick text; const alternatives = [...secrets].sort((a, b) => b.length - a.length) .map(value => value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')); - return text.replace(new RegExp(alternatives.join('|'), 'g'), 'redacted'); + const pattern = new RegExp(alternatives.join('|'), 'g'); + return (text) => text.replace(pattern, 'redacted'); } /** From ed626b8b453ae82890f8f9e6870219eef3c774ef Mon Sep 17 00:00:00 2001 From: Nathan-2367 Date: Tue, 13 Oct 2026 01:31:45 -0400 Subject: [PATCH 0147/1112] test(prose): keep uncertain failures visible when connector data cannot render The bounded connector display is reporting an already uncertain send. A formatting failure must not replace that structured outcome with a thrown exception, because callers retain unexpected exceptions as bounces. Add a control through deliverSpeech for data that the formatter cannot serialize: the outcome and source value remain intact, and the private failure callback receives an explicit display fallback. Removing the formatter catch turns this into a rejection. This exercises the display guard through the exported request-error and registry APIs, using the existing typed fixture. It is a unit boundary test; wire JSON itself cannot contain the cyclic value. The publish-outcome file passes 25 tests after a clean VPS typecheck/build. Co-authored-by: GPT-6.1 Sol --- test/publish-outcome.test.ts | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/test/publish-outcome.test.ts b/test/publish-outcome.test.ts index 78f85412e..1f94d3366 100644 --- a/test/publish-outcome.test.ts +++ b/test/publish-outcome.test.ts @@ -208,6 +208,20 @@ describe('ChannelRegistry.publish outcomes', () => { assert.equal(traces.find((e) => e.type === 'mcpl:speech-route-failed')?.detail, info.detail); }); + it('unrenderable connector data keeps an unknown speech failure visible rather than throwing', async () => { + const data: Record = { posted: ['111'] }; + data.self = data; + const failures: unknown[] = []; + const { registry, seed } = registryWith({ + discord: { publish: async () => { throw new McplRequestError('part 2 failed', 'error-response', -32000, data); } }, + }, failures); + seed('discord', 'c', '#c'); + const outcome = await registry.deliverSpeech('agent', 'words', 'c'); + assert.equal(outcome.status, 'unknown'); + assert.equal(outcome.detail, data, 'the formatter does not change the source value'); + assert.equal((failures[0] as { detail?: string }).detail, '[unserializable connector value]'); + }); + it('treats a timeout or a lost connection as unknown', async () => { const { registry, seed } = registryWith({ discord: { publish: async () => { throw new McplRequestError('MCPL server "discord" did not respond to channels/publish (id=7) within 10ms', 'no-response'); } }, From 4bb693928b84d6d4694cf3ec33437270ab6a0ec5 Mon Sep 17 00:00:00 2001 From: Nathan-2367 Date: Tue, 13 Oct 2026 01:47:14 -0400 Subject: [PATCH 0148/1112] test(mcpl): reject late handling for requests whose write failed Strengthen the synchronous-write fault test with a deferred render request and a controlled watchdog clock. After the write is rejected, its former deadline must not register an orphan that can surface a later response. Inject a state-less response for that unwritten id and verify that no orphaned-render event is emitted, then restore the writer and confirm that a real subsequent request still works. This protects timer cleanup through the response-handling behavior rather than a private timer flag. Also check that the stored plain-speech failure marker carries the same bounded connector display as its trace. The returned publish outcome continues to hold the original structured data. The request-outcome and publish-outcome files pass 31 tests after a clean VPS typecheck/build. Co-authored-by: GPT-6.1 Sol --- test/mcpl-request-outcome.test.ts | 16 ++++++++++++---- test/publish-outcome.test.ts | 2 ++ 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/test/mcpl-request-outcome.test.ts b/test/mcpl-request-outcome.test.ts index 6a681f56e..4d5d79787 100644 --- a/test/mcpl-request-outcome.test.ts +++ b/test/mcpl-request-outcome.test.ts @@ -91,17 +91,25 @@ test('McplRequestError is an Error that keeps its message and name', () => { assert.equal(err.message, 'boom'); }); -test('a synchronous write rejection is not-sent, leaves no pending request, and does not poison the next request', async () => { +test('a synchronous write rejection is not-sent, leaves no pending request, and does not poison the next request', async (t) => { connection = await McplServerConnection.connect(config(), HOST_CAPS); const state = connection as unknown as { - transport: { writeLine(json: string): void }; + transport: { writeLine(json: string): void; emit(event: string, line: string): boolean }; pendingRequests: Map; }; const write = state.transport.writeLine.bind(state.transport); - state.transport.writeLine = () => { throw new Error('synthetic write rejection'); }; - const err = await rejectsWith(connection.sendToolsCall('fail', {}), 'not-sent'); + t.mock.timers.enable({ apis: ['setTimeout'] }); + let unwrittenId: unknown; + const lateRenders: unknown[] = []; + connection.on('orphaned-render-response', (event) => lateRenders.push(event)); + state.transport.writeLine = (json) => { unwrittenId = JSON.parse(json).id; throw new Error('synthetic write rejection'); }; + const err = await rejectsWith(connection.sendPushRender({ featureSet: 'test', key: 'unwritten', eventId: 'e1', notices: [], dropped: 0 }), 'not-sent'); assert.match(err.message, /synthetic write rejection/); assert.equal(state.pendingRequests.size, 0, 'nothing remains awaiting an unwritten request'); + t.mock.timers.tick(5000); + state.transport.emit('line', JSON.stringify({ jsonrpc: '2.0', id: unwrittenId, result: { content: [] } })); + assert.deepEqual(lateRenders, [], 'an unwritten render cannot surface a late-response event after its old watchdog deadline'); + t.mock.timers.reset(); state.transport.writeLine = write; const next = await rejectsWith(connection.sendToolsCall('fail', {}), 'error-response'); assert.deepEqual(next.data, { posted: ['111'] }, 'a subsequent written request still reaches the real server'); diff --git a/test/publish-outcome.test.ts b/test/publish-outcome.test.ts index 1f94d3366..c3721855a 100644 --- a/test/publish-outcome.test.ts +++ b/test/publish-outcome.test.ts @@ -334,6 +334,8 @@ test('the resident\'s failure marker never claims an uncertain delivery did not assert.ok(partial.length < 2500, 'the resident receives a bounded display, not the whole error object'); const trace = traces.find((event) => event.type === 'mcpl:speech-route-failed' && event.detail !== undefined); assert.ok(trace?.detail, 'the same connector account reaches the failure trace'); + const marker = framework.getAgent('scout')!.getContextManager().getAllMessages().find((m) => (m.content[0] as { text: string }).text === partial); + assert.equal((marker?.metadata as { detail?: unknown } | undefined)?.detail, trace?.detail, 'the persisted marker metadata carries the same bounded display'); } finally { await framework.stop(); rmSync(dir, { recursive: true, force: true }); From fa29b6e9a8f5261c7ceed0d366c8204a8130c540 Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Tue, 13 Oct 2026 02:09:17 -0400 Subject: [PATCH 0149/1112] fix(mcp): recorded-text redaction covers the token as a url carries it, credential env values, and any value; it replaces only whole strings From a review of the redaction: - The static token was derived only as written; a transport that puts it in a url sends it percent- or form-encoded, so both forms are derived too. - A stdio server's environment values whose names say they're credentials (the url's parameter rule: token, key, secret, password, auth, signature) are derived as well. - A string is replaced only where no letter or digit adjoins it, so a short name=value such as v=2 no longer changes dev=2 in other text. - The redactor reads any value as text: a connect rejection that isn't an Error gave it undefined, which threw inside the catch and lost the reconnecting stub. - The cleartext warning counts a parameter by its value's shape too, as redaction does, so a credential-shaped value under a plain name is warned about (its name shown) as well as hidden. The helper's doc now lists what it derives and what's outside it. Tests: the token's encoded forms, credential-named env values (and a plain one kept), a short pair kept where another word adjoins it, a ws:// url's userinfo, a url with a stray percent sign, non-string values, and the warning for a credential-shaped value under a plain name. --- src/mcpl/protocol-family.ts | 53 +++++++++++++++++++++++--------- test/mcp-protocol-family.test.ts | 30 ++++++++++++++++++ 2 files changed, 68 insertions(+), 15 deletions(-) diff --git a/src/mcpl/protocol-family.ts b/src/mcpl/protocol-family.ts index 0f021057a..52cb72114 100644 --- a/src/mcpl/protocol-family.ts +++ b/src/mcpl/protocol-family.ts @@ -108,16 +108,23 @@ export function redactUrl(url: string | undefined): string { } /** - * Server/transport diagnostics as the framework records them. Replace only - * known URL credential strings and the configured static token, in their - * encoded and decoded forms. Short standalone strings (under 8 characters) - * are indistinguishable from ordinary words; short query values are removed - * as name=value instead. A single longest-first replacement never scans its - * own replacement text. Unknown secrets, including accessProvider's cached - * bearer token, are outside this contract. JSON-RPC tool answers and errors - * thrown directly to a host caller keep their separate pass-through contract. + * Server and transport text as the framework records it, with the strings a + * server's configuration knows to be credentials replaced: + * - from the url, as written and decoded: credential-like path segments and + * query names, credential-shaped or credential-named query values, every + * other non-empty parameter as its whole name=value, and a ws:// url's + * username and password; + * - the static token, raw and percent- or form-encoded; + * - a stdio server's environment values whose names say they're credentials. + * A string replaced on its own has at least 8 characters, since a shorter + * one can't be told from ordinary text; a short query value is replaced + * with its name, as one name=value. A string is replaced only where no letter + * or digit adjoins it, and one longest-first pass never scans its own + * replacement. A credential the configuration doesn't hold, such as an + * access provider's cached bearer token or another origin's, is outside it. + * JSON-RPC answers and errors thrown to a host's own call pass as they are. */ -export function redactServerText(text: string, config: Pick): string { +export function redactServerText(text: unknown, config: Pick): string { return serverTextRedactor(config)(text); } @@ -125,14 +132,25 @@ export function redactServerText(text: string, config: Pick): (text: string) => string { +export function serverTextRedactor(config: Pick): (text: unknown) => string { const secrets = new Set(); const standalone = (value: string): void => { if (value.length >= 8) secrets.add(value); }; const decode = (value: string, query = false): string => { try { return decodeURIComponent(query ? value.replace(/\+/g, ' ') : value); } catch { return value; } }; - standalone(config.token ?? ''); + // The token as sent: raw in a header, and percent- or form-encoded where a + // transport puts it in a url. + for (const token of [config.token ?? '']) { + standalone(token); + standalone(encodeURIComponent(token)); + standalone(new URLSearchParams({ t: token }).toString().slice(2)); + } + // A stdio server's environment values whose names say they carry a + // credential, by the same rule as the url's parameters. + for (const [name, value] of Object.entries(config.env ?? {})) { + if (CREDENTIAL_PARAM.test(name)) standalone(value); + } let url: URL | undefined; try { url = new URL(config.url ?? ''); } catch { /* Still remove the static token. */ } if (url) { @@ -168,11 +186,14 @@ export function serverTextRedactor(config: Pick text; + const asText = (text: unknown): string => (typeof text === 'string' ? text : String(text)); + if (secrets.size === 0) return asText; const alternatives = [...secrets].sort((a, b) => b.length - a.length) .map(value => value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')); - const pattern = new RegExp(alternatives.join('|'), 'g'); - return (text) => text.replace(pattern, 'redacted'); + // Only where no letter or digit adjoins it, so a short name=value such as + // v=2 leaves dev=2 alone. + const pattern = new RegExp(`(? asText(text).replace(pattern, 'redacted'); } /** @@ -357,7 +378,9 @@ export function serverConfigWarnings(config: McplServerConfig): string[] { if (url.pathname.split('/').some(credentialLikeSegment)) carried.push("a credential-like segment of its url's path"); const entries = [...url.searchParams]; const hidden = entries.filter(([name, value]) => credentialLikeQueryName(name, value)).length; - const params = [...new Set(entries.filter(([name, value]) => !credentialLikeQueryName(name, value) && CREDENTIAL_PARAM.test(name)).map(([name]) => name))]; + // A parameter carries a credential by its name, or by its value's shape, + // as redaction reads it; the name is shown only when it isn't one itself. + const params = [...new Set(entries.filter(([name, value]) => !credentialLikeQueryName(name, value) && (CREDENTIAL_PARAM.test(name) || credentialLikeSegment(value))).map(([name]) => name))]; if (params.length > 0) { carried.push(`its url's query ${params.length === 1 ? 'parameter' : 'parameters'} ${params.map((name) => JSON.stringify(name)).join(', ')}`); } diff --git a/test/mcp-protocol-family.test.ts b/test/mcp-protocol-family.test.ts index d6c490c95..787182c16 100644 --- a/test/mcp-protocol-family.test.ts +++ b/test/mcp-protocol-family.test.ts @@ -320,6 +320,36 @@ test('recorded server text keeps short standalone values and unrelated words, bu assert.equal(redactServerText('12345678', cfg({token:'12345678'})), 'redacted'); }); +test('recorded server text: the token as a url carries it, credential-named env values, only where nothing adjoins, and any value', () => { + // The static token, raw and as a url carries it, percent- or form-encoded. + const token = 'tok+en/123='; + const tokenConfig = cfg({ token }); + for (const form of [token, encodeURIComponent(token), 'tok%2Ben%2F123%3D']) { + assert.equal(redactServerText(`refused: ${form}`, tokenConfig), 'refused: redacted', form); + } + // A stdio server's env: a value whose name says it's a credential, and no other. + const env = cfg({ command: 'srv', env: { GITHUB_TOKEN: 'ghp_abcdef123456', API_KEY: 'k3y-abcdefgh', PATH: '/usr/local/bin' } }); + assert.equal(redactServerText('auth ghp_abcdef123456 k3y-abcdefgh /usr/local/bin', env), 'auth redacted redacted /usr/local/bin'); + // Only where no letter or digit adjoins a string: a short pair stays whole elsewhere. + const pair = cfg({ url: 'https://h/mcp?v=2' }); + assert.equal(redactServerText('dev=2 ready; v=2 sent', pair), 'dev=2 ready; redacted sent'); + // A ws:// url's userinfo, as for wss://. + const ws = cfg({ url: 'ws://wsuser1234:wspass5678@h/mcp' }); + assert.equal(redactServerText('connect wsuser1234 wspass5678', ws), 'connect redacted redacted'); + // A url with a stray percent sign: its value is kept as written. + const stray = cfg({ url: 'https://h/mcp?note=100%&key=abcdefgh1234' }); + assert.equal(redactServerText('note=100% key=abcdefgh1234', stray), 'redacted redacted'); + // Any value reads as text: a rejection that isn't an Error doesn't throw. + assert.equal(redactServerText(undefined, tokenConfig), 'undefined'); + assert.equal(redactServerText({ toString: () => `obj ${token}` }, tokenConfig), 'obj redacted'); +}); + +test('the cleartext warning counts a credential-shaped value under a plain name, as redaction does', () => { + const warned = serverConfigWarnings(cfg({ url: 'http://remote.example/mcp?session=Zx81kQ0pL2mN7vB4tY6c' })); + assert.equal(warned.length, 1); + assert.match(warned[0], /its url's query parameter "session" goes to remote\.example unencrypted/); +}); + test('recorded server text uses longest-first, one-pass replacements and is idempotent', () => { const config = cfg({ url: 'https://h/mcp?secret=token0123456789-tail&redacted=eu', token: 'token0123456789' }); const original = 'secret=token0123456789-tail token0123456789 redacted=eu queue full'; From e76f2c9bbe0aad6729e5a793657dade30927f1c7 Mon Sep 17 00:00:00 2001 From: Nathan-2367 Date: Tue, 13 Oct 2026 02:13:34 -0400 Subject: [PATCH 0150/1112] test(mcpl): arm the observer before asserting no late render response A freshly connected MCPL connection buffers data-plane events until ready(). The failed-write test attached an orphaned-render subscriber but left the plane buffered, so its missing-event assertion passed even when timer cancellation was removed. Release that plane after wiring the listener and add a positive event control before clearing the observed events and attempting the rejected write. The fixed file passes all six tests after a clean build. Removing timer cancellation now fails the intended late-response assertion after the controlled watchdog deadline. This changes only the request-outcome test fixture; all production code and other test files are unchanged from the full per-file check. The final focused runtime checks and affected removals supplement that full receipt. Co-authored-by: GPT-6.1 Sol --- test/mcpl-request-outcome.test.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/mcpl-request-outcome.test.ts b/test/mcpl-request-outcome.test.ts index 4d5d79787..4534fd558 100644 --- a/test/mcpl-request-outcome.test.ts +++ b/test/mcpl-request-outcome.test.ts @@ -102,6 +102,11 @@ test('a synchronous write rejection is not-sent, leaves no pending request, and let unwrittenId: unknown; const lateRenders: unknown[] = []; connection.on('orphaned-render-response', (event) => lateRenders.push(event)); + // connect() buffers data-plane events until the caller has wired listeners. + connection.ready(); + connection.emit('orphaned-render-response', { params: { featureSet: 'test', key: 'observer-probe', eventId: 'probe' }, result: { content: [] } }); + assert.equal(lateRenders.length, 1, 'the observer is armed, so absence of a later event is meaningful'); + lateRenders.length = 0; state.transport.writeLine = (json) => { unwrittenId = JSON.parse(json).id; throw new Error('synthetic write rejection'); }; const err = await rejectsWith(connection.sendPushRender({ featureSet: 'test', key: 'unwritten', eventId: 'e1', notices: [], dropped: 0 }), 'not-sent'); assert.match(err.message, /synthetic write rejection/); From a49e74fb0d3f91a69fc7aa8e0236760ea081c626 Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Tue, 13 Oct 2026 02:30:37 -0400 Subject: [PATCH 0151/1112] fix(mcp): every record of server text is redacted, a restart refuses an operation in flight, and a disconnect keeps another server's prefix From a review of the redaction and the restart: - Server text the framework logged raw is redacted with its server's configuration: the close messages after a failed startup and the tool refresh's error (every configured server's credentials, since either can come from any server), the manifest fetch and refresh lines, the tools/observe refusal and the re-registration after a reconnect. - A tool call's failure from either engine: the result the caller gets and, on the model's dispatch, the tool:failed trace's message and stack. A legacy MCPL server's close reason reaches these unredacted otherwise, and so does any error not built by the modern engine's request reading. - A malformed line's diagnostic is redacted before it's cut at 200 characters, so a credential across the cut can't leave its first part (malformedLineDiagnostic, now its own function). - A request refused with 401 or 403 keeps the SDK's status and statusText in the error's data, and leaves out the response body (`text`), which can echo the url. - The event wiring takes the configuration the connection was made for, rather than reading the framework's map with an empty fallback. - restartMcplServer of an id with a connect or disconnect in flight is refused before it tears anything down; its own disconnect would have queued behind the other and been refused after the server was gone. - A disconnect removes its server's tool-prefix claim only while that server holds it, so a server connected since under that prefix keeps it. Tests: a legacy tool call's failure and the model dispatch's trace and result; the startup alert for a protocol-version verdict; a runtime connect's failed cleanup traced redacted while the host's error stays as raised; the malformed-line diagnostic across its cut; a 403's body kept out of the error's data; a restart refused while a disconnect is in flight; a disconnect leaving another server's prefix. The polling ceilings in two record tests go from 5 s to 30 s (each loop ends when its record appears). Removing each piece fails its test (13 removals). No test drives each console line on its own; the helpers they call are tested. --- src/framework.ts | 52 +++++--- src/mcpl/modern-connection.ts | 20 ++- test/mcp-modern-http.test.ts | 23 ++++ test/mcp-recorded-text.test.ts | 114 +++++++++++++++++- test/mcpl-reconnect-reregistration.test.ts | 2 +- test/mcpl-runtime-lifecycle.test.ts | 24 ++++ test/tools-observe-parser-conformance.test.ts | 4 +- 7 files changed, 211 insertions(+), 28 deletions(-) diff --git a/src/framework.ts b/src/framework.ts index 67b8157ce..3b780be8a 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -11333,7 +11333,7 @@ export class AgentFramework { } catch (err) { return { success: false, - error: err instanceof Error ? err.message : String(err), + error: redactServerText(err instanceof Error ? err.message : err, config), isError: true, }; } @@ -13597,7 +13597,7 @@ export class AgentFramework { for (const outcome of outcomes) { if (outcome.status === 'rejected') { console.error('[mcpl] after a failed startup, some MCP connections could not be closed:', - outcome.reason instanceof Error ? outcome.reason.message : outcome.reason); + this.recordedForEveryServer(outcome.reason instanceof Error ? outcome.reason.message : outcome.reason)); } } } @@ -14002,7 +14002,7 @@ export class AgentFramework { // and a second connect for the same id is refused above. const modern = ModernMcpConnection.create(config); this.modernServers().set(config.id, modern); - this.wireModernEvents(modern); + this.wireModernEvents(modern, config); let failure: Error | null = null; try { await modern.start(); @@ -14075,7 +14075,7 @@ export class AgentFramework { // Wire listeners before either startup staging or the runtime global gate // releases control traffic needed for registration and marker service. - this.wireMcplEvents(connection); + this.wireMcplEvents(connection, config); if (deferAwareness) { // Staged startup: both planes are closed, so the §5.3 policy @@ -14189,7 +14189,7 @@ export class AgentFramework { try { fetched = await connection.sendManifestRequest(); } catch (err) { - console.error(`[mcpl] ${connection.id}: mcpl/manifest fetch failed — previous manifest stands (§17.5): ${err instanceof Error ? err.message : err}`); + console.error(`[mcpl] ${connection.id}: mcpl/manifest fetch failed — previous manifest stands (§17.5): ${this.recordedFor(connection.id, err instanceof Error ? err.message : err)}`); return; } if (!fetched || typeof fetched !== 'object') { @@ -14525,7 +14525,9 @@ export class AgentFramework { this.checkpointManager?.removeServer(id); const prefix = config?.toolPrefix ?? `mcpl--${id}`; - this.mcplPrefixMap.delete(prefix); + // Only this server's own claim: a server connected since under the + // prefix it held keeps it. + if (this.mcplPrefixMap.get(prefix) === id) this.mcplPrefixMap.delete(prefix); this.mcplServerConfigs.delete(id); await this.refreshMcplTools(); @@ -14560,6 +14562,11 @@ export class AgentFramework { if (config.id !== id) { const taken = this.mcpServerIdTaken(config.id); if (taken) throw new Error(taken); + } else if (this.mcpServerOps?.get(id)) { + // A connect or disconnect of this id in flight: the restart's own + // disconnect would queue behind it and then be refused, after the + // server was torn down. + throw new Error(`MCP server "${id}" was not restarted: it is being ${this.mcpServerOps.get(id)!.kind === 'disconnect' ? 'disconnected' : 'connected'}; restart it once that has finished`); } const collision = this.mcplPrefixCollision(config, id); if (collision) throw new Error(collision); @@ -14567,6 +14574,18 @@ export class AgentFramework { await this.connectMcplServer(config); } + /** A server's text as the framework records it ({@link serverTextRedactor}). */ + private recordedFor(serverId: string, text: unknown): string { + return redactServerText(text, this.mcplServerConfigs.get(serverId) ?? {}); + } + + /** Text that may come from any configured server, with every server's credentials replaced. */ + private recordedForEveryServer(text: unknown): string { + let recorded = typeof text === 'string' ? text : String(text); + for (const config of this.mcplServerConfigs.values()) recorded = redactServerText(recorded, config); + return recorded; + } + /** Why a connect for this id is refused, or null when the id is free. One * owner per id, across both engines (holdMcpServerId): a connect or * disconnect in flight holds it, and so does a registered connection, @@ -14750,8 +14769,8 @@ export class AgentFramework { * Wire event listeners on an MCPL server connection. * Push events and inference requests are deferred to Steps 6/7. */ - private wireMcplEvents(connection: McplServerConnection): void { - const recorded = serverTextRedactor(this.mcplServerConfigs.get(connection.id) ?? {}); + private wireMcplEvents(connection: McplServerConnection, config: McplServerConfig): void { + const recorded = serverTextRedactor(config); // Forward subprocess stderr lines as trace events so consumers (conhost, // log sinks, TUI badges) can persist and surface them. connection.on('stderr', (params: { line: string }) => { @@ -14825,7 +14844,7 @@ export class AgentFramework { // payload and no authority; everything below acts on FETCHED content. connection.on('manifest-changed', (params: { revision?: string; domains?: string[] }) => { void this.handleManifestChanged(connection, params).catch((err) => { - console.error(`[mcpl] ${connection.id} manifest refresh failed:`, err instanceof Error ? err.message : err); + console.error(`[mcpl] ${connection.id} manifest refresh failed:`, this.recordedFor(connection.id, err instanceof Error ? err.message : err)); }); }); @@ -14848,7 +14867,7 @@ export class AgentFramework { const parsed = parseToolObserveParams(params); if (!parsed.ok) { if (responder?.respondError) responder.respondError(-32602, parsed.message, parsed.data); - else console.error(`[mcpl] ${connection.id}: tools/observe rejected — ${parsed.message}`); + else console.error(`[mcpl] ${connection.id}: tools/observe rejected — ${this.recordedFor(connection.id, parsed.message)}`); return; } connection.toolObserveFilter = parsed.rules; @@ -14940,7 +14959,7 @@ export class AgentFramework { } catch (error) { console.error( `MCPL server "${connection.id}" re-registration after reconnect failed:`, - error instanceof Error ? error.message : error, + this.recordedFor(connection.id, error instanceof Error ? error.message : error), ); } this.releaseMcplDataPlaneGate(awarenessBarrier); @@ -15078,8 +15097,8 @@ export class AgentFramework { * every other wake. It installs no awareness gate, since that gate orders * MCPL data-plane traffic, which this family doesn't carry. */ - private wireModernEvents(connection: ModernMcpConnection): void { - const recorded = serverTextRedactor(this.mcplServerConfigs.get(connection.id) ?? {}); + private wireModernEvents(connection: ModernMcpConnection, config: McplServerConfig): void { + const recorded = serverTextRedactor(config); connection.on('stderr', (params: { line: string }) => { this.emitTrace({ type: 'mcpl:server-stderr', serverId: connection.id, line: recorded(params.line) }); }); @@ -15359,7 +15378,7 @@ export class AgentFramework { this.emitMcplToolDiff(oldToolNames, serverId); }) .catch((error) => { - console.error('MCPL tool refresh error:', error); + console.error('MCPL tool refresh error:', this.recordedForEveryServer(error instanceof Error ? error.message : error)); }) .finally(() => { this.mcplToolRefreshInFlight = false; @@ -15440,7 +15459,10 @@ export class AgentFramework { const args = (call.input && typeof call.input === 'object') ? call.input as Record : {}; const failDispatch = (error: unknown) => { - const err = error instanceof Error ? error : new Error(String(error)); + const raw = error instanceof Error ? error : new Error(String(error)); + // Server and transport text, as every record of it is. + const recorded = (text: unknown) => this.recordedFor(serverId, text); + const err = { message: recorded(raw.message), stack: raw.stack === undefined ? undefined : recorded(raw.stack) }; this.emitTrace({ type: 'tool:failed', module: `mcpl:${serverId}`, tool: toolName, callId: call.id, error: err.message, stack: err.stack }); // RFC-007: transport error, closed connection or timeout — no result. this.toolLifecycleEmitter?.markDispatchFailure(agentName, call.id); diff --git a/src/mcpl/modern-connection.ts b/src/mcpl/modern-connection.ts index 6515b143f..c64096c60 100644 --- a/src/mcpl/modern-connection.ts +++ b/src/mcpl/modern-connection.ts @@ -223,6 +223,17 @@ class ObservedTransport implements Transport { } } +/** + * The diagnostic for a line that is JSON but not a JSON-RPC message: the + * line's first 200 characters, after the configuration's credentials are + * replaced, so a credential across the cut can't leave its first part. + */ +export function malformedLineDiagnostic(config: McplServerConfig, text: string, error: unknown): string { + const shown = redactServerText(text, config); + return `MCP server "${config.id}" sent a malformed JSON-RPC message (${shown.length > 200 ? `${shown.slice(0, 200)}…` : shown}): ` + + (error instanceof Error ? error.message.split('\n')[0] : String(error)); +} + /** * SDK `Transport` over Connectome's stdio spawner. Framing is the same * newline-delimited JSON-RPC as legacy. Lines that are not JSON are ignored, @@ -260,10 +271,7 @@ class SpawnerStdioTransport implements Transport { try { message = parseJSONRPCMessage(parsed); } catch (error) { - this.onerror?.(new Error( - `MCP server "${this.config.id}" sent a malformed JSON-RPC message (${text.length > 200 ? `${text.slice(0, 200)}…` : text}): ` + - (error instanceof Error ? error.message.split('\n')[0] : String(error)), - )); + this.onerror?.(new Error(malformedLineDiagnostic(this.config, text, error))); return; } this.onmessage?.(message); @@ -921,7 +929,9 @@ export class ModernMcpConnection extends EventEmitter { 'no-response', ); } - const data = error.data && typeof error.data === 'object' ? error.data : {}; + // The response body (the SDK's `text`) is the server's text and can + // echo the request's url, so it's left out; the status stays. + const { text: _body, ...data } = (error.data && typeof error.data === 'object' ? error.data : {}) as Record; return new McplRequestError(`${prefix} refused ${what}: HTTP ${error.status} (${message})`, 'error-response', undefined, { ...data, httpStatus: error.status }); } if (deadline) { diff --git a/test/mcp-modern-http.test.ts b/test/mcp-modern-http.test.ts index 30775d702..0c51970f0 100644 --- a/test/mcp-modern-http.test.ts +++ b/test/mcp-modern-http.test.ts @@ -22,6 +22,7 @@ import { fileURLToPath, pathToFileURL } from 'node:url'; import type { AddressInfo } from 'node:net'; import { createMcpHandler } from '@modelcontextprotocol/server'; +import { SdkErrorCode, SdkHttpError } from '@modelcontextprotocol/client'; import { ModernMcpConnection } from '../src/mcpl/modern-connection.js'; import { McplRequestError } from '../src/mcpl/server-connection.js'; @@ -236,3 +237,25 @@ test('HTTP: an echoed path in a tool transport failure is redacted, but a JSON-R rmSync(dir, { recursive: true, force: true }); } }); + +test("HTTP: a 403 refusal's response body, which can echo the url, stays out of the request error's data", () => { + // The SDK attaches the body as `text` to its step-up 403; staging step-up + // takes an OAuth provider, so the refusal is handed to the connection's + // reading of a failed request directly. + const connection = Object.create(ModernMcpConnection.prototype) as { + id: string; config: unknown; + requestFailure(what: string, record: unknown, error: unknown): Error; + }; + connection.id = 'remote'; + connection.config = { id: 'remote', url: 'https://h/api/mcp/s/pathSecret0123456789/mcp' }; + const refusal = new SdkHttpError(SdkErrorCode.ClientHttpForbidden, 'Server returned 403 insufficient_scope', { + status: 403, statusText: 'Forbidden', text: 'denied for /api/mcp/s/pathSecret0123456789/mcp', + }); + const err = connection.requestFailure('tools/call "echo"', { method: 'tools/call', handedOff: true, answer: 'none' }, refusal); + assert.ok(err instanceof McplRequestError); + assert.equal(err.outcome, 'error-response'); + assert.equal((err.data as { httpStatus?: number }).httpStatus, 403); + assert.equal((err.data as { text?: unknown }).text, undefined); + assert.ok(!JSON.stringify(err.data).includes('pathSecret'), JSON.stringify(err.data)); +}); + diff --git a/test/mcp-recorded-text.test.ts b/test/mcp-recorded-text.test.ts index 8350fac45..ceaa4bd7c 100644 --- a/test/mcp-recorded-text.test.ts +++ b/test/mcp-recorded-text.test.ts @@ -9,8 +9,8 @@ import { Client, type Transport } from '@modelcontextprotocol/client'; import { EventEmitter } from 'node:events'; import type { ModuleContext } from '../src/index.js'; import { AgentFramework } from '../src/framework.js'; -import { McplServerConnection, McplUnreapedLaunchError } from '../src/mcpl/server-connection.js'; -import { ModernMcpConnection } from '../src/mcpl/modern-connection.js'; +import { McplProtocolVersionError, McplServerConnection, McplUnreapedLaunchError } from '../src/mcpl/server-connection.js'; +import { ModernMcpConnection, malformedLineDiagnostic } from '../src/mcpl/modern-connection.js'; import { MockMembrane } from './helpers/mock-membrane.js'; /** Discovery succeeds; the inventory request gets a normal echoed-path 404. */ @@ -52,7 +52,7 @@ test('both framework engines scrub failure, error and stderr records before trac framework.emitTrace = (event: unknown) => traces.push(event); framework.opsAlert = (_kind: string, _id: string, text: string) => alerts.push(text); const connection = Object.assign(new EventEmitter(), { id: config.id, willReconnect: true }); - framework[method](connection); + framework[method](connection, config); connection.emit('connect-failed', { error: `Cannot POST ${secret}`, attempt: 0, permanent: true }); connection.emit('reconnect-failed', { error: `Cannot POST ${secret}`, attempt: 5 }); connection.emit('error', new Error(`transport ${secret}`)); @@ -82,7 +82,8 @@ test('an SDK HTTP 404 error keeps its path but hides the secret in connection co modules: [{ name: 'watcher', async start(ctx: ModuleContext) { ctx.onTrace(event => traces.push(event as never)); }, async stop() {}, getTools: () => [], async handleToolCall() { return { success: false }; }, async onProcess() { return {}; } }], mcplServers: [{ id: 'echo', url: `http://127.0.0.1:${port}/api/mcp/s/${secret}/mcp`, reconnect: true, reconnectIntervalMs: 10, reconnectMaxIntervalMs: 20 }], }); - const deadline = Date.now() + 5000; + // The loop ends as soon as the record appears; the ceiling only bounds a hang. + const deadline = Date.now() + 30_000; while (!traces.some(e => e.type === 'ops:alert') && Date.now() < deadline) await new Promise(resolve => setTimeout(resolve, 10)); const failed = traces.filter(e => e.type === 'mcpl:server-connect-failed'); assert.ok(failed.some(e => e.attempt === 0)); @@ -143,7 +144,8 @@ test('legacy initial and reconnect console records redact known credentials', as const capabilities = { version: '0.5', pushEvents: true }; const connection = await McplServerConnection.connectWithReconnect(legacyConfig, capabilities); try { - const deadline = Date.now() + 5000; + // The loop ends as soon as the record appears; the ceiling only bounds a hang. + const deadline = Date.now() + 30_000; while (!lines.some(line => line.includes('reconnect failed:')) && Date.now() < deadline) await new Promise(resolve => setTimeout(resolve, 10)); assert.ok(lines.some(line => line.includes('initial connect failed, will retry: transport redacted'))); assert.ok(lines.some(line => line.includes('reconnect failed: transport redacted'))); @@ -181,3 +183,105 @@ test('a real discover-then-tools/list 404 is redacted in the framework error tra rmSync(dir, { recursive: true, force: true }); } }); + +test("a tool call's failure, from either engine, reaches its caller and its trace without the server's credentials", async () => { + // A legacy MCPL server's call rejected with a close reason that quotes the + // url; the modern engine's transport text is requestFailure's, tested over HTTP. + const traces: Array<{ type: string; error?: string; stack?: string }> = []; + const framework = Object.create(AgentFramework.prototype) as any; + framework.mcplServerConfigs = new Map([[config.id, config]]); + framework.emitTrace = (event: { type: string }) => traces.push(event); + const server = { id: config.id, sendToolsCall: async () => { throw new Error(`socket closed: 1008 /api/mcp/s/${secret}/mcp`); } }; + framework.mcplServerRegistry = { getServer: (id: string) => (id === config.id ? server : null) }; + framework.modernServers = () => new Map(); + const result = await framework.executeMcplToolCall({ id: 'c1', name: 'mcpl--remote--echo', input: {} }, config); + assert.equal(result.success, false); + assert.match(String(result.error), /socket closed: 1008 \/api\/mcp\/s\/redacted\/mcp/); + assert.equal(framework.recordedForEveryServer(`closed ${secret}`), 'closed redacted'); + + // The model's dispatch: the tool:failed trace (message and stack) and the + // result the model gets. + const events: Array<{ type: string; result?: { error?: string } }> = []; + framework.pushEvent = (event: { type: string }) => events.push(event); + framework.toolLifecycleEmitter = null; + const modern = { callTool: async () => { throw new Error(`HTTP 500 at /api/mcp/s/${secret}/mcp with ${config.token}`); } }; + framework.modernServers = () => new Map([[config.id, modern]]); + framework.dispatchMcplToolCall('assistant', { id: 'c2', name: 'mcpl--remote--echo', input: {} }, config.id, 'mcpl--remote'); + await new Promise((resolve) => setTimeout(resolve, 20)); + const failed = traces.find((event) => event.type === 'tool:failed'); + assert.ok(failed, 'the failure is traced'); + for (const text of [failed!.error, failed!.stack, events[0]?.result?.error]) { + assert.ok(text && !text.includes(secret) && !text.includes(config.token), String(text)); + } + assert.match(String(events[0]?.result?.error), /HTTP 500 at \/api\/mcp\/s\/redacted\/mcp with redacted/); +}); + +test("a malformed line's diagnostic is redacted before it is cut, so no part of a credential across the cut is kept", () => { + const envSecret = 'envSecretValue0123456789'; + const line = JSON.stringify({ note: `${'p'.repeat(185)} ${envSecret}` }); + const diagnostic = malformedLineDiagnostic({ id: 'stdio-env', command: 'srv', env: { API_TOKEN: envSecret } } as never, line, new Error('not a message')); + assert.match(diagnostic, /sent a malformed JSON-RPC message/); + assert.ok(!diagnostic.includes(envSecret.slice(0, 6)), diagnostic); + assert.match(diagnostic, / redac/, 'the replacement, cut where the line is cut'); +}); + +test("the startup alert for a server with no common protocol revision carries the server's text without its credentials", async (t) => { + // The version verdict quotes the SDK's error, which is server text. + t.mock.method(ModernMcpConnection.prototype as never, 'open' as never, (async () => { + throw new McplProtocolVersionError(`MCP server "echo" does not support MCP 2025-11-25 (refused at /api/mcp/s/${secret}/mcp).`, 'rejected', null, null); + }) as never); + const dir = mkdtempSync(join(tmpdir(), 'mcp-startup-alert-')); + t.mock.method(console, 'error', () => {}); + const traces: Array<{ type: string; message?: string; error?: string }> = []; + let framework: AgentFramework | undefined; + try { + framework = await AgentFramework.create({ + storePath: join(dir, 'store'), membrane: new MockMembrane().asMembrane(), + agents: [{ name: 'agent', model: 'test', systemPrompt: 'test' }], + modules: [{ name: 'watcher', async start(ctx: ModuleContext) { ctx.onTrace(event => traces.push(event as never)); }, async stop() {}, getTools: () => [], async handleToolCall() { return { success: false }; }, async onProcess() { return {}; } }], + mcplServers: [{ id: 'echo', url: `https://127.0.0.1:9/api/mcp/s/${secret}/mcp` }], + }); + const alert = traces.find(e => e.type === 'ops:alert'); + assert.ok(alert, JSON.stringify(traces.map(e => e.type))); + assert.match(String(alert!.message), /MCPL server unreachable at startup: .*refused at \/api\/mcp\/s\/redacted\/mcp/); + assert.ok(!JSON.stringify(traces).includes(secret)); + } finally { + await framework?.stop(); + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("a runtime connect's failed cleanup is traced without the server's credentials, and the host's error stays as raised", async (t) => { + const prototype = ModernMcpConnection.prototype as unknown as { open(): Promise }; + const open = t.mock.method(prototype, 'open', async () => { + throw new Error('connect refused'); + }); + const close = t.mock.method(ModernMcpConnection.prototype, 'close', async () => { + throw new Error(`reap failed for /api/mcp/s/${secret}/mcp`); + }); + const dir = mkdtempSync(join(tmpdir(), 'mcp-cleanup-trace-')); + t.mock.method(console, 'error', () => {}); + const traces: Array<{ type: string; error?: string }> = []; + let framework: AgentFramework | undefined; + try { + framework = await AgentFramework.create({ + storePath: join(dir, 'store'), membrane: new MockMembrane().asMembrane(), + agents: [{ name: 'agent', model: 'test', systemPrompt: 'test' }], + modules: [{ name: 'watcher', async start(ctx: ModuleContext) { ctx.onTrace(event => traces.push(event as never)); }, async stop() {}, getTools: () => [], async handleToolCall() { return { success: false }; }, async onProcess() { return {}; } }], + }); + await assert.rejects(framework.connectMcplServer({ id: 'echo', url: `https://127.0.0.1:9/api/mcp/s/${secret}/mcp` }), (error: unknown) => { + assert.ok(error instanceof Error && error.message.includes(secret), 'the host gets the error as raised'); + return true; + }); + const traced = traces.find(e => e.type === 'mcpl:server-error'); + assert.ok(traced, JSON.stringify(traces.map(e => e.type))); + assert.match(String(traced!.error), /reap failed for \/api\/mcp\/s\/redacted\/mcp/); + assert.ok(!JSON.stringify(traces).includes(secret)); + } finally { + open.mock.restore(); + close.mock.restore(); + await framework?.stop().catch(() => {}); + rmSync(dir, { recursive: true, force: true }); + } +}); + diff --git a/test/mcpl-reconnect-reregistration.test.ts b/test/mcpl-reconnect-reregistration.test.ts index a7627a77f..6fb8439cf 100644 --- a/test/mcpl-reconnect-reregistration.test.ts +++ b/test/mcpl-reconnect-reregistration.test.ts @@ -133,7 +133,7 @@ async function makeHarness() { }; fw.discordAwarenessBarrier = null; fw.discordAwarenessBarrierGeneration = 0; - fw.wireMcplEvents(connection); + fw.wireMcplEvents(connection, config); await fw.registerMcplServerFeatures(config, connection); // The push handler responds synchronously when no awareness barrier is diff --git a/test/mcpl-runtime-lifecycle.test.ts b/test/mcpl-runtime-lifecycle.test.ts index 4e5befdb5..dfbd89a01 100644 --- a/test/mcpl-runtime-lifecycle.test.ts +++ b/test/mcpl-runtime-lifecycle.test.ts @@ -179,6 +179,30 @@ test('restartMcplServer refuses once stop() has begun, before it disconnects any }); }); +test('restartMcplServer of an id whose disconnect is in flight is refused before it tears anything down', async (t) => { + await withFramework(true, async (framework) => { + const disconnecting = framework.disconnectMcplServer('first'); + const disconnect = t.mock.method(framework, 'disconnectMcplServer'); + await assert.rejects( + framework.restartMcplServer('first'), + /MCP server "first" was not restarted: it is being disconnected; restart it once that has finished/, + ); + assert.equal(disconnect.mock.callCount(), 0, 'the restart started no disconnect of its own'); + disconnect.mock.restore(); + await disconnecting; + }); +}); + +test("a disconnect removes its server's prefix claim, and leaves one another server holds", async () => { + await withFramework(true, async (framework) => { + const internals = framework as unknown as { mcplPrefixMap: Map }; + // As after a restart under a new id: `second` holds the prefix `first` had. + internals.mcplPrefixMap.set('mcpl--first', 'second'); + await framework.disconnectMcplServer('first'); + assert.equal(internals.mcplPrefixMap.get('mcpl--first'), 'second', "the other server's claim stands"); + }); +}); + test('connectMcplServer rejects a duplicate id and a colliding prefix', async () => { await withFramework(true, async (framework) => { await assert.rejects( diff --git a/test/tools-observe-parser-conformance.test.ts b/test/tools-observe-parser-conformance.test.ts index 1a73a70c1..efdac0e66 100644 --- a/test/tools-observe-parser-conformance.test.ts +++ b/test/tools-observe-parser-conformance.test.ts @@ -81,10 +81,10 @@ function wireHarness() { const connection = new Connection('observer', null, transport); const framework = Object.create(AgentFramework.prototype) as { mcplServerConfigs: Map; - wireMcplEvents(connection: McplServerConnection): void; + wireMcplEvents(connection: McplServerConnection, config: McplServerConfig): void; }; framework.mcplServerConfigs = new Map([['observer', { id: 'observer', command: 'unused' }]]); - framework.wireMcplEvents(connection); + framework.wireMcplEvents(connection, { id: 'observer', command: 'unused' }); connection.establishGrant(new CapabilityGrant(new Set(['toolLifecycle.observe']), [])); connection.ready(); let nextId = 1; From 8c75a6790b229379c7925dcc4f4167c6cfc0558d Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Tue, 13 Oct 2026 02:41:31 -0400 Subject: [PATCH 0152/1112] fix(mcp): a record of server text keeps the configuration its operation began under The tool-call failure on the model's dispatch, the manifest fetch line, the startup cleanup's close line and the tool refresh's error read a server's configuration when the record was made, after the operation's awaits. A server disconnected or reconfigured meanwhile then had its credentials read from nothing, or from its new configuration, so a late record could carry the old ones. Each now builds its redactor before it waits: the dispatch from the configuration it was dispatched under, the manifest handler on entry, and the startup cleanup and the tool refresh from every configuration present when they begin (everyServerRedactor). The event wiring's own records already kept the configuration the connection was made for, and its three console lines now use that redactor too. Tests: a dispatch whose server is removed while the call is in flight records its failure redacted; a redactor for every server made before a server's removal still replaces its credentials. Reading the map at record time fails each. --- src/framework.ts | 42 +++++++++++++++++++--------------- test/mcp-recorded-text.test.ts | 10 +++++++- 2 files changed, 33 insertions(+), 19 deletions(-) diff --git a/src/framework.ts b/src/framework.ts index 3b780be8a..b8bac1403 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -13590,6 +13590,7 @@ export class AgentFramework { * rethrows the startup failure itself, so a cleanup failure never takes * its place. */ private async closeMcpConnectionsAfterFailedStartup(): Promise { + const recorded = this.everyServerRedactor(); const outcomes = await Promise.allSettled([ this.mcplServerRegistry?.closeAll() ?? Promise.resolve(), this.closeModernMcpConnections(), @@ -13597,7 +13598,7 @@ export class AgentFramework { for (const outcome of outcomes) { if (outcome.status === 'rejected') { console.error('[mcpl] after a failed startup, some MCP connections could not be closed:', - this.recordedForEveryServer(outcome.reason instanceof Error ? outcome.reason.message : outcome.reason)); + recorded(outcome.reason instanceof Error ? outcome.reason.message : outcome.reason)); } } } @@ -14152,6 +14153,9 @@ export class AgentFramework { connection: McplServerConnection, announce: { revision?: string; domains?: string[] }, ): Promise { + // The server's configuration as it is now, before any await: a record + // made later keeps it, whatever becomes of the server meanwhile. + const recorded = serverTextRedactor(this.mcplServerConfigs.get(connection.id) ?? {}); const st = this.manifestRefreshState.get(connection.id) ?? { inFlight: false, queued: false, lastCompletedAt: 0 }; this.manifestRefreshState.set(connection.id, st); @@ -14189,7 +14193,7 @@ export class AgentFramework { try { fetched = await connection.sendManifestRequest(); } catch (err) { - console.error(`[mcpl] ${connection.id}: mcpl/manifest fetch failed — previous manifest stands (§17.5): ${this.recordedFor(connection.id, err instanceof Error ? err.message : err)}`); + console.error(`[mcpl] ${connection.id}: mcpl/manifest fetch failed — previous manifest stands (§17.5): ${recorded(err instanceof Error ? err.message : err)}`); return; } if (!fetched || typeof fetched !== 'object') { @@ -14574,16 +14578,15 @@ export class AgentFramework { await this.connectMcplServer(config); } - /** A server's text as the framework records it ({@link serverTextRedactor}). */ - private recordedFor(serverId: string, text: unknown): string { - return redactServerText(text, this.mcplServerConfigs.get(serverId) ?? {}); - } - - /** Text that may come from any configured server, with every server's credentials replaced. */ - private recordedForEveryServer(text: unknown): string { - let recorded = typeof text === 'string' ? text : String(text); - for (const config of this.mcplServerConfigs.values()) recorded = redactServerText(recorded, config); - return recorded; + /** + * A redactor for text that may come from any configured server, with every + * server's credentials replaced. It is built from the configurations as + * they are when it's made, so an operation makes it before it waits, and a + * server removed meanwhile still has its credentials replaced. + */ + private everyServerRedactor(): (text: unknown) => string { + const redactors = [...this.mcplServerConfigs.values()].map(config => serverTextRedactor(config)); + return (text) => redactors.reduce((recorded, redact) => redact(recorded), typeof text === 'string' ? text : String(text)); } /** Why a connect for this id is refused, or null when the id is free. One @@ -14844,7 +14847,7 @@ export class AgentFramework { // payload and no authority; everything below acts on FETCHED content. connection.on('manifest-changed', (params: { revision?: string; domains?: string[] }) => { void this.handleManifestChanged(connection, params).catch((err) => { - console.error(`[mcpl] ${connection.id} manifest refresh failed:`, this.recordedFor(connection.id, err instanceof Error ? err.message : err)); + console.error(`[mcpl] ${connection.id} manifest refresh failed:`, recorded(err instanceof Error ? err.message : err)); }); }); @@ -14867,7 +14870,7 @@ export class AgentFramework { const parsed = parseToolObserveParams(params); if (!parsed.ok) { if (responder?.respondError) responder.respondError(-32602, parsed.message, parsed.data); - else console.error(`[mcpl] ${connection.id}: tools/observe rejected — ${this.recordedFor(connection.id, parsed.message)}`); + else console.error(`[mcpl] ${connection.id}: tools/observe rejected — ${recorded(parsed.message)}`); return; } connection.toolObserveFilter = parsed.rules; @@ -14959,7 +14962,7 @@ export class AgentFramework { } catch (error) { console.error( `MCPL server "${connection.id}" re-registration after reconnect failed:`, - this.recordedFor(connection.id, error instanceof Error ? error.message : error), + recorded(error instanceof Error ? error.message : error), ); } this.releaseMcplDataPlaneGate(awarenessBarrier); @@ -15372,13 +15375,14 @@ export class AgentFramework { this.mcplToolRefreshInFlight = true; const oldToolNames = new Set(this.mcplTools.map(t => t.name)); + const recorded = this.everyServerRedactor(); this.refreshMcplTools() .then(() => { this.emitMcplToolDiff(oldToolNames, serverId); }) .catch((error) => { - console.error('MCPL tool refresh error:', this.recordedForEveryServer(error instanceof Error ? error.message : error)); + console.error('MCPL tool refresh error:', recorded(error instanceof Error ? error.message : error)); }) .finally(() => { this.mcplToolRefreshInFlight = false; @@ -15458,10 +15462,12 @@ export class AgentFramework { const startTime = Date.now(); const args = (call.input && typeof call.input === 'object') ? call.input as Record : {}; + const dispatchRedactor = serverTextRedactor(config ?? {}); const failDispatch = (error: unknown) => { const raw = error instanceof Error ? error : new Error(String(error)); - // Server and transport text, as every record of it is. - const recorded = (text: unknown) => this.recordedFor(serverId, text); + // Server and transport text, as every record of it is, with the + // configuration the call was dispatched under. + const recorded = dispatchRedactor; const err = { message: recorded(raw.message), stack: raw.stack === undefined ? undefined : recorded(raw.stack) }; this.emitTrace({ type: 'tool:failed', module: `mcpl:${serverId}`, tool: toolName, callId: call.id, error: err.message, stack: err.stack }); // RFC-007: transport error, closed connection or timeout — no result. diff --git a/test/mcp-recorded-text.test.ts b/test/mcp-recorded-text.test.ts index ceaa4bd7c..8f84d3166 100644 --- a/test/mcp-recorded-text.test.ts +++ b/test/mcp-recorded-text.test.ts @@ -197,7 +197,12 @@ test("a tool call's failure, from either engine, reaches its caller and its trac const result = await framework.executeMcplToolCall({ id: 'c1', name: 'mcpl--remote--echo', input: {} }, config); assert.equal(result.success, false); assert.match(String(result.error), /socket closed: 1008 \/api\/mcp\/s\/redacted\/mcp/); - assert.equal(framework.recordedForEveryServer(`closed ${secret}`), 'closed redacted'); + // A redactor for any server's text is made before an operation waits, so + // a server removed meanwhile still has its credentials replaced. + const everyServer = framework.everyServerRedactor(); + framework.mcplServerConfigs = new Map(); + assert.equal(everyServer(`closed ${secret}`), 'closed redacted'); + framework.mcplServerConfigs = new Map([[config.id, config]]); // The model's dispatch: the tool:failed trace (message and stack) and the // result the model gets. @@ -207,6 +212,9 @@ test("a tool call's failure, from either engine, reaches its caller and its trac const modern = { callTool: async () => { throw new Error(`HTTP 500 at /api/mcp/s/${secret}/mcp with ${config.token}`); } }; framework.modernServers = () => new Map([[config.id, modern]]); framework.dispatchMcplToolCall('assistant', { id: 'c2', name: 'mcpl--remote--echo', input: {} }, config.id, 'mcpl--remote'); + // The server is disconnected while the call is in flight: its failure is + // still recorded with the configuration it was dispatched under. + framework.mcplServerConfigs.delete(config.id); await new Promise((resolve) => setTimeout(resolve, 20)); const failed = traces.find((event) => event.type === 'tool:failed'); assert.ok(failed, 'the failure is traced'); From b2a21eca0a0e47423ced178f7fc02a0516fd5f2c Mon Sep 17 00:00:00 2001 From: Petra-1012 Date: Tue, 13 Oct 2026 02:43:45 -0400 Subject: [PATCH 0153/1112] fix(drafts): short notices with no draft words, one notice per failed delivery, and {{unsent}} that names nothing sends nothing Notices: - The catch-up notice (drafts a crash, an aborted turn or a rollback left unnamed) names each draft by id, with its state and why it was held, and quotes none of its words. Before, it previewed the newest eight. The held-draft notice drops its sentence about drafts staying until acted on; it keeps the ids, the reason, and how to resend or dismiss. A draft's words now appear only in what the drafts tool returns (list and read). The draftPreview helper is the listing's. - A failed delivery that the gateway holds as a bounce draft gets the bounce notice alone. deliverSpeech takes heldOnFailure, and skips the failure marker for a failed outcome then; an unknown outcome still gets its marker, since nothing re-holds those words. {{unsent}}: - An envelope whose {{unsent}} names no held or unconfirmed bounce draft (it was sent, set aside or let go, or there never was one) now sends nothing. Before, the token became empty text, so the envelope posted the rest of its words as a fragment, or dropped silently. Its other words, if any, are held as a bounce draft with its notice; otherwise a short notice says nothing was sent. A hybrid envelope refused this way clears its pin and holds later text, as any refused {{unsent}} does. Deadlines and bounds: - channels/open keeps the same fallback deadline as channels/publish when requestTimeoutMs is 0. A delivery opens a closed channel before it publishes, and the completing turn now waits for its deliveries, so an open that is never answered would hold the turn. - The connector's own error message is bounded, as its error data is, where it becomes a publish outcome's reason (an error response, or a failed open), so the trace and the marker carry a bounded reason. Text: - The completion's comment says the chain settles before the host's own final routing, receipt and draft settlement; the modules above hear the turn's speech itself. - holdSilencedProse's comment: a private think in the same tool round, in any tool mode. - The drafts header says retention lets an older draft go in any state, at the journal's next checkpoint, so a resident whose drafts change rarely keeps its older ones until then; hold() says a draft whose append threw may still be in the store's log, and reappears as held after a reload. - The changelog fragments say the above, scope the completion's receipt to hybrid and locus mode, and say notices quote no draft words. Tests: - prose-drafts: the catch-up names drafts with their state and no words (three tests rewritten from previews); through the real connection, a failed explicit or hybrid delivery is held with one notice and no failure marker. - explicit-prose-routing: {{unsent}} with nothing to name sends nothing and holds the envelope's other words; {{unsent}} after its draft was delivered sends nothing. - mcpl-request-outcome: closing the connection rejects an awaiting request as no-response; channels/open times out with requestTimeoutMs at 0. --- .../prose-drafts-delivery-outcomes.fixed.md | 10 ++-- changelog.d/prose-drafts.added.md | 4 +- src/framework.ts | 54 ++++++++++++++----- src/mcpl/channel-registry.ts | 13 +++-- src/mcpl/server-connection.ts | 12 +++-- src/prose-drafts.ts | 19 ++++--- test/explicit-prose-routing.test.ts | 47 ++++++++++++++++ test/mcpl-request-outcome.test.ts | 15 ++++++ test/prose-drafts.test.ts | 37 ++++++++++--- 9 files changed, 171 insertions(+), 40 deletions(-) diff --git a/changelog.d/prose-drafts-delivery-outcomes.fixed.md b/changelog.d/prose-drafts-delivery-outcomes.fixed.md index 982bffac9..c0bf7a4eb 100644 --- a/changelog.d/prose-drafts-delivery-outcomes.fixed.md +++ b/changelog.d/prose-drafts-delivery-outcomes.fixed.md @@ -1,4 +1,6 @@ -- Explicit prose whose delivery definitely fails is held as a bounce draft with a private notice, matching hybrid envelopes. Unknown delivery outcomes remain failure-marker-only, so their words are not offered for an ordinary resend that might duplicate a post. -- A completing turn waits for its queued speech and hybrid holds even when its final response is empty, so its receipt includes that work and its draft notices settle with the same turn. -- Plain-speech failure notices and traces include a bounded display of the connector's structured error data, such as parts already posted. Contradictory delivery receipts bound the message id in their diagnostic without changing the receipt's id. A synchronous MCPL write rejection reports that the request was not sent and removes its pending state. -- A hybrid `>>>skip_reply {{unsent}}` in a skip-reply round sets the latest bounce aside while keeping publication private; same-round private-think text stays uninterpreted, matching explicit mode. +- Explicit prose whose delivery definitely fails is held as a bounce draft with its notice, as hybrid envelopes are. A delivery whose outcome is unknown gets the failure marker alone, so its words are not offered for an ordinary resend that might duplicate a post. A failed delivery that is held gets the bounce notice alone, not a second marker. +- An envelope whose `{{unsent}}` names no held draft (the latest bounce draft was sent, set aside or let go) sends nothing, rather than the rest of the envelope. Its other words, if any, are held as a bounce draft; otherwise a short notice says so. +- In hybrid and locus mode, a completing turn waits for its queued speech and hybrid holds even when its final response is empty, so its receipt includes that work and its draft notices settle with the same turn. +- Failure markers and traces include a bounded display of the connector's structured error data, such as parts already posted, and the connector's error message is bounded too. Contradictory delivery receipts bound the message id in their diagnostic without changing the receipt's id. +- A synchronous MCPL write rejection reports that the request was not sent and removes its pending state. `channels/open` keeps a deadline when `requestTimeoutMs` is 0, as `channels/publish` does, so an unanswered open can't hold a delivery for good. +- A hybrid `>>>skip_reply {{unsent}}` in a skip-reply round sets the latest bounce aside while keeping publication private; text in a round with a private think stays uninterpreted, matching explicit mode. diff --git a/changelog.d/prose-drafts.added.md b/changelog.d/prose-drafts.added.md index f55b6d5a2..bcc78d337 100644 --- a/changelog.d/prose-drafts.added.md +++ b/changelog.d/prose-drafts.added.md @@ -41,7 +41,9 @@ counts as named while a message naming it is on the current branch, so one no notice reached (a crash, an aborted turn), or whose notices a rollback or branch move took away, is named at the next turn's start: every such - draft by id, the newest eight with a preview. + draft by id, the newest eight with its state and why it was held. Notices + name drafts and say how to resend or dismiss them, and quote none of their + words; the `drafts` tool's `list` and `read` show the words. - A draft is never published except by its resident's explicit resend (or `{{unsent}}`), and only that resident can list or act on it. Its notices go into the resident's own history, which residents sharing one message slot diff --git a/src/framework.ts b/src/framework.ts index 9753c5e60..43e87ca95 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -9044,8 +9044,8 @@ export class AgentFramework { /** * Silenced plain speech: an explicit send holds it as resendable drafts; a - * deliberate silence (skip_reply, a silent turn, a same-round private - * think in a native round, where that policy applies) keeps it private — + * deliberate silence (skip_reply, a silent turn, or a private think in + * the same tool round, where that policy applies) keeps it private — * counted for the receipt, never drafted. `segments` are the runs exactly * as written (splitProseRuns), so a draft keeps the resident's words byte * for byte. @@ -9249,8 +9249,7 @@ export class AgentFramework { `[drafts] ${agentName}: ${risky.length > 0 ? '' : 'not sent — '}` + `${many ? `${held.length} plain-speech segments` : 'a plain-speech segment'} ` + `held as draft${many ? 's' : ''} ${ids.join(', ')} — ${AgentFramework.DRAFT_REASON_TEXT[reason]}. ` + - `Nothing publishes drafts but your own resend, and they stay until you act: ${resend.join('; ')}; ` + - `drafts(action: "dismiss", draftIds: [${quoted(held)}]) sets ${many ? 'them' : 'it'} aside.` + `${resend.join('; ')}; drafts(action: "dismiss", draftIds: [${quoted(held)}]) sets ${many ? 'them' : 'it'} aside.` ); } @@ -9259,6 +9258,7 @@ export class AgentFramework { return formatZonedDateTime(t, this.timeZone).replace(/:\d\d\.\d{3}/, ''); } + /** A draft's opening words, for the drafts tool's listing only. */ private static draftPreview(text: string, max = 80): string { const flat = text.replace(/\s+/g, ' ').trim(); return JSON.stringify(flat.length > max ? `${flat.slice(0, max)}…` : flat); @@ -9338,15 +9338,16 @@ export class AgentFramework { * compile, any open drafts no notice on the current branch names: none * reached the history (a crash, an aborted turn), or a rollback or branch * move took the notices away. Every draft it marks is named by id, newest - * first: the newest eight with their state and a preview, the rest by id. + * first: the newest eight with their state and why each was held, the rest + * by id. Like every notice, it quotes none of their words; the drafts tool + * shows them. */ private noticeUnnoticedDrafts(agent: Agent): void { const cm = agent.getContextManager(); const pending = this.proseDrafts.unnoticed(agent.name, (id) => cm.getMessage(id as MessageId) !== null); if (pending.length === 0) return; const shown = pending.slice(0, 8).map((d) => - `${d.id} (${this.draftStateText(d)}; held ${this.draftTime(d.heldAt)}, ` + - `${AgentFramework.DRAFT_REASON_SHORT[d.reason]}, ${d.text.length} chars: ${AgentFramework.draftPreview(d.text, 60)})`); + `${d.id} (${this.draftStateText(d)}; ${AgentFramework.DRAFT_REASON_SHORT[d.reason]})`); const rest = pending.slice(shown.length).map((d) => d.id); const more = rest.length > 0 ? ` · and ${rest.length} more: ${rest.join(', ')}` : ''; const text = @@ -9996,9 +9997,32 @@ export class AgentFramework { agent: Agent, body: string, hold: { round?: number; notice: 'now' | 'later' }, + prefix: '>>' | '>>>', ): { draft?: Draft; refused?: true } { const draft = this.proseDrafts.latestBounce(agent.name); - if (!draft) return {}; + if (!draft) { + // {{unsent}} names nothing now: the latest bounce draft was sent, set + // aside or let go, or there never was one. Send nothing rather than a + // fragment. The envelope's own words, if it has any, are held as a + // bounce draft, with its notice; otherwise a short notice says so. + const own = body.replaceAll('{{unsent}}', '').trim(); + const reason = '{{unsent}} names no held draft: your latest bounce draft has been sent, set aside or let go'; + if (own) { + this.bounceProse(agent, own, reason, undefined, prefix, hold); + } else { + try { + this.addMessage( + 'user', + [{ type: 'text', text: `[prose-routing] ${reason}, so nothing was sent. drafts(action: "list") shows your open drafts.` }], + { system: true, kind: 'prose-bounce' }, + { forAgent: agent.name }, + ); + } catch (err) { + console.error('[drafts] empty-unsent notice failed:', err); + } + } + return { refused: true }; + } const key = `${agent.name}\u0000${draft.id}`; let why: string; let next: string; @@ -10072,7 +10096,9 @@ export class AgentFramework { } let outcome: PublishOutcome; try { - outcome = await registry.deliverSpeech(agent.name, body, channelId); + // Both callers hold a failed delivery's words as a bounce draft, whose + // notice says what happened, so the registry writes no second marker. + outcome = await registry.deliverSpeech(agent.name, body, channelId, { heldOnFailure: true }); } finally { if (flight) { this.draftsInFlight.delete(flight); @@ -10149,7 +10175,7 @@ export class AgentFramework { } let body = parsed.body; const usedClipboard = body.includes('{{unsent}}'); - const unsent = usedClipboard ? this.takeUnsent(agent, body, hold) : {}; + const unsent = usedClipboard ? this.takeUnsent(agent, body, hold, '>>>') : {}; if (unsent.refused) { this.proseTargetPins.delete(agent.name); this.proseHybridSuppressed.set(agent.name, 'failed'); @@ -10309,7 +10335,7 @@ export class AgentFramework { const envelopeBody = body; const usedClipboard = body.includes('{{unsent}}'); - const unsent = usedClipboard ? this.takeUnsent(agent, body, hold) : {}; + const unsent = usedClipboard ? this.takeUnsent(agent, body, hold, '>>') : {}; if (unsent.refused) return; if (usedClipboard) { body = body.replaceAll('{{unsent}}', unsent.draft?.text ?? ''); @@ -11965,8 +11991,10 @@ export class AgentFramework { } // Queued deliveries and hybrid holds belong to this turn even - // when its final response is empty. Settle them before final - // dispatch, the receipt and the turn-end draft settlement. + // when its final response is empty. Settle them before the + // host's own final routing below, the receipt and the turn-end + // draft settlement. (The modules above hear the turn's speech + // itself; they publish nothing of the queue's.) await turnSpeechChain; // Host-owned output routing (see LOCUS-ROUTING-DESIGN). On a diff --git a/src/mcpl/channel-registry.ts b/src/mcpl/channel-registry.ts index a14ebb70a..f2d327500 100644 --- a/src/mcpl/channel-registry.ts +++ b/src/mcpl/channel-registry.ts @@ -3172,7 +3172,7 @@ export class ChannelRegistry { return { status: 'failed', destination, - reason: `channel is closed and open failed: ${(err as Error).message}`, + reason: `channel is closed and open failed: ${connectorText(err instanceof Error ? err.message : String(err))}`, at: at(), }; } @@ -3186,7 +3186,8 @@ export class ChannelRegistry { content: [{ type: 'text', text }], }); } catch (err) { - const message = err instanceof Error ? err.message : String(err); + // The connector's error message is its own words: bounded, as its data is. + const message = connectorText(err instanceof Error ? err.message : String(err)); // Only a request that never left the host proves nothing was posted. // An error response may follow a partial post (a multi-part send whose // later part failed), so it is unknown, with the connector's own words @@ -3258,6 +3259,10 @@ export class ChannelRegistry { * PR #32, and the 2026-07-22 Sol DM misroute). Explicit `null` means * "this turn is pinned to no locus": fail loudly rather than guess. */ locusChannelId: string | null, + /** `heldOnFailure`: the caller holds a `failed` delivery's words as a + * draft with its own notice, so no failure marker is written for it. + * An `unknown` outcome still gets its marker. */ + opts?: { heldOnFailure?: boolean }, ): Promise { const fail = (channelId: string | null, reason: string, outcome: PublishOutcome): PublishOutcome => { const status = outcome.status === 'unknown' ? 'unknown' : 'failed'; @@ -3273,7 +3278,9 @@ export class ChannelRegistry { outcome: status, ...(detail !== undefined ? { detail } : {}), }); - this.onRouteFailure?.({ conversationId, channelId, reason, textLen: text.length, outcome: status, ...(detail !== undefined ? { detail } : {}) }); + if (status === 'unknown' || !opts?.heldOnFailure) { + this.onRouteFailure?.({ conversationId, channelId, reason, textLen: text.length, outcome: status, ...(detail !== undefined ? { detail } : {}) }); + } return outcome; }; diff --git a/src/mcpl/server-connection.ts b/src/mcpl/server-connection.ts index 9de3b48b2..965db7c6f 100644 --- a/src/mcpl/server-connection.ts +++ b/src/mcpl/server-connection.ts @@ -243,8 +243,8 @@ export class McplServerConnection extends EventEmitter { /** Per-request timeout in ms (0 disables). See McplServerConfig.requestTimeoutMs. */ private requestTimeoutMs = DEFAULT_REQUEST_TIMEOUT_MS; - /** The deadline a channels/publish request keeps when requestTimeoutMs is 0 - * (see sendChannelsPublish). */ + /** The deadline a channels/publish or channels/open request keeps when + * requestTimeoutMs is 0 (see sendChannelsPublish). */ private publishDeadlineMs = DEFAULT_REQUEST_TIMEOUT_MS; // Event buffering: control-plane traffic can be released independently from @@ -754,9 +754,13 @@ export class McplServerConnection extends EventEmitter { return this.sendRequest(McplMethod.StateRollback, params as unknown as Record) as Promise; } - /** Send `channels/open` request and await result. */ + /** Send `channels/open` request and await result. It keeps a deadline as + * channels/publish does: a delivery opens a closed channel before it + * publishes, so an open that is never answered would hold the delivery, + * and the turn that waits for it, for good. */ sendChannelsOpen(params: ChannelsOpenParams): Promise { - return this.sendRequest(McplMethod.ChannelsOpen, params as unknown as Record) as Promise; + const timeoutMs = this.requestTimeoutMs > 0 ? this.requestTimeoutMs : this.publishDeadlineMs; + return this.sendRequest(McplMethod.ChannelsOpen, params as unknown as Record, { timeoutMs }) as Promise; } /** Send `channels/close` request and await result. */ diff --git a/src/prose-drafts.ts b/src/prose-drafts.ts index 33bfcb3e8..ebaad9d05 100644 --- a/src/prose-drafts.ts +++ b/src/prose-drafts.ts @@ -8,7 +8,8 @@ * unresolvable routing prefix bounces. Before drafts, the held words were * kept only as a count (or, in explicit mode, as one in-memory latest-wins * clipboard): the law was right, and the penalty was amnesia. A draft keeps - * the exact words until the resident resends or dismisses them. + * the exact words until the resident resends or dismisses them, or the + * retention below lets it go. * * The boundary is publication and agency, not visibility: a draft is never * published except by its resident's explicit resend (or `{{unsent}}`), and @@ -28,11 +29,12 @@ * rollback or branch move has left unnamed is named again. * * Drafts are kept, not hoarded. Each resident's KEEP_NEWEST newest drafts - * are always kept; an older one is let go once nothing has happened to it - * (held, named, attempted, answered, dismissed) for KEEP_IDLE_MS. The words - * stay in the resident's history either way. The journal checkpoints what - * it keeps every CHECKPOINT_EVERY entries, so opening it replays only what - * came after. + * are always kept; an older one, in any state, is let go at the journal's + * next checkpoint once nothing has happened to it (held, named, attempted, + * answered, dismissed) for KEEP_IDLE_MS. The journal checkpoints what it + * keeps every CHECKPOINT_EVERY entries, so opening it replays only what came + * after, and a resident whose drafts change rarely keeps its older ones + * until then. The words stay in the resident's history either way. * * A delivery attempt is journaled durably BEFORE its request leaves the * host, and its outcome after: an attempt with no recorded outcome (a crash @@ -357,7 +359,10 @@ export class ProseDraftStore { * already written stand and the rest are reported as not held: their words * remain in the resident's history only. A draft whose record reached the * store before its write failed (its sync did) is in the reconciled - * journal, so it is held, and counted so. + * journal, so it is held, and counted so. When the store's append itself + * threw, the record may still be in its log: the draft is reported as not + * held, and the journal refuses further writes until a reload, which + * replays it as held if it is there. */ hold( agent: string, diff --git a/test/explicit-prose-routing.test.ts b/test/explicit-prose-routing.test.ts index f338f5286..7573e22ca 100644 --- a/test/explicit-prose-routing.test.ts +++ b/test/explicit-prose-routing.test.ts @@ -326,6 +326,53 @@ describe('explicit prose routing', () => { }); } + it('{{unsent}} that names no held draft sends nothing, and holds the envelope\'s other words', async () => { + const framework = await createFramework(); + try { + const routed = stubRegistry(framework); + const store = draftsOf(framework); + const gateway = framework as unknown as { + deliverProse(a: unknown, text: string, hold: { notice: 'later' }): Promise; + }; + const agent = framework.getAgent('assistant')!; + // Nothing was ever bounced: a bare {{unsent}} sends nothing and says so. + await gateway.deliverProse(agent, '>>#alpha {{unsent}}', { notice: 'later' }); + assert.deepEqual(routed, [], 'nothing is sent'); + assert.deepEqual(store.open('assistant'), [], 'and nothing is held'); + assert.ok(context(framework).texts.some((t) => t.startsWith('[prose-routing] {{unsent}} names no held draft') && t.includes('nothing was sent'))); + // With words of its own, those are held as a bounce draft rather than + // posted as a fragment. + await gateway.deliverProse(agent, '>>#alpha Here it is again: {{unsent}}', { notice: 'later' }); + assert.deepEqual(routed, [], 'still nothing is sent'); + const [held] = store.open('assistant'); + assert.equal(held?.text, 'Here it is again:'); + assert.equal(store.latestBounce('assistant')?.id, held?.id); + } finally { + await framework.stop(); + } + }); + + it('{{unsent}} after its draft was sent names nothing, so the envelope sends nothing', async () => { + const framework = await createFramework(); + try { + const routed = stubRegistry(framework); + const store = draftsOf(framework); + const gateway = framework as unknown as { + deliverProse(a: unknown, text: string, hold: { notice: 'later' }): Promise; + }; + const agent = framework.getAgent('assistant')!; + await gateway.deliverProse(agent, '>>#nowhere the words', { notice: 'later' }); + const bounce = store.latestBounce('assistant')!; + assert.equal(bounce.text, 'the words'); + await gateway.deliverProse(agent, '>>#alpha {{unsent}}', { notice: 'later' }); + assert.deepEqual(routed.map((r) => r.text), ['the words'], 'the first {{unsent}} sends the bounce'); + await gateway.deliverProse(agent, '>>#alpha {{unsent}}', { notice: 'later' }); + assert.deepEqual(routed.map((r) => r.text), ['the words'], 'a second one, with the draft delivered, sends nothing'); + } finally { + await framework.stop(); + } + }); + it('a thrown explicit unsent delivery re-bounces the authored body with its unresolved attempt risk', async () => { const framework = await createFramework(); try { diff --git a/test/mcpl-request-outcome.test.ts b/test/mcpl-request-outcome.test.ts index 4534fd558..85dadaa1b 100644 --- a/test/mcpl-request-outcome.test.ts +++ b/test/mcpl-request-outcome.test.ts @@ -76,6 +76,21 @@ test('a connection lost while awaiting is no-response', async () => { await rejectsWith(connection.sendToolsCall('exit', {}), 'no-response'); }); +test('closing the connection while a request awaits its answer rejects it as no-response', async () => { + connection = await McplServerConnection.connect(config({ requestTimeoutMs: 10_000 }), HOST_CAPS); + const pending = connection.sendToolsCall('stuck', {}); + await connection.close(); + await rejectsWith(pending, 'no-response'); + connection = null; +}); + +test('channels/open keeps a deadline when requestTimeoutMs is 0, as channels/publish does', async () => { + connection = await McplServerConnection.connect(config({ requestTimeoutMs: 0 }), HOST_CAPS); + (connection as unknown as { publishDeadlineMs: number }).publishDeadlineMs = 200; + const err = await rejectsWith(connection.sendChannelsOpen({ type: 'channel', address: { id: 'nowhere' } } as never), 'no-response'); + assert.match(err.message, /did not respond to channels\/open/); +}); + test('a request on a closed connection is not-sent', async () => { connection = await McplServerConnection.connect(config(), HOST_CAPS); await connection.close(); diff --git a/test/prose-drafts.test.ts b/test/prose-drafts.test.ts index 1ab212b5f..31ac240ac 100644 --- a/test/prose-drafts.test.ts +++ b/test/prose-drafts.test.ts @@ -583,6 +583,24 @@ describe('held prose drafts, end to end', () => { } }); + for (const mode of ['explicit', 'hybrid'] as const) { + it('through the real connection, failed ' + mode + ' speech is held as a bounce draft with one notice, not a second marker', async () => { + const h = await harness({ agents: [{ name: 'scout', proseRouting: mode }] }); + try { + h.command({ op: 'publish-mode', mode: 'not-delivered' }); + await h.turn([createMockResponse([text((mode === 'explicit' ? '>>' : '>>>') + ROOM + ' refused words')])]); + assert.deepEqual(h.publishes().map((p) => p.text), ['refused words']); + const [draft] = h.drafts(); + assert.ok(draft, 'the words are held'); + assert.equal(draft!.text, 'refused words'); + assert.ok(h.texts().some((t) => t.startsWith('[prose-routing]') && t.includes(`It is held as draft ${draft!.id}`)), 'the bounce notice names it'); + assert.deepEqual(h.texts().filter((t) => t.startsWith('[discord-send-failed]')), [], 'no second notice for the same failure'); + } finally { + await h.close(); + } + }); + } + for (const mode of ['explicit', 'hybrid'] as const) { it('through the real connection, unknown ' + mode + ' speech is marked but not held for an ordinary resend', async () => { const h = await harness({ agents: [{ name: 'scout', proseRouting: mode }] }); @@ -778,9 +796,10 @@ describe('held prose drafts, end to end', () => { await h.turn([createMockResponse([])]); const notice = h.texts().find((t) => t.startsWith('[drafts]'))!; assert.match(notice, /^\[drafts\] scout: 2 held drafts of yours are unnamed on this branch of your history: /); - assert.match(notice, /d-[a-z2-9]{5} \(held \(not sent\); held .*"orphaned words"\)/); - assert.match(notice, new RegExp(`${midSend.id} \\(UNCONFIRMED — an attempt to #room \\(Guild One\\) \\(${ROOM}\\) at .* may have been posted; .*"half-sent words"`)); + assert.match(notice, /d-[a-z2-9]{5} \(held \(not sent\); explicit send in the same round\)/); + assert.match(notice, new RegExp(`${midSend.id} \\(UNCONFIRMED — an attempt to #room \\(Guild One\\) \\(${ROOM}\\) at .* may have been posted; .*explicit send in the same round\\)`)); assert.doesNotMatch(notice, /They were not sent/); + assert.ok(!notice.includes('orphaned words') && !notice.includes('half-sent words'), `the notice quotes no draft's words: ${notice}`); await h.turn([createMockResponse([])]); assert.equal(h.texts().filter((t) => t.startsWith('[drafts]')).length, 1, 'named once'); } finally { @@ -788,18 +807,19 @@ describe('held prose drafts, end to end', () => { } }); - it('the catch-up names every draft it marks: the newest eight with previews, the rest by id', async () => { + it('the catch-up names every draft it marks: the newest eight with their state, the rest by id, and no words', async () => { const h = await harness(); try { const held = Array.from({ length: 10 }, (_, i) => h.store().hold('scout', [{ text: `orphan ${i}`, source }], 'explicit-send').held[0]!); await h.turn([createMockResponse([])]); const notice = h.texts().find((t) => t.startsWith('[drafts]'))!; assert.match(notice, /^\[drafts\] scout: 10 held drafts of yours are unnamed on this branch of your history: /); - // Newest first: orphan 9 down to orphan 2 with their previews, then the - // two oldest by id alone. - const previews = [...notice.matchAll(/"(orphan \d)"/g)].map((m) => m[1]); - assert.deepEqual(previews, [9, 8, 7, 6, 5, 4, 3, 2].map((i) => `orphan ${i}`)); + // Newest first: the eight newest with their state, then the two oldest + // by id alone. No draft's words appear. + const named = [...notice.matchAll(/(d-[a-z2-9]{5}) \(held \(not sent\); explicit send in the same round\)/g)].map((m) => m[1]); + assert.deepEqual(named, [9, 8, 7, 6, 5, 4, 3, 2].map((i) => held[i]!.id)); assert.ok(notice.includes(` · and 2 more: ${held[1]!.id}, ${held[0]!.id}.`), notice); + assert.ok(!/orphan \d/.test(notice), `the notice quotes no draft's words: ${notice}`); await h.turn([createMockResponse([])]); assert.equal(h.texts().filter((t) => t.startsWith('[drafts]')).length, 1, 'all ten named, so none is named again'); } finally { @@ -826,7 +846,8 @@ describe('held prose drafts, end to end', () => { assert.ok(!h.texts().some((t) => t.includes(draft!.id)), 'nothing on the branch names it now'); await h.turn([createMockResponse([])]); assert.equal(catchUps().length, 1); - assert.match(catchUps()[0]!, new RegExp(`a held draft of yours is unnamed on this branch of your history: ${draft!.id} \\(held \\(not sent\\);.*"words a send held back"`)); + assert.match(catchUps()[0]!, new RegExp(`a held draft of yours is unnamed on this branch of your history: ${draft!.id} \\(held \\(not sent\\); explicit send in the same round\\)`)); + assert.ok(!catchUps()[0]!.includes('words a send held back'), "the notice quotes none of the draft's words"); await h.turn([createMockResponse([])]); assert.equal(catchUps().length, 1, 'named once on this branch'); } finally { From 264704ed7c6acf7fc845a6b95b13b9d6c68e67b8 Mon Sep 17 00:00:00 2001 From: Petra-1012 Date: Tue, 13 Oct 2026 02:44:54 -0400 Subject: [PATCH 0154/1112] test(mcpl): bound the channels/open deadline test by its own timeout Without the deadline it tests, the open would wait forever; the test's own timeout turns that into a failure instead of a hung run. --- test/mcpl-request-outcome.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/mcpl-request-outcome.test.ts b/test/mcpl-request-outcome.test.ts index 85dadaa1b..0389ca1fb 100644 --- a/test/mcpl-request-outcome.test.ts +++ b/test/mcpl-request-outcome.test.ts @@ -84,7 +84,7 @@ test('closing the connection while a request awaits its answer rejects it as no- connection = null; }); -test('channels/open keeps a deadline when requestTimeoutMs is 0, as channels/publish does', async () => { +test('channels/open keeps a deadline when requestTimeoutMs is 0, as channels/publish does', { timeout: 10_000 }, async () => { connection = await McplServerConnection.connect(config({ requestTimeoutMs: 0 }), HOST_CAPS); (connection as unknown as { publishDeadlineMs: number }).publishDeadlineMs = 200; const err = await rejectsWith(connection.sendChannelsOpen({ type: 'channel', address: { id: 'nowhere' } } as never), 'no-response'); From 2836849039a27d721b38de0e445dfbf7475a1ef2 Mon Sep 17 00:00:00 2001 From: Petra-1012 Date: Tue, 13 Oct 2026 02:59:50 -0400 Subject: [PATCH 0155/1112] docs(drafts): say what the notices and changelog claim exactly; a test that locus speech keeps its marker From a review of the final round. Text, one notice's wording, and a test: - The catch-up notice says "open drafts" (held or unconfirmed), and the changelog that every such open draft is named. - The refusal for an {{unsent}} that names nothing says it names no held bounce draft, without claiming there was one. - holdSilencedProse's comment: a private think in the same native round, since an XML-mode round has no verbatim content to route live and its prose is routed at the turn's end as ordinary speech. The earlier round's 'any tool mode' was wrong. - connectorText's doc: an outcome keeps the connector's structured data whole, while its reason and every display are bounded. - requestTimeoutMs's doc names channels/open beside channels/publish. - The drafts changelog's opening says what is held (speech routing holds back, and bounces, a definitely failed delivery among them) and that locus speech whose delivery fails keeps its marker; the round's fragment names publishDeadlineMs. - Stale 'clipboard' wording in the explicit routing tests and the gateway's comment now says held draft. Test (prose-drafts): through the real connection, a failed locus delivery keeps its failure marker and holds nothing. With the earlier round's tests, that pins all three rules of heldOnFailure. The {{unsent}} test after a delivered draft now asserts its notice too. --- .../prose-drafts-delivery-outcomes.fixed.md | 4 ++-- changelog.d/prose-drafts.added.md | 13 ++++++------ src/framework.ts | 10 ++++++---- src/mcpl/channel-registry.ts | 3 ++- src/mcpl/types.ts | 7 ++++--- test/explicit-prose-routing.test.ts | 10 ++++++---- test/prose-drafts.test.ts | 20 ++++++++++++++++--- 7 files changed, 44 insertions(+), 23 deletions(-) diff --git a/changelog.d/prose-drafts-delivery-outcomes.fixed.md b/changelog.d/prose-drafts-delivery-outcomes.fixed.md index c0bf7a4eb..2a2c284f2 100644 --- a/changelog.d/prose-drafts-delivery-outcomes.fixed.md +++ b/changelog.d/prose-drafts-delivery-outcomes.fixed.md @@ -1,6 +1,6 @@ - Explicit prose whose delivery definitely fails is held as a bounce draft with its notice, as hybrid envelopes are. A delivery whose outcome is unknown gets the failure marker alone, so its words are not offered for an ordinary resend that might duplicate a post. A failed delivery that is held gets the bounce notice alone, not a second marker. -- An envelope whose `{{unsent}}` names no held draft (the latest bounce draft was sent, set aside or let go) sends nothing, rather than the rest of the envelope. Its other words, if any, are held as a bounce draft; otherwise a short notice says so. +- An envelope whose `{{unsent}}` names no held draft (the latest bounce draft was sent, set aside or let go) sends nothing, rather than the rest of the envelope. Its other words, if any, are held as a bounce draft; otherwise a short notice says nothing was sent. - In hybrid and locus mode, a completing turn waits for its queued speech and hybrid holds even when its final response is empty, so its receipt includes that work and its draft notices settle with the same turn. - Failure markers and traces include a bounded display of the connector's structured error data, such as parts already posted, and the connector's error message is bounded too. Contradictory delivery receipts bound the message id in their diagnostic without changing the receipt's id. -- A synchronous MCPL write rejection reports that the request was not sent and removes its pending state. `channels/open` keeps a deadline when `requestTimeoutMs` is 0, as `channels/publish` does, so an unanswered open can't hold a delivery for good. +- A synchronous MCPL write rejection reports that the request was not sent and removes its pending state. `channels/open` keeps the same deadline as `channels/publish` when `requestTimeoutMs` is 0 (`publishDeadlineMs`, 60 s by default), so an unanswered open can't hold a delivery for good. - A hybrid `>>>skip_reply {{unsent}}` in a skip-reply round sets the latest bounce aside while keeping publication private; text in a round with a private think stays uninterpreted, matching explicit mode. diff --git a/changelog.d/prose-drafts.added.md b/changelog.d/prose-drafts.added.md index bcc78d337..da3bf953a 100644 --- a/changelog.d/prose-drafts.added.md +++ b/changelog.d/prose-drafts.added.md @@ -1,11 +1,12 @@ - **Held prose drafts: suppressed speech is kept, not destroyed.** Plain - speech that is not sent is now held as a draft with its exact words, - instead of surviving only as a count: speech suppressed by an + speech that routing holds back is now held as a draft with its exact + words, instead of surviving only as a count: speech suppressed by an explicit send in the same round (locus and hybrid modes, mid-turn, trailing and text-only, native and XML tool modes), hybrid prose left without a - destination after a failed envelope, and explicit/hybrid bounces (which - replace the in-memory latest-wins clipboard; `{{unsent}}` keeps meaning the - latest bounce). A draft held beside a send keeps its run exactly as + destination after a failed envelope, and explicit/hybrid bounces, a + delivery that definitely failed among them (which replace the in-memory + latest-wins clipboard; `{{unsent}}` keeps meaning the latest bounce). Locus + speech whose delivery fails keeps its failure marker, and isn't held. A draft held beside a send keeps its run exactly as written, indentation and surrounding newlines included; routing still publishes trimmed segments. Deliberate privacy is never drafted: `skip_reply` (the tool, or a hybrid `>>>skip_reply` envelope, which keeps @@ -41,7 +42,7 @@ counts as named while a message naming it is on the current branch, so one no notice reached (a crash, an aborted turn), or whose notices a rollback or branch move took away, is named at the next turn's start: every such - draft by id, the newest eight with its state and why it was held. Notices + open draft by id, the newest eight with its state and why it was held. Notices name drafts and say how to resend or dismiss them, and quote none of their words; the `drafts` tool's `list` and `read` show the words. - A draft is never published except by its resident's explicit resend (or diff --git a/src/framework.ts b/src/framework.ts index 43e87ca95..601f00b35 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -9045,7 +9045,9 @@ export class AgentFramework { /** * Silenced plain speech: an explicit send holds it as resendable drafts; a * deliberate silence (skip_reply, a silent turn, or a private think in - * the same tool round, where that policy applies) keeps it private — + * the same native round, where that policy applies: an XML-mode round has + * no verbatim content to route live, so its prose is routed at the turn's + * end as ordinary speech) keeps it private — * counted for the receipt, never drafted. `segments` are the runs exactly * as written (splitProseRuns), so a draft keeps the resident's words byte * for byte. @@ -9351,7 +9353,7 @@ export class AgentFramework { const rest = pending.slice(shown.length).map((d) => d.id); const more = rest.length > 0 ? ` · and ${rest.length} more: ${rest.join(', ')}` : ''; const text = - `[drafts] ${agent.name}: ${pending.length === 1 ? 'a held draft' : `${pending.length} held drafts`} of yours ` + + `[drafts] ${agent.name}: ${pending.length === 1 ? 'an open draft' : `${pending.length} open drafts`} of yours ` + `${pending.length === 1 ? 'is' : 'are'} unnamed on this branch of your history: ${shown.join(' · ')}${more}. ` + 'drafts(action: "list") shows your open drafts; resend or dismiss them by id.'; let mid: string; @@ -9942,7 +9944,7 @@ export class AgentFramework { * Explicit-prose-routing delivery gateway (docs/explicit-prose-routing.md). * Every prose segment of an `explicit`-mode agent flows through here: * parse the `>>` prefix, resolve the target, deliver — or bounce to the - * clipboard with a resend notice. Misdelivery is structurally impossible: + * held bounce draft with a resend notice. Misdelivery is structurally impossible: * nothing is ever sent to a destination the model did not name (directly * or via the turn's sticky target). */ @@ -10006,7 +10008,7 @@ export class AgentFramework { // fragment. The envelope's own words, if it has any, are held as a // bounce draft, with its notice; otherwise a short notice says so. const own = body.replaceAll('{{unsent}}', '').trim(); - const reason = '{{unsent}} names no held draft: your latest bounce draft has been sent, set aside or let go'; + const reason = '{{unsent}} names no held bounce draft'; if (own) { this.bounceProse(agent, own, reason, undefined, prefix, hold); } else { diff --git a/src/mcpl/channel-registry.ts b/src/mcpl/channel-registry.ts index f2d327500..bebfc41b7 100644 --- a/src/mcpl/channel-registry.ts +++ b/src/mcpl/channel-registry.ts @@ -460,7 +460,8 @@ const CHANNEL_TOOL_DEFINITIONS: ToolDefinition[] = [ // Constructor Options // ============================================================================ -/** Bound connector values for notices/logs; the PublishOutcome keeps the original. */ +/** Bound connector values for notices, logs and an outcome's reason. A + * PublishOutcome keeps the connector's structured data (`detail`) whole. */ function connectorText(value: unknown): string { let text: string; try { diff --git a/src/mcpl/types.ts b/src/mcpl/types.ts index fbc848860..49fe7e8c6 100644 --- a/src/mcpl/types.ts +++ b/src/mcpl/types.ts @@ -374,9 +374,10 @@ export interface McplServerConfig { * accepts a request and never responds would otherwise freeze the awaiting * agent turn forever; with the timeout the pending request rejects with a * descriptive error, which the framework surfaces as a normal isError - * tool_result. Set 0 to disable, except for a channels/publish request, - * which keeps the 60 s default: its outcome is reported to the resident and - * recorded for its draft, so it must end. Default: 60000 (60 seconds). + * tool_result. Set 0 to disable, except for channels/publish and the + * channels/open a delivery may send first, which keep the 60 s default: a + * publish's outcome is reported to the resident and recorded for its + * draft, so it must end. Default: 60000 (60 seconds). */ requestTimeoutMs?: number; diff --git a/test/explicit-prose-routing.test.ts b/test/explicit-prose-routing.test.ts index 7573e22ca..fd253ec31 100644 --- a/test/explicit-prose-routing.test.ts +++ b/test/explicit-prose-routing.test.ts @@ -2,7 +2,7 @@ * Explicit prose routing (docs/explicit-prose-routing.md). * * The model names every destination in-band (`>>` prefixes); unprefixed prose - * is NEVER delivered — it bounces to a clipboard for a prefixed resend. The + * is NEVER delivered — it bounces to a held draft for a prefixed resend. The * laria regression (2026-07-24): a DM reply generated while the host-inferred * locus pointed at a guild channel was misdelivered there. In explicit mode * that class is structurally impossible: the worst case is a bounce. @@ -169,7 +169,7 @@ describe('explicit prose routing', () => { }); } - it('unprefixed prose NEVER delivers — bounces to clipboard, then a prefixed {{unsent}} resend delivers verbatim (laria regression)', async () => { + it('unprefixed prose NEVER delivers — bounces to a held draft, then a prefixed {{unsent}} resend delivers verbatim (laria regression)', async () => { const original = 'laria.\n\nYes, I remember. Two words through the keyhole.'; membrane.pushResponse(createMockResponse([{ type: 'text', text: original }] as ContentBlock[])); @@ -191,7 +191,7 @@ describe('explicit prose routing', () => { assert.equal(bounceNotices.length, 1, 'exactly one bounce notice'); assert.equal(routed.length, 1, 'only the prefixed resend delivered'); assert.equal(routed[0]!.locus, 'discord:dm:99', 'delivered to the DM the model named'); - assert.equal(routed[0]!.text, original, 'clipboard substitution is verbatim'); + assert.equal(routed[0]!.text, original, '{{unsent}} substitution is verbatim'); await framework.stop(); }); @@ -339,7 +339,7 @@ describe('explicit prose routing', () => { await gateway.deliverProse(agent, '>>#alpha {{unsent}}', { notice: 'later' }); assert.deepEqual(routed, [], 'nothing is sent'); assert.deepEqual(store.open('assistant'), [], 'and nothing is held'); - assert.ok(context(framework).texts.some((t) => t.startsWith('[prose-routing] {{unsent}} names no held draft') && t.includes('nothing was sent'))); + assert.ok(context(framework).texts.some((t) => t.startsWith('[prose-routing] {{unsent}} names no held bounce draft') && t.includes('nothing was sent'))); // With words of its own, those are held as a bounce draft rather than // posted as a fragment. await gateway.deliverProse(agent, '>>#alpha Here it is again: {{unsent}}', { notice: 'later' }); @@ -368,6 +368,8 @@ describe('explicit prose routing', () => { assert.deepEqual(routed.map((r) => r.text), ['the words'], 'the first {{unsent}} sends the bounce'); await gateway.deliverProse(agent, '>>#alpha {{unsent}}', { notice: 'later' }); assert.deepEqual(routed.map((r) => r.text), ['the words'], 'a second one, with the draft delivered, sends nothing'); + assert.ok(context(framework).texts.some((t) => t.startsWith('[prose-routing] {{unsent}} names no held bounce draft')), + 'and says why'); } finally { await framework.stop(); } diff --git a/test/prose-drafts.test.ts b/test/prose-drafts.test.ts index 31ac240ac..b06364b4c 100644 --- a/test/prose-drafts.test.ts +++ b/test/prose-drafts.test.ts @@ -583,6 +583,20 @@ describe('held prose drafts, end to end', () => { } }); + it('through the real connection, failed locus speech keeps its failure marker, since nothing holds it', async () => { + const h = await harness(); + try { + h.command({ op: 'publish-mode', mode: 'not-delivered' }); + await h.turn([createMockResponse([text('words a refusing connector drops')])]); + await h.until(() => h.texts().some((t) => t.startsWith('[discord-send-failed]')), 'the marker'); + const marker = h.texts().find((t) => t.startsWith('[discord-send-failed]'))!; + assert.match(marker, /could not be delivered to .* Nothing was posted/); + assert.deepEqual(h.drafts(), [], 'locus speech is not held'); + } finally { + await h.close(); + } + }); + for (const mode of ['explicit', 'hybrid'] as const) { it('through the real connection, failed ' + mode + ' speech is held as a bounce draft with one notice, not a second marker', async () => { const h = await harness({ agents: [{ name: 'scout', proseRouting: mode }] }); @@ -795,7 +809,7 @@ describe('held prose drafts, end to end', () => { h.store().beginAttempt('scout', midSend.id, dest(), 'resend', false); await h.turn([createMockResponse([])]); const notice = h.texts().find((t) => t.startsWith('[drafts]'))!; - assert.match(notice, /^\[drafts\] scout: 2 held drafts of yours are unnamed on this branch of your history: /); + assert.match(notice, /^\[drafts\] scout: 2 open drafts of yours are unnamed on this branch of your history: /); assert.match(notice, /d-[a-z2-9]{5} \(held \(not sent\); explicit send in the same round\)/); assert.match(notice, new RegExp(`${midSend.id} \\(UNCONFIRMED — an attempt to #room \\(Guild One\\) \\(${ROOM}\\) at .* may have been posted; .*explicit send in the same round\\)`)); assert.doesNotMatch(notice, /They were not sent/); @@ -813,7 +827,7 @@ describe('held prose drafts, end to end', () => { const held = Array.from({ length: 10 }, (_, i) => h.store().hold('scout', [{ text: `orphan ${i}`, source }], 'explicit-send').held[0]!); await h.turn([createMockResponse([])]); const notice = h.texts().find((t) => t.startsWith('[drafts]'))!; - assert.match(notice, /^\[drafts\] scout: 10 held drafts of yours are unnamed on this branch of your history: /); + assert.match(notice, /^\[drafts\] scout: 10 open drafts of yours are unnamed on this branch of your history: /); // Newest first: the eight newest with their state, then the two oldest // by id alone. No draft's words appear. const named = [...notice.matchAll(/(d-[a-z2-9]{5}) \(held \(not sent\); explicit send in the same round\)/g)].map((m) => m[1]); @@ -846,7 +860,7 @@ describe('held prose drafts, end to end', () => { assert.ok(!h.texts().some((t) => t.includes(draft!.id)), 'nothing on the branch names it now'); await h.turn([createMockResponse([])]); assert.equal(catchUps().length, 1); - assert.match(catchUps()[0]!, new RegExp(`a held draft of yours is unnamed on this branch of your history: ${draft!.id} \\(held \\(not sent\\); explicit send in the same round\\)`)); + assert.match(catchUps()[0]!, new RegExp(`an open draft of yours is unnamed on this branch of your history: ${draft!.id} \\(held \\(not sent\\); explicit send in the same round\\)`)); assert.ok(!catchUps()[0]!.includes('words a send held back'), "the notice quotes none of the draft's words"); await h.turn([createMockResponse([])]); assert.equal(catchUps().length, 1, 'named once on this branch'); From f62c02e561f90dad46056e9b3da19853c88e1cd5 Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Tue, 13 Oct 2026 03:05:48 -0400 Subject: [PATCH 0156/1112] fix(mcp): only a short pair needs its boundary; a 401 or 403 is a refusal however it's raised; the last unredacted records From a review of the final round: - The letter-or-digit boundary kept a credential raw after a percent escape (%2F), a JSON escape (\n) or an ANSI code. Only a short name=value pair (value under eight characters) needs it, so v=2 leaves dev=2 alone; every other string is replaced wherever it occurs. - A stdio server's env value is a credential when a part of its name ends with a credential word (GITHUB_TOKEN, API_KEY, PGPASSWORD, ACCESSTOKEN), not when the word sits inside a part (GIT_AUTHOR_NAME, KEYBOARD_LAYOUT, TOKENIZERS_PARALLELISM). - A token that can't be url-encoded (an unpaired surrogate) has no encoded form rather than throwing, and a value that can't be printed reads as a placeholder rather than throwing inside a catch. - A 401 or 403 is the server's refusal however the HTTP client raised it: a plain 403 (another SDK code than its step-up limit's) and an insufficient_scope 403 (InsufficientScopeError) were no-response with no status; they're error-response with data.httpStatus now. - The policy refusal's reason, in its log line and its mcpl:policy-refused trace, and the malformed-line diagnostic's parser message are redacted; the transport builds its redactor once; the tool refresh's error line prints the redacted stack it printed before. - restartMcplServer refuses a connect or disconnect in flight for the server it restarts whatever id it moves it to, not only a same-id one. The changelog fragments and README say what's derived now (env values, the token's url forms, both engines' tool-call failures and the trace), that a credential-shaped value under a plain name is warned about, and that a restart refuses before tearing the server down. Tests: escapes before a credential, env names, a surrogate token, unprintable values, a plain and an insufficient_scope 403 over HTTP, the policy reason, the parser's message, the cut's exact kept prefix, two servers' redactor, a stdio env value through both engines' wiring, a connect in flight for a renamed restart, and a disconnect releasing its own claim. Removing each piece fails a test (nine removals). --- README.md | 2 +- .../cleartext-mcp-credential-warning.added.md | 5 +- .../mcp-standard-tool-results.changed.md | 7 +++ .../redact-recorded-mcp-credentials.fixed.md | 2 +- src/framework.ts | 29 +++++---- src/mcpl/modern-connection.ts | 43 +++++++++---- src/mcpl/protocol-family.ts | 63 ++++++++++++------- test/mcp-modern-http.test.ts | 23 +++++++ test/mcp-protocol-family.test.ts | 31 ++++++--- test/mcp-recorded-text.test.ts | 35 ++++++++++- test/mcpl-config-diagnostics.test.ts | 29 +++++++++ test/mcpl-runtime-lifecycle.test.ts | 19 ++++++ 12 files changed, 227 insertions(+), 61 deletions(-) diff --git a/README.md b/README.md index 27cff15bb..3a6320f54 100644 --- a/README.md +++ b/README.md @@ -314,7 +314,7 @@ supports, or its error text when it lists none, and when the list includes `2026-07-28` it says to configure the server with `protocol: 'modern'` (stdio) or reach it over http(s). -Server and transport diagnostics recorded by the framework remove known credentials from the configured URL and static `token`. This covers connection traces, console errors, stderr traces and ops alerts, plus transport failures returned by modern tool calls. Credential-like URL path segments and query names, credential-shaped or credential-named query values, and WebSocket userinfo are replaced in their encoded and decoded forms. Other non-empty query parameters are replaced as whole `name=value` strings. Standalone strings shorter than eight characters stay unchanged because they may be ordinary words; short query values are still removed with their parameter names. The cached token from an `accessProvider` and secrets the framework does not know remain outside this filtering. JSON-RPC tool answers and direct host connection errors pass through as sent. +Server and transport text the framework records has the server's configured credentials replaced, in both engines. That covers connection traces, console lines, stderr traces, ops alerts, and a tool call's failure as its caller, the model and the `tool:failed` trace get it. The credentials are those the configuration holds: from the url, credential-like path segments and query names, credential-shaped or credential-named query values, and a `ws://` url's userinfo, in their encoded and decoded forms, and every other non-empty query parameter as its whole `name=value`; the static `token`, raw and url-encoded; and a stdio server's `env` values whose names end a part with a credential word, such as `GITHUB_TOKEN` or `PGPASSWORD`. Standalone strings shorter than eight characters stay unchanged because they may be ordinary words; short query values are still removed with their parameter names. The token an `accessProvider` fetches, and secrets the configuration doesn't hold, remain outside this filtering. JSON-RPC tool answers, and the error a host's own connect raises, pass through as sent. #### Feature sets diff --git a/changelog.d/cleartext-mcp-credential-warning.added.md b/changelog.d/cleartext-mcp-credential-warning.added.md index 6b801f0c2..3a58775c4 100644 --- a/changelog.d/cleartext-mcp-credential-warning.added.md +++ b/changelog.d/cleartext-mcp-credential-warning.added.md @@ -3,8 +3,9 @@ machine: its `token` or `accessProvider`, or a secret the url carries itself (a `ws://` url's userinfo, a query parameter whose name says credential: `token`, `key`, `secret`, `password`, `auth` or `signature`, - a query parameter name that may itself be one, or a path segment that - looks like one, as the redacted url judges it). A name that may itself be + or whose value is shaped like one under a plain name, a query parameter + name that may itself be one, or a path segment that looks like one, as + the redacted url judges it). A name that may itself be a credential is counted in the warning, never quoted. An `http(s)://` url with userinfo is refused instead, since the HTTP client can't send it and its error would quote it. Loopback hosts are diff --git a/changelog.d/mcp-standard-tool-results.changed.md b/changelog.d/mcp-standard-tool-results.changed.md index e4dfb7fd2..39f9077f0 100644 --- a/changelog.d/mcp-standard-tool-results.changed.md +++ b/changelog.d/mcp-standard-tool-results.changed.md @@ -102,6 +102,13 @@ up when its handshake settles (a modern connect in flight, or such a legacy one, fails with "was not connected"); then it removes what was registered. A teardown never removes what a later connection registered. + `restartMcplServer` refuses, before it tears the running server down, what + its connect would refuse: a configuration with no usable binding, a new id + another server holds, a tool prefix another server or a module holds, a + framework whose `stop()` has begun, and a connect or disconnect of the + server in flight. On main a restart disconnected first, so a refused + connect left the server gone; now the server keeps running. A disconnect + releases its tool-prefix claim only while its own server holds it. - A server configuration naming no usable transport is now an error before anything is spawned or dialed. That covers a `transport` that doesn't match the url's scheme, `transport: 'http'` without a url, and an diff --git a/changelog.d/redact-recorded-mcp-credentials.fixed.md b/changelog.d/redact-recorded-mcp-credentials.fixed.md index 85d447944..6b2ed989d 100644 --- a/changelog.d/redact-recorded-mcp-credentials.fixed.md +++ b/changelog.d/redact-recorded-mcp-credentials.fixed.md @@ -1 +1 @@ -- Server and transport diagnostics recorded by the framework redact known URL credentials and the configured static token, including encoded and decoded forms. An HTTP error that echoes a credential-bearing request path no longer copies the credential into connection traces, console errors, ops alerts or modern tool-call failures. Stderr traces use the same filtering. Short standalone strings remain unchanged; short query values are removed as whole `name=value` strings. Access-provider cached tokens and unknown secrets are outside this filtering, and JSON-RPC tool answers and direct host connection errors retain their pass-through contract. +- Server and transport text the framework records has the server's configured credentials replaced, in both engines: connection traces, console lines, stderr traces, the ops alert (and so `logs/failures.log` and the webhook), a tool call's failure as its caller and the model get it, and the `tool:failed` trace's message and stack. The credentials are those the configuration holds: from the url, credential-like path segments and query names, credential-shaped or credential-named query values (each as written and decoded), every other non-empty parameter as its whole `name=value`, and a `ws://` url's username and password; the static `token`, raw and as a url carries it, percent- or form-encoded; and a stdio server's `env` values whose names end a part with a credential word (`GITHUB_TOKEN`, `API_KEY`, `PGPASSWORD`). A string replaced on its own has at least eight characters, since a shorter one can't be told from ordinary text; a short query value is replaced only with its name, as one `name=value`, and only where no letter or digit adjoins it. A connection's event records use a redactor built once for it, and an operation's records one built when the operation begins, so a server removed or reconfigured meanwhile doesn't change what they replace. A 401 or 403 is classified as the server's refusal however the HTTP client raised it, and its response body is left out of the error's data (its message, built from the body, is redacted like any record). A token an access provider fetches, and credentials the configuration doesn't hold, are outside this. JSON-RPC answers, and the error a host's own connect raises, pass as they are. diff --git a/src/framework.ts b/src/framework.ts index b8bac1403..dfcd7cd23 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -14133,13 +14133,17 @@ export class AgentFramework { * branch actually taken (anything but `close` is treated as `mcp-only`) * and a non-string `reason` is dropped. */ - private static policyRefusalFields(receipt: { fallback?: unknown; reason?: unknown }): { + private static policyRefusalFields( + receipt: { fallback?: unknown; reason?: unknown }, + recorded: (text: unknown) => string, + ): { fallback: 'close' | 'mcp-only'; reason: string | null; } { return { fallback: receipt.fallback === 'close' ? 'close' : 'mcp-only', - reason: typeof receipt.reason === 'string' ? receipt.reason : null, + // The server's own words, recorded as every server text is. + reason: typeof receipt.reason === 'string' ? recorded(receipt.reason) : null, }; } @@ -14265,7 +14269,7 @@ export class AgentFramework { type: 'mcpl:policy-refused', serverId: connection.id, phase: 'manifest-change', - ...AgentFramework.policyRefusalFields(receipt), + ...AgentFramework.policyRefusalFields(receipt, recorded), }); if (fallback === 'close') { await connection.close(); @@ -14377,13 +14381,13 @@ export class AgentFramework { if (receipt && receipt.accepted === false) { const fallback = receipt.fallback ?? 'mcp-only'; console.error( - `[mcpl] ${config.id} refused initial policy (${receipt.reason ?? 'no reason'}) — fallback: ${fallback}`, + `[mcpl] ${config.id} refused initial policy (${redactServerText(receipt.reason ?? 'no reason', config)}) — fallback: ${fallback}`, ); this.emitTrace({ type: 'mcpl:policy-refused', serverId: config.id, phase: 'initial', - ...AgentFramework.policyRefusalFields(receipt), + ...AgentFramework.policyRefusalFields(receipt, (text) => redactServerText(text, config)), }); if (fallback === 'close') { await connection.close(); @@ -14563,14 +14567,17 @@ export class AgentFramework { throw new Error(`MCPL server "${id}" is not configured`); } AgentFramework.usableConfigFamily(config); + const inFlight = this.mcpServerOps?.get(id); + if (inFlight) { + // A connect or disconnect of the server being restarted, in flight, + // whatever id the restart moves it to: the restart's own disconnect + // would wait behind it, and its connect would then meet whatever that + // operation left, after the server was torn down. + throw new Error(`MCP server "${id}" was not restarted: it is being ${inFlight.kind === 'disconnect' ? 'disconnected' : 'connected'}; restart it once that has finished`); + } if (config.id !== id) { const taken = this.mcpServerIdTaken(config.id); if (taken) throw new Error(taken); - } else if (this.mcpServerOps?.get(id)) { - // A connect or disconnect of this id in flight: the restart's own - // disconnect would queue behind it and then be refused, after the - // server was torn down. - throw new Error(`MCP server "${id}" was not restarted: it is being ${this.mcpServerOps.get(id)!.kind === 'disconnect' ? 'disconnected' : 'connected'}; restart it once that has finished`); } const collision = this.mcplPrefixCollision(config, id); if (collision) throw new Error(collision); @@ -15382,7 +15389,7 @@ export class AgentFramework { this.emitMcplToolDiff(oldToolNames, serverId); }) .catch((error) => { - console.error('MCPL tool refresh error:', recorded(error instanceof Error ? error.message : error)); + console.error('MCPL tool refresh error:', recorded(error instanceof Error ? (error.stack ?? error.message) : error)); }) .finally(() => { this.mcplToolRefreshInFlight = false; diff --git a/src/mcpl/modern-connection.ts b/src/mcpl/modern-connection.ts index c64096c60..e97ed30b8 100644 --- a/src/mcpl/modern-connection.ts +++ b/src/mcpl/modern-connection.ts @@ -43,6 +43,7 @@ import { EventEmitter } from 'node:events'; import { Client, + InsufficientScopeError, SdkError, SdkErrorCode, SdkHttpError, @@ -60,7 +61,7 @@ import { import { StdioTransport, type McplTransport } from './transport.js'; import { McplProtocolVersionError, McplRequestError } from './server-connection.js'; -import { LEGACY_MCP_PROTOCOL_VERSION, MAX_TIMER_MS, MODERN_MCP_PROTOCOL_VERSION, checkServerConfig, describeCredentialError, redactServerText } from './protocol-family.js'; +import { LEGACY_MCP_PROTOCOL_VERSION, MAX_TIMER_MS, MODERN_MCP_PROTOCOL_VERSION, checkServerConfig, describeCredentialError, redactServerText, serverTextRedactor } from './protocol-family.js'; import type { McplServerConfig, McpToolDefinition } from './types.js'; const DEFAULT_REQUEST_TIMEOUT_MS = 60_000; @@ -228,10 +229,16 @@ class ObservedTransport implements Transport { * line's first 200 characters, after the configuration's credentials are * replaced, so a credential across the cut can't leave its first part. */ -export function malformedLineDiagnostic(config: McplServerConfig, text: string, error: unknown): string { - const shown = redactServerText(text, config); - return `MCP server "${config.id}" sent a malformed JSON-RPC message (${shown.length > 200 ? `${shown.slice(0, 200)}…` : shown}): ` + - (error instanceof Error ? error.message.split('\n')[0] : String(error)); +export function malformedLineDiagnostic( + config: McplServerConfig, + text: string, + error: unknown, + recorded: (text: unknown) => string = serverTextRedactor(config), +): string { + // The parser's message can quote the line, so it's redacted too. + const shown = recorded(text); + const why = recorded(error instanceof Error ? error.message.split('\n')[0] : error); + return `MCP server "${config.id}" sent a malformed JSON-RPC message (${shown.length > 200 ? `${shown.slice(0, 200)}…` : shown}): ${why}`; } /** @@ -248,10 +255,15 @@ class SpawnerStdioTransport implements Transport { onmessage?: (message: T, extra?: MessageExtraInfo) => void; private line: McplTransport | null = null; + /** Built once for the transport: every malformed line uses it. */ + private readonly recorded: (text: unknown) => string; + constructor( private readonly config: McplServerConfig, private readonly onStderr: (line: string) => void, - ) {} + ) { + this.recorded = serverTextRedactor(config); + } async start(): Promise { const line = StdioTransport.spawn(this.config); @@ -271,7 +283,7 @@ class SpawnerStdioTransport implements Transport { try { message = parseJSONRPCMessage(parsed); } catch (error) { - this.onerror?.(new Error(malformedLineDiagnostic(this.config, text, error))); + this.onerror?.(new Error(malformedLineDiagnostic(this.config, text, error, this.recorded))); return; } this.onmessage?.(message); @@ -921,18 +933,25 @@ export class ModernMcpConnection extends EventEmitter { 'no-response', ); } - if (error instanceof SdkHttpError && (error.code === SdkErrorCode.ClientHttpAuthentication || error.code === SdkErrorCode.ClientHttpForbidden)) { + // An HTTP refusal of the credential: 401 or 403, however the SDK raised + // it. It raises a plain 403 under another code than its step-up limit's, + // and an insufficient_scope 403 as InsufficientScopeError when no OAuth + // provider is configured. + const refusedWith = error instanceof SdkHttpError && (error.status === 401 || error.status === 403) + ? error.status + : error instanceof InsufficientScopeError ? 403 : null; + if (refusedWith !== null) { if (record.continued) { return new McplRequestError( `${prefix} did not finish ${what}: it asked for another round (input_required), and the server refused a later one: ` + - `HTTP ${error.status} (${message}). The outcome is unknown: the server may already have acted on the call. ${advice}`, + `HTTP ${refusedWith} (${message}). The outcome is unknown: the server may already have acted on the call. ${advice}`, 'no-response', ); } // The response body (the SDK's `text`) is the server's text and can - // echo the request's url, so it's left out; the status stays. - const { text: _body, ...data } = (error.data && typeof error.data === 'object' ? error.data : {}) as Record; - return new McplRequestError(`${prefix} refused ${what}: HTTP ${error.status} (${message})`, 'error-response', undefined, { ...data, httpStatus: error.status }); + // echo the request's url, so it's left out of the data; the status stays. + const { text: _body, ...data } = (error instanceof SdkHttpError && error.data && typeof error.data === 'object' ? error.data : {}) as Record; + return new McplRequestError(`${prefix} refused ${what}: HTTP ${refusedWith} (${message})`, 'error-response', undefined, { ...data, httpStatus: refusedWith }); } if (deadline) { const outcome = record.method === 'tools/list' diff --git a/src/mcpl/protocol-family.ts b/src/mcpl/protocol-family.ts index 52cb72114..3c8bcafa2 100644 --- a/src/mcpl/protocol-family.ts +++ b/src/mcpl/protocol-family.ts @@ -133,23 +133,30 @@ export function redactServerText(text: unknown, config: Pick): (text: unknown) => string { + // Strings replaced wherever they occur, and short name=value pairs, which + // are replaced only where no letter or digit adjoins them, so a pair such as + // v=2 leaves dev=2 alone. const secrets = new Set(); + const shortPairs = new Set(); const standalone = (value: string): void => { if (value.length >= 8) secrets.add(value); }; const decode = (value: string, query = false): string => { try { return decodeURIComponent(query ? value.replace(/\+/g, ' ') : value); } catch { return value; } }; // The token as sent: raw in a header, and percent- or form-encoded where a - // transport puts it in a url. - for (const token of [config.token ?? '']) { - standalone(token); - standalone(encodeURIComponent(token)); - standalone(new URLSearchParams({ t: token }).toString().slice(2)); - } + // transport puts it in a url (a token that can't be encoded, such as one + // with an unpaired surrogate, has no encoded form). + const token = config.token ?? ''; + standalone(token); + try { standalone(encodeURIComponent(token)); } catch { /* no percent-encoded form */ } + try { standalone(new URLSearchParams({ t: token }).toString().slice(2)); } catch { /* no form-encoded form */ } // A stdio server's environment values whose names say they carry a - // credential, by the same rule as the url's parameters. + // credential: a part of the name that ends with a credential word + // (GITHUB_TOKEN, API_KEY, PGPASSWORD, ACCESSTOKEN), and not one where the + // word only begins or sits inside a part (GIT_AUTHOR_NAME, KEYBOARD_LAYOUT, + // TOKENIZERS_PARALLELISM). for (const [name, value] of Object.entries(config.env ?? {})) { - if (CREDENTIAL_PARAM.test(name)) standalone(value); + if (name.split(/[_\-.]/).some((part) => CREDENTIAL_NAME_PART.test(part))) standalone(value); } let url: URL | undefined; try { url = new URL(config.url ?? ''); } catch { /* Still remove the static token. */ } @@ -175,8 +182,9 @@ export function serverTextRedactor(config: Pick= 8 ? secrets : shortPairs).add(pair); + } } } if (url.protocol === 'ws:' || url.protocol === 'wss:') { @@ -186,16 +194,25 @@ export function serverTextRedactor(config: Pick (typeof text === 'string' ? text : String(text)); - if (secrets.size === 0) return asText; - const alternatives = [...secrets].sort((a, b) => b.length - a.length) - .map(value => value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')); - // Only where no letter or digit adjoins it, so a short name=value such as - // v=2 leaves dev=2 alone. - const pattern = new RegExp(`(? value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + const alternatives = [ + ...[...secrets].map(value => ({ value, source: escape(value) })), + ...[...shortPairs].map(value => ({ value, source: `(? b.value.length - a.value.length).map(({ source }) => source); + const pattern = new RegExp(alternatives.join('|'), 'g'); return (text) => asText(text).replace(pattern, 'redacted'); } +/** Any value as text: a value that can't be printed reads as a placeholder. */ +function asText(text: unknown): string { + if (typeof text === 'string') return text; + try { return String(text); } catch { return '[a value that cannot be printed]'; } +} + +/** A part of an environment variable's name that ends with a credential word. */ +const CREDENTIAL_NAME_PART = /(?:token|key|secret|password|passwd|auth|signature|credentials?)$/i; + /** * A credential provider's failure as a log line may show it: the error's * class and its message's length, never the text, which may quote the @@ -353,12 +370,12 @@ function isLoopbackHost(hostname: string): boolean { * `ws://` url whose host isn't loopback crosses the network in cleartext: * `token` or `accessProvider` (in the `Authorization` header, or the * WebSocket `?token=` query), and a secret the url carries itself (a ws:// - * url's userinfo, a query parameter whose name says it's a credential or may - * itself be one, or a path segment that looks like one, as {@link redactUrl} - * judges it). A name that may itself be a credential is counted, never - * quoted. That can be - * deliberate, on a private network or behind a TLS-terminating proxy, so it - * is a warning and never a refusal. + * url's userinfo, a query parameter whose name says it's a credential, or + * whose value is shaped like one under a plain name, a query parameter name + * that may itself be one, or a path segment that looks like one, as + * {@link redactUrl} judges it). A name that may itself be a credential is + * counted, never quoted. That can be deliberate, on a private network or + * behind a TLS-terminating proxy, so it is a warning and never a refusal. */ export function serverConfigWarnings(config: McplServerConfig): string[] { let transport: McpTransportKind; diff --git a/test/mcp-modern-http.test.ts b/test/mcp-modern-http.test.ts index 0c51970f0..d43a70d72 100644 --- a/test/mcp-modern-http.test.ts +++ b/test/mcp-modern-http.test.ts @@ -259,3 +259,26 @@ test("HTTP: a 403 refusal's response body, which can echo the url, stays out of assert.ok(!JSON.stringify(err.data).includes('pathSecret'), JSON.stringify(err.data)); }); +for (const [what, headers] of [ + ['a plain 403', {}], + ['an insufficient_scope 403', { 'www-authenticate': 'Bearer error="insufficient_scope", scope="tools"' }], +] as const) { + test(`HTTP: ${what} on a first-round call is an error-response with the status in data, over the wire`, async () => { + const { url, server } = await startServer(() => true, (message, res) => { + if (message.method !== 'tools/call') return false; + res.writeHead(403, { 'content-type': 'text/plain', ...headers }).end('forbidden here'); + return true; + }); + cleanups.push(() => new Promise((resolve) => { server.closeAllConnections?.(); server.close(() => resolve()); })); + const connection = await ModernMcpConnection.connect({ id: 'remote', url, token: 't' }); + cleanups.push(() => connection.close()); + await assert.rejects(connection.callTool('echo', { text: 'x' }), (err: unknown) => { + assert.ok(err instanceof McplRequestError); + assert.equal(err.outcome, 'error-response', err.message); + assert.equal((err.data as { httpStatus?: number }).httpStatus, 403); + assert.equal((err.data as { text?: unknown }).text, undefined); + return true; + }); + }); +} + diff --git a/test/mcp-protocol-family.test.ts b/test/mcp-protocol-family.test.ts index 787182c16..170f9fc6c 100644 --- a/test/mcp-protocol-family.test.ts +++ b/test/mcp-protocol-family.test.ts @@ -322,26 +322,41 @@ test('recorded server text keeps short standalone values and unrelated words, bu test('recorded server text: the token as a url carries it, credential-named env values, only where nothing adjoins, and any value', () => { // The static token, raw and as a url carries it, percent- or form-encoded. - const token = 'tok+en/123='; + const token = 'tok en+x/123='; const tokenConfig = cfg({ token }); - for (const form of [token, encodeURIComponent(token), 'tok%2Ben%2F123%3D']) { + // Percent-encoded (%20 for the space) and form-encoded (+ for it) differ here. + for (const form of [token, 'tok%20en%2Bx%2F123%3D', 'tok+en%2Bx%2F123%3D']) { assert.equal(redactServerText(`refused: ${form}`, tokenConfig), 'refused: redacted', form); } - // A stdio server's env: a value whose name says it's a credential, and no other. - const env = cfg({ command: 'srv', env: { GITHUB_TOKEN: 'ghp_abcdef123456', API_KEY: 'k3y-abcdefgh', PATH: '/usr/local/bin' } }); - assert.equal(redactServerText('auth ghp_abcdef123456 k3y-abcdefgh /usr/local/bin', env), 'auth redacted redacted /usr/local/bin'); - // Only where no letter or digit adjoins a string: a short pair stays whole elsewhere. - const pair = cfg({ url: 'https://h/mcp?v=2' }); + // A stdio server's env: a value whose name has a credential word as a whole + // part of it, and no other (GIT_AUTHOR_NAME holds 'auth' inside a word). + const env = cfg({ command: 'srv', env: { + GITHUB_TOKEN: 'ghp_abcdef123456', API_KEY: 'k3y-abcdefgh', PGPASSWORD: 'pgpass-abcdefgh', ACCESSTOKEN: 'acc-abcdefgh12', + PATH: '/usr/local/bin', GIT_AUTHOR_NAME: 'Somebody Longname', KEYBOARD_LAYOUT: 'dvorak-programmer', TOKENIZERS_PARALLELISM: 'parallel-false', + } }); + assert.equal( + redactServerText('auth ghp_abcdef123456 k3y-abcdefgh pgpass-abcdefgh acc-abcdefgh12 /usr/local/bin Somebody Longname dvorak-programmer parallel-false', env), + 'auth redacted redacted redacted redacted /usr/local/bin Somebody Longname dvorak-programmer parallel-false'); + // A short pair only where no letter or digit adjoins it; any other string + // wherever it occurs, after a percent escape, an ANSI code or a JSON escape too. + const pair = cfg({ url: 'https://h/api/s/pathSecret0123456789/mcp?v=2' }); assert.equal(redactServerText('dev=2 ready; v=2 sent', pair), 'dev=2 ready; redacted sent'); + assert.equal(redactServerText('a%2FpathSecret0123456789 \u001b[31mpathSecret0123456789 "\\npathSecret0123456789"', pair), + 'a%2Fredacted \u001b[31mredacted "\\nredacted"'); // A ws:// url's userinfo, as for wss://. const ws = cfg({ url: 'ws://wsuser1234:wspass5678@h/mcp' }); assert.equal(redactServerText('connect wsuser1234 wspass5678', ws), 'connect redacted redacted'); // A url with a stray percent sign: its value is kept as written. const stray = cfg({ url: 'https://h/mcp?note=100%&key=abcdefgh1234' }); assert.equal(redactServerText('note=100% key=abcdefgh1234', stray), 'redacted redacted'); - // Any value reads as text: a rejection that isn't an Error doesn't throw. + // Any value reads as text: a rejection that isn't an Error doesn't throw, + // and neither does one that can't be printed. assert.equal(redactServerText(undefined, tokenConfig), 'undefined'); assert.equal(redactServerText({ toString: () => `obj ${token}` }, tokenConfig), 'obj redacted'); + assert.equal(redactServerText(Object.create(null), tokenConfig), '[a value that cannot be printed]'); + assert.equal(redactServerText({ toString() { throw new Error('no'); } }, tokenConfig), '[a value that cannot be printed]'); + // A token that can't be url-encoded (an unpaired surrogate) is still replaced as written. + assert.equal(redactServerText('x abcdefgh\ud800ij', cfg({ token: 'abcdefgh\ud800ij' })), 'x redacted'); }); test('the cleartext warning counts a credential-shaped value under a plain name, as redaction does', () => { diff --git a/test/mcp-recorded-text.test.ts b/test/mcp-recorded-text.test.ts index 8f84d3166..76d160244 100644 --- a/test/mcp-recorded-text.test.ts +++ b/test/mcp-recorded-text.test.ts @@ -65,6 +65,22 @@ test('both framework engines scrub failure, error and stderr records before trac } }); +test("both engines' wiring redacts a stdio server's credential env value in its stderr records", () => { + const stdio = { id: 'local', command: 'srv', env: { API_TOKEN: 'envToken0123456789', PATH: '/usr/local/bin' } }; + for (const method of ['wireMcplEvents', 'wireModernEvents']) { + const traces: unknown[] = []; + const framework = Object.create(AgentFramework.prototype) as any; + framework.mcplServerConfigs = new Map([[stdio.id, stdio]]); + framework.emitTrace = (event: unknown) => traces.push(event); + framework.opsAlert = () => {}; + const connection = Object.assign(new EventEmitter(), { id: stdio.id, willReconnect: false }); + framework[method](connection, stdio); + connection.emit('stderr', { line: 'auth with envToken0123456789 from /usr/local/bin' }); + assert.equal(traces.length, 1, method); + assert.match(JSON.stringify(traces[0]), /auth with redacted from \/usr\/local\/bin/, method); + } +}); + test('an SDK HTTP 404 error keeps its path but hides the secret in connection console, traces and ops alerts', async (t) => { failConnectWithHttpBody(t); const dir = mkdtempSync(join(tmpdir(), 'mcp-recorded-text-')); @@ -199,9 +215,11 @@ test("a tool call's failure, from either engine, reaches its caller and its trac assert.match(String(result.error), /socket closed: 1008 \/api\/mcp\/s\/redacted\/mcp/); // A redactor for any server's text is made before an operation waits, so // a server removed meanwhile still has its credentials replaced. + const other = { id: 'other', url: 'https://g/api/s/otherSecret0123456789/mcp' }; + framework.mcplServerConfigs = new Map([[config.id, config], [other.id, other]]); const everyServer = framework.everyServerRedactor(); framework.mcplServerConfigs = new Map(); - assert.equal(everyServer(`closed ${secret}`), 'closed redacted'); + assert.equal(everyServer(`closed ${secret} and otherSecret0123456789`), 'closed redacted and redacted'); framework.mcplServerConfigs = new Map([[config.id, config]]); // The model's dispatch: the tool:failed trace (message and stack) and the @@ -215,7 +233,10 @@ test("a tool call's failure, from either engine, reaches its caller and its trac // The server is disconnected while the call is in flight: its failure is // still recorded with the configuration it was dispatched under. framework.mcplServerConfigs.delete(config.id); - await new Promise((resolve) => setTimeout(resolve, 20)); + const until = Date.now() + 30_000; + while (!(traces.some((event) => event.type === 'tool:failed') && events.length > 0) && Date.now() < until) { + await new Promise((resolve) => setTimeout(resolve, 5)); + } const failed = traces.find((event) => event.type === 'tool:failed'); assert.ok(failed, 'the failure is traced'); for (const text of [failed!.error, failed!.stack, events[0]?.result?.error]) { @@ -227,9 +248,17 @@ test("a tool call's failure, from either engine, reaches its caller and its trac test("a malformed line's diagnostic is redacted before it is cut, so no part of a credential across the cut is kept", () => { const envSecret = 'envSecretValue0123456789'; const line = JSON.stringify({ note: `${'p'.repeat(185)} ${envSecret}` }); + // The credential starts this far into the line; a cut at 200 would keep + // the first (200 - start) of its characters. + const start = line.indexOf(envSecret); + const kept = envSecret.slice(0, 200 - start); + assert.ok(kept.length > 0 && kept.length < envSecret.length, `the cut falls inside the credential (${kept.length})`); const diagnostic = malformedLineDiagnostic({ id: 'stdio-env', command: 'srv', env: { API_TOKEN: envSecret } } as never, line, new Error('not a message')); assert.match(diagnostic, /sent a malformed JSON-RPC message/); - assert.ok(!diagnostic.includes(envSecret.slice(0, 6)), diagnostic); + assert.ok(!diagnostic.includes(kept), diagnostic); + // The parser's own message can quote the line; it's redacted too. + const quoting = malformedLineDiagnostic({ id: 'stdio-env', command: 'srv', env: { API_TOKEN: envSecret } } as never, '{}', new Error(`unexpected ${envSecret}`)); + assert.ok(!quoting.includes(envSecret), quoting); assert.match(diagnostic, / redac/, 'the replacement, cut where the line is cut'); }); diff --git a/test/mcpl-config-diagnostics.test.ts b/test/mcpl-config-diagnostics.test.ts index 70188a519..9e846eaaa 100644 --- a/test/mcpl-config-diagnostics.test.ts +++ b/test/mcpl-config-diagnostics.test.ts @@ -332,6 +332,35 @@ test('FeatureSetManager: each §6.4 disablement is traced beside its console lin assert.equal(quiet.lines.length, 3, 'the console line stays'); }); +test("a refused initial policy's reason is recorded without the server's configured credentials", async () => { + const fw = Object.create(AgentFramework.prototype) as any; + fw.traceListeners = []; + fw.featureSetManager = new FeatureSetManager(); + fw.checkpointManager = null; + const traces: Traced[] = []; + fw.onTrace((e: Traced) => traces.push(e)); + const secret = 'srvSecret0123456789'; + const connection = { + id: 'srv', + capabilities: { version: '0.5', pushEvents: true } as unknown as McplCapabilities, + mcpToolsAdvertised: false, + establishGrant: () => {}, + sendFeatureSetsUpdateRequest: async () => ({ accepted: false, fallback: 'mcp-only', reason: `policy for ${secret} too narrow` }), + close: async () => {}, + }; + const capture = captureConsoleError(); + try { + await fw.registerMcplServerFeatures({ id: 'srv', url: `wss://h/mcp/${secret}` } as McplServerConfig, connection); + } finally { + capture.stop(); + } + const refused = traces.filter((t) => t.type === 'mcpl:policy-refused'); + assert.equal(refused.length, 1); + assert.equal((refused[0] as { reason?: string }).reason, 'policy for redacted too narrow'); + assert.ok(capture.lines.some((l) => l.includes('srv refused initial policy (policy for redacted too narrow)')), capture.lines.join('\n')); + assert.ok(!capture.lines.join('\n').includes(secret)); +}); + test('registerMcplServerFeatures: a refused initial policy is traced, and fallback close closes', async () => { for (const fallback of ['close', 'mcp-only'] as const) { const fw = Object.create(AgentFramework.prototype) as any; diff --git a/test/mcpl-runtime-lifecycle.test.ts b/test/mcpl-runtime-lifecycle.test.ts index dfbd89a01..8bb01254f 100644 --- a/test/mcpl-runtime-lifecycle.test.ts +++ b/test/mcpl-runtime-lifecycle.test.ts @@ -196,13 +196,32 @@ test('restartMcplServer of an id whose disconnect is in flight is refused before test("a disconnect removes its server's prefix claim, and leaves one another server holds", async () => { await withFramework(true, async (framework) => { const internals = framework as unknown as { mcplPrefixMap: Map }; + assert.equal(internals.mcplPrefixMap.get('mcpl--first'), 'first'); + await framework.disconnectMcplServer('first'); + assert.equal(internals.mcplPrefixMap.has('mcpl--first'), false, 'its own claim is released'); // As after a restart under a new id: `second` holds the prefix `first` had. + await framework.connectMcplServer(serverConfig('first') as never); internals.mcplPrefixMap.set('mcpl--first', 'second'); await framework.disconnectMcplServer('first'); assert.equal(internals.mcplPrefixMap.get('mcpl--first'), 'second', "the other server's claim stands"); }); }); +test('restartMcplServer of an id whose connect is in flight is refused before it tears anything down', async (t) => { + // MCPL is up, so the connect holds its id before its first await. + await withFramework(true, async (framework) => { + const connecting = framework.connectMcplServer(serverConfig('later') as never); + const disconnect = t.mock.method(framework, 'disconnectMcplServer'); + await assert.rejects( + framework.restartMcplServer('later', serverConfig('renamed') as never), + /MCP server "later" was not restarted: it is being connected; restart it once that has finished/, + ); + assert.equal(disconnect.mock.callCount(), 0, 'the restart started no disconnect of its own'); + disconnect.mock.restore(); + await connecting; + }); +}); + test('connectMcplServer rejects a duplicate id and a colliding prefix', async () => { await withFramework(true, async (framework) => { await assert.rejects( From 2e316bd10278b5ffcefca41964f72d5898d8c921 Mon Sep 17 00:00:00 2001 From: Petra-1012 Date: Tue, 13 Oct 2026 03:06:11 -0400 Subject: [PATCH 0157/1112] test(drafts): the held-draft and bounce notices quote none of the words they name The catch-up notice's test already asserted that no draft words appear. The held-draft notice (a draft held beside a send) and the bounce notice (an unprefixed explicit envelope) now assert it too, so adding a quoted preview back to either fails a test. --- test/explicit-prose-routing.test.ts | 3 +++ test/prose-drafts.test.ts | 4 +++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/test/explicit-prose-routing.test.ts b/test/explicit-prose-routing.test.ts index fd253ec31..9389594c7 100644 --- a/test/explicit-prose-routing.test.ts +++ b/test/explicit-prose-routing.test.ts @@ -192,6 +192,9 @@ describe('explicit prose routing', () => { assert.equal(routed.length, 1, 'only the prefixed resend delivered'); assert.equal(routed[0]!.locus, 'discord:dm:99', 'delivered to the DM the model named'); assert.equal(routed[0]!.text, original, '{{unsent}} substitution is verbatim'); + const notices = context(framework).texts.filter((t) => t.startsWith('[prose-routing]')); + assert.ok(notices.length > 0, 'the bounce notice is in the context'); + assert.ok(notices.every((t) => !t.includes('Two words through the keyhole')), `the bounce notice quotes none of the words: ${notices.join(' | ')}`); await framework.stop(); }); diff --git a/test/prose-drafts.test.ts b/test/prose-drafts.test.ts index b06364b4c..dc3f21948 100644 --- a/test/prose-drafts.test.ts +++ b/test/prose-drafts.test.ts @@ -512,7 +512,9 @@ describe('held prose drafts, end to end', () => { assert.equal(h.publishes().filter((p) => p.text === 'scout words').length, 0); // Residents of one framework share a message slot (#197), so the // notice is visible there too; it names whose drafts it means. - assert.ok(h.texts('scout').some((t) => t.startsWith(`[drafts] scout: not sent — a plain-speech segment held as draft ${mine.id}`))); + const held = h.texts('scout').find((t) => t.startsWith(`[drafts] scout: not sent — a plain-speech segment held as draft ${mine.id}`)); + assert.ok(held); + assert.ok(!held!.includes('scout words'), `the held-draft notice quotes none of the draft's words: ${held}`); } finally { await h.close(); } From 77cd066399af0cce80e60a840baeaba8a8338255 Mon Sep 17 00:00:00 2001 From: Petra-1012 Date: Tue, 13 Oct 2026 03:09:17 -0400 Subject: [PATCH 0158/1112] fix(drafts): a failed {{unsent}} send re-bounces the draft it substituted, not the bounce that's latest when it fails From a reviewer's reproduction. When a delivery that substituted {{unsent}} definitely fails, or throws, its words are held again as a bounce draft. The re-bounce re-expanded {{unsent}} from the latest bounce at the time of the failure. An explicit turn that a tool ends doesn't hold the next turn for its deliveries, so a later turn could bounce new words while the send waited, and the failure then held the newer bounce's words with the envelope's, words that were never attempted, and lost the source draft and risk of the ones that were. bounceProse and unsentWords now take the draft that was substituted. unsentWords reads it again from the store, so its risk is its state after the attempt (a thrown send's attempt has no outcome, so the copy stays unconfirmed). The explicit and hybrid failure paths, failed and thrown, pass it. Test (explicit-prose-routing): turn one's '>>#alpha {{unsent}} plus a line' is held in delivery after skip_reply ends the turn; turn two bounces 'NEW WORDS'; turn one's send then fails, and its draft holds 'ORIGINAL WORDS plus a line', not 'NEW WORDS plus a line'. Adapted from the reviewer's reproducer. --- src/framework.ts | 26 +++++++++---- test/explicit-prose-routing.test.ts | 60 +++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+), 7 deletions(-) diff --git a/src/framework.ts b/src/framework.ts index 601f00b35..3effe0dac 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -10202,13 +10202,13 @@ export class AgentFramework { // not bounced, so its words are not re-held as a fresh draft that // invites a duplicate. if (outcome.status === 'failed') { - this.bounceProse(agent, parsed.body, `delivery to ${resolved.channelId} failed: ${outcome.reason ?? 'refused'}`, undefined, '>>>', hold); + this.bounceProse(agent, parsed.body, `delivery to ${resolved.channelId} failed: ${outcome.reason ?? 'refused'}`, undefined, '>>>', hold, unsent.draft); } } } catch (err) { this.proseTargetPins.delete(agent.name); this.proseHybridSuppressed.set(agent.name, 'failed'); - this.bounceProse(agent, parsed.body, `delivery to ${resolved.channelId} failed: ${err instanceof Error ? err.message : String(err)}`, undefined, '>>>', hold); + this.bounceProse(agent, parsed.body, `delivery to ${resolved.channelId} failed: ${err instanceof Error ? err.message : String(err)}`, undefined, '>>>', hold, unsent.draft); } continue; } @@ -10354,10 +10354,10 @@ export class AgentFramework { } else if (outcome.status === 'failed') { // Unknown deliveries are marker-only: offering a fresh draft would // invite a duplicate. Keep the authored body for the re-bounce rule. - this.bounceProse(agent, envelopeBody, `delivery to ${targetChannel} failed: ${outcome.reason ?? 'refused'}`, undefined, '>>', hold); + this.bounceProse(agent, envelopeBody, `delivery to ${targetChannel} failed: ${outcome.reason ?? 'refused'}`, undefined, '>>', hold, unsent.draft); } } catch (err) { - this.bounceProse(agent, envelopeBody, `delivery to ${targetChannel} failed: ${err instanceof Error ? err.message : String(err)}`, undefined, '>>', hold); + this.bounceProse(agent, envelopeBody, `delivery to ${targetChannel} failed: ${err instanceof Error ? err.message : String(err)}`, undefined, '>>', hold, unsent.draft); } } @@ -10377,10 +10377,20 @@ export class AgentFramework { * the risk that draft itself inherited, else (truly) that it was in flight * when copied. */ - private unsentWords(agent: Agent, text: string): { draft: Draft } | { text: string; inheritedRisk?: InheritedRisk } { + private unsentWords( + agent: Agent, + text: string, + /** The draft `{{unsent}}` was substituted with, when the words are held + * after that send: the latest bounce may have changed while it waited. */ + substituted?: Draft, + ): { draft: Draft } | { text: string; inheritedRisk?: InheritedRisk } { if (!text.includes('{{unsent}}')) return { text }; const name = agent.name; - const previous = this.proseDrafts.latestBounce(name); + // A substituted draft is read again here, so its risk is its state after + // the attempt, not before it. + const previous = substituted + ? this.proseDrafts.get(name, substituted.id) ?? substituted + : this.proseDrafts.latestBounce(name); if (previous && text.trim() === '{{unsent}}') return { draft: previous }; const expanded = text.replaceAll('{{unsent}}', previous?.text ?? ''); if (!previous || !(draftState(previous) === 'unconfirmed' || this.draftsInFlight.has(`${name}\u0000${previous.id}`))) { @@ -10425,9 +10435,11 @@ export class AgentFramework { candidates?: string[], prefix: '>>' | '>>>' = '>>', hold: { round?: number; notice: 'now' | 'later' } = { notice: 'later' }, + /** The draft a failed send substituted for `{{unsent}}` (unsentWords). */ + substituted?: Draft, ): void { const name = agent.name; - const words = this.unsentWords(agent, text); + const words = this.unsentWords(agent, text, substituted); let draft: Draft | undefined; let inherited: InheritedRisk | undefined; if ('draft' in words) { diff --git a/test/explicit-prose-routing.test.ts b/test/explicit-prose-routing.test.ts index 9389594c7..601f55618 100644 --- a/test/explicit-prose-routing.test.ts +++ b/test/explicit-prose-routing.test.ts @@ -378,6 +378,66 @@ describe('explicit prose routing', () => { } }); + it('a failed {{unsent}} send that a later turn overtook holds the words it attempted, not the newer bounce', async () => { + // Turn one queues '>>#alpha {{unsent}} plus a line' and skip_reply ends + // it while that publish is held. Turn two bounces new words. When turn + // one's send then fails, its draft must hold what it attempted. + membrane.pushResponse(createMockResponse([ + { type: 'text', text: '>>#alpha {{unsent}} plus a line' }, + { type: 'tool_use', id: 'delayed-1', name: 'robot--move', input: { dir: 'north' } }, + ] as ContentBlock[], 'tool_use')); + membrane.pushResponse(createMockResponse([ + { type: 'tool_use', id: 'delayed-2', name: 'skip_reply', input: {} }, + ] as ContentBlock[], 'tool_use')); + const framework = await createFramework(); + const routed = stubRegistry(framework, ['false']); + answerSkipReply(framework); + const store = draftsOf(framework); + const original = store.hold('assistant', [{ text: 'ORIGINAL WORDS', source: { branch: 'main', turn: 0, round: 0, segment: 0 } }], 'bounced').held[0]!; + const internals = framework as unknown as { + channelRegistry: { deliverSpeech: (...args: unknown[]) => Promise }; + proseSpeechChains: Map>; + }; + const deliver = internals.channelRegistry.deliverSpeech; + let release!: () => void; + const held = new Promise((resolve) => { release = resolve; }); + let entered!: () => void; + const reached = new Promise((resolve) => { entered = resolve; }); + internals.channelRegistry.deliverSpeech = async (...args: unknown[]) => { + const outcome = await deliver(...args); + entered(); + await held; + return outcome; + }; + let starts = 0; + framework.onTrace((e) => { + if ((e as { type: string }).type !== 'inference:started') return; + starts++; + if (starts === 2) membrane.pushResponse(createMockResponse([{ type: 'text', text: '>>#nowhere NEW WORDS' }])); + if (starts >= 3) membrane.pushResponse(createMockResponse([{ type: 'text', text: '>>skip_reply' }])); + }); + try { + trigger(framework); + await framework.runUntilIdle(); + await reached; + const firstChain = internals.proseSpeechChains.get('assistant')!; + assert.deepEqual(routed.map((r) => r.text), ['ORIGINAL WORDS plus a line']); + trigger(framework); + await framework.runUntilIdle(); + assert.equal(store.latestBounce('assistant')?.text, 'NEW WORDS', 'the next turn bounced before the earlier send answered'); + release(); + await firstChain; + const heldTexts = store.open('assistant').map((d) => d.text); + assert.ok(heldTexts.includes('ORIGINAL WORDS plus a line'), `the failed send holds the words it attempted: ${JSON.stringify(heldTexts)}`); + assert.ok(!heldTexts.includes('NEW WORDS plus a line'), 'not the newer bounce'); + const copy = store.open('assistant').find((d) => d.text === 'ORIGINAL WORDS plus a line')!; + assert.notEqual(copy.id, original.id); + } finally { + release(); + await framework.stop(); + } + }); + it('a thrown explicit unsent delivery re-bounces the authored body with its unresolved attempt risk', async () => { const framework = await createFramework(); try { From 5dc695f5ae17ac9b3dab95278f8861e43535243b Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Tue, 13 Oct 2026 03:13:39 -0400 Subject: [PATCH 0159/1112] fix(mcp): the policy request's records use one redactor taken before it waits, and log the fallback the host takes Initial and post-manifest policy requests recorded server text through sibling paths the earlier rounds left: an unanswered request's error, a refusal's fallback string as the server sent it, a degraded acceptance's unavailable-feature effect, and (for the reason) a redactor built after the await from a configuration the host could change meanwhile. registerMcplServerFeatures now takes one redactor at entry and uses it for every line and trace it makes of what the server said. A refusal logs and traces the fallback the host actually takes (close or mcp-only), not the server's string. The post-manifest refusal decides and logs by the same normalized fallback. redactServerText's doc says which strings carry the adjacency rule (a short name=value pair alone) and that it's a host's own connect whose error passes as raised. Test: an unanswered request quoting the url, a refusal with a fallback the host doesn't know, a degraded acceptance quoting it, and a url replaced while the request waits: no record carries the credential, and the odd fallback is logged as mcp-only. Removing each piece fails it (four removals). --- src/framework.ts | 24 ++++++++++++------- src/mcpl/protocol-family.ts | 14 ++++++----- test/mcpl-config-diagnostics.test.ts | 36 ++++++++++++++++++++++++++++ 3 files changed, 60 insertions(+), 14 deletions(-) diff --git a/src/framework.ts b/src/framework.ts index dfcd7cd23..486d0edeb 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -14263,13 +14263,14 @@ export class AgentFramework { try { const receipt = await connection.sendFeatureSetsUpdateRequest(updateParams); if (receipt && receipt.accepted === false) { - const fallback = receipt.fallback ?? 'mcp-only'; + const refusal = AgentFramework.policyRefusalFields(receipt, recorded); + const fallback = refusal.fallback; console.error(`[mcpl] ${connection.id} refused post-manifest policy — fallback: ${fallback}`); this.emitTrace({ type: 'mcpl:policy-refused', serverId: connection.id, phase: 'manifest-change', - ...AgentFramework.policyRefusalFields(receipt, recorded), + ...refusal, }); if (fallback === 'close') { await connection.close(); @@ -14349,6 +14350,10 @@ export class AgentFramework { connection: McplServerConnection, ): Promise { if (!connection.capabilities) return; + // The server's configuration as it is now, before any await: every + // record of what the server says below keeps it, whatever the host + // changes in the configuration meanwhile. + const recorded = serverTextRedactor(config); // §5.4: compute the effective grant from the (config-masked) // advertisement. Not yet active — expansion activates only after the @@ -14379,15 +14384,18 @@ export class AgentFramework { try { const receipt = await connection.sendFeatureSetsUpdateRequest(updateParams); if (receipt && receipt.accepted === false) { - const fallback = receipt.fallback ?? 'mcp-only'; + // The fallback the host takes, which is the one logged; the server's + // reason is its own text. + const refusal = AgentFramework.policyRefusalFields(receipt, recorded); + const fallback = refusal.fallback; console.error( - `[mcpl] ${config.id} refused initial policy (${redactServerText(receipt.reason ?? 'no reason', config)}) — fallback: ${fallback}`, + `[mcpl] ${config.id} refused initial policy (${refusal.reason ?? 'no reason'}) — fallback: ${fallback}`, ); this.emitTrace({ type: 'mcpl:policy-refused', serverId: config.id, phase: 'initial', - ...AgentFramework.policyRefusalFields(receipt, (text) => redactServerText(text, config)), + ...refusal, }); if (fallback === 'close') { await connection.close(); @@ -14402,9 +14410,9 @@ export class AgentFramework { connection.establishGrant(grant); if (receipt?.accepted === true && receipt.mode === 'degraded') { for (const uf of receipt.unavailableFeatures ?? []) { - console.error( + console.error(recorded( `[mcpl] ${config.id} degraded: ${uf.featureSet} (${uf.effect}; missing ${uf.missingCapabilities.join(', ')})`, - ); + )); } } } catch (err) { @@ -14416,7 +14424,7 @@ export class AgentFramework { console.error( `[mcpl] ${config.id} did not answer the initial policy Request — ` + `MCPL privileged surface stays disabled (§6.7 unanswered expansion): ` + - (err instanceof Error ? err.message : String(err)), + recorded(err instanceof Error ? err.message : err), ); return; } diff --git a/src/mcpl/protocol-family.ts b/src/mcpl/protocol-family.ts index 3c8bcafa2..f408a5d55 100644 --- a/src/mcpl/protocol-family.ts +++ b/src/mcpl/protocol-family.ts @@ -117,12 +117,14 @@ export function redactUrl(url: string | undefined): string { * - the static token, raw and percent- or form-encoded; * - a stdio server's environment values whose names say they're credentials. * A string replaced on its own has at least 8 characters, since a shorter - * one can't be told from ordinary text; a short query value is replaced - * with its name, as one name=value. A string is replaced only where no letter - * or digit adjoins it, and one longest-first pass never scans its own - * replacement. A credential the configuration doesn't hold, such as an - * access provider's cached bearer token or another origin's, is outside it. - * JSON-RPC answers and errors thrown to a host's own call pass as they are. + * one can't be told from ordinary text, and it is replaced wherever it + * occurs, after an escape or inside a longer token too. A short query value + * is replaced only with its name, as one name=value, and only where no + * letter or digit adjoins the pair, so v=2 leaves dev=2 alone. One + * longest-first pass never scans its own replacement. A credential the + * configuration doesn't hold, such as the token an access provider fetches + * or another origin's, is outside it. JSON-RPC answers, and the error a + * host's own connect raises, pass as they are. */ export function redactServerText(text: unknown, config: Pick): string { return serverTextRedactor(config)(text); diff --git a/test/mcpl-config-diagnostics.test.ts b/test/mcpl-config-diagnostics.test.ts index 9e846eaaa..bf8a8f1d8 100644 --- a/test/mcpl-config-diagnostics.test.ts +++ b/test/mcpl-config-diagnostics.test.ts @@ -361,6 +361,42 @@ test("a refused initial policy's reason is recorded without the server's configu assert.ok(!capture.lines.join('\n').includes(secret)); }); +test("the initial policy's other records keep the credentials out too, with the configuration as it was when the request went", async () => { + const secret = 'srvSecret0123456789'; + const run = async (answer: () => Promise, mutate?: (config: McplServerConfig) => void) => { + const fw = Object.create(AgentFramework.prototype) as any; + fw.traceListeners = []; + fw.featureSetManager = new FeatureSetManager(); + fw.checkpointManager = null; + const traces: Traced[] = []; + fw.onTrace((e: Traced) => traces.push(e)); + const config = { id: 'srv', url: `wss://h/mcp/${secret}` } as McplServerConfig; + const connection = { + id: 'srv', + capabilities: { version: '0.5', pushEvents: true } as unknown as McplCapabilities, + mcpToolsAdvertised: false, + establishGrant: () => {}, + sendFeatureSetsUpdateRequest: async () => { mutate?.(config); return answer(); }, + close: async () => {}, + }; + const capture = captureConsoleError(); + try { await fw.registerMcplServerFeatures(config, connection); } finally { capture.stop(); } + const text = capture.lines.join('\n') + JSON.stringify(traces); + assert.ok(!text.includes(secret), text); + return { lines: capture.lines, traces }; + }; + // An unanswered request whose error quotes the url. + await run(async () => { throw new Error(`timed out talking to wss://h/mcp/${secret}`); }); + // A refusal whose fallback isn't one the host knows: it logs what it does. + const odd = await run(async () => ({ accepted: false, fallback: `close-${secret}`, reason: 'no' })); + assert.ok(odd.lines.some((l) => l.includes('refused initial policy (no) — fallback: mcp-only')), odd.lines.join('\n')); + // A degraded acceptance whose effect quotes the url. + await run(async () => ({ accepted: true, mode: 'degraded', unavailableFeatures: [{ featureSet: 'fs', effect: `see ${secret}`, missingCapabilities: ['x'] }] })); + // The host replaces the url while the request waits: the record still + // knows the credential the request went with. + await run(async () => ({ accepted: false, fallback: 'mcp-only', reason: `bad ${secret}` }), (config) => { (config as { url?: string }).url = 'wss://h/mcp/rotatedSecret0123456789'; }); +}); + test('registerMcplServerFeatures: a refused initial policy is traced, and fallback close closes', async () => { for (const fallback of ['close', 'mcp-only'] as const) { const fw = Object.create(AgentFramework.prototype) as any; From 9169bc99eea70ba220f08added6c9662361cbdee Mon Sep 17 00:00:00 2001 From: Petra-1012 Date: Tue, 13 Oct 2026 03:21:50 -0400 Subject: [PATCH 0160/1112] fix(drafts): a bounce notice offers {{unsent}} only when {{unsent}} would send that draft; draft words go in exactly From the review of the late-bounce fix (77cd0663): - A bare {{unsent}} whose send failed re-bounces the draft it sent. When a later turn bounced new words meanwhile, that newer draft is the latest bounce, so the notice's advice to reply with {{unsent}} would send the newer words. And {{unsent}} refuses a draft that may already have been posted, though the notice offered it beside the risk warning. The notice now offers {{unsent}} (and '>>skip_reply {{unsent}}') only when the draft is the latest bounce and carries no risk; otherwise it names the drafts tool's resend (with confirmDuplicate: true when at risk) and dismiss for that draft. - A bare {{unsent}} whose send threw leaves its draft unconfirmed, and the re-bounce notice said nothing of it: it now says the draft may already have been posted and asks for confirmDuplicate. - {{unsent}} expansion used a string replacement, so `$&`, `$$` and `$1` in a draft's words were rewritten on send and in a re-held copy. All three expansions now use a function replacement. Tests (explicit-prose-routing): - the overtaken-send test runs for '{{unsent}} plus a line' and a bare '{{unsent}}': the re-held words are what was attempted, carry no risk after a definite failure, and the bare case's notice offers only the drafts tool while the newer bounce stays the latest; - a thrown bare {{unsent}} says its draft may already have been posted, with confirmDuplicate, and offers no {{unsent}}; the thrown test with other words asserts the same of its copy's notice; - hybrid: a >>> envelope whose {{unsent}} names nothing sends nothing and holds its other words; - {{unsent}} keeps `$$`, `$&` and `$1` in a draft's words. --- src/framework.ts | 36 ++++++++---- test/explicit-prose-routing.test.ts | 87 +++++++++++++++++++++++++++-- 2 files changed, 107 insertions(+), 16 deletions(-) diff --git a/src/framework.ts b/src/framework.ts index 3effe0dac..01c51ce98 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -10183,7 +10183,8 @@ export class AgentFramework { this.proseHybridSuppressed.set(agent.name, 'failed'); continue; } - if (usedClipboard) body = body.replaceAll('{{unsent}}', unsent.draft?.text ?? ''); + // A function replacement: a draft's words go in exactly, `$&` and `$$` included. + if (usedClipboard) body = body.replaceAll('{{unsent}}', () => unsent.draft?.text ?? ''); this.proseTargetPins.set(agent.name, resolved.channelId); this.proseHybridSuppressed.delete(agent.name); if (!body.trim()) { @@ -10340,7 +10341,8 @@ export class AgentFramework { const unsent = usedClipboard ? this.takeUnsent(agent, body, hold, '>>') : {}; if (unsent.refused) return; if (usedClipboard) { - body = body.replaceAll('{{unsent}}', unsent.draft?.text ?? ''); + // A function replacement: a draft's words go in exactly, `$&` and `$$` included. + body = body.replaceAll('{{unsent}}', () => unsent.draft?.text ?? ''); } if (!body.trim()) { console.error(`[prose] ${name}: empty body after prefix/substitution — nothing to send`); @@ -10392,7 +10394,7 @@ export class AgentFramework { ? this.proseDrafts.get(name, substituted.id) ?? substituted : this.proseDrafts.latestBounce(name); if (previous && text.trim() === '{{unsent}}') return { draft: previous }; - const expanded = text.replaceAll('{{unsent}}', previous?.text ?? ''); + const expanded = text.replaceAll('{{unsent}}', () => previous?.text ?? ''); if (!previous || !(draftState(previous) === 'unconfirmed' || this.draftsInFlight.has(`${name}\u0000${previous.id}`))) { return { text: expanded }; } @@ -10457,16 +10459,30 @@ export class AgentFramework { const streak = (this.proseBounceStreaks.get(name) ?? 0) + 1; this.proseBounceStreaks.set(name, streak); const cand = candidates?.length ? ` Known channels it could mean: ${candidates.join(', ')}.` : ''; - const kept = draft - ? `It is held as draft ${draft.id}; nothing is lost. To deliver it unchanged, reply with a destination plus the ` + - `token {{unsent}}, e.g. "${prefix}#channel {{unsent}}" or "${prefix}@person {{unsent}}", or use ` + - `drafts(action: "resend", draftIds: ["${draft.id}"], destination: "#channel"). ` + - `"${prefix}skip_reply {{unsent}}" sets it aside.` - : 'It could not be held as a draft; the words remain in your history.'; + // A bare {{unsent}} whose send failed re-bounces the draft it sent, which + // a later bounce may have displaced as the latest; and {{unsent}} refuses + // a draft that may already have been posted. Then only the drafts tool + // reaches this one. + const ownRisk = !!draft && !inherited && draftState(draft) === 'unconfirmed'; + const byToken = !!draft && !inherited && !ownRisk && this.proseDrafts.latestBounce(name)?.id === draft.id; + const kept = !draft + ? 'It could not be held as a draft; the words remain in your history.' + : byToken + ? `It is held as draft ${draft.id}; nothing is lost. To deliver it unchanged, reply with a destination plus the ` + + `token {{unsent}}, e.g. "${prefix}#channel {{unsent}}" or "${prefix}@person {{unsent}}", or use ` + + `drafts(action: "resend", draftIds: ["${draft.id}"], destination: "#channel"). ` + + `"${prefix}skip_reply {{unsent}}" sets it aside.` + : `It is held as draft ${draft.id}; nothing is lost. To deliver it unchanged, use ` + + `drafts(action: "resend", draftIds: ["${draft.id}"], destination: "#channel"` + + `${inherited || ownRisk ? ', confirmDuplicate: true' : ''}); ` + + `drafts(action: "dismiss", draftIds: ["${draft.id}"]) sets it aside.`; const risky = inherited && draft ? ` It includes the words of draft ${inherited.draftId}, which may already have been posted, so ${draft.id} ` + 'can be resent only with the drafts tool and confirmDuplicate: true, after checking that channel.' - : ''; + : ownRisk + ? ` Its last send's outcome isn't known, so it may already have been posted: resend it only with ` + + 'confirmDuplicate: true, after checking that channel.' + : ''; const notice = `[prose-routing] Your text (${text.length} chars) was not delivered — ${reason}.${cand} ${kept}${risky}` + (prefix === '>>' ? ' The prose_help tool shows the full syntax.' : ''); diff --git a/test/explicit-prose-routing.test.ts b/test/explicit-prose-routing.test.ts index 601f55618..a54ed503a 100644 --- a/test/explicit-prose-routing.test.ts +++ b/test/explicit-prose-routing.test.ts @@ -378,12 +378,13 @@ describe('explicit prose routing', () => { } }); - it('a failed {{unsent}} send that a later turn overtook holds the words it attempted, not the newer bounce', async () => { + for (const sent of ['{{unsent}} plus a line', '{{unsent}}'] as const) { + it(`a failed '${sent}' send that a later turn overtook holds the words it attempted, not the newer bounce`, async () => { // Turn one queues '>>#alpha {{unsent}} plus a line' and skip_reply ends // it while that publish is held. Turn two bounces new words. When turn // one's send then fails, its draft must hold what it attempted. membrane.pushResponse(createMockResponse([ - { type: 'text', text: '>>#alpha {{unsent}} plus a line' }, + { type: 'text', text: `>>#alpha ${sent}` }, { type: 'tool_use', id: 'delayed-1', name: 'robot--move', input: { dir: 'north' } }, ] as ContentBlock[], 'tool_use')); membrane.pushResponse(createMockResponse([ @@ -421,22 +422,37 @@ describe('explicit prose routing', () => { await framework.runUntilIdle(); await reached; const firstChain = internals.proseSpeechChains.get('assistant')!; - assert.deepEqual(routed.map((r) => r.text), ['ORIGINAL WORDS plus a line']); + const attempted = sent === '{{unsent}}' ? 'ORIGINAL WORDS' : 'ORIGINAL WORDS plus a line'; + assert.deepEqual(routed.map((r) => r.text), [attempted]); trigger(framework); await framework.runUntilIdle(); assert.equal(store.latestBounce('assistant')?.text, 'NEW WORDS', 'the next turn bounced before the earlier send answered'); release(); await firstChain; const heldTexts = store.open('assistant').map((d) => d.text); - assert.ok(heldTexts.includes('ORIGINAL WORDS plus a line'), `the failed send holds the words it attempted: ${JSON.stringify(heldTexts)}`); + assert.ok(heldTexts.includes(attempted), `the failed send holds the words it attempted: ${JSON.stringify(heldTexts)}`); assert.ok(!heldTexts.includes('NEW WORDS plus a line'), 'not the newer bounce'); - const copy = store.open('assistant').find((d) => d.text === 'ORIGINAL WORDS plus a line')!; - assert.notEqual(copy.id, original.id); + const copy = store.open('assistant').find((d) => d.text === attempted)!; + assert.equal(copy.inheritedRisk, undefined, 'a definite failure leaves no risk to inherit'); + if (sent === '{{unsent}}') { + // The bare send re-bounces the draft it sent, which the newer bounce + // displaced as the latest: {{unsent}} now names that one, so the + // notice offers only the drafts tool for this one. + assert.equal(copy.id, original.id); + assert.equal(store.latestBounce('assistant')?.text, 'NEW WORDS'); + const notice = context(framework).texts.filter((t) => t.startsWith('[prose-routing]') && t.includes(original.id)).at(-1)!; + assert.ok(notice.includes(`drafts(action: "resend", draftIds: ["${original.id}"]`), notice); + assert.ok(!notice.includes('{{unsent}}'), `the notice doesn't offer {{unsent}}, which names the newer bounce: ${notice}`); + } else { + assert.notEqual(copy.id, original.id); + assert.equal(store.latestBounce('assistant')?.id, copy.id, 'the re-held words are the latest bounce, as {{unsent}} names'); + } } finally { release(); await framework.stop(); } }); + } it('a thrown explicit unsent delivery re-bounces the authored body with its unresolved attempt risk', async () => { const framework = await createFramework(); @@ -455,6 +471,65 @@ describe('explicit prose routing', () => { assert.equal(copy.text, 'earlier words plus a line'); assert.equal(copy.inheritedRisk?.draftId, original.id); assert.equal(draftState(copy), 'unconfirmed', 'a fresh draft cannot erase its source attempt risk'); + const notice = context(framework).texts.find((t) => t.startsWith('[prose-routing]') && t.includes(copy.id))!; + assert.ok(notice.includes(`draftIds: ["${copy.id}"], destination: "#channel", confirmDuplicate: true`), notice); + assert.ok(!notice.includes('{{unsent}}'), `{{unsent}} would refuse it, so the notice doesn't offer it: ${notice}`); + } finally { + await framework.stop(); + } + }); + + it("{{unsent}} puts a draft's words in exactly, `$&` and `$$` included", async () => { + const framework = await createFramework(); + try { + const routed = stubRegistry(framework); + const store = draftsOf(framework); + store.hold('assistant', [{ text: 'costs $$5, see $& and $1', source: { branch: 'main', turn: 1, round: 1, segment: 0 } }], 'bounced'); + const gateway = framework as unknown as { + deliverProse(a: unknown, text: string, hold: { notice: 'later' }): Promise; + }; + await gateway.deliverProse(framework.getAgent('assistant')!, '>>#alpha {{unsent}} (again)', { notice: 'later' }); + assert.deepEqual(routed.map((r) => r.text), ['costs $$5, see $& and $1 (again)']); + } finally { + await framework.stop(); + } + }); + + it('a thrown bare {{unsent}} delivery says its draft may already have been posted', async () => { + const framework = await createFramework(); + try { + stubRegistry(framework, ['throw']); + const store = draftsOf(framework); + const original = store.hold('assistant', [{ + text: 'earlier words', source: { branch: 'main', turn: 1, round: 1, segment: 0 }, + }], 'bounced').held[0]!; + const gateway = framework as unknown as { + deliverProse(a: unknown, text: string, hold: { notice: 'later' }): Promise; + }; + await gateway.deliverProse(framework.getAgent('assistant')!, '>>#alpha {{unsent}}', { notice: 'later' }); + assert.deepEqual(store.open('assistant').map((d) => d.id), [original.id], 'a bare {{unsent}} holds no copy'); + assert.equal(draftState(store.get('assistant', original.id)!), 'unconfirmed'); + const notice = context(framework).texts.find((t) => t.startsWith('[prose-routing]') && t.includes(original.id))!; + assert.match(notice, /may already have been posted/); + assert.ok(notice.includes(`draftIds: ["${original.id}"], destination: "#channel", confirmDuplicate: true`), notice); + assert.ok(!notice.includes('{{unsent}}'), notice); + } finally { + await framework.stop(); + } + }); + + it('hybrid: a >>> envelope whose {{unsent}} names no held bounce sends nothing, and holds its other words', async () => { + const framework = await createFramework('hybrid'); + try { + const routed = stubRegistry(framework); + const store = draftsOf(framework); + const gateway = framework as unknown as { + deliverHybridProse(a: unknown, text: string, locus: string | null, allowLocus: boolean, hold: { notice: 'later' }): Promise; + }; + await gateway.deliverHybridProse(framework.getAgent('assistant')!, '>>>#alpha {{unsent}} and a note', null, true, { notice: 'later' }); + assert.deepEqual(routed, [], 'no fragment goes out'); + assert.deepEqual(store.open('assistant').map((d) => d.text), ['and a note']); + assert.ok(context(framework).texts.some((t) => t.startsWith('[prose-routing]') && t.includes('{{unsent}} names no held bounce draft'))); } finally { await framework.stop(); } From 665ea4e755dd9e99987a61dc05d0440a95ae5ac9 Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Tue, 13 Oct 2026 03:22:55 -0400 Subject: [PATCH 0161/1112] docs(mcp): a submitted configuration is the caller's to keep unchanged; the texts say removed or replaced A server's configuration is retained by reference: the wire reads it again at a 401 refresh or a legacy redial, and the records read it when an operation begins. Changing a connected server's configuration in place isn't supported, and the one real host doesn't: it builds a fresh configuration and restarts or connects with it, and rotates credentials through accessProvider. McplServerConfig's doc, restartMcplServer's doc and the changelog now say so as the caller's obligation: treat a submitted configuration and its nested values as immutable while the server uses them; apply changed settings with restartMcplServer(id, newConfig); use accessProvider for rotating credentials. The texts that said an operation's records hold whatever the host changes meanwhile now say a server removed, or replaced by a restart; the policy test drops its in-place rotation case. The redaction fragment says the response body leaves the error's data and the message is redacted where the client builds it from the body; the restart lines say "Before" and name what the configuration check refuses. Env names ending a part with 'pass' or 'passphrase' (DB_PASS, SSH_KEY_PASSPHRASE) count as credentials too. --- changelog.d/mcp-standard-tool-results.changed.md | 16 ++++++++++------ .../redact-recorded-mcp-credentials.fixed.md | 2 +- src/framework.ts | 11 ++++++++--- src/mcpl/protocol-family.ts | 2 +- src/mcpl/types.ts | 5 +++++ test/mcp-protocol-family.test.ts | 5 +++-- test/mcpl-config-diagnostics.test.ts | 7 +++---- 7 files changed, 31 insertions(+), 17 deletions(-) diff --git a/changelog.d/mcp-standard-tool-results.changed.md b/changelog.d/mcp-standard-tool-results.changed.md index 39f9077f0..0c197bb76 100644 --- a/changelog.d/mcp-standard-tool-results.changed.md +++ b/changelog.d/mcp-standard-tool-results.changed.md @@ -103,12 +103,16 @@ legacy one, fails with "was not connected"); then it removes what was registered. A teardown never removes what a later connection registered. `restartMcplServer` refuses, before it tears the running server down, what - its connect would refuse: a configuration with no usable binding, a new id - another server holds, a tool prefix another server or a module holds, a - framework whose `stop()` has begun, and a connect or disconnect of the - server in flight. On main a restart disconnected first, so a refused - connect left the server gone; now the server keeps running. A disconnect - releases its tool-prefix claim only while its own server holds it. + its connect would refuse: whatever the configuration check refuses (such as + an unusable binding, userinfo on an http(s) url, or MCPL policy on a modern + server), a new id another server holds, a tool prefix another server or a + module holds, a framework whose `stop()` has begun, and a connect or + disconnect of the server in flight. Before, a restart disconnected first, + so a refused connect left the server gone; now the server keeps running. A + disconnect releases its tool-prefix claim only while its own server holds + it. Treat a submitted configuration as immutable while the server uses it: + change settings with `restartMcplServer(id, newConfig)`, and rotate + credentials through `accessProvider`. - A server configuration naming no usable transport is now an error before anything is spawned or dialed. That covers a `transport` that doesn't match the url's scheme, `transport: 'http'` without a url, and an diff --git a/changelog.d/redact-recorded-mcp-credentials.fixed.md b/changelog.d/redact-recorded-mcp-credentials.fixed.md index 6b2ed989d..a4a1bc37a 100644 --- a/changelog.d/redact-recorded-mcp-credentials.fixed.md +++ b/changelog.d/redact-recorded-mcp-credentials.fixed.md @@ -1 +1 @@ -- Server and transport text the framework records has the server's configured credentials replaced, in both engines: connection traces, console lines, stderr traces, the ops alert (and so `logs/failures.log` and the webhook), a tool call's failure as its caller and the model get it, and the `tool:failed` trace's message and stack. The credentials are those the configuration holds: from the url, credential-like path segments and query names, credential-shaped or credential-named query values (each as written and decoded), every other non-empty parameter as its whole `name=value`, and a `ws://` url's username and password; the static `token`, raw and as a url carries it, percent- or form-encoded; and a stdio server's `env` values whose names end a part with a credential word (`GITHUB_TOKEN`, `API_KEY`, `PGPASSWORD`). A string replaced on its own has at least eight characters, since a shorter one can't be told from ordinary text; a short query value is replaced only with its name, as one `name=value`, and only where no letter or digit adjoins it. A connection's event records use a redactor built once for it, and an operation's records one built when the operation begins, so a server removed or reconfigured meanwhile doesn't change what they replace. A 401 or 403 is classified as the server's refusal however the HTTP client raised it, and its response body is left out of the error's data (its message, built from the body, is redacted like any record). A token an access provider fetches, and credentials the configuration doesn't hold, are outside this. JSON-RPC answers, and the error a host's own connect raises, pass as they are. +- Server and transport text the framework records has the server's configured credentials replaced, in both engines: connection traces, console lines, stderr traces, the ops alert (and so `logs/failures.log` and the webhook), a tool call's failure as its caller and the model get it, and the `tool:failed` trace's message and stack. The credentials are those the configuration holds: from the url, credential-like path segments and query names, credential-shaped or credential-named query values (each as written and decoded), every other non-empty parameter as its whole `name=value`, and a `ws://` url's username and password; the static `token`, raw and as a url carries it, percent- or form-encoded; and a stdio server's `env` values whose names end a part with a credential word (`GITHUB_TOKEN`, `API_KEY`, `PGPASSWORD`, `DB_PASS`, `SSH_KEY_PASSPHRASE`). A string replaced on its own has at least eight characters, since a shorter one can't be told from ordinary text; a short query value is replaced only with its name, as one `name=value`, and only where no letter or digit adjoins it. A connection's event records use a redactor built once for it, and an operation's records one built when the operation begins, so a server removed, or replaced by a restart, while the operation waits doesn't change what they replace. A 401 or 403's response body is left out of the request error's data; where the HTTP client builds the error's message from the body, as for a plain 403, the message is redacted like any record. A token an access provider fetches, and credentials the configuration doesn't hold, are outside this. JSON-RPC answers, and the error a host's own connect raises, pass as they are. diff --git a/src/framework.ts b/src/framework.ts index 486d0edeb..1472ee6f6 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -14350,9 +14350,9 @@ export class AgentFramework { connection: McplServerConnection, ): Promise { if (!connection.capabilities) return; - // The server's configuration as it is now, before any await: every - // record of what the server says below keeps it, whatever the host - // changes in the configuration meanwhile. + // Taken before any await, so every record of what the server says below + // uses one derivation, whatever becomes of the server meanwhile (removed, + // or replaced by a restart). const recorded = serverTextRedactor(config); // §5.4: compute the effective grant from the (config-masked) @@ -14558,6 +14558,11 @@ export class AgentFramework { * same (or an updated) config. This actually respawns a stdio child — * unlike `reconnect: true`, which only retries after transport-level * failures. Throws if the server has no stored config and none is given. + * + * Treat a submitted configuration and its nested values as immutable for + * as long as the server uses them. Apply changed settings with + * `restartMcplServer(id, newConfig)`; use `accessProvider` for rotating + * credentials. */ async restartMcplServer( id: string, diff --git a/src/mcpl/protocol-family.ts b/src/mcpl/protocol-family.ts index f408a5d55..c7bd59e45 100644 --- a/src/mcpl/protocol-family.ts +++ b/src/mcpl/protocol-family.ts @@ -213,7 +213,7 @@ function asText(text: unknown): string { } /** A part of an environment variable's name that ends with a credential word. */ -const CREDENTIAL_NAME_PART = /(?:token|key|secret|password|passwd|auth|signature|credentials?)$/i; +const CREDENTIAL_NAME_PART = /(?:token|key|secret|password|passwd|pass|passphrase|auth|signature|credentials?)$/i; /** * A credential provider's failure as a log line may show it: the error's diff --git a/src/mcpl/types.ts b/src/mcpl/types.ts index 5ca5f0ff8..41049aa59 100644 --- a/src/mcpl/types.ts +++ b/src/mcpl/types.ts @@ -211,6 +211,11 @@ export interface McplChannelCapabilities { /** * Configuration for connecting to a single MCPL server. * Provided in FrameworkConfig.mcplServers[]. + * + * Treat a submitted configuration and its nested values as immutable for as + * long as the server uses them. Apply changed settings with + * `restartMcplServer(id, newConfig)`; use `accessProvider` for rotating + * credentials. */ export interface McplServerConfig { /** Unique server identifier */ diff --git a/test/mcp-protocol-family.test.ts b/test/mcp-protocol-family.test.ts index 170f9fc6c..02b5d6b8a 100644 --- a/test/mcp-protocol-family.test.ts +++ b/test/mcp-protocol-family.test.ts @@ -332,11 +332,12 @@ test('recorded server text: the token as a url carries it, credential-named env // part of it, and no other (GIT_AUTHOR_NAME holds 'auth' inside a word). const env = cfg({ command: 'srv', env: { GITHUB_TOKEN: 'ghp_abcdef123456', API_KEY: 'k3y-abcdefgh', PGPASSWORD: 'pgpass-abcdefgh', ACCESSTOKEN: 'acc-abcdefgh12', + DB_PASS: 'dbpass-abcdefgh', SSH_KEY_PASSPHRASE: 'phrase-abcdefgh', PATH: '/usr/local/bin', GIT_AUTHOR_NAME: 'Somebody Longname', KEYBOARD_LAYOUT: 'dvorak-programmer', TOKENIZERS_PARALLELISM: 'parallel-false', } }); assert.equal( - redactServerText('auth ghp_abcdef123456 k3y-abcdefgh pgpass-abcdefgh acc-abcdefgh12 /usr/local/bin Somebody Longname dvorak-programmer parallel-false', env), - 'auth redacted redacted redacted redacted /usr/local/bin Somebody Longname dvorak-programmer parallel-false'); + redactServerText('auth ghp_abcdef123456 k3y-abcdefgh pgpass-abcdefgh acc-abcdefgh12 dbpass-abcdefgh phrase-abcdefgh /usr/local/bin Somebody Longname dvorak-programmer parallel-false', env), + 'auth redacted redacted redacted redacted redacted redacted /usr/local/bin Somebody Longname dvorak-programmer parallel-false'); // A short pair only where no letter or digit adjoins it; any other string // wherever it occurs, after a percent escape, an ANSI code or a JSON escape too. const pair = cfg({ url: 'https://h/api/s/pathSecret0123456789/mcp?v=2' }); diff --git a/test/mcpl-config-diagnostics.test.ts b/test/mcpl-config-diagnostics.test.ts index bf8a8f1d8..303c3c1f2 100644 --- a/test/mcpl-config-diagnostics.test.ts +++ b/test/mcpl-config-diagnostics.test.ts @@ -361,7 +361,7 @@ test("a refused initial policy's reason is recorded without the server's configu assert.ok(!capture.lines.join('\n').includes(secret)); }); -test("the initial policy's other records keep the credentials out too, with the configuration as it was when the request went", async () => { +test("the initial policy's other records keep the credentials out too", async () => { const secret = 'srvSecret0123456789'; const run = async (answer: () => Promise, mutate?: (config: McplServerConfig) => void) => { const fw = Object.create(AgentFramework.prototype) as any; @@ -392,9 +392,8 @@ test("the initial policy's other records keep the credentials out too, with the assert.ok(odd.lines.some((l) => l.includes('refused initial policy (no) — fallback: mcp-only')), odd.lines.join('\n')); // A degraded acceptance whose effect quotes the url. await run(async () => ({ accepted: true, mode: 'degraded', unavailableFeatures: [{ featureSet: 'fs', effect: `see ${secret}`, missingCapabilities: ['x'] }] })); - // The host replaces the url while the request waits: the record still - // knows the credential the request went with. - await run(async () => ({ accepted: false, fallback: 'mcp-only', reason: `bad ${secret}` }), (config) => { (config as { url?: string }).url = 'wss://h/mcp/rotatedSecret0123456789'; }); + // A refusal's reason, recorded with the derivation the operation began with. + await run(async () => ({ accepted: false, fallback: 'mcp-only', reason: `bad ${secret}` })); }); test('registerMcplServerFeatures: a refused initial policy is traced, and fallback close closes', async () => { From 0f4d8a8c11e59d33df61167501e52dd9a2cd391d Mon Sep 17 00:00:00 2001 From: Petra-1012 Date: Tue, 13 Oct 2026 03:26:38 -0400 Subject: [PATCH 0162/1112] fix(drafts): a thrown send's bounce notice says it may not have been delivered; the overtaken case at every failure call site From the readers of 9169bc99: - A send that threw has no known outcome, and its draft reads unconfirmed, but its bounce notice opened "Your text ... was not delivered" and closed "it may already have been posted". bounceProse takes { substituted, outcome: 'unknown' } after a send; for an unknown outcome the notice opens "may not have been delivered", the reason says the send threw rather than failed, and with no other risk named it asks to check the channel before resending. A definite failure keeps "was not delivered". - Only the explicit failed path had the overtaken case under test, so dropping the substituted draft from the hybrid failed, hybrid thrown or explicit thrown call stayed green. A direct test now runs each of the four: the send is held, a newer bounce lands, and the failure re-holds the original expansion (with its source's risk when it threw). The thrown tests assert the notice doesn't say "was not delivered". --- src/framework.ts | 24 +++++++++------ test/explicit-prose-routing.test.ts | 48 +++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 9 deletions(-) diff --git a/src/framework.ts b/src/framework.ts index 01c51ce98..bf5b9ba77 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -10203,13 +10203,13 @@ export class AgentFramework { // not bounced, so its words are not re-held as a fresh draft that // invites a duplicate. if (outcome.status === 'failed') { - this.bounceProse(agent, parsed.body, `delivery to ${resolved.channelId} failed: ${outcome.reason ?? 'refused'}`, undefined, '>>>', hold, unsent.draft); + this.bounceProse(agent, parsed.body, `delivery to ${resolved.channelId} failed: ${outcome.reason ?? 'refused'}`, undefined, '>>>', hold, { substituted: unsent.draft }); } } } catch (err) { this.proseTargetPins.delete(agent.name); this.proseHybridSuppressed.set(agent.name, 'failed'); - this.bounceProse(agent, parsed.body, `delivery to ${resolved.channelId} failed: ${err instanceof Error ? err.message : String(err)}`, undefined, '>>>', hold, unsent.draft); + this.bounceProse(agent, parsed.body, `sending to ${resolved.channelId} threw: ${err instanceof Error ? err.message : String(err)}`, undefined, '>>>', hold, { substituted: unsent.draft, outcome: 'unknown' }); } continue; } @@ -10356,10 +10356,10 @@ export class AgentFramework { } else if (outcome.status === 'failed') { // Unknown deliveries are marker-only: offering a fresh draft would // invite a duplicate. Keep the authored body for the re-bounce rule. - this.bounceProse(agent, envelopeBody, `delivery to ${targetChannel} failed: ${outcome.reason ?? 'refused'}`, undefined, '>>', hold, unsent.draft); + this.bounceProse(agent, envelopeBody, `delivery to ${targetChannel} failed: ${outcome.reason ?? 'refused'}`, undefined, '>>', hold, { substituted: unsent.draft }); } } catch (err) { - this.bounceProse(agent, envelopeBody, `delivery to ${targetChannel} failed: ${err instanceof Error ? err.message : String(err)}`, undefined, '>>', hold, unsent.draft); + this.bounceProse(agent, envelopeBody, `sending to ${targetChannel} threw: ${err instanceof Error ? err.message : String(err)}`, undefined, '>>', hold, { substituted: unsent.draft, outcome: 'unknown' }); } } @@ -10437,11 +10437,12 @@ export class AgentFramework { candidates?: string[], prefix: '>>' | '>>>' = '>>', hold: { round?: number; notice: 'now' | 'later' } = { notice: 'later' }, - /** The draft a failed send substituted for `{{unsent}}` (unsentWords). */ - substituted?: Draft, + /** After a send: the draft it substituted for `{{unsent}}` + * (unsentWords), and whether its outcome is unknown (it threw). */ + after: { substituted?: Draft; outcome?: 'unknown' } = {}, ): void { const name = agent.name; - const words = this.unsentWords(agent, text, substituted); + const words = this.unsentWords(agent, text, after.substituted); let draft: Draft | undefined; let inherited: InheritedRisk | undefined; if ('draft' in words) { @@ -10482,9 +10483,14 @@ export class AgentFramework { : ownRisk ? ` Its last send's outcome isn't known, so it may already have been posted: resend it only with ` + 'confirmDuplicate: true, after checking that channel.' - : ''; + : after.outcome === 'unknown' + ? ' The send may have gone out, so check that channel before resending.' + : ''; + // A send that threw has no known outcome: the notice doesn't say it + // wasn't delivered. + const delivered = after.outcome === 'unknown' ? 'may not have been delivered' : 'was not delivered'; const notice = - `[prose-routing] Your text (${text.length} chars) was not delivered — ${reason}.${cand} ${kept}${risky}` + + `[prose-routing] Your text (${text.length} chars) ${delivered} — ${reason}.${cand} ${kept}${risky}` + (prefix === '>>' ? ' The prose_help tool shows the full syntax.' : ''); try { // Through framework.addMessage, NOT the context manager directly: while diff --git a/test/explicit-prose-routing.test.ts b/test/explicit-prose-routing.test.ts index a54ed503a..7a4eb4a5a 100644 --- a/test/explicit-prose-routing.test.ts +++ b/test/explicit-prose-routing.test.ts @@ -472,6 +472,7 @@ describe('explicit prose routing', () => { assert.equal(copy.inheritedRisk?.draftId, original.id); assert.equal(draftState(copy), 'unconfirmed', 'a fresh draft cannot erase its source attempt risk'); const notice = context(framework).texts.find((t) => t.startsWith('[prose-routing]') && t.includes(copy.id))!; + assert.doesNotMatch(notice, /was not delivered/, 'an unconfirmed send is not also declared undelivered'); assert.ok(notice.includes(`draftIds: ["${copy.id}"], destination: "#channel", confirmDuplicate: true`), notice); assert.ok(!notice.includes('{{unsent}}'), `{{unsent}} would refuse it, so the notice doesn't offer it: ${notice}`); } finally { @@ -479,6 +480,51 @@ describe('explicit prose routing', () => { } }); + for (const mode of ['explicit', 'hybrid'] as const) { + for (const failure of ['false', 'throw'] as const) { + it(`${mode}: a ${failure === 'false' ? 'failed' : 'thrown'} {{unsent}} send re-holds the draft it substituted, though a newer bounce landed while it waited`, async () => { + const framework = await createFramework(mode); + try { + stubRegistry(framework, [failure]); + const store = draftsOf(framework); + const original = store.hold('assistant', [{ text: 'ORIGINAL WORDS', source: { branch: 'main', turn: 1, round: 1, segment: 0 } }], 'bounced').held[0]!; + const internals = framework as unknown as { + channelRegistry: { deliverSpeech: (...args: unknown[]) => Promise }; + deliverProse(a: unknown, text: string, hold: { notice: 'later' }): Promise; + deliverHybridProse(a: unknown, text: string, locus: string | null, allowLocus: boolean, hold: { notice: 'later' }): Promise; + }; + const deliver = internals.channelRegistry.deliverSpeech; + let release!: () => void; + const held = new Promise((resolve) => { release = resolve; }); + let entered!: () => void; + const reached = new Promise((resolve) => { entered = resolve; }); + internals.channelRegistry.deliverSpeech = async (...args: unknown[]) => { + entered(); + await held; + return deliver(...args); + }; + const agent = framework.getAgent('assistant')!; + const sending = mode === 'explicit' + ? internals.deliverProse(agent, '>>#alpha {{unsent}} plus a line', { notice: 'later' }) + : internals.deliverHybridProse(agent, '>>>#alpha {{unsent}} plus a line', null, true, { notice: 'later' }); + await reached; + // A newer bounce becomes the latest while the send waits. + store.hold('assistant', [{ text: 'NEW WORDS', source: { branch: 'main', turn: 2, round: 1, segment: 0 } }], 'bounced'); + release(); + await sending; + const texts = store.open('assistant').map((d) => d.text); + assert.ok(texts.includes('ORIGINAL WORDS plus a line'), `it re-holds the words it attempted: ${JSON.stringify(texts)}`); + assert.ok(!texts.includes('NEW WORDS plus a line'), 'not the newer bounce'); + const copy = store.open('assistant').find((d) => d.text === 'ORIGINAL WORDS plus a line')!; + if (failure === 'throw') assert.equal(copy.inheritedRisk?.draftId, original.id, "a thrown send's copy carries its source's risk"); + else assert.equal(copy.inheritedRisk, undefined); + } finally { + await framework.stop(); + } + }); + } + } + it("{{unsent}} puts a draft's words in exactly, `$&` and `$$` included", async () => { const framework = await createFramework(); try { @@ -511,6 +557,8 @@ describe('explicit prose routing', () => { assert.equal(draftState(store.get('assistant', original.id)!), 'unconfirmed'); const notice = context(framework).texts.find((t) => t.startsWith('[prose-routing]') && t.includes(original.id))!; assert.match(notice, /may already have been posted/); + assert.doesNotMatch(notice, /was not delivered/, 'an unconfirmed send is not also declared undelivered'); + assert.match(notice, /may not have been delivered/); assert.ok(notice.includes(`draftIds: ["${original.id}"], destination: "#channel", confirmDuplicate: true`), notice); assert.ok(!notice.includes('{{unsent}}'), notice); } finally { From 1479d48a938b86b8b4f82e2ab32ed43fae97e2d6 Mon Sep 17 00:00:00 2001 From: Petra-1012 Date: Tue, 13 Oct 2026 03:27:16 -0400 Subject: [PATCH 0163/1112] test(drafts): the overtaken-send test reloads the store while the send waits The draft {{unsent}} substituted is captured before the send. When the store reloads its projection meanwhile (as it does after an ambiguous append), the captured object is no longer the stored one, and only reading the draft again after the attempt sees the definite failure recorded there. The four-site test now reloads while the send is held, so taking the captured draft's state as it was fails the failed-send cases (a reviewer's pointer). --- test/explicit-prose-routing.test.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/test/explicit-prose-routing.test.ts b/test/explicit-prose-routing.test.ts index 7a4eb4a5a..84aaaaa08 100644 --- a/test/explicit-prose-routing.test.ts +++ b/test/explicit-prose-routing.test.ts @@ -508,8 +508,12 @@ describe('explicit prose routing', () => { ? internals.deliverProse(agent, '>>#alpha {{unsent}} plus a line', { notice: 'later' }) : internals.deliverHybridProse(agent, '>>>#alpha {{unsent}} plus a line', null, true, { notice: 'later' }); await reached; - // A newer bounce becomes the latest while the send waits. + // A newer bounce becomes the latest while the send waits, and the + // store reloads its projection (as after an ambiguous append), so + // the draft captured before the send is no longer the stored one. store.hold('assistant', [{ text: 'NEW WORDS', source: { branch: 'main', turn: 2, round: 1, segment: 0 } }], 'bounced'); + store.reload(); + assert.notEqual(store.get('assistant', original.id), original, 'the reload replaced the stored draft'); release(); await sending; const texts = store.open('assistant').map((d) => d.text); @@ -517,7 +521,7 @@ describe('explicit prose routing', () => { assert.ok(!texts.includes('NEW WORDS plus a line'), 'not the newer bounce'); const copy = store.open('assistant').find((d) => d.text === 'ORIGINAL WORDS plus a line')!; if (failure === 'throw') assert.equal(copy.inheritedRisk?.draftId, original.id, "a thrown send's copy carries its source's risk"); - else assert.equal(copy.inheritedRisk, undefined); + else assert.equal(copy.inheritedRisk, undefined, 'read after the attempt, the failed draft carries no risk to inherit'); } finally { await framework.stop(); } From a2bd86fa6da1df2cec87b11dd5b185598fb72684 Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Tue, 13 Oct 2026 03:27:54 -0400 Subject: [PATCH 0164/1112] fix(mcp): every name=value pair keeps its boundary; env names in the plural; the every-server redactor covers servers connected meanwhile From a review of the round before: - A long name=value pair was replaced inside a longer name (dev=12345678 for a configured v=12345678). Every pair now keeps the letter-or-digit boundary; a credential value is still replaced on its own wherever it occurs, as a standalone string. - Env names: a part ending in a credential word in the plural (API_KEYS, SECRETS), with digits after it (TOKEN2), AUTHORIZATION or AUTHENTICATION, or PAT (GITHUB_PAT) marks a credential; a credential env value is derived percent-encoded too. - The every-server redactor (startup cleanup, tool refresh) applies the configurations present when its operation began and those present when it's used, so a server connected meanwhile is covered as well as one removed; it reads a value as the redactor does, so one that can't be printed doesn't throw there. - The manifest handler does nothing for a server with no configuration (it was removed), rather than recording with an empty one. - URLSearchParams replaces a lone surrogate rather than throwing, so the token's form-encoding needs no guard; the comments say which rule holds for strings and which for pairs. The policy test helper loses its unused mutate parameter. Tests: a long pair inside a longer name kept, the env names above, a server connected after the every-server redactor was made, and an unprintable value through it. Removing each piece fails a test (four removals). --- src/framework.ts | 18 ++++++++--- src/mcpl/protocol-family.ts | 47 +++++++++++++++------------- test/mcp-protocol-family.test.ts | 13 +++++--- test/mcp-recorded-text.test.ts | 5 +++ test/mcpl-config-diagnostics.test.ts | 4 +-- 5 files changed, 55 insertions(+), 32 deletions(-) diff --git a/src/framework.ts b/src/framework.ts index 1472ee6f6..262ccee64 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -394,7 +394,7 @@ import type { SessionUsageSnapshot, UsageUpdatedEvent } from './usage/types.js'; import type { McplServerConnection } from './mcpl/server-connection.js'; import { McplProtocolVersionError, McplUnreapedLaunchError } from './mcpl/server-connection.js'; import { ModernMcpConnection } from './mcpl/modern-connection.js'; -import { redactUrl, redactServerText, resolveServerBinding, serverConfigProblems, serverTextRedactor } from './mcpl/protocol-family.js'; +import { asText, redactUrl, redactServerText, resolveServerBinding, serverConfigProblems, serverTextRedactor } from './mcpl/protocol-family.js'; import { MALFORMED_CONTENT_NOTICE, normalizeStandardToolResult } from './mcpl/tool-result-normalize.js'; import type { McplServerConfig, @@ -14158,8 +14158,12 @@ export class AgentFramework { announce: { revision?: string; domains?: string[] }, ): Promise { // The server's configuration as it is now, before any await: a record - // made later keeps it, whatever becomes of the server meanwhile. - const recorded = serverTextRedactor(this.mcplServerConfigs.get(connection.id) ?? {}); + // made later keeps it, whatever becomes of the server meanwhile. A server + // with no configuration has been removed, and its manifest isn't + // refreshed. + const config = this.mcplServerConfigs.get(connection.id); + if (!config) return; + const recorded = serverTextRedactor(config); const st = this.manifestRefreshState.get(connection.id) ?? { inFlight: false, queued: false, lastCompletedAt: 0 }; this.manifestRefreshState.set(connection.id, st); @@ -14605,8 +14609,12 @@ export class AgentFramework { * server removed meanwhile still has its credentials replaced. */ private everyServerRedactor(): (text: unknown) => string { - const redactors = [...this.mcplServerConfigs.values()].map(config => serverTextRedactor(config)); - return (text) => redactors.reduce((recorded, redact) => redact(recorded), typeof text === 'string' ? text : String(text)); + const snapshot = [...this.mcplServerConfigs.values()].map(config => serverTextRedactor(config)); + // The configurations present when the operation began, then those present + // now: a server removed meanwhile and one connected meanwhile are both + // covered. + return (text) => [...snapshot, ...[...this.mcplServerConfigs.values()].map(config => serverTextRedactor(config))] + .reduce((recorded, redact) => redact(recorded), asText(text)); } /** Why a connect for this id is refused, or null when the id is free. One diff --git a/src/mcpl/protocol-family.ts b/src/mcpl/protocol-family.ts index c7bd59e45..72f86388c 100644 --- a/src/mcpl/protocol-family.ts +++ b/src/mcpl/protocol-family.ts @@ -118,9 +118,10 @@ export function redactUrl(url: string | undefined): string { * - a stdio server's environment values whose names say they're credentials. * A string replaced on its own has at least 8 characters, since a shorter * one can't be told from ordinary text, and it is replaced wherever it - * occurs, after an escape or inside a longer token too. A short query value - * is replaced only with its name, as one name=value, and only where no - * letter or digit adjoins the pair, so v=2 leaves dev=2 alone. One + * occurs, after an escape or inside a longer token too. A query parameter is + * also replaced as its whole name=value, only where no letter or digit + * adjoins the pair, so v=2 leaves dev=2 alone; a short value goes only that + * way. One * longest-first pass never scans its own replacement. A credential the * configuration doesn't hold, such as the token an access provider fetches * or another origin's, is outside it. JSON-RPC answers, and the error a @@ -135,30 +136,35 @@ export function redactServerText(text: unknown, config: Pick): (text: unknown) => string { - // Strings replaced wherever they occur, and short name=value pairs, which - // are replaced only where no letter or digit adjoins them, so a pair such as - // v=2 leaves dev=2 alone. + // Strings replaced wherever they occur (after an escape or inside a longer + // token too), and name=value pairs, replaced only where no letter or digit + // adjoins them, so a pair such as v=2 leaves dev=2 alone. A pair's value is + // also replaced on its own when it's a credential, wherever it occurs. const secrets = new Set(); - const shortPairs = new Set(); + const pairs = new Set(); const standalone = (value: string): void => { if (value.length >= 8) secrets.add(value); }; const decode = (value: string, query = false): string => { try { return decodeURIComponent(query ? value.replace(/\+/g, ' ') : value); } catch { return value; } }; + // A string as written and percent-encoded (a string encodeURIComponent + // refuses, such as one with an unpaired surrogate, has no encoded form). + const withEncoded = (value: string): void => { + standalone(value); + try { standalone(encodeURIComponent(value)); } catch { /* no percent-encoded form */ } + }; // The token as sent: raw in a header, and percent- or form-encoded where a - // transport puts it in a url (a token that can't be encoded, such as one - // with an unpaired surrogate, has no encoded form). + // transport puts it in a url. const token = config.token ?? ''; - standalone(token); - try { standalone(encodeURIComponent(token)); } catch { /* no percent-encoded form */ } - try { standalone(new URLSearchParams({ t: token }).toString().slice(2)); } catch { /* no form-encoded form */ } + withEncoded(token); + standalone(new URLSearchParams({ t: token }).toString().slice(2)); // A stdio server's environment values whose names say they carry a // credential: a part of the name that ends with a credential word // (GITHUB_TOKEN, API_KEY, PGPASSWORD, ACCESSTOKEN), and not one where the // word only begins or sits inside a part (GIT_AUTHOR_NAME, KEYBOARD_LAYOUT, // TOKENIZERS_PARALLELISM). for (const [name, value] of Object.entries(config.env ?? {})) { - if (name.split(/[_\-.]/).some((part) => CREDENTIAL_NAME_PART.test(part))) standalone(value); + if (name.split(/[_\-.]/).some((part) => CREDENTIAL_NAME_PART.test(part))) withEncoded(value); } let url: URL | undefined; try { url = new URL(config.url ?? ''); } catch { /* Still remove the static token. */ } @@ -184,9 +190,8 @@ export function serverTextRedactor(config: Pick= 8 ? secrets : shortPairs).add(pair); - } + pairs.add(`${encodedName}=${encodedValue}`); + pairs.add(`${name}=${value}`); } } if (url.protocol === 'ws:' || url.protocol === 'wss:') { @@ -196,24 +201,24 @@ export function serverTextRedactor(config: Pick value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); const alternatives = [ ...[...secrets].map(value => ({ value, source: escape(value) })), - ...[...shortPairs].map(value => ({ value, source: `(? ({ value, source: `(? b.value.length - a.value.length).map(({ source }) => source); const pattern = new RegExp(alternatives.join('|'), 'g'); return (text) => asText(text).replace(pattern, 'redacted'); } /** Any value as text: a value that can't be printed reads as a placeholder. */ -function asText(text: unknown): string { +export function asText(text: unknown): string { if (typeof text === 'string') return text; try { return String(text); } catch { return '[a value that cannot be printed]'; } } -/** A part of an environment variable's name that ends with a credential word. */ -const CREDENTIAL_NAME_PART = /(?:token|key|secret|password|passwd|pass|passphrase|auth|signature|credentials?)$/i; +/** A part of an environment variable's name that ends with a credential word, in the singular or plural, with any digits after it. */ +const CREDENTIAL_NAME_PART = /(?:tokens?|keys?|secrets?|passwords?|passwd|pass|passphrases?|pat|auth|authorization|authentication|signatures?|credentials?)\d*$/i; /** * A credential provider's failure as a log line may show it: the error's diff --git a/test/mcp-protocol-family.test.ts b/test/mcp-protocol-family.test.ts index 02b5d6b8a..393825496 100644 --- a/test/mcp-protocol-family.test.ts +++ b/test/mcp-protocol-family.test.ts @@ -328,20 +328,25 @@ test('recorded server text: the token as a url carries it, credential-named env for (const form of [token, 'tok%20en%2Bx%2F123%3D', 'tok+en%2Bx%2F123%3D']) { assert.equal(redactServerText(`refused: ${form}`, tokenConfig), 'refused: redacted', form); } - // A stdio server's env: a value whose name has a credential word as a whole - // part of it, and no other (GIT_AUTHOR_NAME holds 'auth' inside a word). + // A stdio server's env: a value whose name has a part ending in a credential + // word (plural, or with digits after it, too), and no other + // (GIT_AUTHOR_NAME's part AUTHOR doesn't end in 'auth'). const env = cfg({ command: 'srv', env: { GITHUB_TOKEN: 'ghp_abcdef123456', API_KEY: 'k3y-abcdefgh', PGPASSWORD: 'pgpass-abcdefgh', ACCESSTOKEN: 'acc-abcdefgh12', DB_PASS: 'dbpass-abcdefgh', SSH_KEY_PASSPHRASE: 'phrase-abcdefgh', + AUTHORIZATION: 'Bearer-abcdefgh', API_KEYS: 'k1-abcdefgh9', TOKEN2: 'tok2-abcdefgh', API_KEY_ID: 'keyid-abcdefgh', GITHUB_PAT: 'pat-abcdefgh12', PATH: '/usr/local/bin', GIT_AUTHOR_NAME: 'Somebody Longname', KEYBOARD_LAYOUT: 'dvorak-programmer', TOKENIZERS_PARALLELISM: 'parallel-false', } }); assert.equal( - redactServerText('auth ghp_abcdef123456 k3y-abcdefgh pgpass-abcdefgh acc-abcdefgh12 dbpass-abcdefgh phrase-abcdefgh /usr/local/bin Somebody Longname dvorak-programmer parallel-false', env), - 'auth redacted redacted redacted redacted redacted redacted /usr/local/bin Somebody Longname dvorak-programmer parallel-false'); + redactServerText('auth ghp_abcdef123456 k3y-abcdefgh pgpass-abcdefgh acc-abcdefgh12 dbpass-abcdefgh phrase-abcdefgh Bearer-abcdefgh k1-abcdefgh9 tok2-abcdefgh keyid-abcdefgh pat-abcdefgh12 /usr/local/bin Somebody Longname dvorak-programmer parallel-false', env), + 'auth redacted redacted redacted redacted redacted redacted redacted redacted redacted redacted redacted /usr/local/bin Somebody Longname dvorak-programmer parallel-false'); // A short pair only where no letter or digit adjoins it; any other string // wherever it occurs, after a percent escape, an ANSI code or a JSON escape too. const pair = cfg({ url: 'https://h/api/s/pathSecret0123456789/mcp?v=2' }); assert.equal(redactServerText('dev=2 ready; v=2 sent', pair), 'dev=2 ready; redacted sent'); + // A long pair too: only as its own name=value. + const longPair = cfg({ url: 'https://h/mcp?v=12345678' }); + assert.equal(redactServerText('dev=12345678 ready; v=12345678 sent', longPair), 'dev=12345678 ready; redacted sent'); assert.equal(redactServerText('a%2FpathSecret0123456789 \u001b[31mpathSecret0123456789 "\\npathSecret0123456789"', pair), 'a%2Fredacted \u001b[31mredacted "\\nredacted"'); // A ws:// url's userinfo, as for wss://. diff --git a/test/mcp-recorded-text.test.ts b/test/mcp-recorded-text.test.ts index 76d160244..ef5e74b58 100644 --- a/test/mcp-recorded-text.test.ts +++ b/test/mcp-recorded-text.test.ts @@ -220,6 +220,11 @@ test("a tool call's failure, from either engine, reaches its caller and its trac const everyServer = framework.everyServerRedactor(); framework.mcplServerConfigs = new Map(); assert.equal(everyServer(`closed ${secret} and otherSecret0123456789`), 'closed redacted and redacted'); + // A server connected after it was made is covered too. + framework.mcplServerConfigs = new Map([['late', { id: 'late', url: 'https://k/api/s/lateSecret0123456789/mcp' }]]); + assert.equal(everyServer('closed lateSecret0123456789'), 'closed redacted'); + // A value that can't be printed reads as a placeholder. + assert.equal(everyServer(Object.create(null)), '[a value that cannot be printed]'); framework.mcplServerConfigs = new Map([[config.id, config]]); // The model's dispatch: the tool:failed trace (message and stack) and the diff --git a/test/mcpl-config-diagnostics.test.ts b/test/mcpl-config-diagnostics.test.ts index 303c3c1f2..65e066b89 100644 --- a/test/mcpl-config-diagnostics.test.ts +++ b/test/mcpl-config-diagnostics.test.ts @@ -363,7 +363,7 @@ test("a refused initial policy's reason is recorded without the server's configu test("the initial policy's other records keep the credentials out too", async () => { const secret = 'srvSecret0123456789'; - const run = async (answer: () => Promise, mutate?: (config: McplServerConfig) => void) => { + const run = async (answer: () => Promise) => { const fw = Object.create(AgentFramework.prototype) as any; fw.traceListeners = []; fw.featureSetManager = new FeatureSetManager(); @@ -376,7 +376,7 @@ test("the initial policy's other records keep the credentials out too", async () capabilities: { version: '0.5', pushEvents: true } as unknown as McplCapabilities, mcpToolsAdvertised: false, establishGrant: () => {}, - sendFeatureSetsUpdateRequest: async () => { mutate?.(config); return answer(); }, + sendFeatureSetsUpdateRequest: async () => answer(), close: async () => {}, }; const capture = captureConsoleError(); From 082d9013ad654426029e2821b1353184467bf15e Mon Sep 17 00:00:00 2001 From: Petra-1012 Date: Tue, 13 Oct 2026 03:38:00 -0400 Subject: [PATCH 0165/1112] fix(drafts): a thrown send with no draft behind it is marker-only, as an unknown outcome is From the readers of 1479d48a. A plain envelope (no {{unsent}}) whose send threw was held as a fresh bounce draft with no attempt and no inherited risk: the notice now said it may have gone out, but the draft read as held, the notice offered {{unsent}}, and {{unsent}} or a resend would post it again with no confirmDuplicate gate. The connector's own unknown outcome is already marker-only for such words: holding them would invite a duplicate. A send that threw has no known outcome either, so bounceProse now writes a notice for it and holds nothing: the words may not have been delivered, they aren't held because a resend could post them twice, they remain in the history, and the channel is worth checking. It doesn't wake the resident, as the unknown-outcome marker doesn't. A thrown send that substituted {{unsent}} is unchanged: the substituted draft carries its own unresolved attempt, and an expanded copy inherits that risk, so its state, listing and resend gate already agree. Test (explicit-prose-routing): a thrown explicit envelope holds no draft, and its notice says it may not have been delivered and why it isn't held, offering neither {{unsent}} nor "was not delivered". --- src/framework.ts | 17 +++++++++++++++++ test/explicit-prose-routing.test.ts | 12 +++++++++++- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/src/framework.ts b/src/framework.ts index bf5b9ba77..fd53942b7 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -10442,6 +10442,23 @@ export class AgentFramework { after: { substituted?: Draft; outcome?: 'unknown' } = {}, ): void { const name = agent.name; + if (after.outcome === 'unknown' && !after.substituted) { + // Words with no draft behind them, whose send threw, may have gone + // out. Like an unknown outcome, that is marker-only: holding them as a + // draft would invite a duplicate post. + const notice = + `[prose-routing] Your text (${text.length} chars) may not have been delivered — ${reason}. ` + + "It isn't held as a draft, since a resend could post it twice; the words remain in your history. " + + 'Check that channel before writing them again.'; + try { + const id = this.addMessage('user', [{ type: 'text', text: notice }], { system: true, kind: 'prose-bounce' } as MessageMetadata, { forAgent: name }); + this.emitTrace({ type: 'message:added', messageId: id ?? 'deferred', source: 'prose-bounce' }); + } catch (err) { + console.error('bounceProse: failed to record the unknown-send notice:', err); + } + console.error(`[prose] ${name}: ${text.length} chars may not have been delivered — ${reason}`); + return; + } const words = this.unsentWords(agent, text, after.substituted); let draft: Draft | undefined; let inherited: InheritedRisk | undefined; diff --git a/test/explicit-prose-routing.test.ts b/test/explicit-prose-routing.test.ts index 84aaaaa08..b0fe6f8fb 100644 --- a/test/explicit-prose-routing.test.ts +++ b/test/explicit-prose-routing.test.ts @@ -287,7 +287,7 @@ describe('explicit prose routing', () => { (framework as unknown as { proseDrafts: DraftsView }).proseDrafts; for (const failure of ['false', 'throw'] as const) { - it('an explicit ' + failure + ' delivery holds its body as a bounce and names it privately', async () => { + it('an explicit ' + failure + ' delivery holds its body as a bounce and names it privately, unless the send threw', async () => { const framework = await createFramework(); try { const routed = stubRegistry(framework, [failure]); @@ -298,6 +298,16 @@ describe('explicit prose routing', () => { await gateway.deliverProse(agent, '>>#alpha keep these words', { notice: 'later' }); assert.deepEqual(routed, [{ text: 'keep these words', locus: 'chan-alpha' }]); const held = draftsOf(framework).open('assistant'); + if (failure === 'throw') { + // The send threw, so the words may have gone out: marker-only, as + // an unknown outcome is, with no draft to resend. + assert.deepEqual(held, [], 'words whose send threw are not held'); + const notice = context(framework).texts.find((t) => t.startsWith('[prose-routing]'))!; + assert.match(notice, /may not have been delivered/); + assert.match(notice, /isn't held as a draft, since a resend could post it twice/); + assert.doesNotMatch(notice, /\{\{unsent\}\}|was not delivered/); + return; + } assert.equal(held.length, 1, 'the failed envelope became one draft'); assert.equal(held[0]!.reason, 'bounced'); assert.equal(held[0]!.text, 'keep these words'); From c8fc8317591badc1745df62e4e2fb564718eff91 Mon Sep 17 00:00:00 2001 From: Petra-1012 Date: Tue, 13 Oct 2026 03:40:42 -0400 Subject: [PATCH 0166/1112] test(drafts): the hybrid throw test follows the marker-only rule A hybrid plain send that throws now holds nothing and its notice offers no {{unsent}} (082d9013); the test still expected the held-draft advice. Its throw variant now asserts the marker-only notice and that no draft holds the words; the failed variant keeps its assertion. --- test/explicit-prose-routing.test.ts | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/test/explicit-prose-routing.test.ts b/test/explicit-prose-routing.test.ts index b0fe6f8fb..af6cef2bb 100644 --- a/test/explicit-prose-routing.test.ts +++ b/test/explicit-prose-routing.test.ts @@ -1099,7 +1099,17 @@ describe('explicit prose routing', () => { assert.deepEqual(routed.filter(r => r.text !== 'fails'), [{ text: 'first', locus: 'discord:g:cafe' }, { text: 'recovered', locus: 'world:commons' }]); const cm = framework.getAgent('assistant')!.getContextManager(); const texts = cm.getAllMessages().flatMap(m => m.content).filter(b => b.type === 'text').map(b => (b as { text: string }).text); - assert.ok(texts.some(t => t.includes('[prose-routing]') && t.includes('>>>#channel {{unsent}}'))); + if (failure === 'throw') { + // A plain send that threw is marker-only: nothing is held, and the + // notice offers no {{unsent}}. + const notice = texts.find(t => t.startsWith('[prose-routing]') && t.includes('fails'.length + ' chars'))!; + assert.match(notice, /may not have been delivered/); + assert.match(notice, /isn't held as a draft/); + assert.doesNotMatch(notice, /\{\{unsent\}\}/); + assert.equal(draftsOf(framework).open('assistant').some(d => d.text === 'fails'), false); + } else { + assert.ok(texts.some(t => t.includes('[prose-routing]') && t.includes('>>>#channel {{unsent}}'))); + } await framework.stop(); }); } From 5d27d07d59be662a06f95b8b2b5ec001d61ae6b1 Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Tue, 13 Oct 2026 03:51:52 -0400 Subject: [PATCH 0167/1112] fix(mcp): only string env values are derived; more credential words; a refresh redacts with the configurations it began with From a review of the round before: - A JavaScript host can pass an env value that isn't a string; the redactor threw on it inside the catch handlers that record failures. Only string values are derived now. - Env name parts ending in passcode, bearer, cookie or creds mark a credential too; PWD stays out, since its value is a path. - The modern tools/list failure took its server's configuration as its turn came, after earlier servers' listings had waited; the refresh now takes every configuration as it began. - A degraded acceptance's entry without a capability list threw inside the policy request's try, was logged as an unanswered request, and skipped the rest of registration; the line reads it as unknown. - The manifest handler has one guard for a server with no configuration, before its coalescing state, instead of two. Texts: accessProvider is for a network server's rotating credentials (a stdio server's changed env goes through a restart); the every-server redactor's doc says it applies the snapshot, then the configurations present when it's used; the fragment and README say every query pair keeps the adjacency rule, credential values are replaced wherever they occur, the env words, that a stdio server's arguments aren't derived, and that the HTTP client builds the message from the body for most statuses. Tests: an env credential percent-encoded, non-string env values, the new words, each policy record present (the unanswered and degraded lines, and an entry with no capability list), and a removed server's manifest change fetching nothing. Removing each piece fails a test (five removals). --- README.md | 2 +- .../mcp-standard-tool-results.changed.md | 5 +++-- .../redact-recorded-mcp-credentials.fixed.md | 2 +- src/framework.ts | 22 ++++++++++--------- src/mcpl/protocol-family.ts | 5 +++-- src/mcpl/types.ts | 5 +++-- test/mcp-protocol-family.test.ts | 9 ++++++-- test/mcpl-config-diagnostics.test.ts | 14 +++++++++--- test/mcpl-manifest-changed.test.ts | 7 ++++++ 9 files changed, 48 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index 3a6320f54..6a9d216ac 100644 --- a/README.md +++ b/README.md @@ -314,7 +314,7 @@ supports, or its error text when it lists none, and when the list includes `2026-07-28` it says to configure the server with `protocol: 'modern'` (stdio) or reach it over http(s). -Server and transport text the framework records has the server's configured credentials replaced, in both engines. That covers connection traces, console lines, stderr traces, ops alerts, and a tool call's failure as its caller, the model and the `tool:failed` trace get it. The credentials are those the configuration holds: from the url, credential-like path segments and query names, credential-shaped or credential-named query values, and a `ws://` url's userinfo, in their encoded and decoded forms, and every other non-empty query parameter as its whole `name=value`; the static `token`, raw and url-encoded; and a stdio server's `env` values whose names end a part with a credential word, such as `GITHUB_TOKEN` or `PGPASSWORD`. Standalone strings shorter than eight characters stay unchanged because they may be ordinary words; short query values are still removed with their parameter names. The token an `accessProvider` fetches, and secrets the configuration doesn't hold, remain outside this filtering. JSON-RPC tool answers, and the error a host's own connect raises, pass through as sent. +Server and transport text the framework records has the server's configured credentials replaced, in both engines. That covers connection traces, console lines, stderr traces, ops alerts, and a tool call's failure as its caller, the model and the `tool:failed` trace get it. The credentials are those the configuration holds: from the url, credential-like path segments and query names, credential-shaped or credential-named query values, and a `ws://` url's userinfo, in their encoded and decoded forms, replaced wherever they occur; every non-empty query parameter as its whole `name=value`, where no letter or digit adjoins it; the static `token`, raw and url-encoded; and a stdio server's `env` values whose names have a part ending in a credential word, such as `GITHUB_TOKEN`, `PGPASSWORD` or `SESSION_COOKIE`, raw and url-encoded. A stdio server's command-line arguments aren't derived. Standalone strings shorter than eight characters stay unchanged because they may be ordinary words; short query values are still removed with their parameter names. Treat a submitted configuration as immutable while the server uses it: apply changes with `restartMcplServer(id, newConfig)`. The token an `accessProvider` fetches, and secrets the configuration doesn't hold, remain outside this filtering. JSON-RPC tool answers, and the error a host's own connect raises, pass through as sent. #### Feature sets diff --git a/changelog.d/mcp-standard-tool-results.changed.md b/changelog.d/mcp-standard-tool-results.changed.md index 0c197bb76..c561b1ea7 100644 --- a/changelog.d/mcp-standard-tool-results.changed.md +++ b/changelog.d/mcp-standard-tool-results.changed.md @@ -111,8 +111,9 @@ so a refused connect left the server gone; now the server keeps running. A disconnect releases its tool-prefix claim only while its own server holds it. Treat a submitted configuration as immutable while the server uses it: - change settings with `restartMcplServer(id, newConfig)`, and rotate - credentials through `accessProvider`. + change settings with `restartMcplServer(id, newConfig)`, and rotate a + network server's credentials through `accessProvider` (a stdio server's + changed `env` goes through a restart with the new configuration). - A server configuration naming no usable transport is now an error before anything is spawned or dialed. That covers a `transport` that doesn't match the url's scheme, `transport: 'http'` without a url, and an diff --git a/changelog.d/redact-recorded-mcp-credentials.fixed.md b/changelog.d/redact-recorded-mcp-credentials.fixed.md index a4a1bc37a..05e8d920f 100644 --- a/changelog.d/redact-recorded-mcp-credentials.fixed.md +++ b/changelog.d/redact-recorded-mcp-credentials.fixed.md @@ -1 +1 @@ -- Server and transport text the framework records has the server's configured credentials replaced, in both engines: connection traces, console lines, stderr traces, the ops alert (and so `logs/failures.log` and the webhook), a tool call's failure as its caller and the model get it, and the `tool:failed` trace's message and stack. The credentials are those the configuration holds: from the url, credential-like path segments and query names, credential-shaped or credential-named query values (each as written and decoded), every other non-empty parameter as its whole `name=value`, and a `ws://` url's username and password; the static `token`, raw and as a url carries it, percent- or form-encoded; and a stdio server's `env` values whose names end a part with a credential word (`GITHUB_TOKEN`, `API_KEY`, `PGPASSWORD`, `DB_PASS`, `SSH_KEY_PASSPHRASE`). A string replaced on its own has at least eight characters, since a shorter one can't be told from ordinary text; a short query value is replaced only with its name, as one `name=value`, and only where no letter or digit adjoins it. A connection's event records use a redactor built once for it, and an operation's records one built when the operation begins, so a server removed, or replaced by a restart, while the operation waits doesn't change what they replace. A 401 or 403's response body is left out of the request error's data; where the HTTP client builds the error's message from the body, as for a plain 403, the message is redacted like any record. A token an access provider fetches, and credentials the configuration doesn't hold, are outside this. JSON-RPC answers, and the error a host's own connect raises, pass as they are. +- Server and transport text the framework records has the server's configured credentials replaced, in both engines: connection traces, console lines, stderr traces, the ops alert (and so `logs/failures.log` and the webhook), a tool call's failure as its caller and the model get it, and the `tool:failed` trace's message and stack. The credentials are those the configuration holds: from the url, credential-like path segments and query names, credential-shaped or credential-named query values (each as written and decoded), every non-empty parameter as its whole `name=value`, and a `ws://` url's username and password; the static `token`, raw and as a url carries it, percent- or form-encoded; and a stdio server's `env` values whose names have a part ending in a credential word, singular or plural, with any digits after it (`GITHUB_TOKEN`, `API_KEYS`, `PGPASSWORD`, `DB_PASS`, `AUTHORIZATION`, `SESSION_COOKIE`, `TOKEN2`), raw and percent-encoded. A stdio server's command-line arguments are not derived. A string replaced on its own has at least eight characters, since a shorter one can't be told from ordinary text, and is replaced wherever it occurs. Every query parameter is also replaced as its whole `name=value`, only where no letter or digit adjoins the pair (so `v=2` leaves `dev=2` alone); a short value goes only that way. A connection's event records use a redactor built once for it, and an operation's records one built when the operation begins, so a server removed, or replaced by a restart, while the operation waits doesn't change what they replace. A 401 or 403's response body is left out of the request error's data; where the HTTP client builds the error's message from the body, as it does for most statuses, the message is redacted like any record. A token an access provider fetches, and credentials the configuration doesn't hold, are outside this. JSON-RPC answers, and the error a host's own connect raises, pass as they are. diff --git a/src/framework.ts b/src/framework.ts index 262ccee64..1a672d8c7 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -14191,9 +14191,6 @@ export class AgentFramework { st.inFlight = true; try { - const config = this.mcplServerConfigs.get(connection.id); - if (!config) return; - // §17.4 fetch. A server that cannot answer keeps its previous // manifest in force — an unparseable/failed fetch teaches nothing and // narrows nothing (§17.5 step 1). @@ -14415,7 +14412,7 @@ export class AgentFramework { if (receipt?.accepted === true && receipt.mode === 'degraded') { for (const uf of receipt.unavailableFeatures ?? []) { console.error(recorded( - `[mcpl] ${config.id} degraded: ${uf.featureSet} (${uf.effect}; missing ${uf.missingCapabilities.join(', ')})`, + `[mcpl] ${config.id} degraded: ${uf.featureSet} (${uf.effect}; missing ${Array.isArray(uf.missingCapabilities) ? uf.missingCapabilities.join(', ') : 'unknown'})`, )); } } @@ -14565,8 +14562,9 @@ export class AgentFramework { * * Treat a submitted configuration and its nested values as immutable for * as long as the server uses them. Apply changed settings with - * `restartMcplServer(id, newConfig)`; use `accessProvider` for rotating - * credentials. + * `restartMcplServer(id, newConfig)`; use `accessProvider` for a network + * server's rotating credentials (a stdio server's changed `env` goes + * through a restart with the new configuration). */ async restartMcplServer( id: string, @@ -14604,9 +14602,10 @@ export class AgentFramework { /** * A redactor for text that may come from any configured server, with every - * server's credentials replaced. It is built from the configurations as - * they are when it's made, so an operation makes it before it waits, and a - * server removed meanwhile still has its credentials replaced. + * server's credentials replaced: those of the configurations present when + * it's made, then those present when it's used. An operation makes it + * before it waits, so a server removed meanwhile, and one connected + * meanwhile, both have their credentials replaced. */ private everyServerRedactor(): (text: unknown) => string { const snapshot = [...this.mcplServerConfigs.values()].map(config => serverTextRedactor(config)); @@ -15254,9 +15253,12 @@ export class AgentFramework { // Modern servers: the same namespacing, tool policy and RFC-008 class // hints. Feature sets are MCPL and don't apply. A server whose listing // fails stays unlisted (unknown), never "listed nothing". + // Each server's configuration as the refresh began, so a server removed + // while an earlier listing waited still has its record redacted. + const configsAtStart = new Map(this.mcplServerConfigs); for (const server of this.modernServers().values()) { if (!server.isConnected) continue; - const config = this.mcplServerConfigs.get(server.id); + const config = configsAtStart.get(server.id); const prefix = config?.toolPrefix ?? `mcpl--${server.id}`; try { const listed = await server.listTools(); diff --git a/src/mcpl/protocol-family.ts b/src/mcpl/protocol-family.ts index 72f86388c..410732751 100644 --- a/src/mcpl/protocol-family.ts +++ b/src/mcpl/protocol-family.ts @@ -164,7 +164,8 @@ export function serverTextRedactor(config: Pick CREDENTIAL_NAME_PART.test(part))) withEncoded(value); + // A JavaScript host can pass a value that isn't a string; only strings are derived. + if (typeof value === 'string' && name.split(/[_\-.]/).some((part) => CREDENTIAL_NAME_PART.test(part))) withEncoded(value); } let url: URL | undefined; try { url = new URL(config.url ?? ''); } catch { /* Still remove the static token. */ } @@ -218,7 +219,7 @@ export function asText(text: unknown): string { } /** A part of an environment variable's name that ends with a credential word, in the singular or plural, with any digits after it. */ -const CREDENTIAL_NAME_PART = /(?:tokens?|keys?|secrets?|passwords?|passwd|pass|passphrases?|pat|auth|authorization|authentication|signatures?|credentials?)\d*$/i; +const CREDENTIAL_NAME_PART = /(?:tokens?|keys?|secrets?|passwords?|passwd|pass|passphrases?|passcodes?|pat|auth|authorization|authentication|bearer|cookies?|creds|signatures?|credentials?)\d*$/i; /** * A credential provider's failure as a log line may show it: the error's diff --git a/src/mcpl/types.ts b/src/mcpl/types.ts index 41049aa59..e78b51680 100644 --- a/src/mcpl/types.ts +++ b/src/mcpl/types.ts @@ -214,8 +214,9 @@ export interface McplChannelCapabilities { * * Treat a submitted configuration and its nested values as immutable for as * long as the server uses them. Apply changed settings with - * `restartMcplServer(id, newConfig)`; use `accessProvider` for rotating - * credentials. + * `restartMcplServer(id, newConfig)`; use `accessProvider` for a network + * server's rotating credentials (a stdio server's changed `env` goes through + * a restart with the new configuration). */ export interface McplServerConfig { /** Unique server identifier */ diff --git a/test/mcp-protocol-family.test.ts b/test/mcp-protocol-family.test.ts index 393825496..fb1e7fe75 100644 --- a/test/mcp-protocol-family.test.ts +++ b/test/mcp-protocol-family.test.ts @@ -335,11 +335,16 @@ test('recorded server text: the token as a url carries it, credential-named env GITHUB_TOKEN: 'ghp_abcdef123456', API_KEY: 'k3y-abcdefgh', PGPASSWORD: 'pgpass-abcdefgh', ACCESSTOKEN: 'acc-abcdefgh12', DB_PASS: 'dbpass-abcdefgh', SSH_KEY_PASSPHRASE: 'phrase-abcdefgh', AUTHORIZATION: 'Bearer-abcdefgh', API_KEYS: 'k1-abcdefgh9', TOKEN2: 'tok2-abcdefgh', API_KEY_ID: 'keyid-abcdefgh', GITHUB_PAT: 'pat-abcdefgh12', + SESSION_COOKIE: 'cookie-abcdefgh', CLOUD_CREDS: 'creds-abcdefgh', PATH: '/usr/local/bin', GIT_AUTHOR_NAME: 'Somebody Longname', KEYBOARD_LAYOUT: 'dvorak-programmer', TOKENIZERS_PARALLELISM: 'parallel-false', } }); assert.equal( - redactServerText('auth ghp_abcdef123456 k3y-abcdefgh pgpass-abcdefgh acc-abcdefgh12 dbpass-abcdefgh phrase-abcdefgh Bearer-abcdefgh k1-abcdefgh9 tok2-abcdefgh keyid-abcdefgh pat-abcdefgh12 /usr/local/bin Somebody Longname dvorak-programmer parallel-false', env), - 'auth redacted redacted redacted redacted redacted redacted redacted redacted redacted redacted redacted /usr/local/bin Somebody Longname dvorak-programmer parallel-false'); + redactServerText('auth ghp_abcdef123456 k3y-abcdefgh pgpass-abcdefgh acc-abcdefgh12 dbpass-abcdefgh phrase-abcdefgh Bearer-abcdefgh k1-abcdefgh9 tok2-abcdefgh keyid-abcdefgh pat-abcdefgh12 cookie-abcdefgh creds-abcdefgh /usr/local/bin Somebody Longname dvorak-programmer parallel-false', env), + 'auth redacted redacted redacted redacted redacted redacted redacted redacted redacted redacted redacted redacted redacted /usr/local/bin Somebody Longname dvorak-programmer parallel-false'); + // A credential env value as a url would carry it, percent-encoded. + assert.equal(redactServerText('GET /x?pw=p%40ss%2Fw0rd!x', cfg({ command: 'srv', env: { DB_PASSWORD: 'p@ss/w0rd!x' } })), 'GET /x?pw=redacted'); + // A value that isn't a string (a JavaScript host can pass one) is skipped, not thrown on. + assert.equal(redactServerText('fine', cfg({ command: 'srv', env: { API_TOKEN: null as unknown as string, OTHER_KEY: 7 as unknown as string } })), 'fine'); // A short pair only where no letter or digit adjoins it; any other string // wherever it occurs, after a percent escape, an ANSI code or a JSON escape too. const pair = cfg({ url: 'https://h/api/s/pathSecret0123456789/mcp?v=2' }); diff --git a/test/mcpl-config-diagnostics.test.ts b/test/mcpl-config-diagnostics.test.ts index 65e066b89..2ea09491d 100644 --- a/test/mcpl-config-diagnostics.test.ts +++ b/test/mcpl-config-diagnostics.test.ts @@ -386,12 +386,20 @@ test("the initial policy's other records keep the credentials out too", async () return { lines: capture.lines, traces }; }; // An unanswered request whose error quotes the url. - await run(async () => { throw new Error(`timed out talking to wss://h/mcp/${secret}`); }); + const unanswered = await run(async () => { throw new Error(`timed out talking to wss://h/mcp/${secret}`); }); + assert.ok(unanswered.lines.some((l) => l.includes('did not answer the initial policy Request') && l.includes('timed out talking to wss://h/mcp/redacted')), unanswered.lines.join('\n')); // A refusal whose fallback isn't one the host knows: it logs what it does. const odd = await run(async () => ({ accepted: false, fallback: `close-${secret}`, reason: 'no' })); assert.ok(odd.lines.some((l) => l.includes('refused initial policy (no) — fallback: mcp-only')), odd.lines.join('\n')); - // A degraded acceptance whose effect quotes the url. - await run(async () => ({ accepted: true, mode: 'degraded', unavailableFeatures: [{ featureSet: 'fs', effect: `see ${secret}`, missingCapabilities: ['x'] }] })); + // A degraded acceptance whose effect quotes the url, and an entry with no + // capability list, which is logged rather than read as an unanswered request. + const degraded = await run(async () => ({ accepted: true, mode: 'degraded', unavailableFeatures: [ + { featureSet: 'fs', effect: `see ${secret}`, missingCapabilities: ['x'] }, + { featureSet: 'odd', effect: 'none' }, + ] })); + assert.ok(degraded.lines.some((l) => l.includes('srv degraded: fs (see redacted; missing x)')), degraded.lines.join('\n')); + assert.ok(degraded.lines.some((l) => l.includes('srv degraded: odd (none; missing unknown)')), degraded.lines.join('\n')); + assert.ok(!degraded.lines.some((l) => l.includes('did not answer')), degraded.lines.join('\n')); // A refusal's reason, recorded with the derivation the operation began with. await run(async () => ({ accepted: false, fallback: 'mcp-only', reason: `bad ${secret}` })); }); diff --git a/test/mcpl-manifest-changed.test.ts b/test/mcpl-manifest-changed.test.ts index 9196cf44b..34f2b21fd 100644 --- a/test/mcpl-manifest-changed.test.ts +++ b/test/mcpl-manifest-changed.test.ts @@ -99,6 +99,13 @@ function makeHarness() { const settle = () => new Promise((r) => setImmediate(r)); +test('a manifest change from a server with no configuration (removed) fetches nothing', async () => { + const { fw, connection } = makeHarness(); + (fw as unknown as { mcplServerConfigs: Map }).mcplServerConfigs = new Map(); + await fw.handleManifestChanged(connection, { revision: 'sha256:gone', domains: ['capabilities'] }); + assert.deepEqual(connection.events.map((e) => e.kind), [], 'no fetch, grant or policy request'); +}); + test('reduction applies BEFORE the policy Request; receipt names it applied', async () => { const { fw, connection, traces } = makeHarness(); // New manifest drops channels.publish (and the chat set with it, since From a36f8ea1e8798181d79279c91858e0b950c68817 Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Tue, 13 Oct 2026 03:54:47 -0400 Subject: [PATCH 0168/1112] fix(mcp): a refresh's tool policy and prefix read the live configuration; only its failure record uses the snapshot 5d27d07d took every modern server's configuration from a snapshot made as the refresh began, for its tools/list failure record. The same variable gave the server's tool prefix and tool policy, and the refresh's loop iterates the live server map, so a server connected while an earlier listing waited was listed with no configuration: its default prefix and every tool allowed. The prefix and policy read the live configuration again, as before 5d27d07d; the failure record alone redacts with the snapshot, falling back to the live configuration. Test: a server connected under its own prefix while a refresh holds another server's listing keeps that prefix in the held refresh's list, which finishes last. Taking the policy from the snapshot fails it. --- src/framework.ts | 10 ++++++---- test/mcp-server-ownership.test.ts | 27 +++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/src/framework.ts b/src/framework.ts index 1a672d8c7..ce922210d 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -15253,12 +15253,14 @@ export class AgentFramework { // Modern servers: the same namespacing, tool policy and RFC-008 class // hints. Feature sets are MCPL and don't apply. A server whose listing // fails stays unlisted (unknown), never "listed nothing". - // Each server's configuration as the refresh began, so a server removed - // while an earlier listing waited still has its record redacted. + // Each server's configuration as the refresh began, for its failure + // record alone, so a server removed while an earlier listing waited still + // has its record redacted. Its tool policy and prefix read the live + // configuration, as a server connected meanwhile has one only there. const configsAtStart = new Map(this.mcplServerConfigs); for (const server of this.modernServers().values()) { if (!server.isConnected) continue; - const config = configsAtStart.get(server.id); + const config = this.mcplServerConfigs.get(server.id); const prefix = config?.toolPrefix ?? `mcpl--${server.id}`; try { const listed = await server.listTools(); @@ -15279,7 +15281,7 @@ export class AgentFramework { this.emitTrace({ type: 'mcpl:server-error', serverId: server.id, - error: `tools/list failed: ${redactServerText(error instanceof Error ? error.message : String(error), config ?? {})}`, + error: `tools/list failed: ${redactServerText(error instanceof Error ? error.message : String(error), configsAtStart.get(server.id) ?? config ?? {})}`, }); } } diff --git a/test/mcp-server-ownership.test.ts b/test/mcp-server-ownership.test.ts index b48f11380..44debcc03 100644 --- a/test/mcp-server-ownership.test.ts +++ b/test/mcp-server-ownership.test.ts @@ -436,6 +436,33 @@ test("a refresh in flight when a server is disconnected doesn't bring its tools }); }); +test("a server connected while a refresh waits keeps its own tool prefix in that refresh's list", async () => { + await withFramework(async ({ framework, dir }) => { + await framework.connectMcplServer(modern('m', '', join(dir, 'raw.jsonl'))); + const internals = framework as unknown as { + modernServers(): Map }>; + refreshMcplTools(): Promise; + }; + const connection = internals.modernServers().get('m')!; + const realList = connection.listTools.bind(connection); + let release!: () => void; + const gate = new Promise((resolve) => { release = resolve; }); + let listed = false; + connection.listTools = async () => { const tools = await realList(); listed = true; await gate; return tools; }; + const refreshing = internals.refreshMcplTools(); + await until(() => listed, "the refresh holds the first server's list"); + connection.listTools = realList; + // Connected while the first refresh waits, under a prefix of its own. + await framework.connectMcplServer({ ...modern('n', '', join(dir, 'raw-n.jsonl')), toolPrefix: 'custom' }); + // The held refresh finishes last, so its list is the one that stands. + release(); + await refreshing; + const names = framework.getAllTools().map((t) => t.name); + assert.ok(names.some((n) => n.startsWith('custom--')), names.join(', ')); + assert.ok(!names.some((n) => n.startsWith('mcpl--n--')), names.join(', ')); + }); +}); + test('a server lost for good, legacy or modern, leaves the tool list', async () => { await withFramework(async ({ framework, dir }) => { await framework.connectMcplServer(legacy('l', 'ok', join(dir, 'legacy.log'))); From 69ce7ec79a3410892bf737f2726f8a791c241cc1 Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Tue, 13 Oct 2026 03:55:55 -0400 Subject: [PATCH 0169/1112] fix(mcp): the modern tools/list refresh reads each server's configuration as before a36f8ea1 kept a snapshot of every configuration, taken as the refresh began, for the tools/list failure record. It isn't needed: each server's configuration is read before that server's own listing waits, and a server removed while an earlier listing waits leaves the live server map too, so the refresh doesn't list it. The snapshot is gone; the record redacts with the configuration the server's tools are listed under, as before 5d27d07d. The test from a36f8ea1 stays: a server connected while a refresh holds another server's listing keeps its own prefix in that refresh's list. --- src/framework.ts | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/framework.ts b/src/framework.ts index ce922210d..2653a1e13 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -15253,11 +15253,6 @@ export class AgentFramework { // Modern servers: the same namespacing, tool policy and RFC-008 class // hints. Feature sets are MCPL and don't apply. A server whose listing // fails stays unlisted (unknown), never "listed nothing". - // Each server's configuration as the refresh began, for its failure - // record alone, so a server removed while an earlier listing waited still - // has its record redacted. Its tool policy and prefix read the live - // configuration, as a server connected meanwhile has one only there. - const configsAtStart = new Map(this.mcplServerConfigs); for (const server of this.modernServers().values()) { if (!server.isConnected) continue; const config = this.mcplServerConfigs.get(server.id); @@ -15281,7 +15276,7 @@ export class AgentFramework { this.emitTrace({ type: 'mcpl:server-error', serverId: server.id, - error: `tools/list failed: ${redactServerText(error instanceof Error ? error.message : String(error), configsAtStart.get(server.id) ?? config ?? {})}`, + error: `tools/list failed: ${redactServerText(error instanceof Error ? error.message : String(error), config ?? {})}`, }); } } From 7c76a2e178d87634071731bb035dd69c711a6196 Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Tue, 13 Oct 2026 05:10:53 -0400 Subject: [PATCH 0170/1112] fix(mcp): a server request handler's failure is recorded with the connection's redactor, before it's cut The merge of #263 brought refuseServerItem, which logs and answers a server request whose handler threw (push/event, inference/request, channels/register, channels/changed, channels/incoming, host/command). Its failure text quotes the handler's error, which can quote what the server sent, and it bypassed this branch's redaction: the console line and the -32000/-32603 answer carried a configured credential whole, or its first characters where it straddled the 500-character cut. refuseServerItem takes the connection's redactor, which wireMcplEvents builds once; thrownText redacts the error's text before cutting it, inside the guard that reads it, so an unreadable value still reads as unreadable and a request is still answered once. Test: a push/event handler that throws an error quoting the server's configured token, once whole and once with it across the cut (nine characters kept), is logged and answered without it. Removing the redaction fails it, and so does cutting before redacting. --- src/framework.ts | 40 +++++++++++++++------------ test/stop-with-work-in-flight.test.ts | 28 ++++++++++++++++++- 2 files changed, 49 insertions(+), 19 deletions(-) diff --git a/src/framework.ts b/src/framework.ts index 3f55bbe52..e9a1306e1 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -1044,10 +1044,12 @@ const THROWN_TEXT_LIMIT = 500; * The text is bounded, since a provider or handler error can carry a whole * request body. */ -function thrownText(error: unknown): string { +function thrownText(error: unknown, recorded: (text: unknown) => string = asText): string { let text: string; try { - text = String(error instanceof Error ? error.message : error); + // Redacted before it's cut, so no part of a credential across the cut + // is kept. + text = recorded(String(error instanceof Error ? error.message : error)); } catch { text = '(a value that could not be read)'; } @@ -1059,15 +1061,15 @@ function thrownText(error: unknown): string { * handler failed" and what it threw (nothing, for `undefined`). Telling an * Error apart is itself guarded, since instanceof throws on a revoked Proxy. */ -function handlerFailureText(error: unknown): string { +function handlerFailureText(error: unknown, recorded?: (text: unknown) => string): string { let isError = false; try { isError = error instanceof Error; } catch { // Not an Error that can be read; thrownText says so. } - if (isError) return thrownText(error); - return error === undefined ? 'the handler failed' : `the handler failed: ${thrownText(error)}`; + if (isError) return thrownText(error, recorded); + return error === undefined ? 'the handler failed' : `the handler failed: ${thrownText(error, recorded)}`; } /** A server's responder, and whether the request it belongs to has been answered yet. */ @@ -2626,10 +2628,12 @@ export class AgentFramework { serverId: string, answer: AnswerOnce, outcome: { kind: 'refused' } | { kind: 'failed'; error: unknown }, + /** The connection's redactor: a handler's error can quote what the server sent. */ + recorded: (text: unknown) => string, ): void { // Optional chaining: prototype-built harnesses leave the queue unset. const stopping = outcome.kind === 'refused' || !!this.queue?.isClosed; - const what = outcome.kind === 'refused' ? '' : handlerFailureText(outcome.error); + const what = outcome.kind === 'refused' ? '' : handlerFailureText(outcome.error, recorded); const reason = outcome.kind === 'refused' ? 'the host is stopping' : stopping @@ -15206,12 +15210,12 @@ export class AgentFramework { // Checked here, after the barrier and before anything records the // push as accepted: a stopped host refuses it. if (this.queue?.isClosed) { - this.refuseServerItem('push/event', connection.id, answer, { kind: 'refused' }); + this.refuseServerItem('push/event', connection.id, answer, { kind: 'refused' }, recorded); return; } await this.pushHandler?.handlePushEvent(connection.id, params, answer.responder as never); } catch (error) { - this.refuseServerItem('push/event', connection.id, answer, { kind: 'failed', error }); + this.refuseServerItem('push/event', connection.id, answer, { kind: 'failed', error }, recorded); } }); @@ -15237,7 +15241,7 @@ export class AgentFramework { // A stopping host starts no inference for a server: refused, like a // push, before the router runs. if (this.queue?.isClosed) { - this.refuseServerItem('inference/request', connection.id, answer, { kind: 'refused' }); + this.refuseServerItem('inference/request', connection.id, answer, { kind: 'refused' }, recorded); return; } await this.inferenceRouter.handleInferenceRequest(connection.id, params, { @@ -15246,7 +15250,7 @@ export class AgentFramework { requestId: responder.id, }); } catch (error) { - this.refuseServerItem('inference/request', connection.id, answer, { kind: 'failed', error }); + this.refuseServerItem('inference/request', connection.id, answer, { kind: 'failed', error }, recorded); } } }); @@ -15259,12 +15263,12 @@ export class AgentFramework { const answer = answerOnce(responder, 'channels/register', connection.id); try { if (this.queue?.isClosed) { - this.refuseServerItem('channels/register', connection.id, answer, { kind: 'refused' }); + this.refuseServerItem('channels/register', connection.id, answer, { kind: 'refused' }, recorded); return; } await this.channelRegistry?.handleRegister(connection.id, params, answer.responder as never); } catch (error) { - this.refuseServerItem('channels/register', connection.id, answer, { kind: 'failed', error }); + this.refuseServerItem('channels/register', connection.id, answer, { kind: 'failed', error }, recorded); } }); @@ -15279,12 +15283,12 @@ export class AgentFramework { const answer = answerOnce(responder, 'channels/changed', connection.id); try { if (this.queue?.isClosed) { - this.refuseServerItem('channels/changed', connection.id, answer, { kind: 'refused' }); + this.refuseServerItem('channels/changed', connection.id, answer, { kind: 'refused' }, recorded); return; } await this.channelRegistry?.handleChanged(connection.id, params, answer.responder as never); } catch (error) { - this.refuseServerItem('channels/changed', connection.id, answer, { kind: 'failed', error }); + this.refuseServerItem('channels/changed', connection.id, answer, { kind: 'failed', error }, recorded); } }); @@ -15351,12 +15355,12 @@ export class AgentFramework { const barrier = this.discordAwarenessBarrier; if (barrier) await barrier.promise; if (this.queue?.isClosed) { - this.refuseServerItem('channels/incoming', connection.id, answer, { kind: 'refused' }); + this.refuseServerItem('channels/incoming', connection.id, answer, { kind: 'refused' }, recorded); return; } await this.channelRegistry?.handleIncoming(connection.id, params, answer.responder as never); } catch (error) { - this.refuseServerItem('channels/incoming', connection.id, answer, { kind: 'failed', error }); + this.refuseServerItem('channels/incoming', connection.id, answer, { kind: 'failed', error }, recorded); } }); @@ -15373,13 +15377,13 @@ export class AgentFramework { // A stopping host runs no host command: refused like the requests // above, before its handler runs. if (this.queue?.isClosed) { - this.refuseServerItem('host/command', connection.id, answer, { kind: 'refused' }); + this.refuseServerItem('host/command', connection.id, answer, { kind: 'refused' }, recorded); return; } const result = await this.handleHostCommand(connection.id, params ?? {}); answer.responder!.respond(result); } catch (error) { - this.refuseServerItem('host/command', connection.id, answer, { kind: 'failed', error }); + this.refuseServerItem('host/command', connection.id, answer, { kind: 'failed', error }, recorded); } }); diff --git a/test/stop-with-work-in-flight.test.ts b/test/stop-with-work-in-flight.test.ts index 2674a1f4f..81fc71d60 100644 --- a/test/stop-with-work-in-flight.test.ts +++ b/test/stop-with-work-in-flight.test.ts @@ -375,7 +375,7 @@ describe('stop() with work still in flight', () => { modules: [], }); const connection = Object.assign(new EventEmitter(), { id: 'srv' }); - (framework as unknown as { wireMcplEvents(c: unknown, config: unknown): void }).wireMcplEvents(connection, { id: 'srv', command: 'unused' }); + (framework as unknown as { wireMcplEvents(c: unknown, config: unknown): void }).wireMcplEvents(connection, { id: 'srv', url: 'wss://h/mcp?token=wireToken0123456789' }); return { framework, connection }; } @@ -596,6 +596,32 @@ describe('stop() with work still in flight', () => { } }); + it("a handler's error quoting the server's configured credential is recorded and answered without it, also across the cut", async () => { + const { framework, connection } = await wiredFramework(); + try { + // The credential starts just before the 500-character cut, so a cut + // made before redacting would keep its first characters. + // Once whole, and once with the credential across the 500-character + // cut, where a cut made before redacting keeps its first nine. + const errors = ['refused wireToken0123456789', `${'x'.repeat(490)} wireToken0123456789 and more`]; + for (const [i, error] of errors.entries()) { + (framework as unknown as { pushHandler: unknown }).pushHandler = { + handlePushEvent: async () => { throw new Error(error); }, + }; + const push = responder(); + connection.emit('push-event', { featureSet: 'chat', eventId: `e-cred-${i}`, payload: { content: [{ type: 'text', text: 'x' }] } }, push); + await answered(push); + const message = push.answers[0]!.error!.message; + await until(() => logged.filter((line) => line.includes('push/event from srv failed')).length > i, 'the log line'); + for (const text of [message, ...logged]) { + assert.ok(!text.includes('wireToken'), text.slice(-120)); + } + } + } finally { + await framework.stop(); + } + }); + it('a request the handler already answered is not answered again when the handler then throws', async () => { const { framework, connection } = await wiredFramework(); try { From 07d622315b33610d589e51a8ce73195f83421a06 Mon Sep 17 00:00:00 2001 From: Petra-1012 Date: Tue, 13 Oct 2026 06:39:30 -0400 Subject: [PATCH 0171/1112] fix(drafts): a logical turn's drafts settle at its end, and the catch-up names only drafts no notice names Follow-up to the merge of main (#312's logical turns). The drafts now ride in the LogicalTurn, so they follow it the way its deliveries do. - settleTurnDrafts runs in driveStream's finally only once the stream's logical turn has ended: openLogical no longer holds it (it settled, or it was a silent start's own). A stream ends owning the agent with its turn still open when a provider-capacity hold defers the turn, whose later compile continues it. Settling there spent the drafts, so that compile's receipt left out the first stream's drafts, its kept-private count and its draft failures, though it reports the first stream's deliveries. Now the stream that ends the turn settles them, after its receipt names them. - The turn-start catch-up leaves the open logical turn's own drafts to that turn: its receipt names them and its settlement records them. Before the continuing start's compile it named the ones no notice had named yet as unnamed, and the receipt then named them again. - The catch-up records, instead of naming, a draft that a message in the recent history already names (draftsNamedInHistory, the same window settleTurnDrafts reads). A notice can reach the history after its turn's drafts were settled: in explicit mode a turn a tool ends doesn't wait for its queued deliveries, so an envelope of it that bounces later is held into that turn's settled logical turn, and its bounce notice names a draft nothing records. The catch-up then said "unnamed on this branch" of a draft a notice names. It applies equally to a notice whose recording failed. recordDraftNotices is the marking both use. - changelog: the receipt names every draft held in the turn, in any of its streams. Tests: - explicit-prose-routing: a capacity-held hybrid turn's later compile writes one receipt with its own delivery and the draft its first stream held, and no catch-up names it (an acceleration 429 replaces stream one's completion; 50 ms cooldown). An explicit turn that skip_reply ends with a queued '>>#nowhere' envelope behind a held post: the bounce notice names the draft, the next turn's catch-up doesn't, and the draft is recorded as noticed. - prose-drafts: the catch-up records a draft whose notice is in the history and names only the other; it leaves the open logical turn's draft unnamed and unrecorded. --- changelog.d/prose-drafts.added.md | 4 +- src/framework.ts | 33 +++++++-- test/explicit-prose-routing.test.ts | 100 +++++++++++++++++++++++++++- test/prose-drafts.test.ts | 58 ++++++++++++++++ 4 files changed, 186 insertions(+), 9 deletions(-) diff --git a/changelog.d/prose-drafts.added.md b/changelog.d/prose-drafts.added.md index da3bf953a..ba9944f97 100644 --- a/changelog.d/prose-drafts.added.md +++ b/changelog.d/prose-drafts.added.md @@ -33,7 +33,9 @@ stream presents mid-turn messages (a draft holding words copied from one that may already have been posted is named with that risk and the `confirmDuplicate` its resend needs), and the turn-end `[delivered]` receipt - names every draft held in the turn by its state at turn end: still held + names every draft held in the turn, in any of its streams (a framework + retry, or the compile a provider-capacity hold defers the turn to), by its + state at turn end: still held ("held as drafts d-… (not sent …)"), delivered by the resident's resend (listed where it landed), unconfirmed, or dismissed. A dismissed draft whose own attempt may already have been posted is named with that risk, diff --git a/src/framework.ts b/src/framework.ts index a533cf24c..cf64b472f 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -9551,11 +9551,21 @@ export class AgentFramework { * move took the notices away. Every draft it marks is named by id, newest * first: the newest eight with their state and why each was held, the rest * by id. Like every notice, it quotes none of their words; the drafts tool - * shows them. + * shows them. A draft a recent message already names is recorded as + * noticed rather than named again: its notice landed after its turn ended, + * as a bounce does when a delivery runs after a tool ends an explicit turn. + * The open logical turn's own drafts are left to its receipt and its + * settlement (settleTurnDrafts). */ private noticeUnnoticedDrafts(agent: Agent): void { const cm = agent.getContextManager(); - const pending = this.proseDrafts.unnoticed(agent.name, (id) => cm.getMessage(id as MessageId) !== null); + const own = new Set(this.openLogical.get(agent.name)?.drafts.map((d) => d.id)); + const unnoticed = this.proseDrafts.unnoticed(agent.name, (id) => cm.getMessage(id as MessageId) !== null) + .filter((d) => !own.has(d.id)); + if (unnoticed.length === 0) return; + const named = this.draftsNamedInHistory(agent, unnoticed.map((d) => d.id)); + this.recordDraftNotices(agent, named); + const pending = unnoticed.filter((d) => !named.has(d.id)); if (pending.length === 0) return; const shown = pending.slice(0, 8).map((d) => `${d.id} (${this.draftStateText(d)}; ${AgentFramework.DRAFT_REASON_SHORT[d.reason]})`); @@ -9580,7 +9590,7 @@ export class AgentFramework { try { this.proseDrafts.markNoticed(agent.name, pending.map((d) => d.id), mid); } catch (err) { - // Named again by the next turn's catch-up. + // The next turn's catch-up finds this notice and records it. console.error('[drafts] could not record the catch-up notice:', err); } } @@ -9598,15 +9608,21 @@ export class AgentFramework { logical.keptPrivate = 0; delete logical.draftFailures; if (held.length === 0) return; + this.recordDraftNotices(agent, this.draftsNamedInHistory(agent, held.map((d) => d.id))); + } + + /** Record each named draft as noticed, with the message that names it + * (draftsNamedInHistory). */ + private recordDraftNotices(agent: Agent, named: Map): void { const byMessage = new Map(); - for (const [draftId, messageId] of this.draftsNamedInHistory(agent, held.map((d) => d.id))) { + for (const [draftId, messageId] of named) { byMessage.set(messageId, [...(byMessage.get(messageId) ?? []), draftId]); } for (const [messageId, ids] of byMessage) { try { this.proseDrafts.markNoticed(agent.name, ids, messageId); } catch (err) { - // Unmarked drafts are named again by the next turn's catch-up. + // The next turn's catch-up looks for their notices again. console.error('[drafts] could not record notice:', err); } } @@ -12973,8 +12989,11 @@ export class AgentFramework { } // The logical turn has ended and its notices are written: record the - // drafts they name as noticed (the store is synced first). - if (frameReachedTerminal && ownsPhysicalStream) { + // drafts they name as noticed (the store is synced first). A stream + // whose logical turn is still open leaves its drafts to the stream + // that ends it (openLogical: the compile a provider-capacity hold + // defers the turn to), whose receipt names them. + if (frameReachedTerminal && ownsPhysicalStream && this.openLogical.get(agent.name) !== streamRouting.logical) { try { this.settleTurnDrafts(agent, streamRouting.logical); } catch (err) { diff --git a/test/explicit-prose-routing.test.ts b/test/explicit-prose-routing.test.ts index 49f1cfc8a..c54d24bd7 100644 --- a/test/explicit-prose-routing.test.ts +++ b/test/explicit-prose-routing.test.ts @@ -28,7 +28,7 @@ import { AgentFramework } from '../src/index.js'; import { MockMembrane, createMockResponse } from './helpers/mock-membrane.js'; import { ProseStreamRouter } from '../src/mcpl/prose-stream-router.js'; import { draftState, type ProseDraftStore } from '../src/prose-drafts.js'; -import type { ContentBlock } from '@animalabs/membrane'; +import { MembraneError, type ContentBlock } from '@animalabs/membrane'; class ToolboxModule implements Module { readonly name = 'robot'; @@ -1190,6 +1190,104 @@ describe('explicit prose routing', () => { }); } + it("hybrid: a capacity-held turn's later compile receipts the drafts its first stream held, and the catch-up leaves them to it", async () => { + // Stream one bounces an envelope into a draft, then its next request + // meets an organization acceleration 429: the turn is held and its later + // compile continues the same logical turn, whose receipt names the draft. + membrane.pushResponse(createMockResponse([ + { type: 'text', text: '>>>#nowhere held words' }, + { type: 'tool_use', id: 'capacity-1', name: 'robot--move', input: { dir: 'north' } }, + ] as ContentBlock[], 'tool_use')); + membrane.pushResponse(createMockResponse([] as ContentBlock[])); + const framework = await createFramework('hybrid'); + stubRegistry(framework); + const internals = framework as unknown as { + providerAccelerationDefaultCooldownMs: number; + providerAccelerationJitterMs: number; + openLogical: Map; + }; + internals.providerAccelerationDefaultCooldownMs = 50; + internals.providerAccelerationJitterMs = 0; + let starts = 0; + framework.onTrace((event) => { + if (event.type !== 'inference:started') return; + starts++; + if (starts === 2) membrane.pushResponse(createMockResponse([{ type: 'text', text: 'carrying on' }])); + if (starts >= 3) membrane.pushResponse(createMockResponse([{ type: 'text', text: '>>>skip_reply' }])); + }); + const makeStream = membrane.streamYielding.bind(membrane); + let streams = 0; + membrane.streamYielding = (request, options) => { + const stream = makeStream(request, options); + const index = ++streams; + const iterate = stream[Symbol.asyncIterator].bind(stream); + stream[Symbol.asyncIterator] = async function* () { + for await (const event of { [Symbol.asyncIterator]: iterate }) { + if (event.type === 'complete' && index === 1) { + yield { + type: 'error' as const, + error: new MembraneError({ + type: 'rate_limit', retryable: true, httpStatus: 429, + message: "This request would exceed your organization's maximum usage increase rate for input tokens per minute", + rawError: { status: 429 }, + }), + }; + return; + } + yield event; + } + }; + return stream; + }; + try { + trigger(framework); + await framework.runUntilIdle(); + const draft = draftsOf(framework).open('assistant').find((d) => d.text === 'held words'); + assert.ok(draft, 'the bounced envelope is held as a draft'); + const open = internals.openLogical.get('assistant'); + assert.ok(open?.drafts.some((d) => d.id === draft.id), "the held turn's logical turn is open, its draft unsettled"); + for (const end = Date.now() + 5000; starts < 2 && Date.now() < end;) { + await new Promise((resolve) => setTimeout(resolve, 10)); + await framework.runUntilIdle(); + } + assert.equal(starts, 2, 'the held turn was compiled again after its cooldown'); + await framework.runUntilIdle(); + const texts = context(framework).texts; + const receipts = texts.filter((t) => t.startsWith('[delivered]')); + assert.equal(receipts.length, 1, `one receipt for the logical turn: ${receipts.join(' | ')}`); + assert.match(receipts[0]!, /world:commons/, "it reports the later compile's delivery"); + assert.ok(receipts[0]!.includes(draft.id), `and the draft the first stream held: ${receipts[0]}`); + assert.ok(!texts.some((t) => t.includes('unnamed on this branch')), 'the catch-up left the draft to the receipt'); + assert.equal(internals.openLogical.has('assistant'), false, 'the logical turn settled'); + } finally { + await framework.stop(); + } + }); + + it("explicit: a draft bounced after a tool ended its turn is recorded from its notice, not named again by the catch-up", async () => { + // The ended turn's queued envelope bounces once the held post before it + // settles, after that turn's drafts were settled. Its bounce notice names + // it; the next turn's catch-up records that notice rather than naming + // the draft as unnamed. + await withQueuedTurns('explicit', '>>#nowhere late words', [ + createMockResponse([{ type: 'text', text: '>>#cafe next' }]), + ], async ({ framework, release, drain, turns }) => { + release(); + for (const end = Date.now() + 5000; turns() < 3 && Date.now() < end;) { + await new Promise((resolve) => setTimeout(resolve, 5)); + } + await drain(); + assert.equal(turns(), 3, "the bounce's wake started a turn"); + const store = draftsOf(framework); + const late = store.open('assistant').find((d) => d.text === 'late words'); + assert.ok(late, 'the late envelope bounced into a draft'); + const texts = context(framework).texts; + assert.ok(texts.some((t) => t.startsWith('[prose-routing]') && t.includes(late.id)), 'its bounce notice names it'); + assert.ok(!texts.some((t) => t.includes('unnamed on this branch')), "the catch-up doesn't name it again"); + assert.ok((store.get('assistant', late.id)?.noticedIn ?? []).length > 0, 'it is recorded as noticed'); + }); + }); + it("the non-streaming fallback leaves the scheduling turn's logical turn open for the re-inference", async () => { // Adapted from a reviewer's producer test: the agent's direct inference // API leaves tools pending with no yielding stream, and the tool result diff --git a/test/prose-drafts.test.ts b/test/prose-drafts.test.ts index 52c9245b6..7b450ee61 100644 --- a/test/prose-drafts.test.ts +++ b/test/prose-drafts.test.ts @@ -1127,6 +1127,64 @@ describe('draft notices and turn-end settlement', () => { rmSync(dir, { recursive: true, force: true }); } }); + + const catchUpFixture = async () => { + const dir = mkdtempSync(join(tmpdir(), 'prose-drafts-catchup-')); + const framework = await AgentFramework.create({ + storePath: join(dir, 'store'), + membrane: new MockMembrane().asMembrane(), + agents: [{ name: 'scout', model: 'test-model', systemPrompt: 'You are scout.' }], + modules: [], + }); + const internals = framework as unknown as { + proseDrafts: ProseDraftStore; + routingOf(name: string): { logical: { drafts: unknown[] } }; + openLogical: Map; + noticeUnnoticedDrafts(agent: unknown): void; + }; + const agent = framework.getAgent('scout')!; + const cm = agent.getContextManager(); + const catchUps = () => cm.getAllMessages() + .flatMap((m) => m.content).filter((b) => b.type === 'text').map((b) => (b as { text: string }).text) + .filter((t) => t.includes('unnamed on this branch')); + return { framework, internals, agent, cm, catchUps, close: async () => { await framework.stop(); rmSync(dir, { recursive: true, force: true }); } }; + }; + + it("the catch-up records a draft a notice already names, rather than naming it again", async () => { + const t = await catchUpFixture(); + try { + const [late, lost] = t.internals.proseDrafts.hold('scout', [{ text: 'late', source }, { text: 'lost', source }], 'bounced').held; + // A bounce notice reached the history after its turn's drafts were + // settled, so nothing recorded it. + const notice = t.cm.addMessage('user', [{ type: 'text', text: '[prose-routing] …' }], { system: true, kind: 'prose-bounce', draftOwner: 'scout', draftIds: [late!.id] } as never); + t.internals.noticeUnnoticedDrafts(t.agent); + assert.deepEqual(t.internals.proseDrafts.get('scout', late!.id)!.noticedIn, [String(notice)], 'recorded with the notice that names it'); + const [catchUp] = t.catchUps(); + assert.ok(catchUp?.includes(lost!.id), 'the draft no notice names is named'); + assert.ok(!catchUp!.includes(late!.id), `the one its notice names is not: ${catchUp}`); + } finally { + await t.close(); + } + }); + + it("the catch-up leaves the open logical turn's own drafts to its receipt", async () => { + const t = await catchUpFixture(); + try { + const [own, other] = t.internals.proseDrafts.hold('scout', [{ text: 'own', source }, { text: 'other', source }], 'explicit-send').held; + // A later start continues this logical turn, whose receipt names its + // drafts at the turn's end. + const logical = t.internals.routingOf('scout').logical; + logical.drafts.push(own); + t.internals.openLogical.set('scout', logical); + t.internals.noticeUnnoticedDrafts(t.agent); + const [catchUp] = t.catchUps(); + assert.ok(catchUp?.includes(other!.id), "another turn's unnamed draft is named"); + assert.ok(!catchUp!.includes(own!.id), `the open turn's own draft is not: ${catchUp}`); + assert.equal(t.internals.proseDrafts.get('scout', own!.id)!.noticedIn, undefined, 'and stays for its settlement'); + } finally { + await t.close(); + } + }); }); describe('draft identity and risk evidence', () => { From 2f50a4b38b47c7aa4a451d5492bffa259de86908 Mon Sep 17 00:00:00 2001 From: Petra-1012 Date: Tue, 13 Oct 2026 06:52:07 -0400 Subject: [PATCH 0172/1112] test(drafts): a hold beside a send writes its own round's routing; an envelope after >>>skip_reply counts as kept private Two pieces no test held in J6360's red checks at 07d62231: - I6: holdSilencedProse writing the live turn's routing instead of its round's stayed green. A direct test now holds a hybrid round (a failed '>>>#nowhere' envelope and the text after it) under a queued routing while another is live: the failed hold and both drafts are the queued round's, and the live turn is untouched. - I10: deliverHybridProse's count of an unprefixed envelope after a '>>>skip_reply' hold stayed green when removed. A direct test sends '>>>skip_reply an aside' then 'still private': nothing is sent, the hold is 'private', the second envelope counts as kept private, and nothing is drafted. --- test/explicit-prose-routing.test.ts | 46 +++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/test/explicit-prose-routing.test.ts b/test/explicit-prose-routing.test.ts index c54d24bd7..24627f647 100644 --- a/test/explicit-prose-routing.test.ts +++ b/test/explicit-prose-routing.test.ts @@ -1721,6 +1721,52 @@ describe('explicit prose routing', () => { } }); + it('hybrid: a hold beside a send writes the routing and drafts of its own round, not the live turn\'s', async () => { + const framework = await createFramework('hybrid'); + stubRegistry(framework); + const gateway = framework as unknown as { + holdSilencedProse(agent: unknown, segments: string[], cause: 'send', privateThink: boolean, round: { routing: RoutingState; notice: 'later' }): void; + turnRouting: Map; + }; + const agent = framework.getAgent('assistant')!; + try { + const queued: RoutingState = { destination: null, + logical: { continuation: false, deliveries: [], suppressed: 0, drafts: [], keptPrivate: 0 } }; + const live: RoutingState = { ...queued, logical: { continuation: false, deliveries: [], suppressed: 0, drafts: [], keptPrivate: 0 } }; + gateway.turnRouting.set('assistant', live); + // A held round that a later start has overtaken: its envelopes move + // its own turn's routing, and its drafts are that turn's. + gateway.holdSilencedProse(agent, ['>>>#nowhere held words', 'after it'], 'send', false, { routing: queued, notice: 'later' }); + assert.equal(queued.hybridHold, 'failed', 'the round that held it records the failed envelope'); + assert.deepEqual(queued.logical.drafts.map((d) => d.text), ['held words', 'after it'], "and holds the round's words as its drafts"); + assert.equal(live.hybridHold, undefined, 'the live turn is untouched'); + assert.deepEqual(live.logical.drafts, [], 'and holds no draft of that round'); + } finally { + await framework.stop(); + } + }); + + it('hybrid: after >>>skip_reply, the next unprefixed envelope stays in the context and counts as kept private', async () => { + const framework = await createFramework('hybrid'); + const routed = stubRegistry(framework); + const gateway = framework as unknown as { + deliverHybridProse(agent: unknown, text: string, locus: string, allowLocus: boolean, round: { routing: RoutingState; notice: 'later' }): Promise; + }; + const agent = framework.getAgent('assistant')!; + try { + const routing: RoutingState = { destination: null, + logical: { continuation: false, deliveries: [], suppressed: 0, drafts: [], keptPrivate: 0 } }; + await gateway.deliverHybridProse(agent, '>>>skip_reply an aside', 'chan-locus', true, { routing, notice: 'later' }); + await gateway.deliverHybridProse(agent, 'still private', 'chan-locus', true, { routing, notice: 'later' }); + assert.deepEqual(routed, [], 'nothing is sent'); + assert.equal(routing.hybridHold, 'private'); + assert.equal(routing.logical.keptPrivate, 1, 'the unprefixed envelope after it is counted as kept private'); + assert.deepEqual(routing.logical.drafts, [], 'and is never drafted'); + } finally { + await framework.stop(); + } + }); + it('admission turns a resolver that throws into an envelope that bounces', async () => { const framework = await createFramework(); const routed = stubRegistry(framework); From 78ce9aa42c1c26b3f874bd847567a0ab9378a28a Mon Sep 17 00:00:00 2001 From: Petra-1012 Date: Tue, 13 Oct 2026 07:02:10 -0400 Subject: [PATCH 0173/1112] fix(drafts): a round carries its turn token; the catch-up says what it checks and leaves out notices on their way; the receipt names every draft From the review of the merge round (sw-af247q, judged) and the readers' notes (Arthur-2378, room-203 #84910 and #84923, J6382; Agnes-1009, #84913): - Provenance (sw-af247q I14; Arthur, Agnes): holdProseDrafts took a draft's source.turn from the agent's current turn token when it held, so a late hold or bounce journaled a later turn's token beside the old round's number. ProseRound now carries `turn`, the stream's token captured at driveStream entry, and every hold takes it from its round. The test-only deliverProse entry gives the live token, as a live round would. DraftSource's docs now say what the fields hold: `turn` is the token of the stream whose round wrote the words (a restart or retry has its own), and `segment` is the position among the words held together. - The catch-up's wording (I18; Arthur, Agnes): it reads only the newest 500 messages, so "unnamed on this branch of your history" claimed more than it checks. It now names open drafts "that no recent notice on this branch names". - The catch-up leaves out drafts that a notice still waiting in the deferred queue names (I21). A bounce that lands while a turn's start awaits push assembly is deferred, and the catch-up then named its draft just before the bounce notice landed and named it again. - The receipt names every draft by id (I20). Its metadata names them all, so settleTurnDrafts records each one as noticed, but its text named eight and counted the rest. The catch-up already names every draft it marks (2c198b5). - A refused {{unsent}} says when the envelope's own words couldn't be held (I7). Explicit mode writes no receipt to report that failed hold. - The receipt's kept-private count says "(skip_reply or a private think)": a same-round private think beside a send is counted there too. - changelog (I13, I27): a silent turn writes no receipt, so its private words aren't counted; the receipt is written in locus and hybrid modes, and explicit mode's drafts are named by their notices; the catch-up's window, and what it records and leaves out. Tests: - explicit-prose-routing: the ordered held-send test asserts the copy's source.turn is the turn that attempted it (Arthur-2378's probe), and for the bare variant that the re-bounce notice offers {{unsent}}, so the order is pinned without the timed check (I11). The hold beside a send records its round's turn. A refused {{unsent}} whose hold fails says so. A receipt with eleven held drafts names all eleven. - prose-drafts: a draft a deferred notice names is left out of the catch-up. The kept-private label and the catch-up's wording are updated where tests pin them, and the private-think turn's receipt is now asserted. --- changelog.d/prose-drafts.added.md | 27 ++++++----- src/framework.ts | 62 +++++++++++++++++-------- src/prose-drafts.ts | 6 ++- test/explicit-prose-routing.test.ts | 71 ++++++++++++++++++++++++++--- test/prose-drafts.test.ts | 42 +++++++++++++---- 5 files changed, 161 insertions(+), 47 deletions(-) diff --git a/changelog.d/prose-drafts.added.md b/changelog.d/prose-drafts.added.md index ba9944f97..393da491b 100644 --- a/changelog.d/prose-drafts.added.md +++ b/changelog.d/prose-drafts.added.md @@ -13,8 +13,8 @@ the rest private until a destination is named), a same-round private `think` in a native round (where that policy applies: XML tool mode routes such prose as ordinary speech, so a send holds it as a draft), silent turns - and `proseRouting: disabled` narration stay private and are counted in the - receipt. + and `proseRouting: disabled` narration stay private. The turn's receipt + counts them, except a silent turn's: a silent turn writes no receipt. - In hybrid mode a send withholds publication, not what an envelope does: a held `>>>` envelope moves the routing state as live routing would (a destination it names becomes the sticky target), `>>>skip_reply {{unsent}}` @@ -32,19 +32,24 @@ - A private notice names held drafts at the next tool boundary when the live stream presents mid-turn messages (a draft holding words copied from one that may already have been posted is named with that risk and the - `confirmDuplicate` its resend needs), and the turn-end `[delivered]` receipt - names every draft held in the turn, in any of its streams (a framework - retry, or the compile a provider-capacity hold defers the turn to), by its - state at turn end: still held + `confirmDuplicate` its resend needs). In locus and hybrid modes the + turn-end `[delivered]` receipt names every draft held in the turn, in any + of its streams (a framework retry, or the compile a provider-capacity hold + defers the turn to), by id and by its state at turn end: still held ("held as drafts d-… (not sent …)"), delivered by the resident's resend (listed where it landed), unconfirmed, or dismissed. A dismissed draft whose own attempt may already have been posted is named with that risk, and counts like an unconfirmed one: a receipt that names one, with nothing - confirmed delivered, reads "nothing confirmed", not "nothing". A draft - counts as named while a message naming it is on the current branch, so one - no notice reached (a crash, an aborted turn), or whose notices a rollback - or branch move took away, is named at the next turn's start: every such - open draft by id, the newest eight with its state and why it was held. Notices + confirmed delivered, reads "nothing confirmed", not "nothing". Explicit + mode writes no receipt: its drafts are named by their bounce and refusal + notices. A draft counts as named while a message naming it is on the + current branch, so one no notice reached (a crash, an aborted turn), or + whose notices a rollback or branch move took away, is named at the next + turn's start: every such open draft by id, the newest eight with its state + and why it was held. That catch-up reads the recent history (the newest + 500 messages): it records a notice it finds there, including one that + landed after its turn had ended, and it leaves out the drafts of a turn + still open and those a notice waiting to be written names. Notices name drafts and say how to resend or dismiss them, and quote none of their words; the `drafts` tool's `list` and `read` show the words. - A draft is never published except by its resident's explicit resend (or diff --git a/src/framework.ts b/src/framework.ts index cf64b472f..5095c7ccd 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -637,6 +637,8 @@ function freshTurnRouting(): TurnRouting { interface ProseRound { /** The routing state of the turn that queued the round. */ routing: TurnRouting; + /** The turn token current when the round was built: a held draft's provenance (DraftSource.turn). */ + turn: number; round?: number; notice: 'now' | 'later'; keepPrivate?: boolean; @@ -9369,7 +9371,7 @@ export class AgentFramework { agent: Agent, texts: Array, reason: DraftReason, - opts: { routing: TurnRouting; round?: number; note?: string; notice: 'now' | 'later' }, + opts: { routing: TurnRouting; turn: number; round?: number; note?: string; notice: 'now' | 'later' }, ): Draft[] { const segments = texts .map((t) => (typeof t === 'string' @@ -9377,7 +9379,7 @@ export class AgentFramework { : { text: t.text, note: t.note ?? opts.note, inheritedRisk: t.inheritedRisk })) .filter((t) => t.text.trim().length > 0); if (segments.length === 0) return []; - const turn = this.logicalTurnToolCalls.get(agent)?.turnToken ?? 0; + const turn = opts.turn; let branch = 'unknown'; try { branch = this.store.currentBranch().name; @@ -9554,14 +9556,22 @@ export class AgentFramework { * shows them. A draft a recent message already names is recorded as * noticed rather than named again: its notice landed after its turn ended, * as a bounce does when a delivery runs after a tool ends an explicit turn. - * The open logical turn's own drafts are left to its receipt and its - * settlement (settleTurnDrafts). + * It reads only the recent history (draftsNamedInHistory), so its notice + * claims no more than that. Left out are the open logical turn's own + * drafts, which its receipt names and its settlement records + * (settleTurnDrafts), and drafts a notice still waiting in the deferred + * queue names. */ private noticeUnnoticedDrafts(agent: Agent): void { const cm = agent.getContextManager(); - const own = new Set(this.openLogical.get(agent.name)?.drafts.map((d) => d.id)); + const leftOut = new Set(this.openLogical.get(agent.name)?.drafts.map((d) => d.id)); + for (const msg of this.deferredMessages) { + const meta = msg.metadata as { draftOwner?: unknown; draftIds?: unknown } | undefined; + if (meta?.draftOwner !== agent.name || !Array.isArray(meta.draftIds)) continue; + for (const id of meta.draftIds) if (typeof id === 'string') leftOut.add(id); + } const unnoticed = this.proseDrafts.unnoticed(agent.name, (id) => cm.getMessage(id as MessageId) !== null) - .filter((d) => !own.has(d.id)); + .filter((d) => !leftOut.has(d.id)); if (unnoticed.length === 0) return; const named = this.draftsNamedInHistory(agent, unnoticed.map((d) => d.id)); this.recordDraftNotices(agent, named); @@ -9573,7 +9583,7 @@ export class AgentFramework { const more = rest.length > 0 ? ` · and ${rest.length} more: ${rest.join(', ')}` : ''; const text = `[drafts] ${agent.name}: ${pending.length === 1 ? 'an open draft' : `${pending.length} open drafts`} of yours ` + - `${pending.length === 1 ? 'is' : 'are'} unnamed on this branch of your history: ${shown.join(' · ')}${more}. ` + + `that no recent notice on this branch names: ${shown.join(' · ')}${more}. ` + 'drafts(action: "list") shows your open drafts; resend or dismiss them by id.'; let mid: string; try { @@ -10001,8 +10011,10 @@ export class AgentFramework { // said only of drafts still held. const current = drafts.map((d) => this.proseDrafts.get(agent.name, d.id) ?? d); const inState = (state: DraftState): Draft[] => current.filter((d) => draftState(d) === state); - const idList = (ds: Draft[]): string => - ds.slice(0, 8).map((d) => d.id).join(', ') + (ds.length > 8 ? `, and ${ds.length - 8} more` : ''); + // Every draft by id: the receipt's metadata names them all, so + // settleTurnDrafts records each one as noticed, and no later catch-up + // names one the text left out. + const idList = (ds: Draft[]): string => ds.map((d) => d.id).join(', '); const held = inState('held'); if (held.length > 0) { notes.push( @@ -10040,7 +10052,7 @@ export class AgentFramework { 'the words remain only in your history', ); } - if (kept > 0) notes.push(`${kept} plain-speech segment(s) kept private (skip_reply)`); + if (kept > 0) notes.push(`${kept} plain-speech segment(s) kept private (skip_reply or a private think)`); if (suppressed > 0) { notes.push(agent.proseRouting === 'disabled' ? `${suppressed} plain-speech segment(s) suppressed (proseRouting=disabled — publish only with an explicit send tool)` @@ -10216,9 +10228,11 @@ export class AgentFramework { rawText: string, round: { notice?: 'now' | 'later'; keepPrivate?: boolean } = {}, ): Promise { - // The direct entry delivers under the live turn's routing. + // The direct entry delivers as a live round does: under the live turn's + // routing and turn token. await this.deliverAdmittedProse(agent, this.admitProse(rawText), { routing: this.routingOf(agent.name), + turn: this.logicalTurnToolCalls.get(agent)?.turnToken ?? 0, notice: round.notice ?? 'later', keepPrivate: round.keepPrivate, }); @@ -10317,12 +10331,17 @@ export class AgentFramework { agent, [{ text: words.text, ...(words.inheritedRisk ? { inheritedRisk: words.inheritedRisk } : {}) }], 'bounced', - { routing: round.routing, round: round.round, note: `{{unsent}} was draft ${draft.id}, which could not be sent`, notice: 'later' }, + { routing: round.routing, turn: round.turn, round: round.round, note: `{{unsent}} was draft ${draft.id}, which could not be sent`, notice: 'later' }, )[0]; - const kept = held - ? ` Your message is held as draft ${held.id}, with {{unsent}} written out; since it includes ${draft.id}'s words, it ` + - 'can be resent only with confirmDuplicate: true, after that.' - : ''; + // A bare {{unsent}} holds nothing new; otherwise say whether the + // envelope's own words were held, since explicit mode writes no receipt + // to report a hold that failed. + const kept = 'draft' in words + ? '' + : held + ? ` Your message is held as draft ${held.id}, with {{unsent}} written out; since it includes ${draft.id}'s words, it ` + + 'can be resent only with confirmDuplicate: true, after that.' + : ' Your message could not be held as a draft; its words remain in your history.'; const text = `[prose-routing] {{unsent}} is draft ${draft.id}, ${why.startsWith('which') ? why : `and ${why}`}. ` + `Nothing was sent; ${next}.${kept}`; try { @@ -10743,7 +10762,7 @@ export class AgentFramework { agent, [{ text: words.text, ...(inherited ? { inheritedRisk: inherited } : {}) }], 'bounced', - { routing: round.routing, round: round.round, note: reason, notice: 'later' }, + { routing: round.routing, turn: round.turn, round: round.round, note: reason, notice: 'later' }, )[0]; } const streak = (this.proseBounceStreaks.get(name) ?? 0) + 1; @@ -11467,6 +11486,9 @@ export class AgentFramework { this.guardRetryTurnSilenced.delete(agent.name); /** Provider rounds of this physical stream, for draft provenance. */ let draftRound = 0; + /** This stream's turn token, for draft provenance: a round queued here + * keeps it even when its hold runs after a later turn has started. */ + const draftTurn = this.logicalTurnToolCalls.get(agent)?.turnToken ?? 0; // Live routing is only trusted when the membrane provides verbatim // round-scoped blocks (roundContent, native tool mode, membrane ≥0.5.64). @@ -11807,7 +11829,7 @@ export class AgentFramework { const presentsInjections = (event.context as { supportsInjectedMessages?: boolean }).supportsInjectedMessages ?? true; // Queued under this turn's routing, whenever it posts. - const holdOpts: ProseRound = { routing: streamRouting, round: draftRound, notice: presentsInjections ? 'now' : 'later' }; + const holdOpts: ProseRound = { routing: streamRouting, turn: draftTurn, round: draftRound, notice: presentsInjections ? 'now' : 'later' }; // Routing publishes trimmed segments; held drafts keep the // runs exactly as written. const roundRuns = splitProseRuns(assistantBlocks); @@ -12346,7 +12368,7 @@ export class AgentFramework { if (speechContent.length > 0 && this.channelRegistry) { const speechRuns = finalSpeechRuns; const speechSegments = finalSpeechSegments; - const textOnlyHold: ProseRound = { routing: finalRouting, round: draftRound + 1, notice: 'later' }; + const textOnlyHold: ProseRound = { routing: finalRouting, turn: draftTurn, round: draftRound + 1, notice: 'later' }; if (speechSegments.length > 0) { if (turnProseRouting === 'disabled') { console.error(`[routing] ${agent.name}: text-only prose NOT routed (proseRouting=disabled)`); @@ -12424,7 +12446,7 @@ export class AgentFramework { : turnSilenced || fallbackCause !== null; const trailingCause: 'send' | 'private' | null = silenceCause === 'private' || fallbackCause === 'private' ? 'private' : (silenceCause ?? fallbackCause); - const trailingHold: ProseRound = { routing: finalRouting, round: draftRound + 1, notice: 'later' }; + const trailingHold: ProseRound = { routing: finalRouting, turn: draftTurn, round: draftRound + 1, notice: 'later' }; const runs = finalRuns; const segments = finalSegments; diff --git a/src/prose-drafts.ts b/src/prose-drafts.ts index ebaad9d05..601ed2578 100644 --- a/src/prose-drafts.ts +++ b/src/prose-drafts.ts @@ -68,11 +68,13 @@ export type DraftReason = export interface DraftSource { /** Chronicle branch current when the words were held. */ branch: string; - /** The resident's logical turn (process-local counter). */ + /** The turn token (a process-local counter) of the stream whose round + * wrote the words; a budget restart or a retry has a token of its own. */ turn: number; /** Round within this stream; numbering restarts at a budget restart. */ round: number; - /** Position of the segment within its round's speech. */ + /** Position of these words among those held together, in one hold of a + * round's speech; a bounce holds its words alone, at 0. */ segment: number; } diff --git a/test/explicit-prose-routing.test.ts b/test/explicit-prose-routing.test.ts index 24627f647..1661ecc07 100644 --- a/test/explicit-prose-routing.test.ts +++ b/test/explicit-prose-routing.test.ts @@ -410,6 +410,7 @@ describe('explicit prose routing', () => { const original = store.hold('assistant', [{ text: 'ORIGINAL WORDS', source: { branch: 'main', turn: 0, round: 0, segment: 0 } }], 'bounced').held[0]!; const internals = framework as unknown as { channelRegistry: { deliverSpeech: (...args: unknown[]) => Promise }; + logicalTurnToolCalls: { get(agent: unknown): { turnToken: number } | undefined }; }; const deliver = internals.channelRegistry.deliverSpeech; let release!: () => void; @@ -423,9 +424,11 @@ describe('explicit prose routing', () => { return outcome; }; let starts = 0; + let firstTurn = 0; framework.onTrace((e) => { if ((e as { type: string }).type !== 'inference:started') return; starts++; + if (starts === 1) firstTurn = internals.logicalTurnToolCalls.get(framework.getAgent('assistant')!)!.turnToken; if (starts === 2) membrane.pushResponse(createMockResponse([{ type: 'text', text: '>>#nowhere NEW WORDS' }])); if (starts >= 3) membrane.pushResponse(createMockResponse([{ type: 'text', text: '>>skip_reply' }])); }); @@ -450,8 +453,21 @@ describe('explicit prose routing', () => { assert.ok(heldTexts.includes(attempted), `the failed send holds the words it attempted: ${JSON.stringify(heldTexts)}`); const copy = store.open('assistant').find((d) => d.text === attempted)!; assert.equal(copy.inheritedRisk, undefined, 'a definite failure leaves no risk to inherit'); - if (sent === '{{unsent}}') assert.equal(copy.id, original.id, 'the bare send re-bounces the draft it sent'); - else assert.notEqual(copy.id, original.id); + if (sent === '{{unsent}}') { + assert.equal(copy.id, original.id, 'the bare send re-bounces the draft it sent'); + // Independent of timing: the re-bounce notice offers {{unsent}} only + // when its draft is the latest bounce, so it does only if the next + // turn's bounce waited for this failure. + const notice = context(framework).texts.filter((t) => t.startsWith('[prose-routing]') && t.includes(original.id)).at(-1)!; + assert.ok(notice.includes('{{unsent}}'), `the failure came back before the next turn's bounce: ${notice}`); + } else { + // Held after the failure's copy, the next turn's bounce is the latest. + assert.notEqual(copy.id, original.id); + // Arthur-2378's probe: held while turn two runs, the copy keeps the + // turn that attempted its words. + assert.ok(firstTurn > 0, 'the first turn token was observed'); + assert.equal(copy.source.turn, firstTurn, "the copy's provenance is the turn that attempted the words"); + } assert.equal(store.latestBounce('assistant')?.text, 'NEW WORDS', "the next turn's bounce came after it, and is the latest"); } finally { release(); @@ -1257,7 +1273,7 @@ describe('explicit prose routing', () => { assert.equal(receipts.length, 1, `one receipt for the logical turn: ${receipts.join(' | ')}`); assert.match(receipts[0]!, /world:commons/, "it reports the later compile's delivery"); assert.ok(receipts[0]!.includes(draft.id), `and the draft the first stream held: ${receipts[0]}`); - assert.ok(!texts.some((t) => t.includes('unnamed on this branch')), 'the catch-up left the draft to the receipt'); + assert.ok(!texts.some((t) => t.includes('that no recent notice on this branch names')), 'the catch-up left the draft to the receipt'); assert.equal(internals.openLogical.has('assistant'), false, 'the logical turn settled'); } finally { await framework.stop(); @@ -1283,7 +1299,7 @@ describe('explicit prose routing', () => { assert.ok(late, 'the late envelope bounced into a draft'); const texts = context(framework).texts; assert.ok(texts.some((t) => t.startsWith('[prose-routing]') && t.includes(late.id)), 'its bounce notice names it'); - assert.ok(!texts.some((t) => t.includes('unnamed on this branch')), "the catch-up doesn't name it again"); + assert.ok(!texts.some((t) => t.includes('that no recent notice on this branch names')), "the catch-up doesn't name it again"); assert.ok((store.get('assistant', late.id)?.noticedIn ?? []).length > 0, 'it is recorded as noticed'); }); }); @@ -1725,7 +1741,7 @@ describe('explicit prose routing', () => { const framework = await createFramework('hybrid'); stubRegistry(framework); const gateway = framework as unknown as { - holdSilencedProse(agent: unknown, segments: string[], cause: 'send', privateThink: boolean, round: { routing: RoutingState; notice: 'later' }): void; + holdSilencedProse(agent: unknown, segments: string[], cause: 'send', privateThink: boolean, round: { routing: RoutingState; turn: number; notice: 'later' }): void; turnRouting: Map; }; const agent = framework.getAgent('assistant')!; @@ -1736,9 +1752,10 @@ describe('explicit prose routing', () => { gateway.turnRouting.set('assistant', live); // A held round that a later start has overtaken: its envelopes move // its own turn's routing, and its drafts are that turn's. - gateway.holdSilencedProse(agent, ['>>>#nowhere held words', 'after it'], 'send', false, { routing: queued, notice: 'later' }); + gateway.holdSilencedProse(agent, ['>>>#nowhere held words', 'after it'], 'send', false, { routing: queued, turn: 7, notice: 'later' }); assert.equal(queued.hybridHold, 'failed', 'the round that held it records the failed envelope'); assert.deepEqual(queued.logical.drafts.map((d) => d.text), ['held words', 'after it'], "and holds the round's words as its drafts"); + assert.deepEqual(draftsOf(framework).open('assistant').map((d) => d.source.turn), [7, 7], "each draft's provenance is the round's turn, not the live one"); assert.equal(live.hybridHold, undefined, 'the live turn is untouched'); assert.deepEqual(live.logical.drafts, [], 'and holds no draft of that round'); } finally { @@ -1767,6 +1784,48 @@ describe('explicit prose routing', () => { } }); + it("a refused {{unsent}} says when the envelope's own words couldn't be held", async () => { + const framework = await createFramework(); + try { + const routed = stubRegistry(framework); + const store = draftsOf(framework); + const original = store.hold('assistant', [{ text: 'earlier words', source: { branch: 'main', turn: 1, round: 1, segment: 0 } }], 'bounced').held[0]!; + // An attempt with no outcome: the latest bounce may already have been + // posted, so {{unsent}} refuses it and holds the envelope's words. + store.beginAttempt('assistant', original.id, { serverId: 'stub', channelId: 'chan-alpha' }, 'unsent-token', false); + // The journal refuses the envelope's words. + store.hold = ((_agent: string, segments: unknown[]) => + ({ held: [], notHeld: { count: segments.length, error: 'the journal refused the write' } })) as typeof store.hold; + const gateway = framework as unknown as { deliverProse(a: unknown, text: string): Promise }; + await gateway.deliverProse(framework.getAgent('assistant')!, '>>#alpha {{unsent}} and a line'); + assert.deepEqual(routed, [], 'nothing is sent'); + const notice = context(framework).texts.find((t) => t.startsWith('[prose-routing] {{unsent}} is draft'))!; + assert.ok(notice.includes('Your message could not be held as a draft; its words remain in your history.'), notice); + } finally { + await framework.stop(); + } + }); + + it('the receipt names every draft it records, past eight', async () => { + const framework = await createFramework('hybrid'); + try { + stubRegistry(framework); + const store = draftsOf(framework); + const held = store.hold('assistant', Array.from({ length: 11 }, (_, segment) => ({ + text: `held ${segment}`, source: { branch: 'main', turn: 1, round: 1, segment }, + })), 'explicit-send').held; + const gateway = framework as unknown as { appendProseDeliveryReceipt(agent: unknown, routing: RoutingState): void }; + const routing: RoutingState = { destination: null, + logical: { continuation: false, deliveries: [], suppressed: 0, drafts: held, keptPrivate: 0 } }; + gateway.appendProseDeliveryReceipt(framework.getAgent('assistant')!, routing); + const receipt = context(framework).texts.find((t) => t.startsWith('[delivered]'))!; + for (const draft of held) assert.ok(receipt.includes(draft.id), `${draft.id} is named: ${receipt}`); + assert.doesNotMatch(receipt, /and \d+ more/, 'none is left as a count'); + } finally { + await framework.stop(); + } + }); + it('admission turns a resolver that throws into an envelope that bounces', async () => { const framework = await createFramework(); const routed = stubRegistry(framework); diff --git a/test/prose-drafts.test.ts b/test/prose-drafts.test.ts index 7b450ee61..054f4d866 100644 --- a/test/prose-drafts.test.ts +++ b/test/prose-drafts.test.ts @@ -465,13 +465,16 @@ describe('held prose drafts, end to end', () => { createMockResponse([]), ]); assert.deepEqual(h.drafts(), []); - assert.ok(h.texts().some((t) => t === '[delivered] nothing — 1 plain-speech segment(s) kept private (skip_reply)')); + assert.ok(h.texts().some((t) => t === '[delivered] nothing — 1 plain-speech segment(s) kept private (skip_reply or a private think)')); await h.turn([ createMockResponse([text('Prose beside a private think.'), call('t1', 'think', { thought: 'hmm' }), explicitSend('s1')], 'tool_use'), createMockResponse([]), ]); assert.deepEqual(h.drafts(), [], 'a same-round private think keeps its prose private, even beside a send'); assert.deepEqual(h.publishes().map((p) => p.text), ['an explicit note elsewhere']); + assert.equal(h.texts().filter((t) => t.startsWith('[delivered]')).at(-1), + '[delivered] nothing — 1 plain-speech segment(s) kept private (skip_reply or a private think)', + 'the receipt counts it, and says a private think may be why'); } finally { await h.close(); } @@ -811,7 +814,7 @@ describe('held prose drafts, end to end', () => { h.store().beginAttempt('scout', midSend.id, dest(), 'resend', false); await h.turn([createMockResponse([])]); const notice = h.texts().find((t) => t.startsWith('[drafts]'))!; - assert.match(notice, /^\[drafts\] scout: 2 open drafts of yours are unnamed on this branch of your history: /); + assert.match(notice, /^\[drafts\] scout: 2 open drafts of yours that no recent notice on this branch names: /); assert.match(notice, /d-[a-z2-9]{5} \(held \(not sent\); explicit send in the same round\)/); assert.match(notice, new RegExp(`${midSend.id} \\(UNCONFIRMED — an attempt to #room \\(Guild One\\) \\(${ROOM}\\) at .* may have been posted; .*explicit send in the same round\\)`)); assert.doesNotMatch(notice, /They were not sent/); @@ -829,7 +832,7 @@ describe('held prose drafts, end to end', () => { const held = Array.from({ length: 10 }, (_, i) => h.store().hold('scout', [{ text: `orphan ${i}`, source }], 'explicit-send').held[0]!); await h.turn([createMockResponse([])]); const notice = h.texts().find((t) => t.startsWith('[drafts]'))!; - assert.match(notice, /^\[drafts\] scout: 10 open drafts of yours are unnamed on this branch of your history: /); + assert.match(notice, /^\[drafts\] scout: 10 open drafts of yours that no recent notice on this branch names: /); // Newest first: the eight newest with their state, then the two oldest // by id alone. No draft's words appear. const named = [...notice.matchAll(/(d-[a-z2-9]{5}) \(held \(not sent\); explicit send in the same round\)/g)].map((m) => m[1]); @@ -854,7 +857,7 @@ describe('held prose drafts, end to end', () => { ]); const [draft] = h.drafts(); assert.ok(h.texts().some((t) => t.includes(`held as draft ${draft!.id}`)), 'the turn-end receipt names it'); - const catchUps = () => h.texts().filter((t) => t.includes('unnamed on this branch of your history')); + const catchUps = () => h.texts().filter((t) => t.includes('that no recent notice on this branch names')); assert.deepEqual(catchUps(), []); // The rollback takes the turn, and its notices, off the branch; the // draft stays, as drafts do. @@ -862,7 +865,7 @@ describe('held prose drafts, end to end', () => { assert.ok(!h.texts().some((t) => t.includes(draft!.id)), 'nothing on the branch names it now'); await h.turn([createMockResponse([])]); assert.equal(catchUps().length, 1); - assert.match(catchUps()[0]!, new RegExp(`an open draft of yours is unnamed on this branch of your history: ${draft!.id} \\(held \\(not sent\\); explicit send in the same round\\)`)); + assert.match(catchUps()[0]!, new RegExp(`an open draft of yours that no recent notice on this branch names: ${draft!.id} \\(held \\(not sent\\); explicit send in the same round\\)`)); assert.ok(!catchUps()[0]!.includes('words a send held back'), "the notice quotes none of the draft's words"); await h.turn([createMockResponse([])]); assert.equal(catchUps().length, 1, 'named once on this branch'); @@ -1146,7 +1149,7 @@ describe('draft notices and turn-end settlement', () => { const cm = agent.getContextManager(); const catchUps = () => cm.getAllMessages() .flatMap((m) => m.content).filter((b) => b.type === 'text').map((b) => (b as { text: string }).text) - .filter((t) => t.includes('unnamed on this branch')); + .filter((t) => t.includes('that no recent notice on this branch names')); return { framework, internals, agent, cm, catchUps, close: async () => { await framework.stop(); rmSync(dir, { recursive: true, force: true }); } }; }; @@ -1167,6 +1170,29 @@ describe('draft notices and turn-end settlement', () => { } }); + it('the catch-up leaves out a draft that a notice still waiting to be written names', async () => { + const t = await catchUpFixture(); + const queue = (t.framework as unknown as { deferredMessages: Array> }).deferredMessages; + try { + const [waiting, lost] = t.internals.proseDrafts.hold('scout', [{ text: 'waiting', source }, { text: 'lost', source }], 'bounced').held; + // A bounce notice deferred while a turn is alive: not in the history + // yet, but on its way. + queue.push({ + id: 'deferred-notice', seq: 1, participant: 'user', + content: [{ type: 'text', text: '[prose-routing] …' }], + metadata: { system: true, kind: 'prose-bounce', draftOwner: 'scout', draftIds: [waiting!.id] }, + forAgent: 'scout', + }); + t.internals.noticeUnnoticedDrafts(t.agent); + const [catchUp] = t.catchUps(); + assert.ok(catchUp?.includes(lost!.id), 'the draft no notice names is named'); + assert.ok(!catchUp!.includes(waiting!.id), `the one a waiting notice names is not: ${catchUp}`); + } finally { + queue.length = 0; + await t.close(); + } + }); + it("the catch-up leaves the open logical turn's own drafts to its receipt", async () => { const t = await catchUpFixture(); try { @@ -1600,7 +1626,7 @@ describe('held drafts: exact runs, hybrid envelopes beside a send, and turn-end ]); assert.equal((h.framework as unknown as { turnRouting: Map }).turnRouting.get('scout')?.pin, ROOM, 'the named destination is the sticky target again, as live routing would leave it'); - assert.match(h.texts().find((t) => t.startsWith('[delivered]'))!, / · 3 plain-speech segment\(s\) kept private \(skip_reply\)$/); + assert.match(h.texts().find((t) => t.startsWith('[delivered]'))!, / · 3 plain-speech segment\(s\) kept private \(skip_reply or a private think\)$/); // A live `>>>skip_reply` in an earlier round binds a later silenced one. const before = h.drafts().length; @@ -1611,7 +1637,7 @@ describe('held drafts: exact runs, hybrid envelopes beside a send, and turn-end assert.equal(h.drafts().length, before, 'nothing private was drafted'); // "still thinking" is the one kept private (live routing does not count // the `>>>skip_reply` envelope itself). - assert.equal(h.texts().filter((t) => t.startsWith('[delivered]')).at(-1), '[delivered] nothing — 1 plain-speech segment(s) kept private (skip_reply)'); + assert.equal(h.texts().filter((t) => t.startsWith('[delivered]')).at(-1), '[delivered] nothing — 1 plain-speech segment(s) kept private (skip_reply or a private think)'); } finally { await h.close(); } From 3a458ee30af7118bcce33ec69b0bd802359a9235 Mon Sep 17 00:00:00 2001 From: Petra-1012 Date: Tue, 13 Oct 2026 07:15:13 -0400 Subject: [PATCH 0174/1112] fix(drafts): notices that name a draft say so in their metadata; the dead 'ambiguous' reason goes; texts say what the code does From the haiku review of 78ce9aa4 (sw-af247r, judged): - The refused-{{unsent}} notice names the refused draft by id in its text, but its metadata listed only the copy it held, or nothing. The unrecorded-outcome notice named its draft with no metadata at all. Since the catch-up and settlement count a draft as named only through a notice's draftIds, both notices now list the drafts they name (I11, I12). - DraftReason 'ambiguous' ("two conversations competed for an unaddressed reply") was produced by no code path. It goes from the type, the reason texts and the module header (I17). - Texts: retention keeps an older draft for at least 7 days, since the journal lets it go at its next checkpoint (changelog, the drafts tool's description and its refusal; I8); `list` shows a preview and `read` the words (I15); a refused {{unsent}} whose copy the journal refuses says the words remain only in the history (changelog, I7); DraftAttempt.via's 'unsent-token' covers hybrid envelopes too (I16); keptPrivate counts a private think beside a send too (I20); ProseRound.turn is read at driveStream's entry (I22). Tests: - explicit-prose-routing: a send's held-back words that the journal refuses are reported in the receipt's "NOT held as drafts" line (I4); the receipt with eleven drafts names all of them in its metadata (I5); the refused {{unsent}} notice's metadata names the refused draft (I11); an {{unsent}} send whose outcome can't be recorded gets a notice naming its draft, in text and metadata (I12); the >>>skip_reply test also checks the journal holds nothing (I21). - prose-drafts: the catch-up finds a notice among the newest 500 messages and names a draft whose only notice is further back (I10). --- changelog.d/prose-drafts.added.md | 14 ++++--- src/framework.ts | 28 ++++++++------ src/prose-drafts.ts | 9 ++--- test/explicit-prose-routing.test.ts | 57 +++++++++++++++++++++++++++++ test/prose-drafts.test.ts | 19 ++++++++++ 5 files changed, 103 insertions(+), 24 deletions(-) diff --git a/changelog.d/prose-drafts.added.md b/changelog.d/prose-drafts.added.md index 393da491b..bb6c83339 100644 --- a/changelog.d/prose-drafts.added.md +++ b/changelog.d/prose-drafts.added.md @@ -24,10 +24,10 @@ - Drafts live in typed Chronicle records (`RecordJournal`) that do not follow branch switches: undo, rollback and checkout never change them, and they survive restarts. Nothing is ever sent from them on its own. Each - resident's 100 newest drafts are kept, and an older one for 7 days after - anything last happened to it (held, named, attempted, answered, dismissed); - the journal checkpoints what it keeps every 500 entries, so opening it - replays only what follows. A request naming a draft that was let go is + resident's 100 newest drafts are kept, and an older one for at least 7 + days after anything last happened to it (held, named, attempted, answered, + dismissed). The journal lets such a draft go when it next checkpoints what + it keeps, every 500 entries, so opening it replays only what follows. A request naming a draft that was let go is refused, and nothing is sent. - A private notice names held drafts at the next tool boundary when the live stream presents mid-turn messages (a draft holding words copied from one @@ -51,7 +51,8 @@ landed after its turn had ended, and it leaves out the drafts of a turn still open and those a notice waiting to be written names. Notices name drafts and say how to resend or dismiss them, and quote none of their - words; the `drafts` tool's `list` and `read` show the words. + words; the `drafts` tool's `read` shows the words, and its `list` a + preview of each. - A draft is never published except by its resident's explicit resend (or `{{unsent}}`), and only that resident can list or act on it. Its notices go into the resident's own history, which residents sharing one message slot @@ -76,7 +77,8 @@ - `{{unsent}}` whose draft is being sent, queued in a resend, or unconfirmed sends nothing, and the envelope's own words are held as a bounce draft with `{{unsent}}` written out and the risk carried, so nothing the - resident wrote has to be typed again. + resident wrote has to be typed again. If the journal refuses that draft, + the notice says the words remain only in the history. - Prose segments also break at XML tool mode's refused attempts and their notices (`tool_attempt`, `tool_notice`): the words before an all-refused call and after its notice are two messages, routed or held separately. diff --git a/src/framework.ts b/src/framework.ts index 5095c7ccd..815ad2fee 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -612,7 +612,7 @@ interface LogicalTurn { * noticed once the turn has ended and its notices are written. */ drafts: Draft[]; - /** Plain-speech segments the turn kept private by the resident's own choice (skip_reply in the same round): counted, never drafted. */ + /** Plain-speech segments the turn kept private by the resident's own choice (skip_reply, or a same-round private think beside a send): counted, never drafted. */ keptPrivate: number; /** Segments that should have been held as drafts but could not be journaled: their words remain in the resident's history only. */ draftFailures?: { count: number; error: string }; @@ -637,7 +637,7 @@ function freshTurnRouting(): TurnRouting { interface ProseRound { /** The routing state of the turn that queued the round. */ routing: TurnRouting; - /** The turn token current when the round was built: a held draft's provenance (DraftSource.turn). */ + /** The turn token of the stream that built the round, read at driveStream's entry: a held draft's provenance (DraftSource.turn). */ turn: number; round?: number; notice: 'now' | 'later'; @@ -1307,10 +1307,10 @@ export class AgentFramework { private lastAnnouncedLocus: Map = new Map(); // ---- Held prose drafts (src/prose-drafts.ts) --------------------------- - /** Every resident's held drafts: suppressed, bounced or ambiguous plain - * speech, never published except by their resident's resend, kept until - * resent or dismissed. A logical turn's own drafts, for its receipt, are - * its LogicalTurn's. */ + /** Every resident's held drafts: plain speech that was suppressed, bounced + * or left without a destination, never published except by their + * resident's resend, kept until resent or dismissed. A logical turn's own + * drafts, for its receipt, are its LogicalTurn's. */ private proseDrafts: ProseDraftStore; // ---- Explicit prose routing (docs/explicit-prose-routing.md) ---------- @@ -5029,7 +5029,7 @@ export class AgentFramework { 'Your held drafts: plain speech of yours that was NOT sent — held back because an explicit send in the ' + 'same round holds plain speech back, because a routing prefix bounced, or because it had no destination. ' + `Drafts are never published except by your own resend, and only you can list or act on them. Your ${KEEP_NEWEST} newest ` + - `drafts are always kept; an older one is let go after ${AgentFramework.DRAFT_KEEP_DAYS} days in which nothing happened to it. ` + + `drafts are always kept; an older one may be let go once ${AgentFramework.DRAFT_KEEP_DAYS} days pass in which nothing happens to it. ` + 'list: your open drafts, newest first. read: drafts in full, with every delivery attempt. ' + 'resend: publish drafts verbatim, in the order given, to one destination ("#channel", "@person" or a ' + 'channel id) — an explicit send, like send_message; it stops at the first draft not confirmed delivered. ' + @@ -9429,14 +9429,12 @@ export class AgentFramework { private static readonly DRAFT_REASON_TEXT: Record = { 'explicit-send': 'an explicit send in the same round holds plain speech back', 'no-destination': 'it had no destination — after a routing envelope failed, plain speech waits for a new valid target', - ambiguous: 'more than one conversation was waiting for you, so unaddressed speech was held rather than guessed', bounced: 'its routing prefix bounced', }; private static readonly DRAFT_REASON_SHORT: Record = { 'explicit-send': 'explicit send in the same round', 'no-destination': 'no destination', - ambiguous: 'competing conversations', bounced: 'routing prefix bounced', }; @@ -9742,7 +9740,7 @@ export class AgentFramework { if (missing.length > 0) { return refuse( `No draft of yours with id ${missing.join(', ')}: there was none, or it was let go (a draft beyond your ${KEEP_NEWEST} newest ` + - `is kept for ${AgentFramework.DRAFT_KEEP_DAYS} days after anything last happened to it). drafts(action: "list") shows your open drafts.`, + `is kept for at least ${AgentFramework.DRAFT_KEEP_DAYS} days after anything last happened to it). drafts(action: "list") shows your open drafts.`, ); } @@ -10348,7 +10346,8 @@ export class AgentFramework { this.addMessage( 'user', [{ type: 'text', text }], - { system: true, kind: 'prose-bounce', ...(held ? { draftOwner: agent.name, draftIds: [held.id] } : {}) } as MessageMetadata, + // It names the refused draft as well as any copy it held. + { system: true, kind: 'prose-bounce', draftOwner: agent.name, draftIds: held ? [draft.id, held.id] : [draft.id] } as MessageMetadata, { forAgent: agent.name }, ); } catch (err) { @@ -10407,7 +10406,12 @@ export class AgentFramework { `[prose-routing] {{unsent}} was draft ${unsent.id}; its send came back ${outcome.status}, but recording that ` + `failed, so the draft reads as ${this.draftStateNow(agent.name, unsent.id)} now. drafts(action: "read") shows it.`; try { - this.addMessage('user', [{ type: 'text', text }], { system: true, kind: 'prose-bounce' }, { forAgent: agent.name }); + this.addMessage( + 'user', + [{ type: 'text', text }], + { system: true, kind: 'prose-bounce', draftOwner: agent.name, draftIds: [unsent.id] } as MessageMetadata, + { forAgent: agent.name }, + ); } catch (noticeErr) { console.error('[drafts] unrecorded-outcome notice failed:', noticeErr); } diff --git a/src/prose-drafts.ts b/src/prose-drafts.ts index 601ed2578..a94086ebb 100644 --- a/src/prose-drafts.ts +++ b/src/prose-drafts.ts @@ -3,9 +3,8 @@ * * Plain speech is held back for reasons that are right in themselves: an * explicit send in the same round suppresses the round's prose (so a "sent - * it" postscript is not posted twice), a turn may have no destination, two - * conversations may compete for an unaddressed reply, and a malformed or - * unresolvable routing prefix bounces. Before drafts, the held words were + * it" postscript is not posted twice), a turn may have no destination, and a + * malformed or unresolvable routing prefix bounces. Before drafts, the held words were * kept only as a count (or, in explicit mode, as one in-memory latest-wins * clipboard): the law was right, and the penalty was amnesia. A draft keeps * the exact words until the resident resends or dismisses them, or the @@ -59,8 +58,6 @@ export type DraftReason = | 'explicit-send' /** The speech had no destination (no route this turn, or a failed envelope). */ | 'no-destination' - /** Two conversations competed for an unaddressed reply. */ - | 'ambiguous' /** An explicit/hybrid routing prefix was missing, malformed or unresolvable. */ | 'bounced'; @@ -90,7 +87,7 @@ export interface DraftAttempt { attemptId: string; at: number; destination: PublishDestination; - /** How the words went out: the drafts tool, or explicit mode's `{{unsent}}`. */ + /** How the words went out: the drafts tool, or a `{{unsent}}` envelope (explicit or hybrid). */ via: 'resend' | 'unsent-token'; /** The resident resent knowing an earlier attempt may have been posted. */ confirmedDuplicate?: true; diff --git a/test/explicit-prose-routing.test.ts b/test/explicit-prose-routing.test.ts index 1661ecc07..b9f7b7b94 100644 --- a/test/explicit-prose-routing.test.ts +++ b/test/explicit-prose-routing.test.ts @@ -282,6 +282,15 @@ describe('explicit prose routing', () => { }; } + /** The metadata of the newest message whose text starts with `prefix`. */ + function noticeOf(framework: AgentFramework, prefix: string): { draftOwner?: string; draftIds?: string[] } | undefined { + const messages = framework.getAgent('assistant')!.getContextManager().getAllMessages() as Array<{ + content: Array<{ type: string; text?: string }>; + metadata?: { draftOwner?: string; draftIds?: string[] }; + }>; + return messages.filter((m) => m.content.some((b) => b.type === 'text' && (b.text ?? '').startsWith(prefix))).at(-1)?.metadata; + } + type DraftsView = ProseDraftStore; const draftsOf = (framework: AgentFramework): DraftsView => (framework as unknown as { proseDrafts: DraftsView }).proseDrafts; @@ -1779,6 +1788,7 @@ describe('explicit prose routing', () => { assert.equal(routing.hybridHold, 'private'); assert.equal(routing.logical.keptPrivate, 1, 'the unprefixed envelope after it is counted as kept private'); assert.deepEqual(routing.logical.drafts, [], 'and is never drafted'); + assert.deepEqual(draftsOf(framework).open('assistant'), [], 'the journal holds nothing either'); } finally { await framework.stop(); } @@ -1801,6 +1811,8 @@ describe('explicit prose routing', () => { assert.deepEqual(routed, [], 'nothing is sent'); const notice = context(framework).texts.find((t) => t.startsWith('[prose-routing] {{unsent}} is draft'))!; assert.ok(notice.includes('Your message could not be held as a draft; its words remain in your history.'), notice); + assert.deepEqual(noticeOf(framework, '[prose-routing] {{unsent}} is draft')?.draftIds, [original.id], + 'the notice names the refused draft in its metadata too, so the catch-up counts it as named'); } finally { await framework.stop(); } @@ -1821,6 +1833,51 @@ describe('explicit prose routing', () => { const receipt = context(framework).texts.find((t) => t.startsWith('[delivered]'))!; for (const draft of held) assert.ok(receipt.includes(draft.id), `${draft.id} is named: ${receipt}`); assert.doesNotMatch(receipt, /and \d+ more/, 'none is left as a count'); + assert.deepEqual([...(noticeOf(framework, '[delivered]')?.draftIds ?? [])].sort(), held.map((d) => d.id).sort(), + 'its metadata names every draft, so settlement records each one'); + } finally { + await framework.stop(); + } + }); + + it("words a send held back that the journal refuses are reported in the receipt", async () => { + const framework = await createFramework('hybrid'); + try { + stubRegistry(framework); + const store = draftsOf(framework); + store.hold = ((_agent: string, segments: unknown[]) => + ({ held: [], notHeld: { count: segments.length, error: 'the journal refused the write' } })) as typeof store.hold; + const gateway = framework as unknown as { + holdSilencedProse(agent: unknown, segments: string[], cause: 'send', privateThink: boolean, round: { routing: RoutingState; turn: number; notice: 'later' }): void; + appendProseDeliveryReceipt(agent: unknown, routing: RoutingState): void; + }; + const agent = framework.getAgent('assistant')!; + const routing: RoutingState = { destination: null, + logical: { continuation: false, deliveries: [], suppressed: 0, drafts: [], keptPrivate: 0 } }; + gateway.holdSilencedProse(agent, ['first words', 'second words'], 'send', false, { routing, turn: 1, notice: 'later' }); + gateway.appendProseDeliveryReceipt(agent, routing); + const receipt = context(framework).texts.find((t) => t.startsWith('[delivered]'))!; + assert.ok(receipt.includes('2 plain-speech segment(s) suppressed and NOT held as drafts (the journal refused the write) — ' + + 'the words remain only in your history'), receipt); + } finally { + await framework.stop(); + } + }); + + it("an {{unsent}} send whose outcome can't be recorded says so in a notice that names its draft", async () => { + const framework = await createFramework(); + try { + const routed = stubRegistry(framework); + const store = draftsOf(framework); + const original = store.hold('assistant', [{ text: 'earlier words', source: { branch: 'main', turn: 1, round: 1, segment: 0 } }], 'bounced').held[0]!; + store.recordOutcome = (() => { throw new Error('the journal refused the write'); }) as typeof store.recordOutcome; + const gateway = framework as unknown as { deliverProse(a: unknown, text: string): Promise }; + await gateway.deliverProse(framework.getAgent('assistant')!, '>>#alpha {{unsent}}'); + assert.deepEqual(routed.map((r) => r.text), ['earlier words'], 'the words were sent'); + const notice = context(framework).texts.find((t) => t.startsWith(`[prose-routing] {{unsent}} was draft ${original.id};`))!; + assert.ok(notice.includes('but recording that failed'), notice); + assert.deepEqual(noticeOf(framework, `[prose-routing] {{unsent}} was draft ${original.id};`)?.draftIds, [original.id], + 'it names the draft in its metadata, so the catch-up counts it as named'); } finally { await framework.stop(); } diff --git a/test/prose-drafts.test.ts b/test/prose-drafts.test.ts index 054f4d866..766bbd75e 100644 --- a/test/prose-drafts.test.ts +++ b/test/prose-drafts.test.ts @@ -1170,6 +1170,25 @@ describe('draft notices and turn-end settlement', () => { } }); + it('the catch-up reads notices among the newest 500 messages, and names a draft whose only notice is further back', async () => { + const t = await catchUpFixture(); + try { + const [old, recent] = t.internals.proseDrafts.hold('scout', [{ text: 'old', source }, { text: 'recent', source }], 'bounced').held; + // Neither notice was recorded: one now sits 500 messages back, the + // other among the newest. + t.cm.addMessage('user', [{ type: 'text', text: '[prose-routing] …' }], { system: true, kind: 'prose-bounce', draftOwner: 'scout', draftIds: [old!.id] } as never); + for (let i = 0; i < 500; i++) t.cm.addMessage('user', [{ type: 'text', text: `filler ${i}` }]); + const notice = t.cm.addMessage('user', [{ type: 'text', text: '[prose-routing] …' }], { system: true, kind: 'prose-bounce', draftOwner: 'scout', draftIds: [recent!.id] } as never); + t.internals.noticeUnnoticedDrafts(t.agent); + assert.deepEqual(t.internals.proseDrafts.get('scout', recent!.id)!.noticedIn, [String(notice)], 'the recent notice is found and recorded'); + const [catchUp] = t.catchUps(); + assert.ok(catchUp?.includes(old!.id), `the draft whose notice is past the window is named: ${catchUp}`); + assert.ok(!catchUp!.includes(recent!.id), 'the recently named one is not'); + } finally { + await t.close(); + } + }); + it('the catch-up leaves out a draft that a notice still waiting to be written names', async () => { const t = await catchUpFixture(); const queue = (t.framework as unknown as { deferredMessages: Array> }).deferredMessages; From 1b9c5724f0cc5be21c2b239ef1837e5da9a465cb Mon Sep 17 00:00:00 2001 From: Petra-1012 Date: Tue, 13 Oct 2026 07:18:46 -0400 Subject: [PATCH 0175/1112] fix(drafts): the drafts-held trace event takes its reason type from DraftReason Agnes-1009's leftover from sw-af247r I17 (room-203 #85015): src/types/trace.ts still typed prose:drafts-held's reason as a literal union that included 'ambiguous', which no path produces. It now imports DraftReason (type-only), so the trace and the store can't drift apart again. --- src/types/trace.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/types/trace.ts b/src/types/trace.ts index aed07dbb5..d7132a414 100644 --- a/src/types/trace.ts +++ b/src/types/trace.ts @@ -1,6 +1,7 @@ import type { ProcessEvent } from './events.js'; import type { ModuleProcessResponse } from './framework.js'; import type { SessionUsage } from '../usage/types.js'; +import type { DraftReason } from '../prose-drafts.js'; /** * Base for all trace events. @@ -585,7 +586,7 @@ export type TraceEvent = | (TraceEventBase & { type: 'prose:drafts-held'; agentName: string; - reason: 'explicit-send' | 'no-destination' | 'ambiguous' | 'bounced'; + reason: DraftReason; draftIds: string[]; textLen: number; }) From 355ade779d3764e74be15df4b9eb93bb23c2b5f8 Mon Sep 17 00:00:00 2001 From: Petra-1012 Date: Tue, 13 Oct 2026 07:35:37 -0400 Subject: [PATCH 0176/1112] fix(drafts): the receipt reports a draft retention let go during the turn as let go, not as held From Arthur-2378's read of 3a458ee3 (room-203 #85122, J6425), with Agnes-1009 agreeing (#85128). appendProseDeliveryReceipt read each of the turn's drafts as `proseDrafts.get(id) ?? d`. When retention had let a draft go since the turn held it, the receipt fell back to the copy the turn captured, and offered that id as held and resendable, though a resend of it is refused. The receipt now reads only the journal's current drafts, and names one it no longer has as "held this turn and since let go by retention (no longer held, so not resendable)". With the shipped constants this takes a turn left open 7 idle days with 100 newer drafts held meanwhile, as a capacity hold could leave it. But the receipt shouldn't claim a state it can't know. The changelog no longer says the receipt counts the private segments it lists: explicit turns write no receipt, and a private think without a send isn't counted (Arthur, Agnes). Test (prose-drafts), from Arthur's probe: with retention shortened (keepNewest 1, keepIdleMs 1, checkpointEvery 1), two holds let the older draft go. The receipt names the newer one as held, and the older one as let go, not as held. --- changelog.d/prose-drafts.added.md | 3 +-- src/framework.ts | 17 +++++++++++++++- test/prose-drafts.test.ts | 32 +++++++++++++++++++++++++++++++ 3 files changed, 49 insertions(+), 3 deletions(-) diff --git a/changelog.d/prose-drafts.added.md b/changelog.d/prose-drafts.added.md index bb6c83339..2414fc092 100644 --- a/changelog.d/prose-drafts.added.md +++ b/changelog.d/prose-drafts.added.md @@ -13,8 +13,7 @@ the rest private until a destination is named), a same-round private `think` in a native round (where that policy applies: XML tool mode routes such prose as ordinary speech, so a send holds it as a draft), silent turns - and `proseRouting: disabled` narration stay private. The turn's receipt - counts them, except a silent turn's: a silent turn writes no receipt. + and `proseRouting: disabled` narration stay private. - In hybrid mode a send withholds publication, not what an envelope does: a held `>>>` envelope moves the routing state as live routing would (a destination it names becomes the sticky target), `>>>skip_reply {{unsent}}` diff --git a/src/framework.ts b/src/framework.ts index 815ad2fee..e46f84fe7 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -10007,7 +10007,16 @@ export class AgentFramework { // Each of the turn's drafts as it stands now: a mid-turn notice lets the // resident resend or dismiss one before the turn ends, so "not sent" is // said only of drafts still held. - const current = drafts.map((d) => this.proseDrafts.get(agent.name, d.id) ?? d); + // Each draft as the journal holds it now. One that retention let go + // since the turn held it is reported as let go: the copy the turn + // captured says nothing about its state now. + const current: Draft[] = []; + const letGo: Draft[] = []; + for (const d of drafts) { + const now = this.proseDrafts.get(agent.name, d.id); + if (now) current.push(now); + else letGo.push(d); + } const inState = (state: DraftState): Draft[] => current.filter((d) => draftState(d) === state); // Every draft by id: the receipt's metadata names them all, so // settleTurnDrafts records each one as noticed, and no later catch-up @@ -10020,6 +10029,12 @@ export class AgentFramework { '(not sent — drafts can resend them unchanged, or dismiss them)', ); } + if (letGo.length > 0) { + notes.push( + `draft${letGo.length === 1 ? '' : 's'} ${idList(letGo)} held this turn and since let go by retention ` + + '(no longer held, so not resendable)', + ); + } const unconfirmed = inState('unconfirmed'); for (const d of unconfirmed) { notes.push(`draft ${d.id} is unconfirmed: ${this.riskText(d)} — check that channel before sending it again (resend needs confirmDuplicate: true)`); diff --git a/test/prose-drafts.test.ts b/test/prose-drafts.test.ts index 766bbd75e..e23be2700 100644 --- a/test/prose-drafts.test.ts +++ b/test/prose-drafts.test.ts @@ -1189,6 +1189,38 @@ describe('draft notices and turn-end settlement', () => { } }); + // Arthur-2378's probe: with retention shortened, the journal lets the + // turn's older draft go before the receipt is written. + it('the receipt reports a draft that retention let go during the turn as let go, not as held', async (t) => { + const h = await catchUpFixture(); + t.mock.timers.enable({ apis: ['Date'], now: 1_000_000 }); + const drafts = new ProseDraftStore(h.framework.getStore(), { keepNewest: 1, keepIdleMs: 1, checkpointEvery: 1 }); + const f = h.framework as unknown as { + proseDrafts: ProseDraftStore; + holdProseDrafts(agent: unknown, texts: Array<{ text: string }>, reason: string, options: unknown): Array<{ id: string }>; + appendProseDeliveryReceipt(agent: unknown, routing: unknown): void; + }; + f.proseDrafts = drafts; + const routing = h.internals.routingOf('scout'); + try { + const old = f.holdProseDrafts(h.agent, [{ text: 'earlier words' }], 'explicit-send', { routing, turn: 1, round: 1, notice: 'later' })[0]!; + t.mock.timers.tick(2); + const newer = f.holdProseDrafts(h.agent, [{ text: 'later words' }], 'explicit-send', { routing, turn: 1, round: 2, notice: 'later' })[0]!; + assert.equal(drafts.get('scout', old.id), undefined, 'the journal let the older draft go'); + f.appendProseDeliveryReceipt(h.agent, routing); + const receipt = h.cm.getAllMessages().flatMap((m) => m.content) + .filter((b) => b.type === 'text').map((b) => (b as { text: string }).text) + .find((text) => text.startsWith('[delivered]'))!; + const held = receipt.match(/held as drafts? ([^(]+)\(not sent/)?.[1] ?? ''; + assert.ok(held.includes(newer.id), `the draft still held is reported as held: ${receipt}`); + assert.ok(!held.includes(old.id), `the let-go draft is not offered as held: ${receipt}`); + assert.ok(receipt.includes(`draft ${old.id} held this turn and since let go by retention (no longer held, so not resendable)`), receipt); + } finally { + t.mock.timers.reset(); + await h.close(); + } + }); + it('the catch-up leaves out a draft that a notice still waiting to be written names', async () => { const t = await catchUpFixture(); const queue = (t.framework as unknown as { deferredMessages: Array> }).deferredMessages; From 89fda1dd39da4a34cc271a6e7024ad014d22a35d Mon Sep 17 00:00:00 2001 From: Petra-1012 Date: Tue, 13 Oct 2026 07:42:19 -0400 Subject: [PATCH 0177/1112] fix(drafts): a receipt with a let-go draft says "nothing confirmed", never "nothing" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit From Arthur-2378's second probe on 355ade77 (room-203 #85176, J6442), with Agnes-1009 agreeing (#85178). A draft that retention let go since the turn held it may have had an attempt whose outcome is unknown, so it may already have been posted. The receipt's headline said "[delivered] nothing" when nothing was shown, though nothing can be known about that draft now. It now treats a let-go draft as it treats an unconfirmed one, and says "nothing confirmed". Test (prose-drafts), from Arthur's probe: the let-go draft first records an attempt with an unknown outcome, and the receipt opens "[delivered] nothing confirmed —". --- src/framework.ts | 7 +++++-- test/prose-drafts.test.ts | 6 ++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/framework.ts b/src/framework.ts index e46f84fe7..8cf487038 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -10073,11 +10073,14 @@ export class AgentFramework { } const suppressedNote = notes.join(' · '); // Nothing is confirmed delivered, but an unconfirmed draft (or a dismissed - // one whose own attempt is uncertain) may have been. + // one whose own attempt is uncertain) may have been, and a draft let go + // since the turn held it may have had such an attempt: its state is + // unknown now. + const uncertain = unconfirmed.length > 0 || dismissedAtRisk.length > 0 || letGo.length > 0; const text = shown.length > 0 ? `[delivered] plain speech → ${shown.join(' · ')}${suppressedNote ? ` · ${suppressedNote}` : ''}` - : `[delivered] nothing${unconfirmed.length > 0 || dismissedAtRisk.length > 0 ? ' confirmed' : ''} — ${suppressedNote}`; + : `[delivered] nothing${uncertain ? ' confirmed' : ''} — ${suppressedNote}`; try { const mid = agent.getContextManager().addMessage( 'user', diff --git a/test/prose-drafts.test.ts b/test/prose-drafts.test.ts index e23be2700..e1643d24b 100644 --- a/test/prose-drafts.test.ts +++ b/test/prose-drafts.test.ts @@ -1204,6 +1204,11 @@ describe('draft notices and turn-end settlement', () => { const routing = h.internals.routingOf('scout'); try { const old = f.holdProseDrafts(h.agent, [{ text: 'earlier words' }], 'explicit-send', { routing, turn: 1, round: 1, notice: 'later' })[0]!; + // An attempt whose outcome is unknown (Arthur-2378's second probe): once + // the draft is let go, nothing says whether it was posted. + const attempt = drafts.beginAttempt('scout', old.id, dest(), 'resend', false); + drafts.recordOutcome('scout', old.id, attempt, { status: 'unknown', reason: 'connection lost', at: Date.now(), destination: dest() }); + assert.equal(draftState(drafts.get('scout', old.id)!), 'unconfirmed'); t.mock.timers.tick(2); const newer = f.holdProseDrafts(h.agent, [{ text: 'later words' }], 'explicit-send', { routing, turn: 1, round: 2, notice: 'later' })[0]!; assert.equal(drafts.get('scout', old.id), undefined, 'the journal let the older draft go'); @@ -1215,6 +1220,7 @@ describe('draft notices and turn-end settlement', () => { assert.ok(held.includes(newer.id), `the draft still held is reported as held: ${receipt}`); assert.ok(!held.includes(old.id), `the let-go draft is not offered as held: ${receipt}`); assert.ok(receipt.includes(`draft ${old.id} held this turn and since let go by retention (no longer held, so not resendable)`), receipt); + assert.ok(receipt.startsWith('[delivered] nothing confirmed —'), `a let-go draft's unknown state makes no claim of non-delivery: ${receipt}`); } finally { t.mock.timers.reset(); await h.close(); From 1e4f6b71a0b251f7b5e277965ade4646bb412860 Mon Sep 17 00:00:00 2001 From: Duncan-2410 Date: Tue, 13 Oct 2026 09:11:43 -0400 Subject: [PATCH 0178/1112] docs(deferral): say where isInjectableMidTurn's exact name check comes from The prefill guard compares a deferred message's participant with the agent's name exactly. That matches the renderer's own test: membrane gives a message the assistant role only when its participant equals the request's assistantParticipant, which Agent sets to its name, so a differently spelled name renders as another speaker rather than as a prefill. A comment at the check now says so. Co-Authored-By: Claude Opus 5.5 --- src/framework.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/framework.ts b/src/framework.ts index 083c11e1e..21b4a03be 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -485,6 +485,8 @@ function withDeferredWriteId(metadata: MessageMetadata | undefined, id: string): * no tool blocks (the membrane's tool cycle) and not named as the agent * itself (it would render as an assistant turn, a prefill). */ function isInjectableMidTurn(msg: { participant: string; content: ContentBlock[] }, agentName: string): boolean { + // Exact, as membrane's role test is: only a participant equal to the + // request's assistantParticipant (Agent's name) renders as the assistant. return msg.participant !== agentName && !msg.content.some((b) => b.type === 'tool_use' || b.type === 'tool_result'); } From 1e95eb21380698df6d484b0fac674ea3281577e3 Mon Sep 17 00:00:00 2001 From: Duncan-2410 Date: Tue, 13 Oct 2026 10:06:45 -0400 Subject: [PATCH 0179/1112] fix(deferral): a failed write cuts the drain; stop() stores past the deferral unless quiesced An internal review of the last round, after main was merged in, found two places where its rules didn't hold. A failed write didn't cut the drain. writeDrained put the entry that threw back into the queue and every caller went on to the next one: stop, host-mode resume and boot recovery, the post-surgery, turn-end, withdrawn-turn and puppet flushes, and the tool boundary. So the failed entry landed after entries queued behind it, and a tool_use that threw left its tool_result to be stored alone. The drain now cuts there, through one helper every site uses (writeDrainedInOrder): the entry that threw and every later entry for that agent go back into the queue under their ids and seq, and the tool boundary injects only what was stored before the cut. Host-mode resume and boot recovery cut per target, so one agent's stuck entry doesn't hold another's. An entry that keeps throwing holds its agent's later deferred writes, and the line each flush logs names it (id, participant, block types). stop()'s drain could meet a live turn token. It drained while a frame that stop() doesn't wait for (a puppet call, or a turn started outside the event loop) still held its target's token, so addMessage deferred each entry again, into memory, where a memory-only queue lost it at exit. Once stop() has begun no stream can register, so no reply can follow a stored entry: the drain now writes past the deferral (bypassDeferralFor) unless the host is quiesced, whose entries are re-deferred into the persisted queue as before. Text: the host-mode flush's doc no longer says a rejected write is acked and dropped; the watermark docs no longer claim a branch switch marks the watermark (no branch switch calls markContextConsumed); the fragment says what a failed write now does, and that a memory-only queue still loses what is queued when the process exits. Tests: - at a tool boundary, a write that throws cuts the drain: neither it nor what follows is injected, and both land at the turn end in order (this replaces the test that pinned the old reorder); - a queued tool_use whose store throws at stop() keeps its tool_result from being stored alone: both stay queued, in order, and boot recovery stores them as a pair; - stop() stores what is deferred for an agent whose turn token is still held by a frame it doesn't wait for; - a turn waiting to retry when the framework stops gives up rather than compile again, and what was deferred for it is stored; - an ephemeral run whose turn stop() refuses ends at once with "the framework is stopping", and one asked for once stop() has begun is refused; - the post-surgery flush lands what waited behind the hold while another agent's tool cycle is pending, and acknowledges it; - host-quiesce's poison-write test now pins that the write behind the poison one stays queued with it, while the one before it lands. Co-Authored-By: Claude Opus 5.5 --- .../mid-turn-window-only-held.fixed.md | 23 ++- src/agent.ts | 7 +- src/framework.ts | 158 ++++++++++++------ test/deferred-window-only.test.ts | 91 +++++++++- test/ephemeral-settle.test.ts | 43 +++++ test/host-quiesce.test.ts | 13 +- test/operator-surgery.test.ts | 31 ++++ 7 files changed, 289 insertions(+), 77 deletions(-) diff --git a/changelog.d/mid-turn-window-only-held.fixed.md b/changelog.d/mid-turn-window-only-held.fixed.md index 0b741f96b..ff15b2e87 100644 --- a/changelog.d/mid-turn-window-only-held.fixed.md +++ b/changelog.d/mid-turn-window-only-held.fixed.md @@ -34,15 +34,22 @@ cycle holds them. - `stop()` now stores what is still deferred for each registered agent before the store closes; before, a memory-only queue lost it. Once `stop()` - begins, no turn starts and no stream registers, for any agent, so nothing - is mid-turn when it does: a turn that was mid-compile when the streams - were cancelled is refused rather than left running, and an ephemeral run - refused this way ends at once rather than at its startup watchdog. A + begins, no turn starts and no stream registers, for any agent: a turn that + was mid-compile when the streams were cancelled, or one waiting to retry, + is refused rather than left running, and an ephemeral run refused this way + ends at once rather than at its startup watchdog. Since no reply can then + follow a stored entry, `stop()` stores each one even while a frame it + doesn't wait for, such as a puppet call, still holds its target's turn. A quiesced host keeps what is deferred in its persisted queue for boot recovery. An entry for a target no longer registered (a disposed fork) is dropped from a memory-only queue at exit, as before. - A deferred write whose store throws goes back into the queue under its - id, and the others continue. Before, it stayed in the un-acked set and the - next acknowledgement dropped it from the persisted queue (at a turn start - or a flush it was lost even from memory). At a tool boundary, such a write - isn't injected either. + id, and so does every entry queued behind it for that agent, so none lands + out of order and a queued `tool_use` and its `tool_result` stay together. + The log line names the entry, and one that keeps throwing holds the + agent's later deferred writes until it can be stored. Before, it stayed in + the un-acked set and the next acknowledgement dropped it from the + persisted queue (at a turn start or a flush it was lost even from memory). + At a tool boundary, nothing from it on is injected. While the framework + runs it is kept; a memory-only queue still loses it when the process + exits. diff --git a/src/agent.ts b/src/agent.ts index 57a4d1a24..da66b1f47 100644 --- a/src/agent.ts +++ b/src/agent.ts @@ -561,15 +561,16 @@ export class Agent { */ private consumedWatermark: { branch: string; sequence: number } | null = null; /** Counts marks: a compile writes its watermark back only if no mark was - * made while it ran (a tool boundary's, a branch switch's). */ + * made while it ran (a tool boundary's, or another compile's). */ private watermarkMarks = 0; getConsumedWatermark(): { branch: string; sequence: number } | null { return this.consumedWatermark; } - /** Mark everything currently stored as consumed (boot, branch switch, and a - * tool boundary that injected messages into the live stream). */ + /** Mark everything currently stored as consumed (the agent's creation, at + * boot or later, and a tool boundary that injected messages into the live + * stream). */ markContextConsumed(): void { // Tolerates partial context-manager doubles (tests build Agents over // stubs): without a watermark nothing is ever treated as unread. diff --git a/src/framework.ts b/src/framework.ts index 21b4a03be..e05481119 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -481,6 +481,11 @@ function withDeferredWriteId(metadata: MessageMetadata | undefined, id: string): return { ...(metadata ?? {}), deferredWriteId: id } as MessageMetadata; } +/** A deferred entry as a log line names it: its id, participant and block types. */ +function describeDeferredWrite(msg: { id: string; participant: string; content: ContentBlock[] }): string { + return `${msg.id} (${msg.participant}: ${[...new Set(msg.content.map((b) => b.type))].join('+') || 'empty'})`; +} + /** Whether a deferred message can join a live stream at a tool boundary: * no tool blocks (the membrane's tool cycle) and not named as the agent * itself (it would render as an assistant turn, a prefill). */ @@ -2272,21 +2277,25 @@ export class AgentFramework { } } - // Every turn is over: from the start of stop() no turn starts and no - // stream registers (providerAdmissionClosed, checked for every agent), - // and every stream registered before was cancelled and awaited above. So - // nothing else will deliver what is still deferred: a message held for - // a turn's end whose flush found the target busy again, or one that - // arrived for an idle agent and was deferred behind its own state. Land - // each at its registered target now, while the store is open; a - // memory-only queue would otherwise lose it. A quiesced host defers it - // again, into the persisted queue, for boot recovery; a write that throws - // stays queued (writeDrained). An entry for a target no longer registered - // (a disposed fork) is left: dropped with a memory-only queue at exit, and - // logged and dropped from a persisted one at boot. + // No stream is left: from the start of stop() none registers and no turn + // starts (providerAdmissionClosed, checked for every agent), and every + // stream registered before was cancelled and awaited above. So nothing + // else will deliver what is still deferred: a message held for a turn's + // end whose flush found the target busy again, or one that arrived for an + // idle agent and was deferred behind its own state. Land each at its + // registered target now, while the store is open; a memory-only queue + // would otherwise lose it at exit. It goes past the deferral: a frame + // stop() doesn't wait for (a puppet call, or a turn started outside the + // event loop) can still hold its target's turn token, but no reply can + // follow the entry in any stream. A quiesced host defers it again + // instead, into the persisted queue, for boot recovery. A write that + // throws stays queued with what follows it (writeDrainedInOrder), and a + // memory-only queue still loses it at exit. An entry for a target no + // longer registered (a disposed fork) is left: dropped with a memory-only + // queue at exit, and logged and dropped from a persisted one at boot. if (this.deferredMessages.length > 0) { for (const name of this.agents.keys()) { - for (const msg of this.drainDeferredFor(name)) this.readdDrained(msg, 'stop'); + this.readdDrained(this.drainDeferredFor(name), 'stop', this.quiesced ? undefined : { bypassDeferralFor: name }); } this.ackDeferredWrites(); } @@ -3654,8 +3663,9 @@ export class AgentFramework { * acknowledging each durably as it lands (the persisted queue is rewritten * after every message). A crash mid-flush therefore replays exactly the * un-acked remainder at the next boot — no loss, no duplicates. A write - * the store REJECTS is logged and acked too: replaying a poison message - * forever would wedge every later boot on it. + * the store rejects stays queued, logged, with what is queued behind it for + * its target (writeDrainedInOrder), so a target's entries never land out of + * order; a boot that meets it again logs it and goes on. * * Per-target: an agent whose turn is still alive keeps ITS messages * deferred for its own turn boundary; everyone else's flush now. @@ -3678,10 +3688,15 @@ export class AgentFramework { flush.sort(bySeq); this.deferredMessages = keep; this.handOffDeferredWrites(flush); - let stored = 0; + // Each target's entries in queue order, cut at the first that can't be + // stored; other targets' entries go on. + const byTarget = new Map(); for (const msg of flush) { - if (this.readdDrained(msg, `host-mode ${label}`)) stored++; + const target = msg.forAgent ?? this.primaryAgentName; + byTarget.set(target, [...(byTarget.get(target) ?? []), msg]); } + let stored = 0; + for (const entries of byTarget.values()) stored += this.readdDrained(entries, `host-mode ${label}`).length; // Ack = sync the chronicle FIRST, then rewrite the durable queue without // the batch. Never the other way round. this.ackDeferredWrites(); @@ -6270,7 +6285,7 @@ export class AgentFramework { if (this.deferredMessages.length > 0) { for (const name of reserved.keys()) { if (this.activeTurnTokens.has(name) || this.pendingAssistantBlocks.has(name)) continue; - for (const msg of this.drainDeferredFor(name)) this.readdDrained(msg, 'post-surgery flush'); + this.readdDrained(this.drainDeferredFor(name), 'post-surgery flush'); } this.ackDeferredWrites(); } @@ -7089,13 +7104,13 @@ export class AgentFramework { const midTurnInjections: Array<{ participant: string; content: ContentBlock[]; metadata?: MessageMetadata }> = []; { const deferred = this.drainDeferredFor(agent.name, (msg) => !isInjectableMidTurn(msg, agent.name)); - for (const msg of deferred) { - // One that can't be stored stays queued and isn't injected; the - // rest are, so the boundary still hands the round's results on. - const stored = this.writeDrained(msg, `tool boundary for ${agent.name}`, () => { - agent.getContextManager().addMessage(msg.participant, msg.content, withDeferredWriteId(msg.metadata, msg.id)); - }); - if (!stored) continue; + // Stored in queue order, cut at the first that can't be stored: it + // and what follows wait for the turn's end, and only what was + // stored is injected (the round's results go on either way). + const stored = this.writeDrainedInOrder(deferred, `tool boundary for ${agent.name}`, (msg) => { + agent.getContextManager().addMessage(msg.participant, msg.content, withDeferredWriteId(msg.metadata, msg.id)); + }); + for (const msg of stored) { midTurnInjections.push({ participant: msg.participant, content: msg.content, @@ -9603,18 +9618,16 @@ export class AgentFramework { this.deferredMessages.length > 0 ) { { - const deferred = this.drainDeferredFor(agent.name); - for (const msg of deferred) { - // One poison message, or a transient store-write failure, must not - // abort the turn or drop the messages behind it in the queue; the - // one that failed stays queued (writeDrained). - this.writeDrained(msg, `turn-start flush for ${agent.name}`, () => { - const id = agent.getContextManager().addMessage( - msg.participant, msg.content, withDeferredWriteId(msg.metadata, msg.id), - ); - this.emitTrace({ type: 'message:added', messageId: id, source: 'deferred-flush:turn-start' }); - }); - } + // One poison message, or a transient store-write failure, must not + // abort the turn or drop the messages behind it in the queue: the + // drain cuts there, and it and what follows stay queued, in order + // (writeDrainedInOrder). + this.writeDrainedInOrder(this.drainDeferredFor(agent.name), `turn-start flush for ${agent.name}`, (msg) => { + const id = agent.getContextManager().addMessage( + msg.participant, msg.content, withDeferredWriteId(msg.metadata, msg.id), + ); + this.emitTrace({ type: 'message:added', messageId: id, source: 'deferred-flush:turn-start' }); + }); this.ackDeferredWrites(); } } @@ -9654,7 +9667,7 @@ export class AgentFramework { } if (!this.activeTurnTokens.has(agent.name) && this.deferredMessages.length > 0 && !this.pendingAssistantBlocks.has(agent.name)) { - for (const msg of this.drainDeferredFor(agent.name)) this.readdDrained(msg, 'withdrawn-turn flush'); + this.readdDrained(this.drainDeferredFor(agent.name), 'withdrawn-turn flush'); this.ackDeferredWrites(); } return false; @@ -11535,7 +11548,7 @@ export class AgentFramework { // prefix is stored in order, or goes back into the queue whole under // its ids when a successor turn has already taken the agent. if (frameReachedTerminal && this.deferredMessages.length > 0 && !this.pendingAssistantBlocks.has(agent.name)) { - for (const msg of this.drainDeferredFor(agent.name)) this.readdDrained(msg, 'turn-end flush'); + this.readdDrained(this.drainDeferredFor(agent.name), 'turn-end flush'); this.ackDeferredWrites(); } } @@ -11720,7 +11733,7 @@ export class AgentFramework { // was queued — nobody else will flush it, so we do. if (!this.activeTurnTokens.has(agentName) && this.deferredMessages.length > 0 && !this.pendingAssistantBlocks.has(agentName)) { - for (const msg of this.drainDeferredFor(agentName)) this.readdDrained(msg, 'puppet flush'); + this.readdDrained(this.drainDeferredFor(agentName), 'puppet flush'); this.ackDeferredWrites(); } } @@ -13228,12 +13241,12 @@ export class AgentFramework { } /** - * Write one drained entry. If the write throws, the entry goes back into - * the queue under its id and seq, and the caller goes on with the rest: - * left in the un-acked set, the next ack would drop it from the durable - * queue, and a memory-only queue would lose it at once. Every site that - * writes a drained entry writes through this. Returns whether it was - * written (stored, or deferred again by addMessage). + * Write one drained entry. If the write throws, it is logged, naming the + * entry, and goes back into the queue under its id and seq + * (requeueDrained): left in the un-acked set, the next ack would drop it + * from the durable queue. Every site writes a drained list through + * writeDrainedInOrder, which cuts there. Returns whether it was written + * (stored, or deferred again by addMessage). */ private writeDrained(msg: DeferredWrite, where: string, write: () => void): boolean { try { @@ -13241,23 +13254,58 @@ export class AgentFramework { return true; } catch (error) { console.error( - `[deferral] ${where}: a deferred write for ${msg.forAgent ?? this.primaryAgentName ?? 'the primary'} ` + - `(participant=${msg.participant}) could not be stored; it stays queued: ` + - `${error instanceof Error ? error.message : String(error)}`, + `[deferral] ${where}: deferred write ${describeDeferredWrite(msg)} for ` + + `${msg.forAgent ?? this.primaryAgentName ?? 'the primary'} could not be stored; it stays queued, ` + + `holding what is queued behind it for that agent: ${error instanceof Error ? error.message : String(error)}`, ); - this.unackedDeferredWrites = this.unackedDeferredWrites.filter((m) => m.id !== msg.id); - if (!this.deferredMessages.some((m) => m.id === msg.id)) this.deferredMessages.push(msg); - if (this.quiesced || this.deferredWritesPersisted) this.persistDeferredWrites(); + this.requeueDrained([msg]); return false; } } - /** Re-add one drained entry through addMessage, which stores it or defers it again. */ - private readdDrained(msg: DeferredWrite, where: string): boolean { - return this.writeDrained(msg, where, () => { + /** + * Put drained entries back into the queue under their ids and seq: out of + * the un-acked set, and persisted when the queue is. A memory-only queue + * keeps them while the framework runs. + */ + private requeueDrained(entries: DeferredWrite[]): void { + if (entries.length === 0) return; + const ids = new Set(entries.map((m) => m.id)); + this.unackedDeferredWrites = this.unackedDeferredWrites.filter((m) => !ids.has(m.id)); + for (const msg of entries) { + if (!this.deferredMessages.some((m) => m.id === msg.id)) this.deferredMessages.push(msg); + } + if (this.quiesced || this.deferredWritesPersisted) this.persistDeferredWrites(); + } + + /** + * Write one agent's drained entries in queue order, each through + * writeDrained. The drain cuts at the first that can't be stored: it and + * every later entry go back into the queue, so nothing lands ahead of an + * earlier entry and a queued pair stays together. An entry that keeps + * failing holds its agent's later writes, logged at each flush, rather than + * letting them land out of order. Returns what was written, a prefix of + * `entries`. + */ + private writeDrainedInOrder(entries: DeferredWrite[], where: string, write: (msg: DeferredWrite) => void): DeferredWrite[] { + for (let i = 0; i < entries.length; i++) { + const msg = entries[i]!; + if (!this.writeDrained(msg, where, () => write(msg))) { + this.requeueDrained(entries.slice(i + 1)); + return entries.slice(0, i); + } + } + return entries; + } + + /** Re-add one agent's drained entries through addMessage, which stores each + * or defers it again, cut at the first that can't be stored. */ + private readdDrained(entries: DeferredWrite[], where: string, opts?: { bypassDeferralFor?: string }): DeferredWrite[] { + return this.writeDrainedInOrder(entries, where, (msg) => { this.addMessage(msg.participant, msg.content, msg.metadata, { deferredWriteId: msg.id, ...(msg.forAgent ? { forAgent: msg.forAgent } : {}), + ...(opts?.bypassDeferralFor ? { bypassDeferralFor: opts.bypassDeferralFor } : {}), }); }); } diff --git a/test/deferred-window-only.test.ts b/test/deferred-window-only.test.ts index e6d93a526..626f433ab 100644 --- a/test/deferred-window-only.test.ts +++ b/test/deferred-window-only.test.ts @@ -125,12 +125,13 @@ describe('mid-turn deliveries keep the window what the stream carried', () => { rmSync(tempDir, { recursive: true, force: true }); }); - async function createFramework(): Promise { + async function createFramework(extra: Partial[0]> = {}): Promise { const framework = await AgentFramework.create({ storePath: join(tempDir, 'test.chronicle'), membrane: membrane.asMembrane(), agents: [{ name: 'assistant', model: 'test-model', systemPrompt: 'You are a robot pilot.' }], modules: [module], + ...extra, }); module.framework = framework; return framework; @@ -397,7 +398,7 @@ describe('mid-turn deliveries keep the window what the stream carried', () => { await reopened.stop(); }); - it('at a tool boundary, a write whose store throws stays queued and is not injected; the rest are', async () => { + it('at a tool boundary, a write whose store throws cuts the drain: it and what follows wait for the turn end, in order', async () => { membrane.pushResponse(createMockResponse([ { type: 'text', text: 'Moving.' }, { type: 'tool_use', id: 'c1', name: 'robot--move', input: {} }, @@ -424,16 +425,96 @@ describe('mid-turn deliveries keep the window what the stream carried', () => { framework.pushEvent({ type: 'external-message', source: 'test', content: 'go' } as unknown as ProcessEvent); await framework.runUntilIdle(); } finally { console.error = error; } - assert.deepEqual(injected(), ['Antra: look right']); + assert.deepEqual(injected(), [], 'nothing from the failed write on is injected'); assert.deepEqual(windowOf(framework).slice(2), [ 'user: ', - 'Antra: look right', 'assistant: Done.', 'Antra: look left', - ], 'the one that failed was kept, and stored at the turn end'); + 'Antra: look right', + ], 'both were kept, and stored at the turn end in queue order'); await framework.stop(); }); + it("a queued tool_use whose store throws keeps its tool_result from being stored alone", async () => { + const framework = await createFramework(); + const internals = internalsOf(framework); + internals.deferredWritesPersisted = true; + // A pair queued for the agent, as puppetToolCall queues one behind a live turn. + const queue = framework as unknown as { + deferredSeq: number; + deferredMessages: Array<{ id: string; seq: number; participant: string; content: ContentBlock[]; forAgent?: string }>; + }; + queue.deferredMessages.push( + { id: 'pair-use', seq: ++queue.deferredSeq, participant: 'assistant', content: [{ type: 'tool_use', id: 'p1', name: 'robot--move', input: {} } as ContentBlock], forAgent: 'assistant' }, + { id: 'pair-result', seq: ++queue.deferredSeq, participant: 'user', content: [{ type: 'tool_result', toolUseId: 'p1', content: 'moved' } as ContentBlock], forAgent: 'assistant' }, + ); + const cm = framework.getAgent('assistant')!.getContextManager(); + const add = cm.addMessage.bind(cm); + (cm as unknown as { addMessage: typeof add }).addMessage = (...args) => { + if ((args[1] as ContentBlock[]).some((b) => b.type === 'tool_use')) throw new Error('disk full'); + return add(...args); + }; + const errors: string[] = []; + const error = console.error; + console.error = (...args: unknown[]) => { errors.push(args.map(String).join(' ')); }; + try { await framework.stop(); } finally { console.error = error; } + assert.ok(errors.some((e) => e.includes('pair-use (assistant: tool_use)') && e.includes('disk full')), errors.join('\n')); + assert.deepEqual(queue.deferredMessages.map((m) => m.id), ['pair-use', 'pair-result'], 'both stay queued, in order'); + + const reopened = await createFramework(); + const window = windowOf(reopened); + assert.deepEqual(window.slice(-2), ['assistant: ', 'user: '], 'boot recovery stored the pair, in order'); + assert.equal(window.filter((line) => line === 'user: ').length, 1, 'no tool_result was stored alone'); + await reopened.stop(); + }); + + it("stop() stores what is deferred for an agent whose turn frame it doesn't wait for", async () => { + const framework = await createFramework(); + const internals = internalsOf(framework); + // A frame stop() doesn't await, such as a puppet call still running, + // holds the agent's turn token. + internals.activeTurnTokens.set('assistant', 1); + assert.equal(internals.addMessage('Antra', text('still to land')), '', 'deferred: the turn is alive'); + await framework.stop(); + + const reopened = await createFramework(); + assert.ok(windowOf(reopened).includes('Antra: still to land'), 'stored before the store closed'); + await reopened.stop(); + }); + + it('a turn waiting to retry when the framework stops gives up rather than compile again', async () => { + let waiting!: () => void; + const inDelay = new Promise((resolve) => { waiting = resolve; }); + const framework = await createFramework({ + errorPolicy: { + maxRetries: 2, + onInferenceError: (_error: Error, _agent: string, attempt: number) => { + waiting(); + return attempt < 2 ? { retry: true as const, delayMs: 50 } : { retry: false as const }; + }, + }, + }); + const internals = internalsOf(framework); + const agent = framework.getAgent('assistant')!; + let compiles = 0; + (agent as unknown as { startStreamWithInjections: () => Promise }).startStreamWithInjections = async () => { + compiles++; + throw new Error('boom'); + }; + framework.start(); + framework.pushEvent({ type: 'external-message', source: 'test', content: 'go' } as unknown as ProcessEvent); + await inDelay; + assert.equal(internals.addMessage('Antra', text('while it waits')), '', 'deferred: the turn is alive'); + const error = console.error; + console.error = () => {}; + try { await framework.stop(); } finally { console.error = error; } + assert.equal(compiles, 1, 'the retry was refused, not compiled'); + + const reopened = await createFramework(); + assert.ok(windowOf(reopened).includes('Antra: while it waits'), 'what was deferred for it was stored'); + await reopened.stop(); + }); + it('a message stored straight into the context during a round, with nothing injected at the boundary, stays unread', async () => { membrane.pushResponse(createMockResponse([ { type: 'text', text: 'Moving.' }, diff --git a/test/ephemeral-settle.test.ts b/test/ephemeral-settle.test.ts index ce029a56c..d3998aa05 100644 --- a/test/ephemeral-settle.test.ts +++ b/test/ephemeral-settle.test.ts @@ -122,6 +122,49 @@ async function stopAndRemove(framework: AgentFramework, tempDir: string): Promis } describe('runEphemeralToCompletion settle signal', () => { + it('a run whose turn stop() refuses ends at once, and a run asked for once stop() has begun is refused', async () => { + const { tempDir, storePath } = tempStorePath('ephemeral-stopping-'); + const membrane = new MockMembrane(); + membrane.pushResponse(createMockResponse([{ type: 'text', text: 'too late' }])); + const framework = await AgentFramework.create({ + storePath, + membrane: membrane.asMembrane(), + agents: [], + modules: [new EphemeralToolModule()], + syncIntervalMs: 0, + }); + try { + const { agent, contextManager } = await createEphemeral(framework); + const later = await createEphemeral(framework, 'later'); + // Hold the run's compile until stop() has begun. + let compiling!: () => void; + const inCompile = new Promise((resolve) => { compiling = resolve; }); + let release!: () => void; + const released = new Promise((resolve) => { release = resolve; }); + const start = agent.startStreamWithInjections.bind(agent); + (agent as unknown as { startStreamWithInjections: typeof start }).startStreamWithInjections = async (...args) => { + compiling(); + await released; + return start(...args); + }; + const run = runStartedEphemeral(framework, agent, contextManager); + await inCompile; + const stopped = framework.stop(); + release(); + const refusedAt = Date.now(); + await assert.rejects(run, /refused: the framework is stopping/); + assert.ok(Date.now() - refusedAt < 5_000, 'ended at once, not at its startup watchdog'); + await stopped; + await assert.rejects( + framework.runEphemeralToCompletion(later.agent, later.contextManager), + /framework is stopping: refusing new ephemeral run/, + ); + later.cleanup(); + } finally { + rmSync(tempDir, { recursive: true, force: true }); + } + }); + it('resolves plain text completion and deregisters the agent', async () => { const { tempDir, storePath } = tempStorePath('ephemeral-plain-'); const membrane = new MockMembrane(); diff --git a/test/host-quiesce.test.ts b/test/host-quiesce.test.ts index c0a08f611..ac24605d8 100644 --- a/test/host-quiesce.test.ts +++ b/test/host-quiesce.test.ts @@ -683,7 +683,7 @@ test('a batch carrying a budget restart re-parks its sibling wakes instead of co }); }); -test('resume flushes deferred writes per message: one poison write neither drops the rest nor skips the reopen, and stays queued', async () => { +test('resume flushes deferred writes in order: a poison write stays queued with what follows it, and the reopen still completes', async () => { const membrane = new MockMembrane(); await withFramework(membrane, async (framework) => { const traces: TraceEvent[] = []; @@ -709,13 +709,14 @@ test('resume flushes deferred writes per message: one poison write neither drops const status = await framework.resume(); assert.equal(status.quiesced, false); - // A drained write whose store throws is never dropped: it goes back into - // the queue under its id, for the next flush. - assert.equal(status.deferredWrites, 1, 'only the write that failed is still queued'); + // A drained write whose store throws is never dropped, and nothing lands + // ahead of it: it and the write queued behind it go back into the queue + // under their ids, for the next flush. + assert.equal(status.deferredWrites, 2, 'the write that failed and the one behind it are still queued'); assert.deepEqual( cm.getAllMessages().slice(before).map((m) => (m.content[0] as { text: string }).text), - ['first', 'third'], - 'the writes behind the poison one still landed', + ['first'], + 'the write queued before the poison one landed', ); assert.ok(traces.some((t) => t.type === 'host:resume'), 'resume completed its follow-through'); }); diff --git a/test/operator-surgery.test.ts b/test/operator-surgery.test.ts index 2be1ce269..7a4d051a0 100644 --- a/test/operator-surgery.test.ts +++ b/test/operator-surgery.test.ts @@ -257,6 +257,37 @@ describe('live operator surgery', () => { } }); + it("the post-surgery flush lands what waited behind the hold while another agent's tool cycle is pending, and acknowledges it", async () => { + const c = cm() as unknown as { switchBranch: (name: string) => Promise }; + const originalSwitch = c.switchBranch; + let releaseSwitch!: () => void; + const gate = new Promise((resolve) => { releaseSwitch = resolve; }); + c.switchBranch = async (name: string) => { await gate; return originalSwitch.call(c, name); }; + const internals = framework as unknown as { + surgeryHold: unknown; + deferredWritesPersisted: boolean; + unackedDeferredWrites: unknown[]; + pendingAssistantBlocks: Map; + addMessage(participant: string, content: unknown[], metadata?: Record): string; + }; + internals.deferredWritesPersisted = true; + try { + const rollback = framework.rollbackToMessage('scout', { messageId: ids[2] }); + await new Promise((resolve) => setImmediate(resolve)); + assert.ok(internals.surgeryHold, 'store hold is up while the switch is awaited'); + assert.equal(internals.addMessage('user', [{ type: 'text', text: 'waited for the hold' }]), '', 'deferred'); + // Another agent's tool cycle says nothing about scout's window. + internals.pendingAssistantBlocks.set('builder', [{ type: 'tool_use', id: 'b1', name: 'x', input: {} }]); + releaseSwitch(); + await rollback; + assert.ok(remainingTexts().includes('waited for the hold'), 'flushed when the hold released'); + assert.equal(internals.unackedDeferredWrites.length, 0, 'and acknowledged, so the recovery queue no longer holds it'); + } finally { + internals.pendingAssistantBlocks.delete('builder'); + c.switchBranch = originalSwitch; + } + }); + // --- strategy-initialization failure ----------------------------------------- // switchBranch()/fork() move the chronicle branch BEFORE awaiting strategy // initialization, so a rejection there leaves the store on the new branch. From 7e1c588ff3eda192d1df05a22f8258d70ad46c9d Mon Sep 17 00:00:00 2001 From: Duncan-2410 Date: Tue, 13 Oct 2026 10:48:53 -0400 Subject: [PATCH 0180/1112] fix(deferral): a stuck write's log line can't drop it; say what the hold covers From the internal review of the previous round. Code: writeDrained puts a failed entry back into the queue before it logs, and describeDeferredWrite no longer throws on a malformed entry (a block that isn't an object with a type reads as "malformed"). Before, a null block made the description throw inside the catch, before the requeue: the entry stayed in the un-acked set for the next acknowledgement to drop, writeDrainedInOrder never requeued what followed it, and the flush threw. addMessage refuses a null block, so only a queue persisted by other code could carry one, but the log line must not be able to lose what it reports. Text, as Ada-1017 decided: the cut holds what is queued, and a write that arrives while its agent is idle isn't deferred, so it is stored at once and can land ahead of a held entry; an entry that keeps throwing holds only what is deferred behind it during its agent's turns, or while the host is quiesced. (A gate that held fresh writes too would blind an agent behind a poison entry: every wake's payload held, each turn run without what woke it.) A queued tool_use that can't be stored keeps its tool_result from being stored alone, but a stored tool_use whose tool_result then fails stays stored without it until the result lands; the only queued pair is puppetToolCall's defensive one. stop() neither refuses nor waits for a puppet call, as before: one still running stores its pair after the drain's entries, and on a closed store that write fails. The fragment no longer says no turn starts "for any agent". Tests: - host-quiesce: two residents, one with a stuck (and malformed) write ahead of the other's in the queue. Resume stores the other's writes and holds the stuck one with what is queued behind it for its own agent; a second flush holds it again; each flush's log line names its id and block types. - ephemeral-settle: a run waiting to retry when stop() begins gives up before compiling again. An ephemeral run doesn't hold the primary's provider gate, so startAgentStream's top refusal is the only check its retry meets; the primary's retry also meets the check after the gate, which is why removing the top refusal passed before. The "ended at once" check drops its 5 s wall-clock bound: the rejection's message already tells the refusal from the startup watchdog's. - deferred-window-only: the primary's retry test asserts the retry was granted and re-entered after stop() began, so a retry never granted can't pass for one refused. - operator-surgery: waits for the switchBranch wrapper to be entered rather than one setImmediate tick. Co-Authored-By: Claude Opus 5.5 --- .../mid-turn-window-only-held.fixed.md | 33 +++++---- src/framework.ts | 62 ++++++++++++----- test/deferred-window-only.test.ts | 13 ++++ test/ephemeral-settle.test.ts | 43 +++++++++++- test/host-quiesce.test.ts | 69 +++++++++++++++++++ test/operator-surgery.test.ts | 6 +- 6 files changed, 191 insertions(+), 35 deletions(-) diff --git a/changelog.d/mid-turn-window-only-held.fixed.md b/changelog.d/mid-turn-window-only-held.fixed.md index ff15b2e87..d45b27636 100644 --- a/changelog.d/mid-turn-window-only-held.fixed.md +++ b/changelog.d/mid-turn-window-only-held.fixed.md @@ -34,20 +34,29 @@ cycle holds them. - `stop()` now stores what is still deferred for each registered agent before the store closes; before, a memory-only queue lost it. Once `stop()` - begins, no turn starts and no stream registers, for any agent: a turn that - was mid-compile when the streams were cancelled, or one waiting to retry, - is refused rather than left running, and an ephemeral run refused this way + begins, no agent turn starts and no stream registers: a turn that was + mid-compile when the streams were cancelled, or one waiting to retry, is + refused rather than left running, and an ephemeral run refused this way ends at once rather than at its startup watchdog. Since no reply can then - follow a stored entry, `stop()` stores each one even while a frame it - doesn't wait for, such as a puppet call, still holds its target's turn. A - quiesced host keeps what is deferred in its persisted queue for boot - recovery. An entry for a target no longer registered (a disposed fork) is - dropped from a memory-only queue at exit, as before. + follow a stored entry in a stream, `stop()` stores each one even while a + frame it doesn't wait for still holds its target's turn: a turn started + outside the event loop, or a puppet call. `stop()` neither refuses nor + waits for a puppet call, as before, so one still running stores its pair + when its tool returns, after those entries, and on a store `stop()` has + closed that write fails. A quiesced host keeps what is deferred in its + persisted queue for boot recovery. An entry for a target no longer + registered (a disposed fork) is dropped from a memory-only queue at exit, + as before. - A deferred write whose store throws goes back into the queue under its - id, and so does every entry queued behind it for that agent, so none lands - out of order and a queued `tool_use` and its `tool_result` stay together. - The log line names the entry, and one that keeps throwing holds the - agent's later deferred writes until it can be stored. Before, it stayed in + id, and so does every entry queued behind it for that agent, so none of + them lands ahead of an earlier one, and a queued `tool_use` that can't be + stored keeps its `tool_result` from being stored alone. The log line names + the entry at each flush, and one that keeps throwing holds what is deferred + behind it for that agent until it can be stored. The hold is the queue's: + a write that arrives while the agent is idle isn't deferred, so it is + stored at once and can land ahead of a held entry, and an entry that keeps + throwing holds only what is deferred during the agent's turns (or while the + host is quiesced). Before, it stayed in the un-acked set and the next acknowledgement dropped it from the persisted queue (at a turn start or a flush it was lost even from memory). At a tool boundary, nothing from it on is injected. While the framework diff --git a/src/framework.ts b/src/framework.ts index e05481119..8ebf455df 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -481,9 +481,19 @@ function withDeferredWriteId(metadata: MessageMetadata | undefined, id: string): return { ...(metadata ?? {}), deferredWriteId: id } as MessageMetadata; } -/** A deferred entry as a log line names it: its id, participant and block types. */ +/** A deferred entry as a log line names it: its id, participant and block + * types. It never throws: it describes the entries the store rejected, which + * may be malformed (a block that isn't an object with a type reads as + * `malformed`). */ function describeDeferredWrite(msg: { id: string; participant: string; content: ContentBlock[] }): string { - return `${msg.id} (${msg.participant}: ${[...new Set(msg.content.map((b) => b.type))].join('+') || 'empty'})`; + const content: unknown = msg.content; + const types = Array.isArray(content) + ? [...new Set(content.map((b: unknown) => { + const type = b && typeof b === 'object' ? (b as { type?: unknown }).type : undefined; + return typeof type === 'string' ? type : 'malformed'; + }))].join('+') || 'empty' + : 'malformed'; + return `${msg.id} (${msg.participant}: ${types})`; } /** Whether a deferred message can join a live stream at a tool boundary: @@ -2277,17 +2287,20 @@ export class AgentFramework { } } - // No stream is left: from the start of stop() none registers and no turn - // starts (providerAdmissionClosed, checked for every agent), and every - // stream registered before was cancelled and awaited above. So nothing - // else will deliver what is still deferred: a message held for a turn's - // end whose flush found the target busy again, or one that arrived for an - // idle agent and was deferred behind its own state. Land each at its + // No stream is left: from the start of stop() none registers and no + // agent turn starts (providerAdmissionClosed, checked for every agent), + // and every stream registered before was cancelled and awaited above. + // What is still deferred (a message held for a turn's end whose flush + // found the target busy again, or one deferred behind its target's own + // state) would wait for a flush that may never come. Land each at its // registered target now, while the store is open; a memory-only queue // would otherwise lose it at exit. It goes past the deferral: a frame - // stop() doesn't wait for (a puppet call, or a turn started outside the - // event loop) can still hold its target's turn token, but no reply can - // follow the entry in any stream. A quiesced host defers it again + // stop() doesn't wait for (a turn started outside the event loop, or a + // puppet call) can still hold its target's turn token, but no reply can + // follow the entry in any stream. stop() neither refuses nor waits for a + // puppet call, as before this drain: one still running stores its pair + // when its tool returns, after these entries, and on a store closed + // below that write fails. A quiesced host defers it again // instead, into the persisted queue, for boot recovery. A write that // throws stays queued with what follows it (writeDrainedInOrder), and a // memory-only queue still loses it at exit. An entry for a target no @@ -3664,8 +3677,10 @@ export class AgentFramework { * after every message). A crash mid-flush therefore replays exactly the * un-acked remainder at the next boot — no loss, no duplicates. A write * the store rejects stays queued, logged, with what is queued behind it for - * its target (writeDrainedInOrder), so a target's entries never land out of - * order; a boot that meets it again logs it and goes on. + * its target (writeDrainedInOrder), so the flush never stores a target's + * later entries ahead of it (a write that arrives while its target is idle + * isn't deferred, though, and can land ahead of it); a boot that meets it + * again logs it and goes on. * * Per-target: an agent whose turn is still alive keeps ITS messages * deferred for its own turn boundary; everyone else's flush now. @@ -13253,12 +13268,13 @@ export class AgentFramework { write(); return true; } catch (error) { + // Back into the queue before anything else can go wrong here. + this.requeueDrained([msg]); console.error( `[deferral] ${where}: deferred write ${describeDeferredWrite(msg)} for ` + `${msg.forAgent ?? this.primaryAgentName ?? 'the primary'} could not be stored; it stays queued, ` + `holding what is queued behind it for that agent: ${error instanceof Error ? error.message : String(error)}`, ); - this.requeueDrained([msg]); return false; } } @@ -13281,11 +13297,19 @@ export class AgentFramework { /** * Write one agent's drained entries in queue order, each through * writeDrained. The drain cuts at the first that can't be stored: it and - * every later entry go back into the queue, so nothing lands ahead of an - * earlier entry and a queued pair stays together. An entry that keeps - * failing holds its agent's later writes, logged at each flush, rather than - * letting them land out of order. Returns what was written, a prefix of - * `entries`. + * every later entry go back into the queue, so none of them lands ahead of + * an earlier one, and a queued tool_use that can't be stored keeps its + * tool_result from being stored alone. (A tool_use that is stored, followed + * by a tool_result that can't be, stays stored without it until the + * tool_result lands at a later flush. The only queued pair is + * puppetToolCall's defensive one, which no path reaches today.) An + * entry that keeps failing holds its agent's later queued writes, logged + * at each flush, rather than letting them land out of order. The hold is + * the queue's alone: a write that arrives while its agent is idle isn't + * deferred (addMessage stores it at once), so it can land ahead of a held + * entry, and an entry that keeps failing holds only what is deferred + * behind it while its agent's turn, stream or tool cycle is alive, or the + * host is quiesced. Returns what was written, a prefix of `entries`. */ private writeDrainedInOrder(entries: DeferredWrite[], where: string, write: (msg: DeferredWrite) => void): DeferredWrite[] { for (let i = 0; i < entries.length; i++) { diff --git a/test/deferred-window-only.test.ts b/test/deferred-window-only.test.ts index 626f433ab..e0039d2d3 100644 --- a/test/deferred-window-only.test.ts +++ b/test/deferred-window-only.test.ts @@ -501,6 +501,18 @@ describe('mid-turn deliveries keep the window what the stream carried', () => { compiles++; throw new Error('boom'); }; + // The retry re-enters startAgentStream after its delay; count those + // re-entries, so a retry that was never granted can't pass for one that + // was refused. + const retries: number[] = []; + const frames = framework as unknown as { + startAgentStream(agent: unknown, trigger?: unknown, attempt?: number, held?: boolean): Promise; + }; + const startAgentStream = frames.startAgentStream.bind(framework); + frames.startAgentStream = (a, trigger, attempt = 0, held = false) => { + if (attempt > 0) retries.push(attempt); + return startAgentStream(a, trigger, attempt, held); + }; framework.start(); framework.pushEvent({ type: 'external-message', source: 'test', content: 'go' } as unknown as ProcessEvent); await inDelay; @@ -508,6 +520,7 @@ describe('mid-turn deliveries keep the window what the stream carried', () => { const error = console.error; console.error = () => {}; try { await framework.stop(); } finally { console.error = error; } + assert.deepEqual(retries, [1], 'the retry was granted, and re-entered once stop() had begun'); assert.equal(compiles, 1, 'the retry was refused, not compiled'); const reopened = await createFramework(); diff --git a/test/ephemeral-settle.test.ts b/test/ephemeral-settle.test.ts index d3998aa05..9d20db562 100644 --- a/test/ephemeral-settle.test.ts +++ b/test/ephemeral-settle.test.ts @@ -151,9 +151,9 @@ describe('runEphemeralToCompletion settle signal', () => { await inCompile; const stopped = framework.stop(); release(); - const refusedAt = Date.now(); + // Ended by the refusal, not by its startup watchdog, whose rejection + // names a zombie instead. await assert.rejects(run, /refused: the framework is stopping/); - assert.ok(Date.now() - refusedAt < 5_000, 'ended at once, not at its startup watchdog'); await stopped; await assert.rejects( framework.runEphemeralToCompletion(later.agent, later.contextManager), @@ -165,6 +165,45 @@ describe('runEphemeralToCompletion settle signal', () => { } }); + it('a run waiting to retry when stop() begins gives up before compiling again', async () => { + // An ephemeral run doesn't hold the primary's provider gate, so the + // refusal at the top of startAgentStream is the only check its retry + // meets before it would compile, and start a stream, again. + const { tempDir, storePath } = tempStorePath('ephemeral-retry-stopping-'); + let waiting!: () => void; + const inDelay = new Promise((resolve) => { waiting = resolve; }); + const framework = await AgentFramework.create({ + storePath, + membrane: new MockMembrane().asMembrane(), + agents: [], + modules: [new EphemeralToolModule()], + syncIntervalMs: 0, + errorPolicy: { + maxRetries: 1, + onInferenceError: (_error: Error, _agent: string, attempt: number): ErrorAction => { + waiting(); + return attempt < 1 ? { retry: true, delayMs: 50 } : { retry: false }; + }, + }, + }); + try { + const { agent, contextManager } = await createEphemeral(framework); + let compiles = 0; + (agent as unknown as { startStreamWithInjections: () => Promise }).startStreamWithInjections = async () => { + compiles++; + throw new Error('boom'); + }; + const run = runStartedEphemeral(framework, agent, contextManager); + await inDelay; + const stopped = framework.stop(); + await assert.rejects(run, /refused: the framework is stopping/); + await stopped; + assert.equal(compiles, 1, 'the retry was refused, not compiled'); + } finally { + rmSync(tempDir, { recursive: true, force: true }); + } + }); + it('resolves plain text completion and deregisters the agent', async () => { const { tempDir, storePath } = tempStorePath('ephemeral-plain-'); const membrane = new MockMembrane(); diff --git a/test/host-quiesce.test.ts b/test/host-quiesce.test.ts index ac24605d8..75ff5fbb6 100644 --- a/test/host-quiesce.test.ts +++ b/test/host-quiesce.test.ts @@ -722,6 +722,75 @@ test('resume flushes deferred writes in order: a poison write stays queued with }); }); +test("resume cuts each target at its own stuck write, naming it at every flush, and other targets' writes land", async () => { + const dir = mkdtempSync(join(tmpdir(), 'host-quiesce-targets-')); + const framework = await AgentFramework.create({ + storePath: join(dir, 'store'), + membrane: new MockMembrane().asMembrane(), + agents: [ + { name: 'agent', model: 'test-model', systemPrompt: 'test' }, + { name: 'scout', model: 'test-model', systemPrompt: 'test' }, + ], + modules: [], + }); + const lines: string[] = []; + const error = console.error; + console.error = (...args: unknown[]) => { lines.push(args.map(String).join(' ')); }; + try { + await framework.quiesce({ reason: 'refold' }); + const internals = framework as unknown as { + deferredSeq: number; + deferredMessages: Array<{ id: string; seq: number; participant: string; content: unknown[]; forAgent?: string }>; + addMessage(participant: string, content: unknown[], metadata?: undefined, opts?: { forAgent?: string }): unknown; + }; + // Residents share one message slot, so each one's context manager reads + // what either stored; each stores through its own. + const agentCm = framework.getAgent('agent')!.getContextManager(); + const before = agentCm.getAllMessages().length; + const stored = () => agentCm.getAllMessages().slice(before).map((m) => (m.content[0] as { text: string }).text); + // A malformed entry the store rejects, as only a queue persisted by other + // code could carry (addMessage refuses a null block): the log line still + // names it, and it still goes back into the queue. + internals.deferredMessages.push({ + id: 'stuck-1', seq: ++internals.deferredSeq, participant: 'user', + content: [{ type: 'text', text: 'stuck' }, null], forAgent: 'agent', + }); + internals.addMessage('user', [{ type: 'text', text: 'for scout' }], undefined, { forAgent: 'scout' }); + internals.addMessage('user', [{ type: 'text', text: 'behind the stuck one' }], undefined, { forAgent: 'agent' }); + internals.addMessage('user', [{ type: 'text', text: 'also for scout' }], undefined, { forAgent: 'scout' }); + assert.equal(framework.getHostModeStatus().deferredWrites, 4); + + const realAdd = agentCm.addMessage.bind(agentCm); + (agentCm as unknown as { addMessage: typeof agentCm.addMessage }).addMessage = ((participant, content, metadata, causedBy) => { + if ((content[0] as { text?: string }).text === 'stuck') throw new Error('store write failed'); + return realAdd(participant, content, metadata, causedBy); + }) as typeof agentCm.addMessage; + const named = () => lines.filter((line) => + line.includes('deferred write stuck-1 (user: text+malformed) for agent could not be stored')).length; + + let status = await framework.resume(); + // Queued after agent's stuck write, scout's land; agent's own later + // write is held behind it. + assert.deepEqual(stored(), ['for scout', 'also for scout'], "scout's writes land; agent's stuck write holds what is queued behind it for agent"); + assert.equal(status.deferredWrites, 2); + assert.equal(named(), 1, 'the flush names the stuck write: its id and block types'); + + // A second flush with the store still refusing it: held again, named again. + await framework.quiesce({ reason: 'refold' }); + status = await framework.resume(); + assert.deepEqual(stored(), ['for scout', 'also for scout'], 'still nothing lands ahead of it'); + assert.equal(status.deferredWrites, 2); + assert.equal(named(), 2, 'every flush names it'); + } finally { + try { + await framework.stop(); + } finally { + console.error = error; + rmSync(dir, { recursive: true, force: true }); + } + } +}); + test('context writes deferred while quiesced survive a restart and land at resume', async () => { const dir = mkdtempSync(join(tmpdir(), 'host-quiesce-deferred-')); const storePath = join(dir, 'store'); diff --git a/test/operator-surgery.test.ts b/test/operator-surgery.test.ts index 7a4d051a0..d38379d17 100644 --- a/test/operator-surgery.test.ts +++ b/test/operator-surgery.test.ts @@ -262,7 +262,9 @@ describe('live operator surgery', () => { const originalSwitch = c.switchBranch; let releaseSwitch!: () => void; const gate = new Promise((resolve) => { releaseSwitch = resolve; }); - c.switchBranch = async (name: string) => { await gate; return originalSwitch.call(c, name); }; + let entered!: () => void; + const inSwitch = new Promise((resolve) => { entered = resolve; }); + c.switchBranch = async (name: string) => { entered(); await gate; return originalSwitch.call(c, name); }; const internals = framework as unknown as { surgeryHold: unknown; deferredWritesPersisted: boolean; @@ -273,7 +275,7 @@ describe('live operator surgery', () => { internals.deferredWritesPersisted = true; try { const rollback = framework.rollbackToMessage('scout', { messageId: ids[2] }); - await new Promise((resolve) => setImmediate(resolve)); + await inSwitch; assert.ok(internals.surgeryHold, 'store hold is up while the switch is awaited'); assert.equal(internals.addMessage('user', [{ type: 'text', text: 'waited for the hold' }]), '', 'deferred'); // Another agent's tool cycle says nothing about scout's window. From 2ab4b0f1b4303e4e75b03c8e498ce5c0f9c67d0b Mon Sep 17 00:00:00 2001 From: Duncan-2410 Date: Tue, 13 Oct 2026 11:24:06 -0400 Subject: [PATCH 0181/1112] test(deferral): a run that stop() should end fails at once if it isn't ended 7e1c588's ephemeral test said a run the refusal didn't end would be ended by its startup watchdog, whose rejection names a zombie. It wouldn't: the turn marks inference started before its compile, so the startup watchdog never fires, and only the idle watchdog would end the run, after fifteen minutes. With the refusal removed, both stop() tests in ephemeral-settle waited out the test runner's timeout, and CI's npm test sets none. (The 5 s bound that 7e1c588 dropped never caught that case either: it was checked only after the rejection, which never came.) The two tests now give their run a 20 s idle watchdog, so a run the refusal doesn't end is rejected as stalled, which the refusal's pattern doesn't match. A refusal that works ends the run within milliseconds of the compile's release, far inside that window. Co-Authored-By: Claude Opus 5.5 --- test/ephemeral-settle.test.ts | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/test/ephemeral-settle.test.ts b/test/ephemeral-settle.test.ts index 9d20db562..21765f833 100644 --- a/test/ephemeral-settle.test.ts +++ b/test/ephemeral-settle.test.ts @@ -110,12 +110,19 @@ async function runStartedEphemeral( framework: AgentFramework, agent: Awaited>['agent'], contextManager: Awaited>['contextManager'], + watchdogs?: Parameters[2], ) { - const promise = framework.runEphemeralToCompletion(agent, contextManager); + const promise = framework.runEphemeralToCompletion(agent, contextManager, watchdogs); framework.start(); return promise; } +// For a run that stop() is to end: its turn has started inference, so if the +// refusal didn't end it, only the idle watchdog would, after fifteen minutes. +// A short one turns that into a "stalled" rejection, which the refusal's +// pattern doesn't match. +const STOPPED_RUN_WATCHDOGS = { idleTimeoutMs: 20_000, idlePollMs: 50 }; + async function stopAndRemove(framework: AgentFramework, tempDir: string): Promise { await framework.stop(); rmSync(tempDir, { recursive: true, force: true }); @@ -147,12 +154,10 @@ describe('runEphemeralToCompletion settle signal', () => { await released; return start(...args); }; - const run = runStartedEphemeral(framework, agent, contextManager); + const run = runStartedEphemeral(framework, agent, contextManager, STOPPED_RUN_WATCHDOGS); await inCompile; const stopped = framework.stop(); release(); - // Ended by the refusal, not by its startup watchdog, whose rejection - // names a zombie instead. await assert.rejects(run, /refused: the framework is stopping/); await stopped; await assert.rejects( @@ -193,7 +198,7 @@ describe('runEphemeralToCompletion settle signal', () => { compiles++; throw new Error('boom'); }; - const run = runStartedEphemeral(framework, agent, contextManager); + const run = runStartedEphemeral(framework, agent, contextManager, STOPPED_RUN_WATCHDOGS); await inDelay; const stopped = framework.stop(); await assert.rejects(run, /refused: the framework is stopping/); From c9d612e98ad6da1a7ff5d489b6be3a4efb3c7773 Mon Sep 17 00:00:00 2001 From: Duncan-2410 Date: Tue, 13 Oct 2026 11:47:16 -0400 Subject: [PATCH 0182/1112] fix(deferral): flushes store in queue order across agents; recovery drops entries the store can't take From the internal review of the previous round, with Ada-1017 deciding the two design points. Queue order across agents. Residents share one message slot, so the order deferred writes arrived in is the order every resident reads. 1e95eb2 grouped resume's and boot recovery's flush by target to cut per target, so even with nothing failing it stored all of one target's entries before the next target's, where 0c5c0b0 stored them in queue order. stop()'s drain and the post-surgery flush went agent by agent, as main's post-surgery flush did. writeDrainedInOrder now takes any targets' entries in queue (seq) order and cuts each target at its own first entry that can't be stored: that entry and every later one for the same target go back into the queue before anything is logged, while other targets' entries go on. Resume, boot recovery, stop() and the post-surgery flush each hand it every agent's entries at once. writeDrained is folded into it. The log line: a thrown value with no string form (Object.create(null)) made String(error) throw inside the catch, before the entries behind the failed one were requeued, so the next ack dropped them. Everything for the cut target now goes back first, and the reason falls back to "a thrown value that has no text". Malformed recovered entries. An entry with a null block can never be stored, and can reach the queue only from the recovery file: a fresh write with a null block throws in addMessage before it is queued. Held, it blocked its agent's later deferred writes for good, and at a tool boundary isInjectableMidTurn threw on it. The loader already dropped, silently, an entry whose content isn't an array; it now also drops one whose blocks aren't all objects, logs each drop by id and participant, and rewrites the recovery file without them. Every reader of the queue can trust its entries' shape, so describeDeferredWrite is back to its plain form. Text: the fragment, the stop() comment and the docs now say that a fresh tool_result doesn't wait; that while quiesced an idle agent's write is deferred too; that an entry that keeps failing holds what is deferred while its agent's turn, stream or tool cycle is alive, or the host is quiesced; that a stored tool_use whose tool_result can't be stored stays alone until the result lands; that a puppet call whose tool returns before stop()'s drain stores its pair first, and one that returns later stores it after the drained entries; and that a refused ephemeral run ends at once rather than waiting for a watchdog. Tests: - host-quiesce: two residents' interleaved writes land through resume in queue order, and a stuck write, refused with a thrown value that has no text, cuts only its own agent and is named by id at each of two flushes. Boot recovery drops a recovered entry with a null block and one whose content isn't blocks, names each, rewrites the file, and restores and lands the rest. - deferred-window-only: stop() stores two residents' interleaved deferred writes in queue order. The retry test records that stop() had begun when the retry re-entered. - operator-surgery: the post-surgery flush stores two residents' interleaved writes in queue order. - The tests that wait on an event (the post-surgery hold, the ephemeral stop tests, the retry delay) take a 60 s timeout, so a missed event fails rather than hangs. Co-Authored-By: Claude Opus 5.5 --- .../mid-turn-window-only-held.fixed.md | 77 +++--- src/framework.ts | 227 ++++++++++-------- test/deferred-window-only.test.ts | 34 ++- test/ephemeral-settle.test.ts | 4 +- test/host-quiesce.test.ts | 97 ++++++-- test/operator-surgery.test.ts | 39 ++- 6 files changed, 317 insertions(+), 161 deletions(-) diff --git a/changelog.d/mid-turn-window-only-held.fixed.md b/changelog.d/mid-turn-window-only-held.fixed.md index d45b27636..698cb1cb1 100644 --- a/changelog.d/mid-turn-window-only-held.fixed.md +++ b/changelog.d/mid-turn-window-only-held.fixed.md @@ -22,43 +22,56 @@ content. That happens while its target's own turn, stream or tool cycle is alive, or the host is quiesced. A `tool_result` used to land regardless, apart from the `tool_use` it answers, which went back into the queue. -- Every write now waits on its own target's state: the target's turn, stream - and tool cycle, or a quiesced host. Before, a write for the primary agent - waited out every agent's tool cycle, while one addressed to an agent waited - only on that agent's. So a turn's end could store a later addressed message - ahead of an earlier one for the primary, and a fork's, subagent's or the +- A write that waits now waits on its own target's state: the target's + turn, stream and tool cycle, or a quiesced host. (A fresh `tool_result` + still doesn't wait.) Before, a write for the primary agent waited out every + agent's tool cycle, while one addressed to an agent waited only on that + agent's. So a turn's end could store a later addressed message ahead of an + earlier one for the primary, and a fork's, subagent's or the subconscious's tools held the primary's writes, which protected nothing in the primary's window. Each flush, at a turn's end, after surgery, after a withdrawn turn and after a puppet call, now delivers the agent's deferred messages while another agent's tool cycle is pending; only the agent's own - cycle holds them. + cycle holds them. The flush after surgery stores several agents' entries in + the order they arrived, where it went agent by agent: residents share one + message slot, so that reordered their shared transcript. - `stop()` now stores what is still deferred for each registered agent - before the store closes; before, a memory-only queue lost it. Once `stop()` - begins, no agent turn starts and no stream registers: a turn that was - mid-compile when the streams were cancelled, or one waiting to retry, is - refused rather than left running, and an ephemeral run refused this way - ends at once rather than at its startup watchdog. Since no reply can then - follow a stored entry in a stream, `stop()` stores each one even while a - frame it doesn't wait for still holds its target's turn: a turn started - outside the event loop, or a puppet call. `stop()` neither refuses nor - waits for a puppet call, as before, so one still running stores its pair - when its tool returns, after those entries, and on a store `stop()` has - closed that write fails. A quiesced host keeps what is deferred in its - persisted queue for boot recovery. An entry for a target no longer - registered (a disposed fork) is dropped from a memory-only queue at exit, - as before. + before the store closes, in the order it arrived; before, a memory-only + queue lost it. Once `stop()` begins, no agent turn starts and no stream + registers: a turn that was mid-compile when the streams were cancelled, or + one waiting to retry, is refused rather than left running, and an ephemeral + run refused this way ends at once rather than waiting for a watchdog. Since + no reply can then follow a stored entry in a stream, `stop()` stores each + one even while a frame it doesn't wait for still holds its target's turn: + a turn started outside the event loop, or a puppet call. `stop()` neither + refuses nor waits for a puppet call, as before. One still running stores + its pair when its tool returns. If that's before `stop()` stores the + deferred entries, the pair lands first and the puppet's own flush stores + what was deferred behind it; if it's later, the pair lands after them, and + on a store `stop()` has closed that write fails. A quiesced host keeps what + is deferred in its persisted queue for boot recovery. An entry for a target + no longer registered (a disposed fork) is dropped from a memory-only queue + at exit, as before. - A deferred write whose store throws goes back into the queue under its id, and so does every entry queued behind it for that agent, so none of them lands ahead of an earlier one, and a queued `tool_use` that can't be - stored keeps its `tool_result` from being stored alone. The log line names - the entry at each flush, and one that keeps throwing holds what is deferred - behind it for that agent until it can be stored. The hold is the queue's: - a write that arrives while the agent is idle isn't deferred, so it is - stored at once and can land ahead of a held entry, and an entry that keeps - throwing holds only what is deferred during the agent's turns (or while the - host is quiesced). Before, it stayed in - the un-acked set and the next acknowledgement dropped it from the - persisted queue (at a turn start or a flush it was lost even from memory). - At a tool boundary, nothing from it on is injected. While the framework - runs it is kept; a memory-only queue still loses it when the process - exits. + stored keeps the `tool_result` queued behind it from being stored alone. (A + stored `tool_use` whose `tool_result` then can't be stored stays alone + until the result lands at a later flush.) Other agents' entries go on, in + the order they arrived. The log line names the entry at each flush, and one + that keeps throwing holds what is deferred behind it for that agent until + it can be stored. The hold is the queue's: a write that isn't deferred is + stored at once and can land ahead of a held entry. That's a fresh + `tool_result`, and any write while the agent's turn, stream and tool cycle + are idle and the host isn't quiesced. So an entry that keeps throwing holds + only what is deferred while the agent's turn, stream or tool cycle is + alive, or the host is quiesced. Before, it stayed in the un-acked set and + the next acknowledgement dropped it from the persisted queue (at a turn + start or a flush it was lost even from memory). At a tool boundary, + nothing from it on is injected. While the framework runs it is kept; a + memory-only queue still loses it when the process exits. +- Boot recovery drops a recovered deferred write whose content isn't an + array of block objects, which the store could never take, and logs it by + id and participant, rather than holding it in the queue. Before, an entry + whose content wasn't an array was dropped without a line, and one with a + null block was kept, where it could never be stored. diff --git a/src/framework.ts b/src/framework.ts index 8ebf455df..227cf80ce 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -482,18 +482,12 @@ function withDeferredWriteId(metadata: MessageMetadata | undefined, id: string): } /** A deferred entry as a log line names it: its id, participant and block - * types. It never throws: it describes the entries the store rejected, which - * may be malformed (a block that isn't an object with a type reads as - * `malformed`). */ + * types. Every queued entry's blocks are objects: a fresh write with a null + * block throws in addMessage before it could be queued, and a recovered + * entry whose content isn't an array of objects is dropped at load + * (restorePersistedDeferredWrites). */ function describeDeferredWrite(msg: { id: string; participant: string; content: ContentBlock[] }): string { - const content: unknown = msg.content; - const types = Array.isArray(content) - ? [...new Set(content.map((b: unknown) => { - const type = b && typeof b === 'object' ? (b as { type?: unknown }).type : undefined; - return typeof type === 'string' ? type : 'malformed'; - }))].join('+') || 'empty' - : 'malformed'; - return `${msg.id} (${msg.participant}: ${types})`; + return `${msg.id} (${msg.participant}: ${[...new Set(msg.content.map((b) => b.type))].join('+') || 'empty'})`; } /** Whether a deferred message can join a live stream at a tool boundary: @@ -2294,22 +2288,29 @@ export class AgentFramework { // found the target busy again, or one deferred behind its target's own // state) would wait for a flush that may never come. Land each at its // registered target now, while the store is open; a memory-only queue - // would otherwise lose it at exit. It goes past the deferral: a frame - // stop() doesn't wait for (a turn started outside the event loop, or a - // puppet call) can still hold its target's turn token, but no reply can - // follow the entry in any stream. stop() neither refuses nor waits for a - // puppet call, as before this drain: one still running stores its pair - // when its tool returns, after these entries, and on a store closed - // below that write fails. A quiesced host defers it again - // instead, into the persisted queue, for boot recovery. A write that - // throws stays queued with what follows it (writeDrainedInOrder), and a - // memory-only queue still loses it at exit. An entry for a target no - // longer registered (a disposed fork) is left: dropped with a memory-only - // queue at exit, and logged and dropped from a persisted one at boot. + // would otherwise lose it at exit. Every registered agent's entries go in + // queue order (residents share one message slot), each target cut at its + // own first entry that can't be stored (writeDrainedInOrder). They go + // past the deferral: a frame stop() doesn't wait for (a turn started + // outside the event loop, or a puppet call) can still hold its target's + // turn token, but no reply can follow an entry in any stream. + // + // stop() neither refuses nor waits for a puppet call, as before this + // drain. One still running stores its pair when its tool returns. If + // that's before this drain runs, the pair lands first, and the puppet's + // own flush stores what was deferred behind it. If it's later, the pair + // lands after these entries, and on a store closed below that write + // fails. + // + // A quiesced host defers each entry again instead, into the persisted + // queue, for boot recovery. A write that throws stays queued with what + // follows it for its target, and a memory-only queue still loses it at + // exit. An entry for a target no longer registered (a disposed fork) is + // left: dropped with a memory-only queue at exit, and logged and dropped + // from a persisted one at boot. if (this.deferredMessages.length > 0) { - for (const name of this.agents.keys()) { - this.readdDrained(this.drainDeferredFor(name), 'stop', this.quiesced ? undefined : { bypassDeferralFor: name }); - } + const entries = [...this.agents.keys()].flatMap((name) => this.drainDeferredFor(name)); + this.readdDrained(entries, 'stop', { bypassDeferral: !this.quiesced }); this.ackDeferredWrites(); } @@ -3675,12 +3676,13 @@ export class AgentFramework { * Land every deferred write whose target has no turn alive, one at a time, * acknowledging each durably as it lands (the persisted queue is rewritten * after every message). A crash mid-flush therefore replays exactly the - * un-acked remainder at the next boot — no loss, no duplicates. A write - * the store rejects stays queued, logged, with what is queued behind it for - * its target (writeDrainedInOrder), so the flush never stores a target's - * later entries ahead of it (a write that arrives while its target is idle - * isn't deferred, though, and can land ahead of it); a boot that meets it - * again logs it and goes on. + * un-acked remainder at the next boot — no loss, no duplicates. The batch + * is stored in queue order, whatever its targets (writeDrainedInOrder). A + * write the store rejects stays queued, logged, with what is queued behind + * it for its target, so the flush never stores that target's later entries + * ahead of it, and other targets' entries go on. (A write that isn't + * deferred can still land ahead of it, such as one for an idle agent once + * the host has resumed.) A boot that meets it again logs it and goes on. * * Per-target: an agent whose turn is still alive keeps ITS messages * deferred for its own turn boundary; everyone else's flush now. @@ -3703,15 +3705,7 @@ export class AgentFramework { flush.sort(bySeq); this.deferredMessages = keep; this.handOffDeferredWrites(flush); - // Each target's entries in queue order, cut at the first that can't be - // stored; other targets' entries go on. - const byTarget = new Map(); - for (const msg of flush) { - const target = msg.forAgent ?? this.primaryAgentName; - byTarget.set(target, [...(byTarget.get(target) ?? []), msg]); - } - let stored = 0; - for (const entries of byTarget.values()) stored += this.readdDrained(entries, `host-mode ${label}`).length; + const stored = this.readdDrained(flush, `host-mode ${label}`).length; // Ack = sync the chronicle FIRST, then rewrite the durable queue without // the batch. Never the other way round. this.ackDeferredWrites(); @@ -3914,10 +3908,27 @@ export class AgentFramework { if (typeof v === 'number' && Number.isFinite(v)) scanFrom.set(k, Math.max(0, Math.floor(v))); } } + // An entry the store could never take has no place in the queue, whose + // readers trust each entry's shape: one whose participant isn't text, + // or whose content isn't an array of block objects, is dropped here + // with a line naming it, rather than held. + const wellFormed = (m: unknown): m is Omit & { id?: string; seq?: number } => + !!m && typeof m === 'object' && typeof (m as { participant?: unknown }).participant === 'string' + && Array.isArray((m as { content?: unknown }).content) + && (m as { content: unknown[] }).content.every((b) => !!b && typeof b === 'object'); + const malformed = data.filter((m) => !wellFormed(m)); + for (const m of malformed) { + const field = (key: string): string => { + const value = m && typeof m === 'object' ? (m as Record)[key] : undefined; + return typeof value === 'string' ? value : `no ${key}`; + }; + console.error( + `[host-mode] a recovered deferred context write is malformed and was dropped: ` + + `${field('id')} (${field('participant')})`, + ); + } const candidates = data - .filter((m): m is Omit & { id?: string; seq?: number } => - !!m && typeof m === 'object' && typeof (m as { participant?: unknown }).participant === 'string' - && Array.isArray((m as { content?: unknown }).content)) + .filter(wellFormed) .map((m, i) => ({ ...m, id: typeof m.id === 'string' ? m.id : randomUUID(), @@ -3942,8 +3953,9 @@ export class AgentFramework { this.deferredMessages.push(...restored); this.deferredWritesPersisted = true; // Rewrite the durable queue without the already-landed entries so a - // second interrupted boot does not re-scan them. - if (landed.size > 0) this.persistDeferredWrites(); + // second interrupted boot does not re-scan them, and without the + // malformed ones. + if (landed.size > 0 || malformed.length > 0) this.persistDeferredWrites(); return restored.length; } catch (err) { console.error('[host-mode] failed to restore persisted deferred context writes:', err); @@ -6296,12 +6308,12 @@ export class AgentFramework { } // Writers that deferred behind the reservation land now, on the branch // the operator chose (or the restored source) — the same end-of-turn - // flush driveStream/puppet perform, under the same guard. + // flush driveStream/puppet perform, under the same guard — for every + // agent free to take them, in queue order (writeDrainedInOrder). if (this.deferredMessages.length > 0) { - for (const name of reserved.keys()) { - if (this.activeTurnTokens.has(name) || this.pendingAssistantBlocks.has(name)) continue; - this.readdDrained(this.drainDeferredFor(name), 'post-surgery flush'); - } + const free = [...reserved.keys()] + .filter((name) => !this.activeTurnTokens.has(name) && !this.pendingAssistantBlocks.has(name)); + this.readdDrained(free.flatMap((name) => this.drainDeferredFor(name)), 'post-surgery flush'); this.ackDeferredWrites(); } }; @@ -13255,30 +13267,6 @@ export class AgentFramework { return drained; } - /** - * Write one drained entry. If the write throws, it is logged, naming the - * entry, and goes back into the queue under its id and seq - * (requeueDrained): left in the un-acked set, the next ack would drop it - * from the durable queue. Every site writes a drained list through - * writeDrainedInOrder, which cuts there. Returns whether it was written - * (stored, or deferred again by addMessage). - */ - private writeDrained(msg: DeferredWrite, where: string, write: () => void): boolean { - try { - write(); - return true; - } catch (error) { - // Back into the queue before anything else can go wrong here. - this.requeueDrained([msg]); - console.error( - `[deferral] ${where}: deferred write ${describeDeferredWrite(msg)} for ` + - `${msg.forAgent ?? this.primaryAgentName ?? 'the primary'} could not be stored; it stays queued, ` + - `holding what is queued behind it for that agent: ${error instanceof Error ? error.message : String(error)}`, - ); - return false; - } - } - /** * Put drained entries back into the queue under their ids and seq: out of * the un-acked set, and persisted when the queue is. A memory-only queue @@ -13295,41 +13283,80 @@ export class AgentFramework { } /** - * Write one agent's drained entries in queue order, each through - * writeDrained. The drain cuts at the first that can't be stored: it and - * every later entry go back into the queue, so none of them lands ahead of - * an earlier one, and a queued tool_use that can't be stored keeps its - * tool_result from being stored alone. (A tool_use that is stored, followed - * by a tool_result that can't be, stays stored without it until the - * tool_result lands at a later flush. The only queued pair is - * puppetToolCall's defensive one, which no path reaches today.) An - * entry that keeps failing holds its agent's later queued writes, logged - * at each flush, rather than letting them land out of order. The hold is - * the queue's alone: a write that arrives while its agent is idle isn't - * deferred (addMessage stores it at once), so it can land ahead of a held - * entry, and an entry that keeps failing holds only what is deferred - * behind it while its agent's turn, stream or tool cycle is alive, or the - * host is quiesced. Returns what was written, a prefix of `entries`. + * Write drained entries in queue (seq) order, whatever their targets: + * residents share one message slot, so the order entries arrived in is the + * order they're stored. Each target is cut at its own first entry that + * can't be stored: that entry and every later one for the same target go + * back into the queue under their ids and seq (requeueDrained) before + * anything is logged, and other targets' entries go on. Left in the + * un-acked set, the next ack would drop them from the durable queue. + * + * So none of a target's entries lands ahead of an earlier one, and a queued + * tool_use that can't be stored keeps the tool_result queued behind it from + * being stored alone. (A tool_use that is stored, followed by a tool_result + * that can't be, stays stored without it until the tool_result lands at a + * later flush. The only queued pair is puppetToolCall's defensive one, + * which no path reaches today.) An entry that keeps failing holds its + * target's later queued writes, logged by id at each flush, rather than + * letting them land out of order. + * + * The hold is the queue's alone. A write that isn't deferred is stored at + * once and can land ahead of a held entry: a fresh tool_result, whatever + * its agent's state, and any write while its agent's turn, stream and tool + * cycle are all idle and the host isn't quiesced. So an entry that keeps + * failing holds only what is deferred behind it. + * + * Returns what was written (stored, or deferred again by addMessage), in + * queue order: for one target, a prefix of its entries. */ private writeDrainedInOrder(entries: DeferredWrite[], where: string, write: (msg: DeferredWrite) => void): DeferredWrite[] { - for (let i = 0; i < entries.length; i++) { - const msg = entries[i]!; - if (!this.writeDrained(msg, where, () => write(msg))) { - this.requeueDrained(entries.slice(i + 1)); - return entries.slice(0, i); + const ordered = [...entries].sort(bySeq); + const targetOf = (msg: DeferredWrite): string | null => msg.forAgent ?? this.primaryAgentName; + const cut = new Set(); + const written: DeferredWrite[] = []; + for (let i = 0; i < ordered.length; i++) { + const msg = ordered[i]!; + const target = targetOf(msg); + if (cut.has(target)) continue; + try { + write(msg); + written.push(msg); + } catch (error) { + cut.add(target); + this.requeueDrained(ordered.slice(i).filter((m) => targetOf(m) === target)); + this.logStuckDeferredWrite(msg, where, error); } } - return entries; + return written; } - /** Re-add one agent's drained entries through addMessage, which stores each - * or defers it again, cut at the first that can't be stored. */ - private readdDrained(entries: DeferredWrite[], where: string, opts?: { bypassDeferralFor?: string }): DeferredWrite[] { + /** The line naming a drained write that couldn't be stored. The entries are + * already back in the queue, and logging mustn't fail the flush that met + * it, so a thrown value that has no text is named as such. */ + private logStuckDeferredWrite(msg: DeferredWrite, where: string, error: unknown): void { + let reason: string; + try { + reason = error instanceof Error ? error.message : String(error); + } catch { + reason = 'a thrown value that has no text'; + } + console.error( + `[deferral] ${where}: deferred write ${describeDeferredWrite(msg)} for ` + + `${msg.forAgent ?? this.primaryAgentName ?? 'the primary'} could not be stored; it stays queued, ` + + `holding what is queued behind it for that agent: ${reason}`, + ); + } + + /** Re-add drained entries through addMessage, which stores each or defers + * it again, in queue order and cut per target (writeDrainedInOrder). With + * `bypassDeferral`, each is stored past its target's deferral (stop()). */ + private readdDrained(entries: DeferredWrite[], where: string, opts?: { bypassDeferral?: boolean }): DeferredWrite[] { return this.writeDrainedInOrder(entries, where, (msg) => { + const target = msg.forAgent ?? this.primaryAgentName; this.addMessage(msg.participant, msg.content, msg.metadata, { deferredWriteId: msg.id, ...(msg.forAgent ? { forAgent: msg.forAgent } : {}), - ...(opts?.bypassDeferralFor ? { bypassDeferralFor: opts.bypassDeferralFor } : {}), + ...(opts?.bypassDeferral && target ? { bypassDeferralFor: target } : {}), }); }); } diff --git a/test/deferred-window-only.test.ts b/test/deferred-window-only.test.ts index e0039d2d3..fe1b9fc47 100644 --- a/test/deferred-window-only.test.ts +++ b/test/deferred-window-only.test.ts @@ -482,7 +482,32 @@ describe('mid-turn deliveries keep the window what the stream carried', () => { await reopened.stop(); }); - it('a turn waiting to retry when the framework stops gives up rather than compile again', async () => { + it("stop() stores several agents' deferred writes in the order they arrived", async () => { + // Residents share one message slot, so the order writes arrived in is the + // order every resident reads. + const agents = [ + { name: 'assistant', model: 'test-model', systemPrompt: 'You are a robot pilot.' }, + { name: 'scout', model: 'test-model', systemPrompt: 'You scout.' }, + ]; + const framework = await createFramework({ agents }); + const internals = internalsOf(framework); + // Frames stop() doesn't wait for hold both agents' turns. + internals.activeTurnTokens.set('assistant', 1); + internals.activeTurnTokens.set('scout', 2); + assert.equal(internals.addMessage('Antra', text('one')), '', 'deferred'); + assert.equal(internals.addMessage('Antra', text('two'), undefined, { forAgent: 'scout' }), '', 'deferred'); + assert.equal(internals.addMessage('Antra', text('three')), '', 'deferred'); + await framework.stop(); + + const reopened = await createFramework({ agents }); + assert.deepEqual( + windowOf(reopened).filter((line) => /^Antra: (one|two|three)$/.test(line)), + ['Antra: one', 'Antra: two', 'Antra: three'], + ); + await reopened.stop(); + }); + + it('a turn waiting to retry when the framework stops gives up rather than compile again', { timeout: 60_000 }, async () => { let waiting!: () => void; const inDelay = new Promise((resolve) => { waiting = resolve; }); const framework = await createFramework({ @@ -504,13 +529,14 @@ describe('mid-turn deliveries keep the window what the stream carried', () => { // The retry re-enters startAgentStream after its delay; count those // re-entries, so a retry that was never granted can't pass for one that // was refused. - const retries: number[] = []; + const retries: Array<{ attempt: number; stopping: boolean }> = []; const frames = framework as unknown as { + providerAdmissionClosed: boolean; startAgentStream(agent: unknown, trigger?: unknown, attempt?: number, held?: boolean): Promise; }; const startAgentStream = frames.startAgentStream.bind(framework); frames.startAgentStream = (a, trigger, attempt = 0, held = false) => { - if (attempt > 0) retries.push(attempt); + if (attempt > 0) retries.push({ attempt, stopping: frames.providerAdmissionClosed }); return startAgentStream(a, trigger, attempt, held); }; framework.start(); @@ -520,7 +546,7 @@ describe('mid-turn deliveries keep the window what the stream carried', () => { const error = console.error; console.error = () => {}; try { await framework.stop(); } finally { console.error = error; } - assert.deepEqual(retries, [1], 'the retry was granted, and re-entered once stop() had begun'); + assert.deepEqual(retries, [{ attempt: 1, stopping: true }], 'the retry was granted, and re-entered once stop() had begun'); assert.equal(compiles, 1, 'the retry was refused, not compiled'); const reopened = await createFramework(); diff --git a/test/ephemeral-settle.test.ts b/test/ephemeral-settle.test.ts index 21765f833..8ebe8d870 100644 --- a/test/ephemeral-settle.test.ts +++ b/test/ephemeral-settle.test.ts @@ -129,7 +129,7 @@ async function stopAndRemove(framework: AgentFramework, tempDir: string): Promis } describe('runEphemeralToCompletion settle signal', () => { - it('a run whose turn stop() refuses ends at once, and a run asked for once stop() has begun is refused', async () => { + it('a run whose turn stop() refuses ends at once, and a run asked for once stop() has begun is refused', { timeout: 60_000 }, async () => { const { tempDir, storePath } = tempStorePath('ephemeral-stopping-'); const membrane = new MockMembrane(); membrane.pushResponse(createMockResponse([{ type: 'text', text: 'too late' }])); @@ -170,7 +170,7 @@ describe('runEphemeralToCompletion settle signal', () => { } }); - it('a run waiting to retry when stop() begins gives up before compiling again', async () => { + it('a run waiting to retry when stop() begins gives up before compiling again', { timeout: 60_000 }, async () => { // An ephemeral run doesn't hold the primary's provider gate, so the // refusal at the top of startAgentStream is the only check its retry // meets before it would compile, and start a stream, again. diff --git a/test/host-quiesce.test.ts b/test/host-quiesce.test.ts index 75ff5fbb6..fe3b81676 100644 --- a/test/host-quiesce.test.ts +++ b/test/host-quiesce.test.ts @@ -722,7 +722,7 @@ test('resume flushes deferred writes in order: a poison write stays queued with }); }); -test("resume cuts each target at its own stuck write, naming it at every flush, and other targets' writes land", async () => { +test("resume stores several agents' writes in the order they arrived, and cuts each agent at its own stuck write, named at every flush", { timeout: 60_000 }, async () => { const dir = mkdtempSync(join(tmpdir(), 'host-quiesce-targets-')); const framework = await AgentFramework.create({ storePath: join(dir, 'store'), @@ -739,46 +739,48 @@ test("resume cuts each target at its own stuck write, naming it at every flush, try { await framework.quiesce({ reason: 'refold' }); const internals = framework as unknown as { - deferredSeq: number; - deferredMessages: Array<{ id: string; seq: number; participant: string; content: unknown[]; forAgent?: string }>; + deferredMessages: Array<{ id: string; content: Array<{ text?: string }> }>; addMessage(participant: string, content: unknown[], metadata?: undefined, opts?: { forAgent?: string }): unknown; }; + const add = (text: string, forAgent: string) => + internals.addMessage('user', [{ type: 'text', text }], undefined, { forAgent }); // Residents share one message slot, so each one's context manager reads // what either stored; each stores through its own. const agentCm = framework.getAgent('agent')!.getContextManager(); const before = agentCm.getAllMessages().length; const stored = () => agentCm.getAllMessages().slice(before).map((m) => (m.content[0] as { text: string }).text); - // A malformed entry the store rejects, as only a queue persisted by other - // code could carry (addMessage refuses a null block): the log line still - // names it, and it still goes back into the queue. - internals.deferredMessages.push({ - id: 'stuck-1', seq: ++internals.deferredSeq, participant: 'user', - content: [{ type: 'text', text: 'stuck' }, null], forAgent: 'agent', - }); - internals.addMessage('user', [{ type: 'text', text: 'for scout' }], undefined, { forAgent: 'scout' }); - internals.addMessage('user', [{ type: 'text', text: 'behind the stuck one' }], undefined, { forAgent: 'agent' }); - internals.addMessage('user', [{ type: 'text', text: 'also for scout' }], undefined, { forAgent: 'scout' }); - assert.equal(framework.getHostModeStatus().deferredWrites, 4); - + add('one', 'agent'); + add('two', 'scout'); + add('three', 'agent'); + add('stuck', 'agent'); + add('for scout', 'scout'); + add('behind the stuck one', 'agent'); + add('also for scout', 'scout'); + assert.equal(framework.getHostModeStatus().deferredWrites, 7); + const stuckId = internals.deferredMessages.find((m) => m.content[0]?.text === 'stuck')!.id; + + // The store refuses agent's 'stuck' write with a thrown value that has no + // text, so the log line can't print it. const realAdd = agentCm.addMessage.bind(agentCm); (agentCm as unknown as { addMessage: typeof agentCm.addMessage }).addMessage = ((participant, content, metadata, causedBy) => { - if ((content[0] as { text?: string }).text === 'stuck') throw new Error('store write failed'); + if ((content[0] as { text?: string }).text === 'stuck') throw Object.create(null); return realAdd(participant, content, metadata, causedBy); }) as typeof agentCm.addMessage; const named = () => lines.filter((line) => - line.includes('deferred write stuck-1 (user: text+malformed) for agent could not be stored')).length; + line.includes(`deferred write ${stuckId} (user: text) for agent could not be stored`) + && line.includes('a thrown value that has no text')).length; let status = await framework.resume(); - // Queued after agent's stuck write, scout's land; agent's own later - // write is held behind it. - assert.deepEqual(stored(), ['for scout', 'also for scout'], "scout's writes land; agent's stuck write holds what is queued behind it for agent"); - assert.equal(status.deferredWrites, 2); + // In the order they arrived, across both agents; from agent's stuck + // write on, agent's are held, and scout's go on. + assert.deepEqual(stored(), ['one', 'two', 'three', 'for scout', 'also for scout']); + assert.equal(status.deferredWrites, 2, "agent's stuck write and the one behind it are still queued"); assert.equal(named(), 1, 'the flush names the stuck write: its id and block types'); // A second flush with the store still refusing it: held again, named again. await framework.quiesce({ reason: 'refold' }); status = await framework.resume(); - assert.deepEqual(stored(), ['for scout', 'also for scout'], 'still nothing lands ahead of it'); + assert.deepEqual(stored(), ['one', 'two', 'three', 'for scout', 'also for scout'], 'still nothing lands ahead of it'); assert.equal(status.deferredWrites, 2); assert.equal(named(), 2, 'every flush names it'); } finally { @@ -791,6 +793,57 @@ test("resume cuts each target at its own stuck write, naming it at every flush, } }); +test('boot recovery drops a recovered deferred write the store could never take, naming it, and restores the rest', { timeout: 60_000 }, async () => { + const dir = mkdtempSync(join(tmpdir(), 'host-quiesce-malformed-')); + const storePath = join(dir, 'store'); + const config = () => ({ + storePath, + membrane: new MockMembrane().asMembrane(), + agents: [{ name: 'agent', model: 'test-model', systemPrompt: 'test' }], + modules: [], + }); + let framework = await AgentFramework.create(config()); + const lines: string[] = []; + const error = console.error; + try { + await framework.quiesce({ reason: 'surgery' }); + const before = framework.getAgent('agent')!.getContextManager().getAllMessages().length; + const add = (framework as unknown as { addMessage(p: string, c: unknown[]): unknown }).addMessage.bind(framework); + add('user', [{ type: 'text', text: 'first' }]); + add('user', [{ type: 'text', text: 'second' }]); + await framework.stop(); + + // Entries no write of the framework's makes, as other code could leave + // them in the recovery file: a null block, and content that isn't blocks. + const file = join(storePath, 'recovery', 'deferred-writes.json'); + const doc = JSON.parse(readFileSync(file, 'utf8')) as { pending: Array> }; + assert.equal(doc.pending.length, 2); + doc.pending.splice(1, 0, + { id: 'null-block', seq: 100, participant: 'user', content: [{ type: 'text', text: 'x' }, null] }, + { id: 'not-blocks', seq: 101, participant: 'user', content: 'x' }, + ); + writeFileSync(file, JSON.stringify(doc)); + + console.error = (...args: unknown[]) => { lines.push(args.map(String).join(' ')); }; + framework = await AgentFramework.create(config()); + console.error = error; + assert.equal(framework.getHostModeStatus().deferredWrites, 2, 'the well-formed entries are restored'); + for (const entry of ['null-block (user)', 'not-blocks (user)']) { + assert.ok(lines.some((line) => line.includes(`malformed and was dropped: ${entry}`)), `${entry} is named:\n${lines.join('\n')}`); + } + const rewritten = readFileSync(file, 'utf8'); + assert.ok(!rewritten.includes('null-block') && !rewritten.includes('not-blocks'), 'the recovery file is rewritten without them'); + await framework.resume(); + const texts = framework.getAgent('agent')!.getContextManager().getAllMessages().slice(before) + .map((m) => (m.content[0] as { text: string }).text); + assert.deepEqual(texts, ['first', 'second']); + } finally { + console.error = error; + await framework.stop(); + rmSync(dir, { recursive: true, force: true }); + } +}); + test('context writes deferred while quiesced survive a restart and land at resume', async () => { const dir = mkdtempSync(join(tmpdir(), 'host-quiesce-deferred-')); const storePath = join(dir, 'store'); diff --git a/test/operator-surgery.test.ts b/test/operator-surgery.test.ts index d38379d17..dd732b96f 100644 --- a/test/operator-surgery.test.ts +++ b/test/operator-surgery.test.ts @@ -257,7 +257,7 @@ describe('live operator surgery', () => { } }); - it("the post-surgery flush lands what waited behind the hold while another agent's tool cycle is pending, and acknowledges it", async () => { + it("the post-surgery flush lands what waited behind the hold while another agent's tool cycle is pending, and acknowledges it", { timeout: 60_000 }, async () => { const c = cm() as unknown as { switchBranch: (name: string) => Promise }; const originalSwitch = c.switchBranch; let releaseSwitch!: () => void; @@ -290,6 +290,43 @@ describe('live operator surgery', () => { } }); + it("the post-surgery flush stores several agents' writes in the order they arrived", { timeout: 60_000 }, async () => { + // Residents share one message slot, so the order writes arrived in is the + // order every resident reads. + await framework.stop(); + framework = await AgentFramework.create({ + storePath, + membrane: membrane.asMembrane(), + agents: [ + { name: 'scout', model: 'test-model', systemPrompt: 'You are scout.' }, + { name: 'builder', model: 'test-model', systemPrompt: 'You are builder.' }, + ], + modules: [], + }); + const c = cm() as unknown as { switchBranch: (name: string) => Promise }; + const originalSwitch = c.switchBranch; + let releaseSwitch!: () => void; + const gate = new Promise((resolve) => { releaseSwitch = resolve; }); + let entered!: () => void; + const inSwitch = new Promise((resolve) => { entered = resolve; }); + c.switchBranch = async (name: string) => { entered(); await gate; return originalSwitch.call(c, name); }; + const add = (framework as unknown as { + addMessage(participant: string, content: unknown[], metadata?: undefined, opts?: { forAgent?: string }): string; + }).addMessage.bind(framework); + try { + const rollback = framework.rollbackToMessage('scout', { messageId: ids[2] }); + await inSwitch; + assert.equal(add('user', [{ type: 'text', text: 'one' }]), '', 'deferred'); + assert.equal(add('user', [{ type: 'text', text: 'two' }], undefined, { forAgent: 'builder' }), '', 'deferred'); + assert.equal(add('user', [{ type: 'text', text: 'three' }]), '', 'deferred'); + releaseSwitch(); + await rollback; + assert.deepEqual(remainingTexts().filter((t) => ['one', 'two', 'three'].includes(t)), ['one', 'two', 'three']); + } finally { + c.switchBranch = originalSwitch; + } + }); + // --- strategy-initialization failure ----------------------------------------- // switchBranch()/fork() move the chronicle branch BEFORE awaiting strategy // initialization, so a rejection there leaves the store on the new branch. From 0ed2f4c48fdeb42ecb8e4708bc0264eb6393cf3c Mon Sep 17 00:00:00 2001 From: Duncan-2410 Date: Tue, 13 Oct 2026 13:20:01 -0400 Subject: [PATCH 0183/1112] fix(deferral): a re-deferred entry keeps its seq whether or not the queue is persisted Found by both readers of the previous round: Tobias-2430 with a helper fixture, Ansel-2428 end to end with a real store and a reopen. A drained entry that addMessage defers again took its seq from the un-acked set, which handOffDeferredWrites fills only for a persisted queue. With the usual memory-only queue it got a fresh seq, so a turn-end flush that met a successor's turn moved the agent's entries behind later arrivals for other agents, and the next flush of several agents' entries (stop(), resume, after surgery) stored them out of the order they arrived. addMessage's deferredWriteId option becomes fromQueue: { id, seq }. A write coming back from the queue is re-deferred under its own id and seq, and leaves the un-acked set as before. readdDrained is its only caller. Tests (deferred-window-only): the two-resident stop() test first re-defers the assistant's entries through a turn-end flush while its turn is held, and still expects one, two, three; the re-deferral test asserts the seq it passed. The fragment says a re-deferred write keeps its place in arrival order. Not yet typechecked or run: written as the house stop began (room-203 #88162), which starts no builds. Co-Authored-By: Claude Opus 5.5 --- .../mid-turn-window-only-held.fixed.md | 7 ++-- src/framework.ts | 35 +++++++++---------- test/deferred-window-only.test.ts | 20 +++++++---- 3 files changed, 35 insertions(+), 27 deletions(-) diff --git a/changelog.d/mid-turn-window-only-held.fixed.md b/changelog.d/mid-turn-window-only-held.fixed.md index 698cb1cb1..3208b9e5d 100644 --- a/changelog.d/mid-turn-window-only-held.fixed.md +++ b/changelog.d/mid-turn-window-only-held.fixed.md @@ -19,9 +19,10 @@ branch the watermark only advances: a compile that started before such a mark doesn't roll it back when it finishes. - A deferred write that goes back into the queue does so whatever its - content. That happens while its target's own turn, stream or tool cycle is - alive, or the host is quiesced. A `tool_result` used to land regardless, - apart from the `tool_use` it answers, which went back into the queue. + content, under its id and in its place in the order writes arrived. That + happens while its target's own turn, stream or tool cycle is alive, or the + host is quiesced. A `tool_result` used to land regardless, apart from the + `tool_use` it answers, which went back into the queue. - A write that waits now waits on its own target's state: the target's turn, stream and tool cycle, or a quiesced host. (A fresh `tool_result` still doesn't wait.) Before, a write for the primary agent waited out every diff --git a/src/framework.ts b/src/framework.ts index 227cf80ce..672bfecbb 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -13135,13 +13135,14 @@ export class AgentFramework { */ forAgent?: string; /** - * Set when the message is being flushed FROM the durable deferred - * queue: it is the entry's one durable identity. A write stamps it - * into `metadata.deferredWriteId` (boot dedup); a re-deferral moves - * the same logical entry back to pending under the same id instead of - * minting a second, independently replayable one. + * Set when the message comes back from the deferred queue: the entry's + * one durable identity and its place in arrival order. A write stamps + * the id into `metadata.deferredWriteId` (boot dedup). A re-deferral + * moves the same logical entry back to pending under the same id and + * seq, instead of minting a second, independently replayable one, or + * moving it behind entries that arrived after it. */ - deferredWriteId?: string; + fromQueue?: { id: string; seq: number }; /** Out-param: where the message landed (RFC-006 needs to find it again). */ placement?: { agent?: string; messageId?: MessageId; deferredId?: string }; /** @@ -13190,12 +13191,12 @@ export class AgentFramework { // append mid-surgery. Deferred here, flushed by resume(). tool_result // still lands (a draining turn's continuation depends on it). // - // A write coming back from the deferred queue (deferredWriteId) is + // A write coming back from the deferred queue (fromQueue) is // re-deferred whatever its content: its tool_use went back into the // queue just before it, in the same drain, and a tool_result stored // without it would split the pair. (A turn-end flush can run after a // successor turn has taken the agent: the 2026-07-31 interleave.) - const freshToolResult = opts?.deferredWriteId === undefined && content.some(b => b.type === 'tool_result'); + const freshToolResult = opts?.fromQueue === undefined && content.some(b => b.type === 'tool_result'); // One rule for every write, primary-bound or targeted: it waits while ITS // TARGET's own tool cycle is pending (pendingAssistantBlocks is keyed by // agent). Another agent's cycle says nothing about this window. The @@ -13215,16 +13216,14 @@ export class AgentFramework { this.activeTurnTokens.has(agent.name) || this.activeStreams.has(agent.name)) ) { - // A re-deferral of a drained entry keeps its id and leaves the un-acked - // set: one logical message, one durable identity — never two entries. - const id = opts?.deferredWriteId ?? randomUUID(); - let seq: number | undefined; - if (opts?.deferredWriteId) { - const prior = this.unackedDeferredWrites.find((m) => m.id === id); - seq = prior?.seq; + // A re-deferral of a drained entry keeps its id and seq, and leaves the + // un-acked set: one logical message, one durable identity, one place + // in arrival order, whether or not the queue is persisted. + const id = opts?.fromQueue?.id ?? randomUUID(); + if (opts?.fromQueue) { this.unackedDeferredWrites = this.unackedDeferredWrites.filter((m) => m.id !== id); } - this.deferredMessages.push({ id, seq: seq ?? ++this.deferredSeq, participant, content, metadata, forAgent: opts?.forAgent }); + this.deferredMessages.push({ id, seq: opts?.fromQueue?.seq ?? ++this.deferredSeq, participant, content, metadata, forAgent: opts?.forAgent }); if (this.quiesced || this.deferredWritesPersisted) this.persistDeferredWrites(); if (opts?.placement) { opts.placement.agent = agent.name; opts.placement.deferredId = id; } return '' as MessageId; // Deferred — flushed at the target's next boundary @@ -13233,7 +13232,7 @@ export class AgentFramework { const stored = agent.getContextManager().addMessage( participant, content, - opts?.deferredWriteId ? withDeferredWriteId(metadata, opts.deferredWriteId) : metadata, + opts?.fromQueue ? withDeferredWriteId(metadata, opts.fromQueue.id) : metadata, ); if (opts?.placement) { opts.placement.agent = agent.name; opts.placement.messageId = stored; } return stored; @@ -13354,7 +13353,7 @@ export class AgentFramework { return this.writeDrainedInOrder(entries, where, (msg) => { const target = msg.forAgent ?? this.primaryAgentName; this.addMessage(msg.participant, msg.content, msg.metadata, { - deferredWriteId: msg.id, + fromQueue: { id: msg.id, seq: msg.seq }, ...(msg.forAgent ? { forAgent: msg.forAgent } : {}), ...(opts?.bypassDeferral && target ? { bypassDeferralFor: target } : {}), }); diff --git a/test/deferred-window-only.test.ts b/test/deferred-window-only.test.ts index fe1b9fc47..485de9487 100644 --- a/test/deferred-window-only.test.ts +++ b/test/deferred-window-only.test.ts @@ -270,10 +270,11 @@ describe('mid-turn deliveries keep the window what the stream carried', () => { pendingAssistantBlocks: Map; activeTurnTokens: Map; activeStreams: Map; - deferredMessages: Array<{ id: string; participant: string }>; + deferredMessages: Array<{ id: string; seq: number; participant: string }>; deferredWritesPersisted: boolean; drainDeferredFor: (name: string, stopAt?: unknown) => unknown[]; - addMessage(participant: string, content: ContentBlock[], metadata?: undefined, opts?: { forAgent?: string; deferredWriteId?: string }): string; + readdDrained: (entries: unknown[], where: string) => unknown[]; + addMessage(participant: string, content: ContentBlock[], metadata?: undefined, opts?: { forAgent?: string; fromQueue?: { id: string; seq: number } }): string; }; } const otherCycle = (): ContentBlock[] => [{ type: 'tool_use', id: 'b1', name: 'robot--move', input: {} } as ContentBlock]; @@ -497,6 +498,11 @@ describe('mid-turn deliveries keep the window what the stream carried', () => { assert.equal(internals.addMessage('Antra', text('one')), '', 'deferred'); assert.equal(internals.addMessage('Antra', text('two'), undefined, { forAgent: 'scout' }), '', 'deferred'); assert.equal(internals.addMessage('Antra', text('three')), '', 'deferred'); + // A turn-end flush of the assistant's entries while a successor holds its + // turn defers them again (the 2026-07-31 interleave). They keep their + // places in arrival order. + internals.readdDrained(internals.drainDeferredFor('assistant'), 'turn-end flush'); + assert.equal(internals.deferredMessages.length, 3, 'deferred again'); await framework.stop(); const reopened = await createFramework({ agents }); @@ -662,12 +668,14 @@ describe('mid-turn deliveries keep the window what the stream carried', () => { // successor turn has started). const requeued = internals.addMessage('user', [ { type: 'tool_result', toolUseId: 'p1', content: 'ok' } as ContentBlock, - ], undefined, { deferredWriteId: 'w1' }); + ], undefined, { fromQueue: { id: 'w1', seq: 41 } }); assert.equal(requeued, ''); - assert.ok(internals.deferredMessages.some((m) => m.id === 'w1')); + // It goes back under its id and its place in arrival order, though this + // queue is held in memory only. + assert.ok(internals.deferredMessages.some((m) => m.id === 'w1' && m.seq === 41)); // So does plain text. - assert.equal(internals.addMessage('Antra', text('plain'), undefined, { deferredWriteId: 'w2' }), ''); - assert.ok(internals.deferredMessages.some((m) => m.id === 'w2')); + assert.equal(internals.addMessage('Antra', text('plain'), undefined, { fromQueue: { id: 'w2', seq: 42 } }), ''); + assert.ok(internals.deferredMessages.some((m) => m.id === 'w2' && m.seq === 42)); // A fresh tool_result write keeps landing as before. const fresh = internals.addMessage('user', [ { type: 'tool_result', toolUseId: 'p2', content: 'ok' } as ContentBlock, From dee13683b4766e45556552bd30c673193e206751 Mon Sep 17 00:00:00 2001 From: Duncan-2410 Date: Tue, 13 Oct 2026 20:56:04 -0400 Subject: [PATCH 0184/1112] fix(deferral): a turn stop() refuses before its compile ends its ephemeral run too #281, merged from main in the previous commit, refuses a turn before its compile once stop() has closed provider admission, beside the existing identity check. That refusal returned without ending an ephemeral run waiting on the turn, so the run waited for its idle watchdog (fifteen minutes by default) where this branch ends a run refused at the top of startAgentStream or after the compile at once, with "the framework is stopping". It now ends there too, through endEphemeralRunRefused, which is a no-op for an agent that was replaced. Test (ephemeral-settle): a module's gatherContext holds the run's turn until stop() has begun; the run rejects with the refusal and the agent never compiled. On the merge's source the run instead ends by its 20 s idle watchdog, whose message the test's pattern doesn't match. Co-Authored-By: Claude Opus 5.5 --- src/framework.ts | 1 + test/ephemeral-settle.test.ts | 41 +++++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/src/framework.ts b/src/framework.ts index a701a9c26..d2f2eeba4 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -10006,6 +10006,7 @@ export class AgentFramework { if (this.agents.get(agent.name) !== agent || this.providerAdmissionClosed) { this.channelRegistry?.stopTyping(); this.eventGate?.onInferenceEnded(agent.name); + if (this.providerAdmissionClosed) this.endEphemeralRunRefused(agent); return false; } diff --git a/test/ephemeral-settle.test.ts b/test/ephemeral-settle.test.ts index 8ebe8d870..dcb73a0db 100644 --- a/test/ephemeral-settle.test.ts +++ b/test/ephemeral-settle.test.ts @@ -170,6 +170,47 @@ describe('runEphemeralToCompletion settle signal', () => { } }); + it('a run whose turn stop() refuses before its compile ends at once', { timeout: 60_000 }, async () => { + // The turn waits on its modules' context when stop() begins, so it is + // refused before it compiles, not after. + const { tempDir, storePath } = tempStorePath('ephemeral-stopping-precompile-'); + let gathering!: () => void; + const inGather = new Promise((resolve) => { gathering = resolve; }); + let release!: () => void; + const released = new Promise((resolve) => { release = resolve; }); + const contextModule: Module = new EphemeralToolModule(); + contextModule.gatherContext = async () => { + gathering(); + await released; + return []; + }; + const framework = await AgentFramework.create({ + storePath, + membrane: new MockMembrane().asMembrane(), + agents: [], + modules: [contextModule], + syncIntervalMs: 0, + }); + try { + const { agent, contextManager } = await createEphemeral(framework); + let compiles = 0; + const start = agent.startStreamWithInjections.bind(agent); + (agent as unknown as { startStreamWithInjections: typeof start }).startStreamWithInjections = async (...args) => { + compiles++; + return start(...args); + }; + const run = runStartedEphemeral(framework, agent, contextManager, STOPPED_RUN_WATCHDOGS); + await inGather; + const stopped = framework.stop(); + release(); + await assert.rejects(run, /refused: the framework is stopping/); + await stopped; + assert.equal(compiles, 0, 'the turn was refused before its compile'); + } finally { + rmSync(tempDir, { recursive: true, force: true }); + } + }); + it('a run waiting to retry when stop() begins gives up before compiling again', { timeout: 60_000 }, async () => { // An ephemeral run doesn't hold the primary's provider gate, so the // refusal at the top of startAgentStream is the only check its retry From bc267743fd99a5e664fb153fb1cbaa09bc90efe2 Mon Sep 17 00:00:00 2001 From: Duncan-2410 Date: Tue, 13 Oct 2026 20:56:04 -0400 Subject: [PATCH 0185/1112] test(deferral): the interleave holds in both queue modes; say which blocks the log line can't read and what a persisted queue does with a stuck write From both readers of c9d612e (Ansel-2428, Tobias-2430). - deferred-window-only: the two-resident stop() test, which first re-defers the assistant's entries through a turn-end flush, now runs with a memory-only queue and with a persisted one. The persisted queue kept each entry's seq before 0ed2f4c too (its un-acked set held it), so that run is the control for the memory-only one. - describeDeferredWrite's doc said every queued entry's blocks are objects. addMessage reads each block's type before it defers a fresh write, which throws only on a null or undefined block; a primitive block gets past it and can be queued, and is named with no type. The doc now says that no queued entry has a null or undefined block, the one kind the log line can't read, and why. No new validation. - The fragment says what a persisted queue (after a quiesce, or once boot recovery restored one) does with a write that keeps failing: it outlives restarts, each boot restores it and tries it again with the rest of the queue, the queue stays persisted while it's held, and only the host log names it. Co-Authored-By: Claude Opus 5.5 --- .../mid-turn-window-only-held.fixed.md | 7 ++- src/framework.ts | 8 +-- test/deferred-window-only.test.ts | 63 ++++++++++--------- 3 files changed, 44 insertions(+), 34 deletions(-) diff --git a/changelog.d/mid-turn-window-only-held.fixed.md b/changelog.d/mid-turn-window-only-held.fixed.md index 3208b9e5d..fa897bbae 100644 --- a/changelog.d/mid-turn-window-only-held.fixed.md +++ b/changelog.d/mid-turn-window-only-held.fixed.md @@ -70,7 +70,12 @@ the next acknowledgement dropped it from the persisted queue (at a turn start or a flush it was lost even from memory). At a tool boundary, nothing from it on is injected. While the framework runs it is kept; a - memory-only queue still loses it when the process exits. + memory-only queue still loses it when the process exits. A persisted queue + (one a quiesce persisted, or boot recovery restored) keeps it across + restarts: each boot restores it and tries it again with the rest of the + queue, at once or at the resume after a quiesced boot, and logs it again. + While it's held the queue stays persisted, so every later deferral is + written to the recovery file. Only the host log names it. - Boot recovery drops a recovered deferred write whose content isn't an array of block objects, which the store could never take, and logs it by id and participant, rather than holding it in the queue. Before, an entry diff --git a/src/framework.ts b/src/framework.ts index d2f2eeba4..ceb4a0998 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -484,10 +484,10 @@ function withDeferredWriteId(metadata: MessageMetadata | undefined, id: string): } /** A deferred entry as a log line names it: its id, participant and block - * types. Every queued entry's blocks are objects: a fresh write with a null - * block throws in addMessage before it could be queued, and a recovered - * entry whose content isn't an array of objects is dropped at load - * (restorePersistedDeferredWrites). */ + * types. Reading a block's type throws only on a null or undefined block, + * and no queued entry has one: addMessage reads every block's type before + * it defers a fresh write, and a recovered entry whose content isn't an + * array of objects is dropped at load (restorePersistedDeferredWrites). */ function describeDeferredWrite(msg: { id: string; participant: string; content: ContentBlock[] }): string { return `${msg.id} (${msg.participant}: ${[...new Set(msg.content.map((b) => b.type))].join('+') || 'empty'})`; } diff --git a/test/deferred-window-only.test.ts b/test/deferred-window-only.test.ts index 485de9487..212e47d4f 100644 --- a/test/deferred-window-only.test.ts +++ b/test/deferred-window-only.test.ts @@ -483,35 +483,40 @@ describe('mid-turn deliveries keep the window what the stream carried', () => { await reopened.stop(); }); - it("stop() stores several agents' deferred writes in the order they arrived", async () => { - // Residents share one message slot, so the order writes arrived in is the - // order every resident reads. - const agents = [ - { name: 'assistant', model: 'test-model', systemPrompt: 'You are a robot pilot.' }, - { name: 'scout', model: 'test-model', systemPrompt: 'You scout.' }, - ]; - const framework = await createFramework({ agents }); - const internals = internalsOf(framework); - // Frames stop() doesn't wait for hold both agents' turns. - internals.activeTurnTokens.set('assistant', 1); - internals.activeTurnTokens.set('scout', 2); - assert.equal(internals.addMessage('Antra', text('one')), '', 'deferred'); - assert.equal(internals.addMessage('Antra', text('two'), undefined, { forAgent: 'scout' }), '', 'deferred'); - assert.equal(internals.addMessage('Antra', text('three')), '', 'deferred'); - // A turn-end flush of the assistant's entries while a successor holds its - // turn defers them again (the 2026-07-31 interleave). They keep their - // places in arrival order. - internals.readdDrained(internals.drainDeferredFor('assistant'), 'turn-end flush'); - assert.equal(internals.deferredMessages.length, 3, 'deferred again'); - await framework.stop(); - - const reopened = await createFramework({ agents }); - assert.deepEqual( - windowOf(reopened).filter((line) => /^Antra: (one|two|three)$/.test(line)), - ['Antra: one', 'Antra: two', 'Antra: three'], - ); - await reopened.stop(); - }); + for (const persisted of [false, true]) { + it(`stop() stores several agents' deferred writes in the order they arrived (${persisted ? 'persisted' : 'memory-only'} queue)`, async () => { + // Residents share one message slot, so the order writes arrived in is + // the order every resident reads. + const agents = [ + { name: 'assistant', model: 'test-model', systemPrompt: 'You are a robot pilot.' }, + { name: 'scout', model: 'test-model', systemPrompt: 'You scout.' }, + ]; + const framework = await createFramework({ agents }); + const internals = internalsOf(framework); + // A queue boot recovery restored, or one a quiesce persisted, stays + // persisted until it empties. + internals.deferredWritesPersisted = persisted; + // Frames stop() doesn't wait for hold both agents' turns. + internals.activeTurnTokens.set('assistant', 1); + internals.activeTurnTokens.set('scout', 2); + assert.equal(internals.addMessage('Antra', text('one')), '', 'deferred'); + assert.equal(internals.addMessage('Antra', text('two'), undefined, { forAgent: 'scout' }), '', 'deferred'); + assert.equal(internals.addMessage('Antra', text('three')), '', 'deferred'); + // A turn-end flush of the assistant's entries while a successor holds + // its turn defers them again (the 2026-07-31 interleave). They keep + // their places in arrival order. + internals.readdDrained(internals.drainDeferredFor('assistant'), 'turn-end flush'); + assert.equal(internals.deferredMessages.length, 3, 'deferred again'); + await framework.stop(); + + const reopened = await createFramework({ agents }); + assert.deepEqual( + windowOf(reopened).filter((line) => /^Antra: (one|two|three)$/.test(line)), + ['Antra: one', 'Antra: two', 'Antra: three'], + ); + await reopened.stop(); + }); + } it('a turn waiting to retry when the framework stops gives up rather than compile again', { timeout: 60_000 }, async () => { let waiting!: () => void; From d32d8cb83044b24950570ddbc11a087ff42e4c17 Mon Sep 17 00:00:00 2001 From: Duncan-2410 Date: Tue, 13 Oct 2026 21:47:01 -0400 Subject: [PATCH 0186/1112] test(deferral): an ephemeral run's retry refused once stop() begins doesn't start its turn The removal run at 411310a found that taking out the refusal at the top of startAgentStream no longer failed a test. #281, merged from main, refuses a turn before its compile once stop() has closed admission, and dee1368 ends a waiting ephemeral run there too, so the ephemeral retry test's checks (the run ends with the refusal, one compile) held without the top refusal. What the top refusal still does is keep the retry's turn from starting at all: no turn token, no turn-start flush, no context gathered and no hooks run while stop() tears down. The test now counts the module's gatherContext calls and expects the first attempt's alone, and is renamed for it. With the top refusal removed, that assertion fails. Co-Authored-By: Claude Opus 5.5 --- test/ephemeral-settle.test.ts | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/test/ephemeral-settle.test.ts b/test/ephemeral-settle.test.ts index dcb73a0db..7d75812a5 100644 --- a/test/ephemeral-settle.test.ts +++ b/test/ephemeral-settle.test.ts @@ -211,18 +211,25 @@ describe('runEphemeralToCompletion settle signal', () => { } }); - it('a run waiting to retry when stop() begins gives up before compiling again', { timeout: 60_000 }, async () => { + it('a run waiting to retry when stop() begins gives up before its turn starts again', { timeout: 60_000 }, async () => { // An ephemeral run doesn't hold the primary's provider gate, so the - // refusal at the top of startAgentStream is the only check its retry - // meets before it would compile, and start a stream, again. + // refusal at the top of startAgentStream is the first check its retry + // meets. The refusal before the compile would stop it too, but only once + // the turn had started again: its context gathered, its hooks run. const { tempDir, storePath } = tempStorePath('ephemeral-retry-stopping-'); let waiting!: () => void; const inDelay = new Promise((resolve) => { waiting = resolve; }); + let gathered = 0; + const contextModule: Module = new EphemeralToolModule(); + contextModule.gatherContext = async () => { + gathered++; + return []; + }; const framework = await AgentFramework.create({ storePath, membrane: new MockMembrane().asMembrane(), agents: [], - modules: [new EphemeralToolModule()], + modules: [contextModule], syncIntervalMs: 0, errorPolicy: { maxRetries: 1, @@ -245,6 +252,7 @@ describe('runEphemeralToCompletion settle signal', () => { await assert.rejects(run, /refused: the framework is stopping/); await stopped; assert.equal(compiles, 1, 'the retry was refused, not compiled'); + assert.equal(gathered, 1, 'the retry was refused before its turn started'); } finally { rmSync(tempDir, { recursive: true, force: true }); } From 4f025627b45161cee1b17a9951eb5458e30fbdfc Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Wed, 14 Oct 2026 00:40:31 -0400 Subject: [PATCH 0187/1112] fix(mcp): a restart holds the id and tool prefix it moves its server to restartMcplServer checked the new id and tool prefix and then awaited its disconnect and connect holding neither, so a connect started during the restart could take them, and the restart's own connect was then refused after the old server was gone. From its checks through its connect, a restart now claims the id and prefix (mcpRestartClaims): another connect or restart is refused them, naming the restart, and the restart's own connect passes its claim through (connectMcplServerClaimed). The claims cover a same-id restart's gap between its disconnect and its connect too, and are released in finally. A restart of an id another restart holds is refused before it tears anything down. Also: - A disconnect releases every prefix the prefix map says its server holds, rather than the one its configuration names now, so a configuration changed after its connect leaves no stale claim. - Recorded server text that may come from any server takes one longest-first pass over every configuration's strings (serverCredentialPatterns + credentialRedactor), so a secret that begins another server's longer one no longer cuts that one short. - The redactor also replaces a token's or credential env value's JSON-escaped slashes and lowercase-hex percent-escapes, a ws:// url's userinfo as one unit before its @ (a short password goes with its username), and the Basic credential a WebSocket client sends for it. - The fixed fragment says the 401/403 body rule is over HTTP, which only the modern engine speaks. --- .../mcp-standard-tool-results.changed.md | 16 ++- .../redact-recorded-mcp-credentials.fixed.md | 2 +- src/framework.ts | 111 ++++++++++++++---- src/mcpl/protocol-family.ts | 84 ++++++++++--- test/mcp-protocol-family.test.ts | 26 ++++ test/mcp-recorded-text.test.ts | 7 ++ test/mcpl-runtime-lifecycle.test.ts | 78 +++++++++++- 7 files changed, 279 insertions(+), 45 deletions(-) diff --git a/changelog.d/mcp-standard-tool-results.changed.md b/changelog.d/mcp-standard-tool-results.changed.md index c561b1ea7..5d0a6972b 100644 --- a/changelog.d/mcp-standard-tool-results.changed.md +++ b/changelog.d/mcp-standard-tool-results.changed.md @@ -108,12 +108,16 @@ server), a new id another server holds, a tool prefix another server or a module holds, a framework whose `stop()` has begun, and a connect or disconnect of the server in flight. Before, a restart disconnected first, - so a refused connect left the server gone; now the server keeps running. A - disconnect releases its tool-prefix claim only while its own server holds - it. Treat a submitted configuration as immutable while the server uses it: - change settings with `restartMcplServer(id, newConfig)`, and rotate a - network server's credentials through `accessProvider` (a stdio server's - changed `env` goes through a restart with the new configuration). + so a refused connect left the server gone; now the server keeps running. + From those checks through its connect, the restart holds the id and the + tool prefix it moves the server to, so a connect or restart started + meanwhile is refused them, naming the restart, rather than taking them + while the old server is torn down. A disconnect releases its tool-prefix + claim only while its own server holds it. Treat a submitted configuration + as immutable while the server uses it: change settings with + `restartMcplServer(id, newConfig)`, and rotate a network server's + credentials through `accessProvider` (a stdio server's changed `env` goes + through a restart with the new configuration). - A server configuration naming no usable transport is now an error before anything is spawned or dialed. That covers a `transport` that doesn't match the url's scheme, `transport: 'http'` without a url, and an diff --git a/changelog.d/redact-recorded-mcp-credentials.fixed.md b/changelog.d/redact-recorded-mcp-credentials.fixed.md index 05e8d920f..e58f63aaa 100644 --- a/changelog.d/redact-recorded-mcp-credentials.fixed.md +++ b/changelog.d/redact-recorded-mcp-credentials.fixed.md @@ -1 +1 @@ -- Server and transport text the framework records has the server's configured credentials replaced, in both engines: connection traces, console lines, stderr traces, the ops alert (and so `logs/failures.log` and the webhook), a tool call's failure as its caller and the model get it, and the `tool:failed` trace's message and stack. The credentials are those the configuration holds: from the url, credential-like path segments and query names, credential-shaped or credential-named query values (each as written and decoded), every non-empty parameter as its whole `name=value`, and a `ws://` url's username and password; the static `token`, raw and as a url carries it, percent- or form-encoded; and a stdio server's `env` values whose names have a part ending in a credential word, singular or plural, with any digits after it (`GITHUB_TOKEN`, `API_KEYS`, `PGPASSWORD`, `DB_PASS`, `AUTHORIZATION`, `SESSION_COOKIE`, `TOKEN2`), raw and percent-encoded. A stdio server's command-line arguments are not derived. A string replaced on its own has at least eight characters, since a shorter one can't be told from ordinary text, and is replaced wherever it occurs. Every query parameter is also replaced as its whole `name=value`, only where no letter or digit adjoins the pair (so `v=2` leaves `dev=2` alone); a short value goes only that way. A connection's event records use a redactor built once for it, and an operation's records one built when the operation begins, so a server removed, or replaced by a restart, while the operation waits doesn't change what they replace. A 401 or 403's response body is left out of the request error's data; where the HTTP client builds the error's message from the body, as it does for most statuses, the message is redacted like any record. A token an access provider fetches, and credentials the configuration doesn't hold, are outside this. JSON-RPC answers, and the error a host's own connect raises, pass as they are. +- Server and transport text the framework records has the server's configured credentials replaced, in both engines: connection traces, console lines, stderr traces, the ops alert (and so `logs/failures.log` and the webhook), a tool call's failure as its caller and the model get it, and the `tool:failed` trace's message and stack. The credentials are those the configuration holds: from the url, credential-like path segments and query names, credential-shaped or credential-named query values (each as written and decoded), every non-empty parameter as its whole `name=value`, and a `ws://` url's username and password, its userinfo as one unit before its `@`, and the Basic credential a WebSocket client sends for it; the static `token`, raw, with its slashes escaped as some JSON encoders write them, and as a url carries it, percent- or form-encoded with either case of hex; and a stdio server's `env` values whose names have a part ending in a credential word, singular or plural, with any digits after it (`GITHUB_TOKEN`, `API_KEYS`, `PGPASSWORD`, `DB_PASS`, `AUTHORIZATION`, `SESSION_COOKIE`, `TOKEN2`), raw, with their slashes escaped as some JSON encoders write them, and percent-encoded with either case of hex. A stdio server's command-line arguments are not derived. A string replaced on its own has at least eight characters, since a shorter one can't be told from ordinary text, and is replaced wherever it occurs. Every query parameter is also replaced as its whole `name=value`, only where no letter or digit adjoins the pair (so `v=2` leaves `dev=2` alone); a short value goes only that way. A connection's event records use a redactor built once for it, and an operation's records one built when the operation begins, so a server removed, or replaced by a restart, while the operation waits doesn't change what they replace. Text that may come from any server takes one pass over every server's strings, so a secret that begins another server's longer one doesn't cut that one short. Over HTTP, which only the modern engine speaks, a 401 or 403's response body is left out of the request error's data; where the HTTP client builds the error's message from the body, as it does for most statuses, the message is redacted like any record. A token an access provider fetches, and credentials the configuration doesn't hold, are outside this. JSON-RPC answers, and the error a host's own connect raises, pass as they are. diff --git a/src/framework.ts b/src/framework.ts index e9a1306e1..d5a67049f 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -394,7 +394,7 @@ import type { SessionUsageSnapshot, UsageUpdatedEvent } from './usage/types.js'; import type { McplServerConnection } from './mcpl/server-connection.js'; import { McplProtocolVersionError, McplUnreapedLaunchError } from './mcpl/server-connection.js'; import { ModernMcpConnection } from './mcpl/modern-connection.js'; -import { asText, redactUrl, redactServerText, resolveServerBinding, serverConfigProblems, serverTextRedactor } from './mcpl/protocol-family.js'; +import { asText, credentialRedactor, redactUrl, redactServerText, resolveServerBinding, serverConfigProblems, serverCredentialPatterns, serverTextRedactor } from './mcpl/protocol-family.js'; import { MALFORMED_CONTENT_NOTICE, normalizeStandardToolResult } from './mcpl/tool-result-normalize.js'; import type { McplServerConfig, @@ -1078,6 +1078,10 @@ type AnswerOnce = { answered: () => boolean; }; +/** A restart's hold on the id and tool prefix it is moving its server to; + * `restarting` names the server it restarts. Compared by identity. */ +type McpRestartClaim = { readonly restarting: string }; + /** * Wrap a server request's responder so the request is answered at most once, * and a later failure can tell whether the handler already answered. @@ -1566,6 +1570,14 @@ export class AgentFramework { /** Connects and disconnects in flight, by server id (holdMcpServerId); * created on first use, like modernMcpConnections. */ private mcpServerOps: Map }> | undefined; + /** The ids and tool prefixes restarts hold, each with the claim of the + * restart holding it, from the restart's checks through its connect + * (restartMcplServer): no other connect or restart takes them while the + * old server is torn down. */ + private readonly mcpRestartClaims = { + ids: new Map(), + prefixes: new Map(), + }; /** Host capabilities advertised during the MCP handshake — stored so servers * can be connected at runtime (connectMcplServer) after initialization. */ private mcplHostCapabilities: McplHostCapabilities | null = null; @@ -14336,6 +14348,7 @@ export class AgentFramework { private async connectMcplServerInternal( config: import('./mcpl/types.js').McplServerConfig, deferAwareness = false, + claim?: McpRestartClaim, ): Promise { if (this.mcpServerAdmissionClosed) { throw new Error(`MCP server "${config.id}" was not connected: the framework is stopping or stopped`); @@ -14350,7 +14363,7 @@ export class AgentFramework { // protocol on a URL, a modern deadline of 0, MCPL policy on a modern // server. Every valid legacy shape resolves exactly as before. const family = AgentFramework.usableConfigFamily(config); - const taken = this.mcpServerIdTaken(config.id); + const taken = this.mcpServerIdTaken(config.id, claim); if (taken) throw new Error(taken); const hold = this.holdMcpServerId(config.id, 'connect'); try { @@ -14843,6 +14856,15 @@ export class AgentFramework { */ async connectMcplServer( config: import('./mcpl/types.js').McplServerConfig, + ): Promise { + await this.connectMcplServerClaimed(config); + } + + /** connectMcplServer, for a restart's connect too: the id and tool prefix + * its `claim` holds are free to it, and refused to everyone else. */ + private async connectMcplServerClaimed( + config: import('./mcpl/types.js').McplServerConfig, + claim?: McpRestartClaim, ): Promise { // Refused before anything initializes, spawns or dials. if (this.mcpServerAdmissionClosed) { @@ -14854,11 +14876,11 @@ export class AgentFramework { await this.initializeMcpl([], this.mcplInferenceRoutingConfig ?? undefined); } - const collision = this.mcplPrefixCollision(config); + const collision = this.mcplPrefixCollision(config, undefined, claim); if (collision) throw new Error(collision); const oldToolNames = new Set(this.mcplTools.map(t => t.name)); - await this.connectMcplServerInternal(config); + await this.connectMcplServerInternal(config, false, claim); await this.refreshMcplTools(); this.emitMcplToolDiff(oldToolNames, config.id); } @@ -14897,7 +14919,6 @@ export class AgentFramework { // What held the id before (a connect still running, an earlier // disconnect) settles first, so what it registered is removed whole. await hold.previous; - const config = this.mcplServerConfigs.get(id); const oldToolNames = new Set(this.mcplTools.map(t => t.name)); try { @@ -14922,10 +14943,13 @@ export class AgentFramework { this.featureSetManager?.removeServer(id); this.checkpointManager?.removeServer(id); - const prefix = config?.toolPrefix ?? `mcpl--${id}`; - // Only this server's own claim: a server connected since under the - // prefix it held keeps it. - if (this.mcplPrefixMap.get(prefix) === id) this.mcplPrefixMap.delete(prefix); + // Only this server's own claims, as the prefix map holds them rather + // than as its configuration reads now: a server connected since under + // a prefix it held keeps it, and a configuration changed after its + // connect leaves no claim behind. + for (const [prefix, owner] of this.mcplPrefixMap) { + if (owner === id) this.mcplPrefixMap.delete(prefix); + } this.mcplServerConfigs.delete(id); await this.refreshMcplTools(); @@ -14940,6 +14964,9 @@ export class AgentFramework { * same (or an updated) config. This actually respawns a stdio child — * unlike `reconnect: true`, which only retries after transport-level * failures. Throws if the server has no stored config and none is given. + * What its connect would refuse is refused before the running server is + * torn down, and until the connect, a connect or restart started meanwhile + * is refused the id and tool prefix the restart moves the server to. * * Treat a submitted configuration and its nested values as immutable for * as long as the server uses them. Apply changed settings with @@ -14954,7 +14981,10 @@ export class AgentFramework { // What the connect would refuse is refused here, before the running // server is torn down for it: a framework whose stop() has begun, the // configuration's binding and settings, a new id another server holds, - // and a tool prefix another server or a module already holds. + // and a tool prefix another server or a module already holds. From these + // checks through its connect, the restart holds the id and the tool + // prefix it moves the server to (mcpRestartClaims), so no connect or + // restart started meanwhile takes them while the old server is torn down. if (this.mcpServerAdmissionClosed) { throw new Error(`MCP server "${id}" was not restarted: the framework is stopping or stopped`); } @@ -14971,14 +15001,32 @@ export class AgentFramework { // operation left, after the server was torn down. throw new Error(`MCP server "${id}" was not restarted: it is being ${inFlight.kind === 'disconnect' ? 'disconnected' : 'connected'}; restart it once that has finished`); } + const claimedBy = this.mcpRestartClaims.ids.get(id); + if (claimedBy) { + // Another restart is moving its server to this id, or restarting this + // server already. + throw new Error(claimedBy.restarting === id + ? `MCP server "${id}" was not restarted: it is being restarted; restart it once that has finished` + : `MCP server "${id}" was not restarted: the restart of "${claimedBy.restarting}" holds its id; restart it once that has finished`); + } if (config.id !== id) { const taken = this.mcpServerIdTaken(config.id); if (taken) throw new Error(taken); } const collision = this.mcplPrefixCollision(config, id); if (collision) throw new Error(collision); - await this.disconnectMcplServer(id); - await this.connectMcplServer(config); + + const claim: McpRestartClaim = { restarting: id }; + const prefix = config.toolPrefix ?? `mcpl--${config.id}`; + this.mcpRestartClaims.ids.set(config.id, claim); + this.mcpRestartClaims.prefixes.set(prefix, claim); + try { + await this.disconnectMcplServer(id); + await this.connectMcplServerClaimed(config, claim); + } finally { + if (this.mcpRestartClaims.ids.get(config.id) === claim) this.mcpRestartClaims.ids.delete(config.id); + if (this.mcpRestartClaims.prefixes.get(prefix) === claim) this.mcpRestartClaims.prefixes.delete(prefix); + } } /** @@ -14989,19 +15037,28 @@ export class AgentFramework { * meanwhile, both have their credentials replaced. */ private everyServerRedactor(): (text: unknown) => string { - const snapshot = [...this.mcplServerConfigs.values()].map(config => serverTextRedactor(config)); + const snapshot = [...this.mcplServerConfigs.values()].flatMap(config => serverCredentialPatterns(config)); // The configurations present when the operation began, then those present // now: a server removed meanwhile and one connected meanwhile are both - // covered. - return (text) => [...snapshot, ...[...this.mcplServerConfigs.values()].map(config => serverTextRedactor(config))] - .reduce((recorded, redact) => redact(recorded), asText(text)); + // covered. One pass over all their strings, so a secret that begins + // another server's longer one can't cut that one short. + return (text) => credentialRedactor([ + ...snapshot, + ...[...this.mcplServerConfigs.values()].flatMap(config => serverCredentialPatterns(config)), + ])(text); } /** Why a connect for this id is refused, or null when the id is free. One * owner per id, across both engines (holdMcpServerId): a connect or * disconnect in flight holds it, and so does a registered connection, - * including one kept because its launch couldn't be confirmed exited. */ - private mcpServerIdTaken(id: string): string | null { + * including one kept because its launch couldn't be confirmed exited, and + * a restart moving its server to the id (mcpRestartClaims), except to + * that restart's own connect (`claim`). */ + private mcpServerIdTaken(id: string, claim?: McpRestartClaim): string | null { + const claimedBy = this.mcpRestartClaims.ids.get(id); + if (claimedBy && claimedBy !== claim) { + return `MCP server "${id}" is held by the restart of "${claimedBy.restarting}"; connect it once that has finished`; + } const op = this.mcpServerOps?.get(id); if (op?.kind === 'disconnect') { return `MCP server "${id}" is still being disconnected; connect it once that has finished`; @@ -15016,14 +15073,24 @@ export class AgentFramework { /** The collision rules create() enforces at startup, as a runtime connect * (or a restart, before its disconnect) meets them: the tool prefix must - * not shadow a module name or another server's prefix. The server's own - * prefix is free to keep, and so is that of the server a restart is - * about to disconnect (`leaving`). */ - private mcplPrefixCollision(config: import('./mcpl/types.js').McplServerConfig, leaving?: string): string | null { + * not shadow a module name or another server's prefix, or be one a + * restart is moving its server to (mcpRestartClaims). The server's own + * prefix is free to keep, so is that of the server a restart is about to + * disconnect (`leaving`), and so is the prefix a restart's own connect + * holds (`claim`). */ + private mcplPrefixCollision( + config: import('./mcpl/types.js').McplServerConfig, + leaving?: string, + claim?: McpRestartClaim, + ): string | null { const prefix = config.toolPrefix ?? `mcpl--${config.id}`; if (this.moduleRegistry.getAllModules().some(m => m.name === prefix)) { return `MCPL server "${config.id}" toolPrefix "${prefix}" collides with module "${prefix}"`; } + const claimedBy = this.mcpRestartClaims.prefixes.get(prefix); + if (claimedBy && claimedBy !== claim) { + return `MCPL server "${config.id}" toolPrefix "${prefix}" is held by the restart of "${claimedBy.restarting}"`; + } const prefixOwner = this.mcplPrefixMap.get(prefix); if (prefixOwner && prefixOwner !== config.id && prefixOwner !== leaving) { return `MCPL server "${config.id}" toolPrefix "${prefix}" collides with server "${prefixOwner}"`; diff --git a/src/mcpl/protocol-family.ts b/src/mcpl/protocol-family.ts index 410732751..7ab46747f 100644 --- a/src/mcpl/protocol-family.ts +++ b/src/mcpl/protocol-family.ts @@ -113,19 +113,25 @@ export function redactUrl(url: string | undefined): string { * - from the url, as written and decoded: credential-like path segments and * query names, credential-shaped or credential-named query values, every * other non-empty parameter as its whole name=value, and a ws:// url's - * username and password; - * - the static token, raw and percent- or form-encoded; - * - a stdio server's environment values whose names say they're credentials. + * username and password, its userinfo as one unit before its @, and the + * Basic credential a WebSocket client sends for that userinfo; + * - the static token, raw, with its slashes escaped as some JSON encoders + * write them, and percent- or form-encoded with either case of hex; + * - a stdio server's environment values whose names say they're + * credentials, raw, with their slashes JSON-escaped, and percent-encoded + * with either case of hex. * A string replaced on its own has at least 8 characters, since a shorter * one can't be told from ordinary text, and it is replaced wherever it * occurs, after an escape or inside a longer token too. A query parameter is * also replaced as its whole name=value, only where no letter or digit * adjoins the pair, so v=2 leaves dev=2 alone; a short value goes only that - * way. One - * longest-first pass never scans its own replacement. A credential the + * way. One longest-first pass never scans its own replacement, and text + * that may come from several servers takes one pass over all their + * configurations' strings (serverCredentialPatterns). A credential the * configuration doesn't hold, such as the token an access provider fetches - * or another origin's, is outside it. JSON-RPC answers, and the error a - * host's own connect raises, pass as they are. + * or another origin's, is outside it, and so are other encodings of the + * ones it holds. JSON-RPC answers, and the error a host's own connect + * raises, pass as they are. */ export function redactServerText(text: unknown, config: Pick): string { return serverTextRedactor(config)(text); @@ -136,28 +142,56 @@ export function redactServerText(text: unknown, config: Pick): (text: unknown) => string { + return credentialRedactor(serverCredentialPatterns(config)); +} + +/** A string {@link redactServerText} replaces, and the pattern that finds it. */ +export interface CredentialPattern { + readonly value: string; + readonly source: string; +} + +/** + * The strings {@link redactServerText} replaces for one configuration, as + * patterns for {@link credentialRedactor}. Text that may come from several + * servers goes through one redactor over all their patterns, so its one + * longest-first pass takes the longest match among them all: a secret that + * begins another configuration's longer one can't cut that one short. + */ +export function serverCredentialPatterns(config: Pick): CredentialPattern[] { // Strings replaced wherever they occur (after an escape or inside a longer // token too), and name=value pairs, replaced only where no letter or digit // adjoins them, so a pair such as v=2 leaves dev=2 alone. A pair's value is // also replaced on its own when it's a credential, wherever it occurs. const secrets = new Set(); const pairs = new Set(); + const userinfo = new Set(); const standalone = (value: string): void => { if (value.length >= 8) secrets.add(value); }; const decode = (value: string, query = false): string => { try { return decodeURIComponent(query ? value.replace(/\+/g, ' ') : value); } catch { return value; } }; - // A string as written and percent-encoded (a string encodeURIComponent - // refuses, such as one with an unpaired surrogate, has no encoded form). + // Percent-escapes in lowercase hex, as some encoders write them. + const lowerEscapes = (value: string): string => value.replace(/%[0-9A-F]{2}/g, (escape) => escape.toLowerCase()); + // A string as written, with its slashes escaped as some JSON encoders + // write them, and percent-encoded in either case of hex (a string + // encodeURIComponent refuses, such as one with an unpaired surrogate, has + // no encoded form). const withEncoded = (value: string): void => { standalone(value); - try { standalone(encodeURIComponent(value)); } catch { /* no percent-encoded form */ } + standalone(value.replace(/\//g, '\\/')); + let encoded: string; + try { encoded = encodeURIComponent(value); } catch { return; /* no percent-encoded form */ } + standalone(encoded); + standalone(lowerEscapes(encoded)); }; // The token as sent: raw in a header, and percent- or form-encoded where a // transport puts it in a url. const token = config.token ?? ''; withEncoded(token); - standalone(new URLSearchParams({ t: token }).toString().slice(2)); + const formEncoded = new URLSearchParams({ t: token }).toString().slice(2); + standalone(formEncoded); + standalone(lowerEscapes(formEncoded)); // A stdio server's environment values whose names say they carry a // credential: a part of the name that ends with a credential word // (GITHUB_TOKEN, API_KEY, PGPASSWORD, ACCESSTOKEN), and not one where the @@ -200,15 +234,35 @@ export function serverTextRedactor(config: Pick= 8) userinfo.add(`${username}:${password}`); + standalone(Buffer.from(`${username}:${password}`).toString('base64')); + } + } } } - if (secrets.size === 0 && pairs.size === 0) return asText; const escape = (value: string): string => value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); - const alternatives = [ + return [ ...[...secrets].map(value => ({ value, source: escape(value) })), ...[...pairs].map(value => ({ value, source: `(? b.value.length - a.value.length).map(({ source }) => source); - const pattern = new RegExp(alternatives.join('|'), 'g'); + ...[...userinfo].map(value => ({ value, source: `${escape(value)}(?=@)` })), + ]; +} + +/** + * One longest-first pass over the given patterns that never scans its own + * replacement: at each place, the longest string that matches there becomes + * `redacted`. + */ +export function credentialRedactor(patterns: readonly CredentialPattern[]): (text: unknown) => string { + if (patterns.length === 0) return asText; + const sources = [...patterns].sort((a, b) => b.value.length - a.value.length).map(({ source }) => source); + const pattern = new RegExp([...new Set(sources)].join('|'), 'g'); return (text) => asText(text).replace(pattern, 'redacted'); } diff --git a/test/mcp-protocol-family.test.ts b/test/mcp-protocol-family.test.ts index fb1e7fe75..64a9564f9 100644 --- a/test/mcp-protocol-family.test.ts +++ b/test/mcp-protocol-family.test.ts @@ -14,6 +14,8 @@ import { checkServerConfig, redactUrl, redactServerText, + credentialRedactor, + serverCredentialPatterns, describeCredentialError, MCPL_ONLY_POLICY_FIELDS, MAX_TIMER_MS, @@ -376,6 +378,30 @@ test('the cleartext warning counts a credential-shaped value under a plain name, assert.match(warned[0], /its url's query parameter "session" goes to remote\.example unencrypted/); }); +test('recorded server text: a slash escaped as JSON may write it, and percent-escapes in lowercase hex', () => { + const config = cfg({ token: 'tok/en+x0123', env: { API_TOKEN: 'env/secret+0123' } } as Partial); + for (const form of ['tok/en+x0123', 'tok\\/en+x0123', 'tok%2Fen%2Bx0123', 'tok%2fen%2bx0123', 'env\\/secret+0123', 'env%2fsecret%2b0123']) { + assert.equal(redactServerText(`refused: ${form}`, config), 'refused: redacted', form); + } +}); + +test("recorded server text: a ws:// url's userinfo as one unit before its @, and the Basic credential sent for it", () => { + // Neither part is long enough to replace on its own. + const config = cfg({ url: 'wss://alice:pw@h/mcp' }); + assert.equal(redactServerText('dial wss://alice:pw@h/mcp failed', config), 'dial wss://redacted@h/mcp failed'); + assert.equal(redactServerText('Authorization: Basic YWxpY2U6cHc=', config), 'Authorization: Basic redacted'); + assert.equal(redactServerText('alice:pw is not followed by its @ here', config), 'alice:pw is not followed by its @ here'); +}); + +test("one redactor over several configurations' strings takes the longest secret among them all", () => { + const short = cfg({ id: 'a', token: 'ghp_SHAREDPREFIX123' }); + const long = cfg({ id: 'b', token: 'ghp_SHAREDPREFIX123_but_longer_TAIL9' }); + const redact = credentialRedactor([...serverCredentialPatterns(short), ...serverCredentialPatterns(long)]); + assert.equal(redact('auth failed for ghp_SHAREDPREFIX123_but_longer_TAIL9'), 'auth failed for redacted'); + assert.equal(redact('auth failed for ghp_SHAREDPREFIX123'), 'auth failed for redacted'); + assert.equal(credentialRedactor([])('nothing to replace'), 'nothing to replace'); +}); + test('recorded server text uses longest-first, one-pass replacements and is idempotent', () => { const config = cfg({ url: 'https://h/mcp?secret=token0123456789-tail&redacted=eu', token: 'token0123456789' }); const original = 'secret=token0123456789-tail token0123456789 redacted=eu queue full'; diff --git a/test/mcp-recorded-text.test.ts b/test/mcp-recorded-text.test.ts index ef5e74b58..90e011105 100644 --- a/test/mcp-recorded-text.test.ts +++ b/test/mcp-recorded-text.test.ts @@ -225,6 +225,13 @@ test("a tool call's failure, from either engine, reaches its caller and its trac assert.equal(everyServer('closed lateSecret0123456789'), 'closed redacted'); // A value that can't be printed reads as a placeholder. assert.equal(everyServer(Object.create(null)), '[a value that cannot be printed]'); + // One pass over every server's strings: a secret that begins another + // server's longer one doesn't cut that one short. + framework.mcplServerConfigs = new Map([ + ['a', { id: 'a', token: 'ghp_SHAREDPREFIX123' }], + ['b', { id: 'b', token: 'ghp_SHAREDPREFIX123_but_longer_TAIL9' }], + ]); + assert.equal(framework.everyServerRedactor()('auth failed for ghp_SHAREDPREFIX123_but_longer_TAIL9'), 'auth failed for redacted'); framework.mcplServerConfigs = new Map([[config.id, config]]); // The model's dispatch: the tool:failed trace (message and stack) and the diff --git a/test/mcpl-runtime-lifecycle.test.ts b/test/mcpl-runtime-lifecycle.test.ts index 8bb01254f..d18f45637 100644 --- a/test/mcpl-runtime-lifecycle.test.ts +++ b/test/mcpl-runtime-lifecycle.test.ts @@ -113,7 +113,7 @@ test('restartMcplServer respawns the server and keeps tools available', async () assert.equal(servers.length, 1); assert.equal(servers[0].connected, true); assert.equal(servers[0].toolCount, 1); - assert.ok(framework.getAllTools().some(t => t.name === 'mcpl--first--echo')); + assert.ok(framework.getAllTools().some(tool => tool.name === 'mcpl--first--echo')); }); }); @@ -207,6 +207,18 @@ test("a disconnect removes its server's prefix claim, and leaves one another ser }); }); +test('a disconnect releases the prefix its server claimed at connect, though its configuration changed since', async () => { + await withFramework(false, async (framework) => { + const config = serverConfig('dyn', { toolPrefix: 'p1' }); + await framework.connectMcplServer(config as never); + // Against the documented rule, the host changes the configuration it submitted. + (config as unknown as { toolPrefix: string }).toolPrefix = 'p2'; + await framework.disconnectMcplServer('dyn'); + await framework.connectMcplServer(serverConfig('other', { toolPrefix: 'p1' }) as never); + assert.deepEqual(framework.listMcplServers().map(s => s.id), ['other']); + }); +}); + test('restartMcplServer of an id whose connect is in flight is refused before it tears anything down', async (t) => { // MCPL is up, so the connect holds its id before its first await. await withFramework(true, async (framework) => { @@ -222,6 +234,70 @@ test('restartMcplServer of an id whose connect is in flight is refused before it }); }); +test('a connect started during a restart is refused the tool prefix the restart moves to, and the restart completes', async () => { + await withFramework(true, async (framework) => { + const restarting = framework.restartMcplServer('first', serverConfig('first', { toolPrefix: 'xx' }) as never); + await assert.rejects( + framework.connectMcplServer(serverConfig('third', { toolPrefix: 'xx' }) as never), + /MCPL server "third" toolPrefix "xx" is held by the restart of "first"/, + ); + await restarting; + assert.deepEqual(framework.listMcplServers().map(s => [s.id, s.connected]), [['first', true]]); + assert.ok(framework.getAllTools().some(t => t.name === 'xx--echo'), 'the restarted server serves the prefix'); + }); +}); + +test('a connect started during a restart is refused the id the restart moves to, and the restart completes', async () => { + await withFramework(true, async (framework) => { + const restarting = framework.restartMcplServer('first', serverConfig('second') as never); + // Another prefix, so only the id is contested. + await assert.rejects( + framework.connectMcplServer(serverConfig('second', { toolPrefix: 'other' }) as never), + /MCP server "second" is held by the restart of "first"/, + ); + await restarting; + assert.deepEqual(framework.listMcplServers().map(s => [s.id, s.connected]), [['second', true]]); + assert.ok(framework.getAllTools().some(t => t.name === 'mcpl--second--echo')); + }); +}); + +test("a connect between a restart's disconnect and its connect is refused the id the restart keeps", async (t) => { + await withFramework(true, async (framework) => { + const disconnect = framework.disconnectMcplServer.bind(framework); + let racing: Promise | undefined; + t.mock.method(framework, 'disconnectMcplServer', async (id: string) => { + await disconnect(id); + // The old server is gone and its disconnect holds nothing now. + racing = framework.connectMcplServer(serverConfig('first', { toolPrefix: 'other' }) as never) + .then(() => 'connected', (error: unknown) => error); + }); + await framework.restartMcplServer('first'); + const outcome = await racing; + assert.ok(outcome instanceof Error, `the racing connect was refused, not ${String(outcome)}`); + assert.match(outcome.message, /MCP server "first" is held by the restart of "first"/); + assert.deepEqual(framework.listMcplServers().map(s => [s.id, s.connected]), [['first', true]]); + assert.ok(framework.getAllTools().some(tool => tool.name === 'mcpl--first--echo')); + }); +}); + +test('a restart that fails releases the id and the tool prefix it held', async (t) => { + await withFramework(true, async (framework) => { + t.mock.method(framework, 'disconnectMcplServer', async () => { + throw new Error('disconnect failed for the test'); + }); + await assert.rejects( + framework.restartMcplServer('first', serverConfig('renamed', { toolPrefix: 'xx' }) as never), + /disconnect failed for the test/, + ); + t.mock.restoreAll(); + await framework.connectMcplServer(serverConfig('renamed', { toolPrefix: 'xx' }) as never); + assert.deepEqual( + framework.listMcplServers().map(s => [s.id, s.connected]).sort(), + [['first', true], ['renamed', true]], + ); + }); +}); + test('connectMcplServer rejects a duplicate id and a colliding prefix', async () => { await withFramework(true, async (framework) => { await assert.rejects( From 379490b66561e94fb1e606876780a9f74f0abccd Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Wed, 14 Oct 2026 00:50:05 -0400 Subject: [PATCH 0188/1112] test(mcp): a restart of an id another restart holds is refused Between a restart's disconnect and its connect, a second restart of the id it keeps, or of the new id it moves its server to, is refused before it tears anything down, naming the restart that holds it. --- test/mcpl-runtime-lifecycle.test.ts | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/test/mcpl-runtime-lifecycle.test.ts b/test/mcpl-runtime-lifecycle.test.ts index d18f45637..a5c4e468d 100644 --- a/test/mcpl-runtime-lifecycle.test.ts +++ b/test/mcpl-runtime-lifecycle.test.ts @@ -280,6 +280,35 @@ test("a connect between a restart's disconnect and its connect is refused the id }); }); +test('a restart of an id another restart holds is refused before it tears anything down', async (t) => { + await withFramework(true, async (framework) => { + await framework.connectMcplServer(serverConfig('second') as never); + const disconnect = framework.disconnectMcplServer.bind(framework); + let during: (() => Promise) | undefined; + const refusals: string[] = []; + t.mock.method(framework, 'disconnectMcplServer', async (id: string) => { + await disconnect(id); + // Between the restart's disconnect and its connect, only its claims hold these ids. + const attempt = during; + during = undefined; + if (attempt) { + await attempt().then(() => refusals.push('restarted'), (error: unknown) => refusals.push((error as Error).message)); + } + }); + // A restart under the same id holds it. + during = () => framework.restartMcplServer('first', serverConfig('first') as never); + await framework.restartMcplServer('first'); + // A restart under a new id holds the new one. + during = () => framework.restartMcplServer('renamed', serverConfig('renamed') as never); + await framework.restartMcplServer('second', serverConfig('renamed') as never); + assert.deepEqual(refusals, [ + 'MCP server "first" was not restarted: it is being restarted; restart it once that has finished', + 'MCP server "renamed" was not restarted: the restart of "second" holds its id; restart it once that has finished', + ]); + assert.deepEqual(framework.listMcplServers().map(s => [s.id, s.connected]).sort(), [['first', true], ['renamed', true]]); + }); +}); + test('a restart that fails releases the id and the tool prefix it held', async (t) => { await withFramework(true, async (framework) => { t.mock.method(framework, 'disconnectMcplServer', async () => { From aafc2a8820efce7825f532d5cbb703657ab7de67 Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Wed, 14 Oct 2026 01:04:55 -0400 Subject: [PATCH 0189/1112] fix(mcp): what main's merge left: a script's reference names its expiry, and only a legacy connect reads the ledger Two things the merge of main took in cleanly but wrongly: - #303 made an unfetched, expired reference's stub say why in place of advising fetch_reference. referenceBlockForScript, which makes the stub's decision for a script, still gave such a reference savedTo: null and nothing else, and for an expiresAt rejected as a field, no expiresAt at all. It now adds expired, the stub's reason, as the stub judges it: a saved copy is still served. The structured-references fixture's refused reference now has a parseable expiry, and a reference whose expiry doesn't parse joins it, unfetched and told why. - #211's read of the awareness ledger landed before the family split in connectHeldMcplServer, and the restart read it for any family. A modern connection has no awareness barrier and no reaction default, so an unreadable ledger refused a modern connect and restart for no reason of its own. Both reads are now the legacy path's. --- src/framework.ts | 14 +++++++------ src/mcpl/references.ts | 7 ++++++- test/mcpl-reference-fields.test.ts | 24 ++++++++++++++++++++-- test/placed-reaction-baseline.test.ts | 17 +++++++++++++++ test/script-structured-references.test.ts | 25 +++++++++++++++-------- 5 files changed, 70 insertions(+), 17 deletions(-) diff --git a/src/framework.ts b/src/framework.ts index b5ae461f8..6e5e381ac 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -15947,13 +15947,14 @@ export class AgentFramework { throw new Error('MCPL subsystem is not initialized'); } - // Every connect needs a readable awareness ledger: a runtime connect's - // awareness barrier reconciles it, and a stdio child taking the + // Every legacy connect needs a readable awareness ledger: a runtime + // connect's awareness barrier reconciles it, and a stdio child taking the // protective default derives that from it. Read it first, so an // unreadable ledger refuses the connect before anything is registered // under its id or spawned (a reconnecting server would otherwise be left // registered as a closed stub). With no outbox configured nothing is read. - this.getPlacedReactionBaseline(); + // A modern connection has no awareness barrier and no reaction default. + if (family === 'legacy') this.getPlacedReactionBaseline(); // Register prefix + config for tool dispatch routing (idempotent with // the pre-registration pass in initializeMcpl). @@ -16674,7 +16675,7 @@ export class AgentFramework { if (!config) { throw new Error(`MCPL server "${id}" is not configured`); } - AgentFramework.usableConfigFamily(config); + const family = AgentFramework.usableConfigFamily(config); const inFlight = this.mcpServerOps?.get(id); if (inFlight) { // A connect or disconnect of the server being restarted, in flight, @@ -16697,12 +16698,13 @@ export class AgentFramework { } const collision = this.mcplPrefixCollision(config, id); if (collision) throw new Error(collision); - // Every connect reads the awareness ledger: its awareness barrier + // A legacy connect reads the awareness ledger: its awareness barrier // reconciles it, and a stdio child taking the protective default derives // that from it. Read it before disconnecting, so an unreadable ledger // refuses the restart and leaves the running child up, whatever the // server's baseline or transport. (No outbox configured: nothing to read.) - this.getPlacedReactionBaseline(); + // A modern connection reads neither. + if (family === 'legacy') this.getPlacedReactionBaseline(); const claim: McpRestartClaim = { restarting: id }; const prefix = config.toolPrefix ?? `mcpl--${config.id}`; diff --git a/src/mcpl/references.ts b/src/mcpl/references.ts index 661bb6345..236bfb4c2 100644 --- a/src/mcpl/references.ts +++ b/src/mcpl/references.ts @@ -356,7 +356,9 @@ function unrecognizedBlockNotice(reason: string): string { * without its URI: the host's reference id, what the server said about it in * this block (its labels stripped and bounded as in the stub), and `savedTo`, * the workspace path its payload was fetched to, or null when it wasn't - * (`fetch_reference` takes the id as `ref_id`). A block the classifier + * (`fetch_reference` takes the id as `ref_id`). An unsaved reference that + * counts as expired also carries `expired`, the reason the stub gives in + * place of advising fetch_reference. A block the classifier * rejects becomes the notice the model's lanes show, and inline data that * also claims a bulk disposition the withheld notice, each as a text block. * A text block, or inline data that claims nothing, is returned as sent. @@ -371,6 +373,8 @@ export function referenceBlockForScript(block: unknown): unknown { case 'reference': { const t = c.testimony; const record = referenceRegistry.register(t); + // As the stub judges it: a saved copy is still served. + const expiry = record.fetchedPath ? null : referenceExpiry(record.testimony); return { type: (block as { type: string }).type, refId: record.refId, @@ -380,6 +384,7 @@ export function referenceBlockForScript(block: unknown): unknown { ...(t.digest !== undefined ? { digest: t.digest } : {}), ...(t.expiresAt !== undefined ? { expiresAt: sanitizeLabel(t.expiresAt, STUB_FIELD_CHARS) } : {}), savedTo: record.fetchedPath ?? null, + ...(expiry ? { expired: EXPIRY_REASON[expiry] } : {}), }; } } diff --git a/test/mcpl-reference-fields.test.ts b/test/mcpl-reference-fields.test.ts index 7732d5062..1a2c8d4a9 100644 --- a/test/mcpl-reference-fields.test.ts +++ b/test/mcpl-reference-fields.test.ts @@ -1,8 +1,8 @@ import { test } from 'node:test'; import assert from 'node:assert/strict'; import { - buildReferenceStub, classifyBlock, isReferenceExpired, sanitizeLabel, REFERENCE_LIMITS, - referenceRegistry, referenceStubOrNull, + buildReferenceStub, classifyBlock, isReferenceExpired, sanitizeLabel, EXPIRY_REASON, REFERENCE_LIMITS, + referenceBlockForScript, referenceRegistry, referenceStubOrNull, } from '../src/mcpl/references.js'; const fields = ['name', 'mimeType', 'expiresAt'] as const; @@ -116,6 +116,26 @@ test('the stub leaves out a type, or a provenance, that sanitizes to nothing', ( assert.equal(originStub, `[${referenceRegistry.findByUri(originGone.uri)!.refId}] image/png — fetch with fetch_reference`); }); +test("an expired reference reaches a script with the reason its stub gives, and a live or saved one without it", () => { + const cases = [['2001-01-01T00:00:00Z', EXPIRY_REASON.passed], ['not-a-date', EXPIRY_REASON.unparseable], + ['x'.repeat(REFERENCE_LIMITS.expiresAt + 1), EXPIRY_REASON.rejected]] as const; + for (const [i, [expiresAt, why]] of cases.entries()) { + const { block, testimony } = reference('resource', { uri: `https://example.test/script-expired-${i}`, expiresAt }); + const view = referenceBlockForScript(block) as { savedTo: unknown; expired?: string }; + assert.equal(view.savedTo, null); + assert.equal(view.expired, why); + assert.ok(buildReferenceStub(testimony).includes(why), 'the stub gives the same reason'); + } + const live = reference('resource', { uri: 'https://example.test/script-live', expiresAt: '2999-01-01T00:00:00Z' }).block; + assert.equal((referenceBlockForScript(live) as { expired?: string }).expired, undefined); + // A copy saved before its expiry passed is still served, and the script gets its path. + const saved = reference('resource', { uri: 'https://example.test/script-expired-0', expiresAt: '2001-01-01T00:00:00Z' }).block; + referenceRegistry.findByUri(saved.uri)!.fetchedPath = 'scratch/refs/kept.png'; + const view = referenceBlockForScript(saved) as { savedTo: unknown; expired?: string }; + assert.equal(view.savedTo, 'scratch/refs/kept.png'); + assert.equal(view.expired, undefined); +}); + test("an expired reference's stub says why, where a live one's advises fetch_reference", () => { const cases = [['2001-01-01T00:00:00Z', /expired: its expiresAt has passed/], ['not-a-date', /treated as expired: its expiresAt does not parse/], ['x'.repeat(REFERENCE_LIMITS.expiresAt + 1), /treated as expired: its expiresAt was rejected as a field/]] as const; diff --git a/test/placed-reaction-baseline.test.ts b/test/placed-reaction-baseline.test.ts index 77c7dbee3..f9ad03ff3 100644 --- a/test/placed-reaction-baseline.test.ts +++ b/test/placed-reaction-baseline.test.ts @@ -12,6 +12,7 @@ import { MockMembrane } from './helpers/mock-membrane.js'; const ENV_KEY = 'DISCORD_SUPPRESSED_REACTIONS_BASELINE'; const FIXTURE = fileURLToPath(new URL('./fixtures/reaction-baseline-mcpl-server.mjs', import.meta.url)); +const MODERN_FIXTURE = fileURLToPath(new URL('./fixtures/modern-mcp-server.mjs', import.meta.url)); function server(overrides: Partial = {}): McplServerConfig { return { id: 'probe', command: process.execPath, args: [FIXTURE], ...overrides }; @@ -290,6 +291,22 @@ describe('placed-reaction baseline', () => { }); } + it('a modern server connects and restarts on a corrupt ledger, which it never reads', async () => { + // A modern connection has no awareness barrier and no reaction default, + // so the ledger is no reason of its own to refuse it. + const fw = await create(); + const modern = (id: string): McplServerConfig => ({ + id, command: process.execPath, args: [MODERN_FIXTURE, join(dir, `${id}-starts.log`), join(dir, `${id}-events.log`)], protocol: 'modern', + }); + await fw.connectMcplServer(modern('modern')); + writeFileSync(outboxPath, '{broken'); + await fw.restartMcplServer('modern'); + await fw.connectMcplServer(modern('second')); + assert.deepEqual(fw.listMcplServers().map((s) => [s.id, s.connected]).sort(), [['modern', true], ['second', true]]); + // A legacy connect still refuses on it. + await assert.rejects(() => fw.connectMcplServer(server()), accountingRefusal); + }); + it('passes a URL-only WebSocket config through unchanged', async () => { const fw = await create(); // Bring up the MCPL subsystem as a first runtime connect does. diff --git a/test/script-structured-references.test.ts b/test/script-structured-references.test.ts index 8942a3854..77ce75550 100644 --- a/test/script-structured-references.test.ts +++ b/test/script-structured-references.test.ts @@ -18,7 +18,7 @@ import { join } from 'node:path'; import { AgentFramework, WorkspaceModule } from '../src/index.js'; import type { ToolResult } from '../src/index.js'; -import { INLINE_WITHHELD_TEXT } from '../src/mcpl/references.js'; +import { EXPIRY_REASON, INLINE_WITHHELD_TEXT } from '../src/mcpl/references.js'; import type { ReferenceRecord } from '../src/mcpl/references.js'; import { MockMembrane } from './helpers/mock-membrane.js'; @@ -30,6 +30,7 @@ const DIGEST = 'sha256:47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU'; const uris = (test: string) => ({ report: `https://files.example.invalid/${test}/report.pdf?sig=secret-token`, big: `https://files.example.invalid/${test}/big.png?sig=other-token`, + old: `https://files.example.invalid/${test}/old.png?sig=old-token`, }); /** A block of each kind the dispatch decides about. */ @@ -41,7 +42,7 @@ const blocks = (test: string) => [ // The same reference again, saying something else about itself. { type: 'resource', uri: uris(test).report, name: 'report-copy.pdf' }, // A reference the fetch doesn't save. - { type: 'image', uri: uris(test).big, mimeType: 'image/png', disposition: 'ref', expiresAt: '2027-01-01T00:00:00Z\u202E' }, + { type: 'image', uri: uris(test).big, mimeType: 'image/png', disposition: 'ref', expiresAt: '2027-01-01T00:00:00Z' }, // Inline data that claims a bulk disposition, and inline data with a uri // beside it: withheld. { type: 'image', data: 'iVBORw0KGgo=', mimeType: 'image/png', disposition: 'ref' }, @@ -55,6 +56,10 @@ const blocks = (test: string) => [ { type: 'resource', resource: { uri: 'https://files.example.invalid/embedded?sig=embedded-token', mimeType: 'text/plain', text: 'embedded' } }, { type: 'resource_link', uri: 'https://files.example.invalid/link?sig=link-token', name: 'link' }, { type: 'text', text: 'two attachments' }, + // A reference that counts as expired: with a bidi override after it, its + // expiresAt doesn't parse, so it fails closed. It isn't fetched, and the + // script is told why. + { type: 'image', uri: uris(test).old, mimeType: 'image/png', expiresAt: '2027-01-01T00:00:00Z\u202E' }, ]; /** A tools/call result as a server may send it, nonconformant blocks included. */ @@ -137,7 +142,7 @@ async function withPeers(fn: (h: Harness) => Promise, opts: { eager?: bool } /** What a script reads for those blocks from an MCPL peer, given each reference's id. */ -const treated = (report: string, big: string) => [ +const treated = (report: string, big: string, old: string) => [ { type: 'resource', refId: report, name: 'report.pdf', mimeType: 'application/pdf', sizeBytes: 1234, digest: DIGEST, expiresAt: '2026-12-31T00:00:00Z', savedTo: SAVED }, { type: 'resource', refId: report, name: 'report-copy.pdf', savedTo: SAVED }, @@ -149,6 +154,7 @@ const treated = (report: string, big: string) => [ { type: 'text', text: '[unrecognized content block: reference without string uri]' }, { type: 'text', text: '[unrecognized content block: unknown block type: resource_link]' }, { type: 'text', text: 'two attachments' }, + { type: 'image', refId: old, mimeType: 'image/png', expiresAt: '2027-01-01T00:00:00Z', savedTo: null, expired: EXPIRY_REASON.unparseable }, ]; test("an MCPL peer's structured result reaches a script with each block treated as the model's lanes treat it", async () => { @@ -156,13 +162,16 @@ test("an MCPL peer's structured result reaches a script with each block treated answer({ content: blocks('treated'), structuredContent: { attachments: 2 } }); const out = await script('mcpl--files--attach'); assert.ok(!out.includes('example.invalid'), `no URI reaches the script: ${out.slice(0, 2000)}`); - assert.deepEqual(fetched, [uris('treated').report, uris('treated').big], 'every reference was fetched for the script before it read the result'); + assert.deepEqual(fetched, [uris('treated').report, uris('treated').big], + 'every reference was fetched for the script before it read the result, except the one that counts as expired'); const object = JSON.parse(out) as { content: Array<{ refId?: string }>; structuredContent: unknown; isError: boolean }; const [report, , big] = object.content; + const old = object.content[10]; assert.match(String(report!.refId), /^ref_/); assert.match(String(big!.refId), /^ref_/); - assert.notEqual(report!.refId, big!.refId); - assert.deepEqual(object, { content: treated(report!.refId!, big!.refId!), structuredContent: { attachments: 2 }, isError: false }); + assert.match(String(old!.refId), /^ref_/); + assert.equal(new Set([report!.refId, big!.refId, old!.refId]).size, 3); + assert.deepEqual(object, { content: treated(report!.refId!, big!.refId!, old!.refId!), structuredContent: { attachments: 2 }, isError: false }); }); }); @@ -224,7 +233,7 @@ test("an MCPL peer's structured result past the script cap is saved with its ref const saved = read.data.toString('utf8'); assert.ok(!saved.includes('example.invalid'), 'no URI in the saved file'); const object = JSON.parse(saved) as { content: Array<{ refId?: string }> }; - assert.deepEqual(object.content, treated(object.content[0]!.refId!, object.content[2]!.refId!)); + assert.deepEqual(object.content, treated(object.content[0]!.refId!, object.content[2]!.refId!, object.content[10]!.refId!)); }); }); @@ -260,7 +269,7 @@ test('an error with structured content is fetched for the script, and reaches it assert.ok(!out.includes('example.invalid'), out); const object = JSON.parse(out) as { content: Array<{ refId?: string }>; structuredContent: unknown; isError: boolean; error: string }; assert.deepEqual(object, { - content: [treated(object.content[0]!.refId!, '')[0], { type: 'text', text: 'the attach failed' }], + content: [treated(object.content[0]!.refId!, '', '')[0], { type: 'text', text: 'the attach failed' }], structuredContent: { partial: true }, isError: true, error: 'the attach failed', From a02525a2de5a6447b5e71d95c31ff3d0546af1eb Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Wed, 14 Oct 2026 01:05:14 -0400 Subject: [PATCH 0190/1112] docs(changelog): a script's expired reference carries expired --- changelog.d/mcp-standard-tool-results.changed.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/changelog.d/mcp-standard-tool-results.changed.md b/changelog.d/mcp-standard-tool-results.changed.md index 5d0a6972b..4e07d2530 100644 --- a/changelog.d/mcp-standard-tool-results.changed.md +++ b/changelog.d/mcp-standard-tool-results.changed.md @@ -41,7 +41,9 @@ bidi characters stripped from the text fields, as in the model's stub), and `savedTo`, the workspace path the framework fetched its payload to for the script (`null` when it couldn't; `fetch_reference` takes the id as its - `ref_id` argument). A block the classifier rejects arrives as the model's + `ref_id` argument). An unsaved reference that counts as expired also + carries `expired`, the reason the model's stub gives, and isn't fetched. + A block the classifier rejects arrives as the model's `[unrecognized content block: …]` notice, and inline data that also claims a bulk disposition as the withheld notice. #181, by @Meganeuridae, first found scripts losing `structuredContent`, and it kept the saved paths of From c5763d3543b52341b3f636d2f2f916d3e087a589 Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Wed, 14 Oct 2026 01:08:24 -0400 Subject: [PATCH 0191/1112] test(baseline): command + URL with no transport is refused, as this branch's binding rule says main's #211 test injected the baseline for a stdio child configured with command and url and no transport, which main launched as stdio. This branch refuses that configuration before anything is spawned or dialed (resolveServerBinding; the legacy boundary's breaking note), so the implicit case now asserts the refusal and that nothing is registered, and the explicit stdio case keeps the injection. --- test/placed-reaction-baseline.test.ts | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/test/placed-reaction-baseline.test.ts b/test/placed-reaction-baseline.test.ts index f9ad03ff3..ef3a91a52 100644 --- a/test/placed-reaction-baseline.test.ts +++ b/test/placed-reaction-baseline.test.ts @@ -141,13 +141,23 @@ describe('placed-reaction baseline', () => { assert.deepEqual(config.env, { BASELINE_TEST_EXTRA: 'preserved' }, 'caller config stays unchanged'); }); - for (const transport of [undefined, 'stdio'] as const) { - it('injects the baseline with command + URL and ' + (transport ?? 'implicit') + ' transport', async () => { - const fw = await create('🔕', [server({ url: 'ws://127.0.0.1:1', transport })]); - const env = await childEnv(fw); - assert.deepEqual(new Set(env.baseline?.split(',')), new Set([...REFUSAL_REACTION_BASELINE, '💤', '🔕'])); - }); - } + it('injects the baseline with command + URL and stdio transport', async () => { + const fw = await create('🔕', [server({ url: 'ws://127.0.0.1:1', transport: 'stdio' })]); + const env = await childEnv(fw); + assert.deepEqual(new Set(env.baseline?.split(',')), new Set([...REFUSAL_REACTION_BASELINE, '💤', '🔕'])); + }); + + it('refuses command + URL with no transport before spawning anything', async () => { + // Which of the two the configuration means is ambiguous, so its binding + // is refused: nothing is launched, and no baseline is derived for it. + const fw = await create('🔕'); + await assert.rejects( + () => fw.connectMcplServer(server({ url: 'ws://127.0.0.1:1' })), + /"command" and "url" are both set, and no "transport" says which one to use/, + ); + const registry = (fw as unknown as { mcplServerRegistry: { getServer(id: string): unknown } | null }).mcplServerRegistry; + assert.equal(registry?.getServer('probe') ?? null, null, 'nothing is registered or spawned'); + }); it('derives the default on runtime connect, including newly retained ledger markers', async () => { const fw = await create('🔕'); From 792ef25a18a72771282da9bde529420977dd65bd Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Wed, 14 Oct 2026 01:16:36 -0400 Subject: [PATCH 0192/1112] fix(mcp): what the full run and the round's readers found From the full run at 379490b6, merge adaptations: - The restart claims are created on first use (restartClaims()), as the modern connections are, so a framework assembled field by field, as the reaction-baseline platform tests build one, needs no setup. - The eager-slot test calls autoFetchReferences with forScript false: the eager pass is unconditional now, and its third argument is the script's demand, which would fetch past the eight slots. - The awareness barrier's inbound-handler tests wire the connection with its configuration, as wireMcplEvents takes it here. - The surgery-marks fixture answers initialize with protocolVersion 2024-11-05, which the legacy engine requires. - A module's result that can't be stored is an error naming the tool by the time the dispatch returns it (main's storableToolResult), so a script reads that error's text, structured or not. The structured results test expects that text, and the docs say so. From the round's read: - addModule refuses a name a server's tool prefix holds, as create() does at startup, and a prefix a restart is moving its server to, which a module added during the restart would otherwise take after the old server was gone. - A ws:// url's userinfo goes as one unit with no length floor, anchored by its @ and by no letter or digit just before it, so a short password goes with its username however short both are. - A test token whose form encoding differs from its percent encoding pins the lowercase form-encoded variant. --- .../mcp-standard-tool-results.changed.md | 11 ++-- .../redact-recorded-mcp-credentials.fixed.md | 2 +- src/framework.ts | 64 ++++++++++++------- src/mcpl/protocol-family.ts | 17 +++-- .../code-execution-structured-results.test.ts | 27 ++++---- test/fixtures/surgery-marks-mcpl-server.mjs | 1 + test/mcp-protocol-family.test.ts | 18 ++++-- test/mcpl-awareness-barrier.test.ts | 6 +- test/mcpl-reference-fetcher.test.ts | 5 +- test/mcpl-runtime-lifecycle.test.ts | 17 +++++ 10 files changed, 109 insertions(+), 59 deletions(-) diff --git a/changelog.d/mcp-standard-tool-results.changed.md b/changelog.d/mcp-standard-tool-results.changed.md index 4e07d2530..f17265a89 100644 --- a/changelog.d/mcp-standard-tool-results.changed.md +++ b/changelog.d/mcp-standard-tool-results.changed.md @@ -112,11 +112,12 @@ disconnect of the server in flight. Before, a restart disconnected first, so a refused connect left the server gone; now the server keeps running. From those checks through its connect, the restart holds the id and the - tool prefix it moves the server to, so a connect or restart started - meanwhile is refused them, naming the restart, rather than taking them - while the old server is torn down. A disconnect releases its tool-prefix - claim only while its own server holds it. Treat a submitted configuration - as immutable while the server uses it: change settings with + tool prefix it moves the server to, so a connect, a restart or a module + added meanwhile is refused them, naming the restart, rather than taking + them while the old server is torn down. `addModule` also refuses a name a + server's tool prefix holds, as `create()` does at startup. A disconnect + releases the tool-prefix claims its own server holds. Treat a submitted + configuration as immutable while the server uses it: change settings with `restartMcplServer(id, newConfig)`, and rotate a network server's credentials through `accessProvider` (a stdio server's changed `env` goes through a restart with the new configuration). diff --git a/changelog.d/redact-recorded-mcp-credentials.fixed.md b/changelog.d/redact-recorded-mcp-credentials.fixed.md index e58f63aaa..62df4f2e8 100644 --- a/changelog.d/redact-recorded-mcp-credentials.fixed.md +++ b/changelog.d/redact-recorded-mcp-credentials.fixed.md @@ -1 +1 @@ -- Server and transport text the framework records has the server's configured credentials replaced, in both engines: connection traces, console lines, stderr traces, the ops alert (and so `logs/failures.log` and the webhook), a tool call's failure as its caller and the model get it, and the `tool:failed` trace's message and stack. The credentials are those the configuration holds: from the url, credential-like path segments and query names, credential-shaped or credential-named query values (each as written and decoded), every non-empty parameter as its whole `name=value`, and a `ws://` url's username and password, its userinfo as one unit before its `@`, and the Basic credential a WebSocket client sends for it; the static `token`, raw, with its slashes escaped as some JSON encoders write them, and as a url carries it, percent- or form-encoded with either case of hex; and a stdio server's `env` values whose names have a part ending in a credential word, singular or plural, with any digits after it (`GITHUB_TOKEN`, `API_KEYS`, `PGPASSWORD`, `DB_PASS`, `AUTHORIZATION`, `SESSION_COOKIE`, `TOKEN2`), raw, with their slashes escaped as some JSON encoders write them, and percent-encoded with either case of hex. A stdio server's command-line arguments are not derived. A string replaced on its own has at least eight characters, since a shorter one can't be told from ordinary text, and is replaced wherever it occurs. Every query parameter is also replaced as its whole `name=value`, only where no letter or digit adjoins the pair (so `v=2` leaves `dev=2` alone); a short value goes only that way. A connection's event records use a redactor built once for it, and an operation's records one built when the operation begins, so a server removed, or replaced by a restart, while the operation waits doesn't change what they replace. Text that may come from any server takes one pass over every server's strings, so a secret that begins another server's longer one doesn't cut that one short. Over HTTP, which only the modern engine speaks, a 401 or 403's response body is left out of the request error's data; where the HTTP client builds the error's message from the body, as it does for most statuses, the message is redacted like any record. A token an access provider fetches, and credentials the configuration doesn't hold, are outside this. JSON-RPC answers, and the error a host's own connect raises, pass as they are. +- Server and transport text the framework records has the server's configured credentials replaced, in both engines: connection traces, console lines, stderr traces, the ops alert (and so `logs/failures.log` and the webhook), a tool call's failure as its caller and the model get it, and the `tool:failed` trace's message and stack. The credentials are those the configuration holds: from the url, credential-like path segments and query names, credential-shaped or credential-named query values (each as written and decoded), every non-empty parameter as its whole `name=value`, and a `ws://` url's username and password, its userinfo as one unit before its `@` (with no length floor, so a short password goes with its username), and the Basic credential a WebSocket client sends for it; the static `token`, raw, with its slashes escaped as some JSON encoders write them, and as a url carries it, percent- or form-encoded with either case of hex; and a stdio server's `env` values whose names have a part ending in a credential word, singular or plural, with any digits after it (`GITHUB_TOKEN`, `API_KEYS`, `PGPASSWORD`, `DB_PASS`, `AUTHORIZATION`, `SESSION_COOKIE`, `TOKEN2`), raw, with their slashes escaped as some JSON encoders write them, and percent-encoded with either case of hex. A stdio server's command-line arguments are not derived. A string replaced on its own has at least eight characters, since a shorter one can't be told from ordinary text, and is replaced wherever it occurs. Every query parameter is also replaced as its whole `name=value`, only where no letter or digit adjoins the pair (so `v=2` leaves `dev=2` alone); a short value goes only that way. A connection's event records use a redactor built once for it, and an operation's records one built when the operation begins, so a server removed, or replaced by a restart, while the operation waits doesn't change what they replace. Text that may come from any server takes one pass over every server's strings, so a secret that begins another server's longer one doesn't cut that one short. Over HTTP, which only the modern engine speaks, a 401 or 403's response body is left out of the request error's data; where the HTTP client builds the error's message from the body, as it does for most statuses, the message is redacted like any record. A token an access provider fetches, and credentials the configuration doesn't hold, are outside this. JSON-RPC answers, and the error a host's own connect raises, pass as they are. diff --git a/src/framework.ts b/src/framework.ts index 6e5e381ac..4e23c3659 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -1960,11 +1960,9 @@ export class AgentFramework { /** The ids and tool prefixes restarts hold, each with the claim of the * restart holding it, from the restart's checks through its connect * (restartMcplServer): no other connect or restart takes them while the - * old server is torn down. */ - private readonly mcpRestartClaims = { - ids: new Map(), - prefixes: new Map(), - }; + * old server is torn down. Created on first use (restartClaims), like + * mcpServerOps. */ + private mcpRestartClaims: { ids: Map; prefixes: Map } | undefined; /** Host capabilities advertised during the MCP handshake — stored so servers * can be connected at runtime (connectMcplServer) after initialization. */ private mcplHostCapabilities: McplHostCapabilities | null = null; @@ -3546,6 +3544,18 @@ export class AgentFramework { * Add a module at runtime. */ async addModule(module: Module): Promise { + // A server's tool prefix and a module's name share one namespace, as + // create() enforces at startup (mcplPrefixCollision): a module isn't + // added under a prefix a server holds, or one a restart is moving its + // server to. + const server = this.mcplPrefixMap.get(module.name); + if (server !== undefined) { + throw new Error(`Module "${module.name}" collides with MCPL server "${server}"'s toolPrefix "${module.name}"`); + } + const claimedBy = this.restartClaims().prefixes.get(module.name); + if (claimedBy) { + throw new Error(`Module "${module.name}" collides with the toolPrefix "${module.name}" the restart of "${claimedBy.restarting}" holds`); + } this.resetCompletedStop(); await this.moduleRegistry.addModule(module); this.emitTrace({ type: 'module:added', moduleName: module.name }); @@ -13723,9 +13733,10 @@ export class AgentFramework { * megabytes of base64 helps nobody. A result with structured content is * one JSON object holding both of the server's views * (scriptStructuredResult), so its content blocks reach the script as the - * dispatch gave them, images and payloads included. A result that can't be - * converted to JSON, such as a cyclic object or a BigInt from an in-process - * module, becomes an error that names the tool, in each path's error form. + * dispatch gave them, images and payloads included. A module's result that + * can't be stored or sent, such as a cyclic object or a BigInt, is already + * an error naming the tool when the dispatch returns it (storableToolResult), + * so a script reads that error's text. */ private async handleScriptToolCall( agentName: string, @@ -13785,9 +13796,10 @@ export class AgentFramework { * `{ isError, oversized: { chars, savedTo } }`. If it can't be saved, the * script gets a failure to materialize the result, saying the tool may * already have completed, never a success missing its result. A result - * that can't be converted to JSON (a cyclic object, a BigInt) gets the - * same object empty, `{ content: [], structuredContent: null, isError: - * true, error }`, the error naming the tool. + * that can't be converted to JSON (a cyclic object, a BigInt), which the + * dispatch's own check makes unlikely here, gets the same object empty, + * `{ content: [], structuredContent: null, isError: true, error }`, the + * error naming the tool. */ private async scriptStructuredResult(result: ToolResult, toolName: string): Promise { const object = { @@ -15454,6 +15466,12 @@ export class AgentFramework { return (this.modernMcpConnections ??= new Map()); } + /** The restarts' claims (mcpRestartClaims), created on first use, as the + * modern connections are. */ + private restartClaims(): { ids: Map; prefixes: Map } { + return (this.mcpRestartClaims ??= { ids: new Map(), prefixes: new Map() }); + } + /** * Hold a server id for one connect or disconnect, until `release()`. * Together with the live connections (the MCPL registry and the modern @@ -16648,8 +16666,9 @@ export class AgentFramework { * unlike `reconnect: true`, which only retries after transport-level * failures. Throws if the server has no stored config and none is given. * What its connect would refuse is refused before the running server is - * torn down, and until the connect, a connect or restart started meanwhile - * is refused the id and tool prefix the restart moves the server to. + * torn down, and until the connect, a connect, a restart or a module added + * meanwhile is refused the id and tool prefix the restart moves the server + * to. * * Treat a submitted configuration and its nested values as immutable for * as long as the server uses them. Apply changed settings with @@ -16666,8 +16685,9 @@ export class AgentFramework { // configuration's binding and settings, a new id another server holds, // and a tool prefix another server or a module already holds. From these // checks through its connect, the restart holds the id and the tool - // prefix it moves the server to (mcpRestartClaims), so no connect or - // restart started meanwhile takes them while the old server is torn down. + // prefix it moves the server to (mcpRestartClaims), so no connect, + // restart or module added meanwhile takes them while the old server is + // torn down. if (this.mcpServerAdmissionClosed) { throw new Error(`MCP server "${id}" was not restarted: the framework is stopping or stopped`); } @@ -16684,7 +16704,7 @@ export class AgentFramework { // operation left, after the server was torn down. throw new Error(`MCP server "${id}" was not restarted: it is being ${inFlight.kind === 'disconnect' ? 'disconnected' : 'connected'}; restart it once that has finished`); } - const claimedBy = this.mcpRestartClaims.ids.get(id); + const claimedBy = this.restartClaims().ids.get(id); if (claimedBy) { // Another restart is moving its server to this id, or restarting this // server already. @@ -16708,14 +16728,14 @@ export class AgentFramework { const claim: McpRestartClaim = { restarting: id }; const prefix = config.toolPrefix ?? `mcpl--${config.id}`; - this.mcpRestartClaims.ids.set(config.id, claim); - this.mcpRestartClaims.prefixes.set(prefix, claim); + this.restartClaims().ids.set(config.id, claim); + this.restartClaims().prefixes.set(prefix, claim); try { await this.disconnectMcplServer(id); await this.connectMcplServerClaimed(config, claim); } finally { - if (this.mcpRestartClaims.ids.get(config.id) === claim) this.mcpRestartClaims.ids.delete(config.id); - if (this.mcpRestartClaims.prefixes.get(prefix) === claim) this.mcpRestartClaims.prefixes.delete(prefix); + if (this.restartClaims().ids.get(config.id) === claim) this.restartClaims().ids.delete(config.id); + if (this.restartClaims().prefixes.get(prefix) === claim) this.restartClaims().prefixes.delete(prefix); } } @@ -16745,7 +16765,7 @@ export class AgentFramework { * a restart moving its server to the id (mcpRestartClaims), except to * that restart's own connect (`claim`). */ private mcpServerIdTaken(id: string, claim?: McpRestartClaim): string | null { - const claimedBy = this.mcpRestartClaims.ids.get(id); + const claimedBy = this.restartClaims().ids.get(id); if (claimedBy && claimedBy !== claim) { return `MCP server "${id}" is held by the restart of "${claimedBy.restarting}"; connect it once that has finished`; } @@ -16777,7 +16797,7 @@ export class AgentFramework { if (this.moduleRegistry.getAllModules().some(m => m.name === prefix)) { return `MCPL server "${config.id}" toolPrefix "${prefix}" collides with module "${prefix}"`; } - const claimedBy = this.mcpRestartClaims.prefixes.get(prefix); + const claimedBy = this.restartClaims().prefixes.get(prefix); if (claimedBy && claimedBy !== claim) { return `MCPL server "${config.id}" toolPrefix "${prefix}" is held by the restart of "${claimedBy.restarting}"`; } diff --git a/src/mcpl/protocol-family.ts b/src/mcpl/protocol-family.ts index 7ab46747f..fe230315f 100644 --- a/src/mcpl/protocol-family.ts +++ b/src/mcpl/protocol-family.ts @@ -113,7 +113,8 @@ export function redactUrl(url: string | undefined): string { * - from the url, as written and decoded: credential-like path segments and * query names, credential-shaped or credential-named query values, every * other non-empty parameter as its whole name=value, and a ws:// url's - * username and password, its userinfo as one unit before its @, and the + * username and password, its userinfo as one unit before its @ (with no + * length floor, so a short password goes with its username), and the * Basic credential a WebSocket client sends for that userinfo; * - the static token, raw, with its slashes escaped as some JSON encoders * write them, and percent- or form-encoded with either case of hex; @@ -235,12 +236,14 @@ export function serverCredentialPatterns(config: Pick= 8) userinfo.add(`${username}:${password}`); + if (password) userinfo.add(`${username}:${password}`); standalone(Buffer.from(`${username}:${password}`).toString('base64')); } } @@ -250,7 +253,7 @@ export function serverCredentialPatterns(config: Pick ({ value, source: escape(value) })), ...[...pairs].map(value => ({ value, source: `(? ({ value, source: `${escape(value)}(?=@)` })), + ...[...userinfo].map(value => ({ value, source: `(? 5000000, large', 'assert "could not be delivered" in large["error"], large', - // Results JSON can't hold: an error naming the tool, in each path's form. - // A structured result's error keeps the documented shape, so a script reading "content" first still works. - 'cyclic = json.loads(await loop__cyclic({}))', - 'assert cyclic["content"] == [] and cyclic["structuredContent"] is None and cyclic["isError"] is True, cyclic', - 'assert "loop--cyclic" in cyclic["error"] and "circular" in cyclic["error"], cyclic', - 'bigint = json.loads(await loop__bigint({}))', - 'assert bigint["content"] == [] and bigint["structuredContent"] is None and bigint["isError"] is True, bigint', - 'assert "loop--bigint" in bigint["error"] and "BigInt" in bigint["error"], bigint', + // Results JSON can't hold: the dispatch makes each an error naming the + // tool (a module's result has to be stored), which a script reads as + // its error text, structured or not. + 'cyclic = await loop__cyclic({})', + 'assert cyclic.startswith("Error: Tool \'loop--cyclic\' ") and "circular" in cyclic, cyclic', + 'bigint = await loop__bigint({})', + 'assert bigint.startswith("Error: Tool \'loop--bigint\' ") and "BigInt" in bigint, bigint', 'cyclic_data = await loop__cyclic_data({})', - 'assert cyclic_data.startswith("Error: tool ") and "loop--cyclic_data" in cyclic_data and "circular" in cyclic_data, cyclic_data', - 'assert all("The call itself completed." in e for e in [cyclic["error"], bigint["error"], cyclic_data]), "completed"', + 'assert cyclic_data.startswith("Error: Tool \'loop--cyclic_data\' ") and "circular" in cyclic_data, cyclic_data', + 'assert all(e.endswith("The call itself completed.") for e in [cyclic, bigint, cyclic_data]), "completed"', // An error result that can't be converted keeps the tool's own error. - 'cyclic_error = json.loads(await loop__cyclic_error({}))', - 'assert cyclic_error["isError"] is True and cyclic_error["error"].endswith("The call itself completed, with this error: db timeout"), cyclic_error', - 'bare_throw = json.loads(await loop__bare_throw({}))', - 'assert bare_throw["isError"] is True and "(a value with no text form)" in bare_throw["error"], bare_throw', + 'cyclic_error = await loop__cyclic_error({})', + 'assert cyclic_error.endswith("The call itself completed, with this error: db timeout"), cyclic_error', + 'bare_throw = await loop__bare_throw({})', + 'assert "(a value with no text form)" in bare_throw, bare_throw', 'print(r["structuredContent"]["runId"], r["structuredContent"]["state"], r["structuredContent"]["output"])', ].join('\n'); membrane.pushResponse(createMockResponse([ diff --git a/test/fixtures/surgery-marks-mcpl-server.mjs b/test/fixtures/surgery-marks-mcpl-server.mjs index 4d2a0a117..1eb76c2ed 100644 --- a/test/fixtures/surgery-marks-mcpl-server.mjs +++ b/test/fixtures/surgery-marks-mcpl-server.mjs @@ -47,6 +47,7 @@ function answer(message) { function handle(message) { if (message.method === 'initialize') { reply(message.id, { + protocolVersion: '2024-11-05', capabilities: { experimental: { mcpl: { diff --git a/test/mcp-protocol-family.test.ts b/test/mcp-protocol-family.test.ts index 64a9564f9..5c58a79e8 100644 --- a/test/mcp-protocol-family.test.ts +++ b/test/mcp-protocol-family.test.ts @@ -383,14 +383,22 @@ test('recorded server text: a slash escaped as JSON may write it, and percent-es for (const form of ['tok/en+x0123', 'tok\\/en+x0123', 'tok%2Fen%2Bx0123', 'tok%2fen%2bx0123', 'env\\/secret+0123', 'env%2fsecret%2b0123']) { assert.equal(redactServerText(`refused: ${form}`, config), 'refused: redacted', form); } + // A token whose form encoding differs from its percent encoding: a space is + there, %20 here. + const spaced = cfg({ token: 'tok en/x0123' }); + for (const form of ['tok+en%2Fx0123', 'tok+en%2fx0123', 'tok%20en%2Fx0123', 'tok%20en%2fx0123']) { + assert.equal(redactServerText(`refused: ${form}`, spaced), 'refused: redacted', form); + } }); test("recorded server text: a ws:// url's userinfo as one unit before its @, and the Basic credential sent for it", () => { - // Neither part is long enough to replace on its own. - const config = cfg({ url: 'wss://alice:pw@h/mcp' }); - assert.equal(redactServerText('dial wss://alice:pw@h/mcp failed', config), 'dial wss://redacted@h/mcp failed'); - assert.equal(redactServerText('Authorization: Basic YWxpY2U6cHc=', config), 'Authorization: Basic redacted'); - assert.equal(redactServerText('alice:pw is not followed by its @ here', config), 'alice:pw is not followed by its @ here'); + // Neither part is long enough to replace on its own, nor is the unit. + const config = cfg({ url: 'wss://bot:pw@h/mcp' }); + assert.equal(redactServerText('dial wss://bot:pw@h/mcp failed', config), 'dial wss://redacted@h/mcp failed'); + assert.equal(redactServerText('Authorization: Basic Ym90OnB3', config), 'Authorization: Basic redacted'); + // The @ and a boundary before it anchor the unit. + assert.equal(redactServerText('bot:pw is not followed by its @ here', config), 'bot:pw is not followed by its @ here'); + assert.equal(redactServerText('robot:pw@h', config), 'robot:pw@h'); + assert.equal(redactServerText('dial wss://alice:pw@h/mcp failed', cfg({ url: 'wss://alice:pw@h/mcp' })), 'dial wss://redacted@h/mcp failed'); }); test("one redactor over several configurations' strings takes the longest secret among them all", () => { diff --git a/test/mcpl-awareness-barrier.test.ts b/test/mcpl-awareness-barrier.test.ts index 6530bee6a..56de8ecac 100644 --- a/test/mcpl-awareness-barrier.test.ts +++ b/test/mcpl-awareness-barrier.test.ts @@ -865,7 +865,7 @@ test('inbound handlers wait out a generation installed while they wait', async ( framework.handleHostCommand = async () => { handled.push('host-command'); return { ok: true }; }; const connection = new EventEmitter() as any; connection.id = 'discord'; - framework.wireMcplEvents(connection); + framework.wireMcplEvents(connection, { id: 'discord' }); const responder = { id: 1, respond: () => {}, respondError: () => {} }; for (const event of ['push-event', 'inference-request', 'channels-incoming', 'host-command']) { connection.emit(event, {}, responder); @@ -897,7 +897,7 @@ test('an inbound handler waiting on a generation that fails refuses its request framework.handleHostCommand = async () => { handled.push('host-command'); return { ok: true }; }; const connection = new EventEmitter() as any; connection.id = 'discord'; - framework.wireMcplEvents(connection); + framework.wireMcplEvents(connection, { id: 'discord' }); const answers: Array<{ event: string; code?: number; result?: unknown }> = []; for (const event of ['push-event', 'inference-request', 'channels-incoming', 'host-command']) { connection.emit(event, {}, { @@ -929,7 +929,7 @@ test('with no generation current, an inbound event is handled in the turn it arr framework.handleHostCommand = async () => { handled.push('host-command'); return { ok: true }; }; const connection = new EventEmitter() as any; connection.id = 'discord'; - framework.wireMcplEvents(connection); + framework.wireMcplEvents(connection, { id: 'discord' }); const responder = { id: 1, respond: () => {}, respondError: () => {} }; for (const event of ['push-event', 'inference-request', 'channels-incoming', 'host-command']) { connection.emit(event, {}, responder); diff --git a/test/mcpl-reference-fetcher.test.ts b/test/mcpl-reference-fetcher.test.ts index 11a55a1f2..9867fc2d3 100644 --- a/test/mcpl-reference-fetcher.test.ts +++ b/test/mcpl-reference-fetcher.test.ts @@ -274,10 +274,11 @@ async function eagerFetch(content: unknown[]): Promise { const fetcher = makeFetcher({ srv: { url: ORIGIN } }); const framework = Object.create(AgentFramework.prototype) as { getReferenceFetcher(): ReferenceFetcher; - autoFetchReferences(content: unknown[], serverId: string, eager: boolean): Promise; + autoFetchReferences(content: unknown[], serverId: string, forScript: boolean): Promise; }; framework.getReferenceFetcher = () => fetcher; - await framework.autoFetchReferences(content, 'srv', true); + // The eager pass alone, as for the model's call: a script's demand would fetch the rest. + await framework.autoFetchReferences(content, 'srv', false); } test('the eager default skips expired references, so the live ones after them get its slots', async (t) => { diff --git a/test/mcpl-runtime-lifecycle.test.ts b/test/mcpl-runtime-lifecycle.test.ts index a5c4e468d..28146a928 100644 --- a/test/mcpl-runtime-lifecycle.test.ts +++ b/test/mcpl-runtime-lifecycle.test.ts @@ -309,6 +309,23 @@ test('a restart of an id another restart holds is refused before it tears anythi }); }); +test('a module is refused a name a server holds as its tool prefix, or a restart is moving its server to', async () => { + await withFramework(true, async (framework) => { + await assert.rejects( + framework.addModule(new NamedModule('mcpl--first')), + /Module "mcpl--first" collides with MCPL server "first"'s toolPrefix "mcpl--first"/, + ); + const restarting = framework.restartMcplServer('first', serverConfig('first', { toolPrefix: 'xx' }) as never); + await assert.rejects( + framework.addModule(new NamedModule('xx')), + /Module "xx" collides with the toolPrefix "xx" the restart of "first" holds/, + ); + await restarting; + assert.deepEqual(framework.listMcplServers().map(s => [s.id, s.connected]), [['first', true]]); + assert.ok(framework.getAllTools().some(t => t.name === 'xx--echo'), 'the restarted server serves the prefix'); + }); +}); + test('a restart that fails releases the id and the tool prefix it held', async (t) => { await withFramework(true, async (framework) => { t.mock.method(framework, 'disconnectMcplServer', async () => { From e773a451c477c6a9b86361239ce7ab20d93c1768 Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Wed, 14 Oct 2026 02:30:58 -0400 Subject: [PATCH 0193/1112] test(baseline): the platform tests' registry stub answers getServer The reaction-baseline platform tests build a framework field by field and stop the connect at addServer. This branch's connect first asks the registry whether the id is already registered (mcpServerIdTaken), so the stub registry answers getServer with nothing registered. Before, every test in the file failed with getServer not a function. --- test/reaction-baseline-platform.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/test/reaction-baseline-platform.test.ts b/test/reaction-baseline-platform.test.ts index 83b9e1163..7611f6e15 100644 --- a/test/reaction-baseline-platform.test.ts +++ b/test/reaction-baseline-platform.test.ts @@ -29,6 +29,7 @@ async function connectionEnv( discordAwarenessOutbox: null, discordAwarenessEmoji: '🔕', mcplServerRegistry: { + getServer: () => undefined, async addServer(config: McplServerConfig) { captured = config; throw stopBeforeSpawn; From 350c6597a46d761aeb20f78235f22675b70d7790 Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Wed, 14 Oct 2026 02:59:07 -0400 Subject: [PATCH 0194/1112] test(mcp): the every-server redactor's one pass spans the configurations it read when made and those present now The every-server redactor reads two sets of configurations: those present when it's made and those present when it's used. Its one longest-first pass over both is what keeps a secret that begins another server's longer one from cutting that one short, but the test put both servers in the first set, so a redactor that ran one set's strings after the other's passed it too. The new case makes the redactor with the shorter secret's server alone, then connects the longer one's, and expects the longer secret redacted whole. --- test/mcp-recorded-text.test.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/mcp-recorded-text.test.ts b/test/mcp-recorded-text.test.ts index 90e011105..fdaa5042d 100644 --- a/test/mcp-recorded-text.test.ts +++ b/test/mcp-recorded-text.test.ts @@ -232,6 +232,12 @@ test("a tool call's failure, from either engine, reaches its caller and its trac ['b', { id: 'b', token: 'ghp_SHAREDPREFIX123_but_longer_TAIL9' }], ]); assert.equal(framework.everyServerRedactor()('auth failed for ghp_SHAREDPREFIX123_but_longer_TAIL9'), 'auth failed for redacted'); + // The same across the two sets it reads: the secret of a server present + // when it was made begins the longer secret of a server connected since. + framework.mcplServerConfigs = new Map([['a', { id: 'a', token: 'ghp_SHAREDPREFIX123' }]]); + const across = framework.everyServerRedactor(); + framework.mcplServerConfigs = new Map([['b', { id: 'b', token: 'ghp_SHAREDPREFIX123_but_longer_TAIL9' }]]); + assert.equal(across('auth failed for ghp_SHAREDPREFIX123_but_longer_TAIL9'), 'auth failed for redacted'); framework.mcplServerConfigs = new Map([[config.id, config]]); // The model's dispatch: the tool:failed trace (message and stack) and the From 69a9b607ce4d6f66aed2a2bd5cc00ac603a99be4 Mon Sep 17 00:00:00 2001 From: Duncan-2410 Date: Wed, 14 Oct 2026 04:54:11 -0400 Subject: [PATCH 0195/1112] fix(deferral): the log line names any queued block; the text says what the code does at its edges From an internal review of the final round. - describeDeferredWrite read each block's type directly. Nothing checks a fresh write's blocks beyond addMessage reading each type, so a block that isn't an object (a string or number) can be queued, and the line named such content "empty", or "text+" beside a typed block. It now names a block that isn't an object by its JavaScript type, and an object whose type isn't a string as untyped. Its doc says so, in place of the argument that no queued entry has a null block. host-quiesce's two-resident resume test queues its stuck write with a bare string beside its text block and expects "text+string"; reading the type directly fails it. - The stop() comment: past the deferral a write that throws stays queued (a quiesced host never stores here), and an entry for a target no longer registered is logged and dropped by the boot or resume flush that meets it, which at a quiesced boot is the resume. - The changelog fragment: - a write a flush defers again is one case of going back into the queue; a write whose store throws goes back too; - only the agent's own state (turn, stream, tool cycle) holds its flushes, not only its tool cycle; - stop(): main's refusals before and after a compile (#281) are main's; this change refuses a fork's, subagent's or retry's turn before it starts, and a new ephemeral run, and ends an ephemeral run whose turn is refused because stop() closed admission; - writes that aren't deferred include a message assembled into a turn before its compile (RFC-006 coalescing); - a persisted queue keeps a stuck write across restarts unless boot drops it as an entry the store could never take, and persists later deferrals up to the recovery file's size cap; - boot also drops an entry whose participant isn't text, as its loader always did. Co-Authored-By: Claude Opus 5.5 --- .../mid-turn-window-only-held.fixed.md | 101 +++++++++--------- src/framework.ts | 26 +++-- test/host-quiesce.test.ts | 7 +- 3 files changed, 74 insertions(+), 60 deletions(-) diff --git a/changelog.d/mid-turn-window-only-held.fixed.md b/changelog.d/mid-turn-window-only-held.fixed.md index fa897bbae..b622b16fa 100644 --- a/changelog.d/mid-turn-window-only-held.fixed.md +++ b/changelog.d/mid-turn-window-only-held.fixed.md @@ -18,11 +18,12 @@ subject could remove a message the agent had heard and answered. Within a branch the watermark only advances: a compile that started before such a mark doesn't roll it back when it finishes. -- A deferred write that goes back into the queue does so whatever its - content, under its id and in its place in the order writes arrived. That - happens while its target's own turn, stream or tool cycle is alive, or the - host is quiesced. A `tool_result` used to land regardless, apart from the - `tool_use` it answers, which went back into the queue. +- A deferred write that a flush defers again does so whatever its content, + under its id and in its place in the order writes arrived. That happens + while its target's own turn, stream or tool cycle is alive, or the host is + quiesced. (A write whose store throws goes back into the queue too; see + below.) A `tool_result` used to land regardless, apart from the `tool_use` + it answers, which went back into the queue. - A write that waits now waits on its own target's state: the target's turn, stream and tool cycle, or a quiesced host. (A fresh `tool_result` still doesn't wait.) Before, a write for the primary agent waited out every @@ -33,51 +34,55 @@ the primary's window. Each flush, at a turn's end, after surgery, after a withdrawn turn and after a puppet call, now delivers the agent's deferred messages while another agent's tool cycle is pending; only the agent's own - cycle holds them. The flush after surgery stores several agents' entries in + state holds them. The flush after surgery stores several agents' entries in the order they arrived, where it went agent by agent: residents share one message slot, so that reordered their shared transcript. - `stop()` now stores what is still deferred for each registered agent before the store closes, in the order it arrived; before, a memory-only - queue lost it. Once `stop()` begins, no agent turn starts and no stream - registers: a turn that was mid-compile when the streams were cancelled, or - one waiting to retry, is refused rather than left running, and an ephemeral - run refused this way ends at once rather than waiting for a watchdog. Since - no reply can then follow a stored entry in a stream, `stop()` stores each - one even while a frame it doesn't wait for still holds its target's turn: - a turn started outside the event loop, or a puppet call. `stop()` neither - refuses nor waits for a puppet call, as before. One still running stores - its pair when its tool returns. If that's before `stop()` stores the - deferred entries, the pair lands first and the puppet's own flush stores - what was deferred behind it; if it's later, the pair lands after them, and - on a store `stop()` has closed that write fails. A quiesced host keeps what - is deferred in its persisted queue for boot recovery. An entry for a target - no longer registered (a disposed fork) is dropped from a memory-only queue - at exit, as before. -- A deferred write whose store throws goes back into the queue under its - id, and so does every entry queued behind it for that agent, so none of - them lands ahead of an earlier one, and a queued `tool_use` that can't be - stored keeps the `tool_result` queued behind it from being stored alone. (A - stored `tool_use` whose `tool_result` then can't be stored stays alone + queue lost it. Once `stop()` begins, no agent turn starts: a fork's, + subagent's or retry's turn is refused before it starts, as a resident's + already was, and a new ephemeral run is refused. An ephemeral run whose + turn is refused because `stop()` has closed admission, at any of these + checks or at the ones before and after a compile, ends at once rather than + waiting for a watchdog. Since no reply can then follow a stored entry in a + stream, `stop()` stores each one even while a frame it doesn't wait for + still holds its target's turn: a turn started outside the event loop, or a + puppet call. `stop()` neither refuses nor waits for a puppet call, as + before. One still running stores its pair when its tool returns. If that's + before `stop()` stores the deferred entries, the pair lands first and the + puppet's own flush stores what was deferred behind it; if it's later, the + pair lands after them, and on a store `stop()` has closed that write + fails. A quiesced host keeps what is deferred in its persisted queue for + boot recovery. An entry for a target no longer registered (a disposed + fork) is dropped from a memory-only queue at exit, as before. +- A deferred write whose store throws goes back into the queue under its id, + and so does every entry queued behind it for that agent, so none of them + lands ahead of an earlier one, and a queued `tool_use` that can't be + stored keeps the `tool_result` queued behind it from being stored alone. + (A stored `tool_use` whose `tool_result` then can't be stored stays alone until the result lands at a later flush.) Other agents' entries go on, in - the order they arrived. The log line names the entry at each flush, and one - that keeps throwing holds what is deferred behind it for that agent until - it can be stored. The hold is the queue's: a write that isn't deferred is - stored at once and can land ahead of a held entry. That's a fresh - `tool_result`, and any write while the agent's turn, stream and tool cycle - are idle and the host isn't quiesced. So an entry that keeps throwing holds - only what is deferred while the agent's turn, stream or tool cycle is - alive, or the host is quiesced. Before, it stayed in the un-acked set and - the next acknowledgement dropped it from the persisted queue (at a turn - start or a flush it was lost even from memory). At a tool boundary, - nothing from it on is injected. While the framework runs it is kept; a - memory-only queue still loses it when the process exits. A persisted queue - (one a quiesce persisted, or boot recovery restored) keeps it across - restarts: each boot restores it and tries it again with the rest of the - queue, at once or at the resume after a quiesced boot, and logs it again. - While it's held the queue stays persisted, so every later deferral is - written to the recovery file. Only the host log names it. -- Boot recovery drops a recovered deferred write whose content isn't an - array of block objects, which the store could never take, and logs it by - id and participant, rather than holding it in the queue. Before, an entry - whose content wasn't an array was dropped without a line, and one with a - null block was kept, where it could never be stored. + the order they arrived. The log line names the entry at each flush, and + one that keeps throwing holds what is deferred behind it for that agent + until it can be stored. The hold is the queue's: a write that isn't + deferred is stored at once and can land ahead of a held entry. That's a + fresh `tool_result`, a message assembled into the agent's turn before its + compile (RFC-006 coalescing), and any write while the agent's turn, stream + and tool cycle are idle and the host isn't quiesced. So an entry that + keeps throwing holds only what is deferred while the agent's turn, stream + or tool cycle is alive, or the host is quiesced. Before, it stayed in the + un-acked set and the next acknowledgement dropped it from the persisted + queue (at a turn start or a flush it was lost even from memory). At a tool + boundary, nothing from it on is injected. While the framework runs it is + kept; a memory-only queue still loses it when the process exits. A + persisted queue (one a quiesce persisted, or boot recovery restored) keeps + it across restarts: each boot restores it and tries it again with the rest + of the queue, at once or at the resume after a quiesced boot, and logs it + again, unless it's one boot drops (below). While it's held the queue stays + persisted, so every later deferral is written to the recovery file, up to + the file's size cap. Only the host log names it. +- Boot recovery drops a recovered deferred write whose participant isn't + text or whose content isn't an array of block objects, which the store + could never take, and logs it by id and participant, rather than holding + it in the queue. Before, such an entry was dropped without a line, except + one whose blocks weren't all objects, such as a null block, which was kept, + where it could never be stored. diff --git a/src/framework.ts b/src/framework.ts index b6a254fdf..034bf6e97 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -484,12 +484,17 @@ function withDeferredWriteId(metadata: MessageMetadata | undefined, id: string): } /** A deferred entry as a log line names it: its id, participant and block - * types. Reading a block's type throws only on a null or undefined block, - * and no queued entry has one: addMessage reads every block's type before - * it defers a fresh write, and a recovered entry whose content isn't an - * array of objects is dropped at load (restorePersistedDeferredWrites). */ + * types. A block that isn't an object is named by its JavaScript type, and + * an object whose type isn't a string as untyped, so the line names whatever + * a queued entry holds. */ function describeDeferredWrite(msg: { id: string; participant: string; content: ContentBlock[] }): string { - return `${msg.id} (${msg.participant}: ${[...new Set(msg.content.map((b) => b.type))].join('+') || 'empty'})`; + const kind = (b: unknown): string => { + if (b === null) return 'null'; + if (typeof b !== 'object') return typeof b; + const type = (b as { type?: unknown }).type; + return typeof type === 'string' ? type : 'untyped'; + }; + return `${msg.id} (${msg.participant}: ${[...new Set(msg.content.map(kind))].join('+') || 'empty'})`; } /** Whether a deferred message can join a live stream at a tool boundary: @@ -2337,11 +2342,12 @@ export class AgentFramework { // fails. // // A quiesced host defers each entry again instead, into the persisted - // queue, for boot recovery. A write that throws stays queued with what - // follows it for its target, and a memory-only queue still loses it at - // exit. An entry for a target no longer registered (a disposed fork) is - // left: dropped with a memory-only queue at exit, and logged and dropped - // from a persisted one at boot. + // queue, for boot recovery. Past the deferral, a write that throws stays + // queued with what follows it for its target, and a memory-only queue + // still loses it at exit. An entry for a target no longer registered (a + // disposed fork) is left: dropped with a memory-only queue at exit, and + // logged and dropped from a persisted one by the boot or resume flush + // that meets it. if (this.deferredMessages.length > 0) { const entries = [...this.agents.keys()].flatMap((name) => this.drainDeferredFor(name)); this.readdDrained(entries, 'stop', { bypassDeferral: !this.quiesced }); diff --git a/test/host-quiesce.test.ts b/test/host-quiesce.test.ts index fe3b81676..812ea60bb 100644 --- a/test/host-quiesce.test.ts +++ b/test/host-quiesce.test.ts @@ -752,7 +752,10 @@ test("resume stores several agents' writes in the order they arrived, and cuts e add('one', 'agent'); add('two', 'scout'); add('three', 'agent'); - add('stuck', 'agent'); + // Nothing checks a fresh write's blocks beyond reading each one's type, + // so a block that isn't an object can be queued; the log line names it + // by its JavaScript type. + internals.addMessage('user', [{ type: 'text', text: 'stuck' }, 'a bare string'], undefined, { forAgent: 'agent' }); add('for scout', 'scout'); add('behind the stuck one', 'agent'); add('also for scout', 'scout'); @@ -767,7 +770,7 @@ test("resume stores several agents' writes in the order they arrived, and cuts e return realAdd(participant, content, metadata, causedBy); }) as typeof agentCm.addMessage; const named = () => lines.filter((line) => - line.includes(`deferred write ${stuckId} (user: text) for agent could not be stored`) + line.includes(`deferred write ${stuckId} (user: text+string) for agent could not be stored`) && line.includes('a thrown value that has no text')).length; let status = await framework.resume(); From 57943f6e5d16cbf21fa4da20343156d5fb67bc22 Mon Sep 17 00:00:00 2001 From: Duncan-2410 Date: Wed, 14 Oct 2026 05:12:46 -0400 Subject: [PATCH 0196/1112] fix(deferral): #251's retry cancellations end a waiting ephemeral run; one stop refusal at startAgentStream's top #251 (89a51fab), merged from main in the previous commit, made stop() end a retry's wait and cancel the retry there, at both retry sites, and added a refusal at the top of startAgentStream for every agent once admission has closed. The merge put that refusal right after this branch's own, which did the same and also ended a waiting ephemeral run, and its two cancellation points left such a run to its idle watchdog. - startAgentStream keeps one refusal, main's, which now also ends a waiting ephemeral run (endEphemeralRunRefused). This branch's copy and its comment are gone. - beginAgentTurn's catch and driveStream's error case end a waiting ephemeral run when stop() has cancelled their retry. Tests: - ephemeral-settle: the retry test waits 60 s, so only stop() ends the wait, and now pins the ending at beginAgentTurn's cancellation (the retry never re-enters). New: a stream that fails and waits to retry ends its run at once when stop() begins (driveStream's cancellation); a turn asked for once stop() has begun, for a run whose request never reached the loop, doesn't start and ends the run at once (the refusal at the top; a store the framework doesn't own, so the run's cleanup can still read it). Removing each of the three endings fails its test. - deferred-window-only: the primary's retry test waits 60 s and now expects no re-entry: stop() ends the wait and cancels the retry. Co-Authored-By: Claude Opus 5.5 --- src/framework.ts | 20 ++++--- test/deferred-window-only.test.ts | 13 ++--- test/ephemeral-settle.test.ts | 88 ++++++++++++++++++++++++++++--- 3 files changed, 98 insertions(+), 23 deletions(-) diff --git a/src/framework.ts b/src/framework.ts index 802196187..e266111cf 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -10903,21 +10903,15 @@ export class AgentFramework { attempt = 0, providerGateAlreadyHeld = false, ): Promise { - // Once stop() has begun, no turn starts, for any agent: a fork's or a - // subagent's retry after its delay, or a wake the loop dequeued before it - // ended, would otherwise begin a turn stop() never cancels, and the - // deferred drain at stop's end would meet it live. - if (this.providerAdmissionClosed) { - if (providerGateAlreadyHeld) this.releasePrimaryProviderGate(agent.name); - this.endEphemeralRunRefused(agent); - return; - } const ownsProviderGate = !this.ephemeralRuns.has(agent.name) && !this.conversationAgentHomes.has(agent.name); // A stopping framework starts no turn, whoever owns provider admission - // (conversation agents and ephemeral runs do not take the gate). + // (conversation agents and ephemeral runs do not take the gate). An + // ephemeral run waiting on the turn ends now, as at every refusal of + // stop()'s. if (this.providerAdmissionClosed) { if (ownsProviderGate && providerGateAlreadyHeld) this.releasePrimaryProviderGate(agent.name); + this.endEphemeralRunRefused(agent); return; } if (ownsProviderGate && !providerGateAlreadyHeld) { @@ -11446,11 +11440,13 @@ export class AgentFramework { if (this.providerAdmissionClosed) { // stop() ended the wait: that cancels the retry, it does not hasten // it. Nothing is written (the store is closing); this frame still - // owns its token, since no retry frame replaced it. + // owns its token, since no retry frame replaced it. An ephemeral + // run waiting on the turn ends now, as at every refusal of stop()'s. if (this.activeTurnTokens.get(agent.name) === turnToken) { this.activeTurnTokens.delete(agent.name); this.activeTurnTriggers.delete(agent.name); } + this.endEphemeralRunRefused(agent); return false; } // The retry re-enters startAgentStream, which replaces this frame's @@ -12672,7 +12668,9 @@ export class AgentFramework { await this.waitForRetry(action.delayMs); // stop() ending the wait cancels the retry (this frame's finally // clears the token); it is never permission to retry at once. + // An ephemeral run waiting on the turn ends now. if (!this.providerAdmissionClosed) await this.startAgentStream(agent, trigger, attempt + 1); + else this.endEphemeralRunRefused(agent); } else { this.settleAgent(agent.name, { stopReason: 'exhausted', diff --git a/test/deferred-window-only.test.ts b/test/deferred-window-only.test.ts index 212e47d4f..e0500a338 100644 --- a/test/deferred-window-only.test.ts +++ b/test/deferred-window-only.test.ts @@ -526,7 +526,8 @@ describe('mid-turn deliveries keep the window what the stream carried', () => { maxRetries: 2, onInferenceError: (_error: Error, _agent: string, attempt: number) => { waiting(); - return attempt < 2 ? { retry: true as const, delayMs: 50 } : { retry: false as const }; + // Long enough that only stop() ends the wait. + return attempt < 2 ? { retry: true as const, delayMs: 60_000 } : { retry: false as const }; }, }, }); @@ -537,9 +538,9 @@ describe('mid-turn deliveries keep the window what the stream carried', () => { compiles++; throw new Error('boom'); }; - // The retry re-enters startAgentStream after its delay; count those - // re-entries, so a retry that was never granted can't pass for one that - // was refused. + // A retry re-enters startAgentStream after its wait. stop() ends the + // wait, and that cancels the retry before it re-enters: count re-entries, + // so a retry that went ahead can't pass for one that was cancelled. const retries: Array<{ attempt: number; stopping: boolean }> = []; const frames = framework as unknown as { providerAdmissionClosed: boolean; @@ -557,8 +558,8 @@ describe('mid-turn deliveries keep the window what the stream carried', () => { const error = console.error; console.error = () => {}; try { await framework.stop(); } finally { console.error = error; } - assert.deepEqual(retries, [{ attempt: 1, stopping: true }], 'the retry was granted, and re-entered once stop() had begun'); - assert.equal(compiles, 1, 'the retry was refused, not compiled'); + assert.deepEqual(retries, [], 'stop() ended the wait, and the retry was cancelled before it re-entered'); + assert.equal(compiles, 1, 'the retry was not compiled'); const reopened = await createFramework(); assert.ok(windowOf(reopened).includes('Antra: while it waits'), 'what was deferred for it was stored'); diff --git a/test/ephemeral-settle.test.ts b/test/ephemeral-settle.test.ts index 7d75812a5..db251e5d0 100644 --- a/test/ephemeral-settle.test.ts +++ b/test/ephemeral-settle.test.ts @@ -17,6 +17,7 @@ import type { ToolResult, } from '../src/index.js'; import { AgentFramework } from '../src/index.js'; +import { JsStore } from '@animalabs/chronicle'; import { createMockResponse, MockMembrane, MockYieldingStream } from './helpers/mock-membrane.js'; class EphemeralToolModule implements Module { @@ -211,11 +212,9 @@ describe('runEphemeralToCompletion settle signal', () => { } }); - it('a run waiting to retry when stop() begins gives up before its turn starts again', { timeout: 60_000 }, async () => { - // An ephemeral run doesn't hold the primary's provider gate, so the - // refusal at the top of startAgentStream is the first check its retry - // meets. The refusal before the compile would stop it too, but only once - // the turn had started again: its context gathered, its hooks run. + it('a run waiting to retry when stop() begins gives up before its turn starts again, and ends at once', { timeout: 60_000 }, async () => { + // stop() ends the retry's wait, which cancels the retry before its turn + // starts again; the run ends then, not when its idle watchdog fires. const { tempDir, storePath } = tempStorePath('ephemeral-retry-stopping-'); let waiting!: () => void; const inDelay = new Promise((resolve) => { waiting = resolve; }); @@ -235,7 +234,8 @@ describe('runEphemeralToCompletion settle signal', () => { maxRetries: 1, onInferenceError: (_error: Error, _agent: string, attempt: number): ErrorAction => { waiting(); - return attempt < 1 ? { retry: true, delayMs: 50 } : { retry: false }; + // Long enough that only stop() ends the wait. + return attempt < 1 ? { retry: true, delayMs: 60_000 } : { retry: false }; }, }, }); @@ -258,6 +258,82 @@ describe('runEphemeralToCompletion settle signal', () => { } }); + it('a run whose stream failed and waits to retry ends at once when stop() begins', { timeout: 60_000 }, async () => { + // The stream's own retry: stop() ends its wait and cancels it there. + const { tempDir, storePath } = tempStorePath('ephemeral-stream-retry-stopping-'); + let waiting!: () => void; + const inDelay = new Promise((resolve) => { waiting = resolve; }); + const membrane = new MockMembrane(); + membrane.streamYielding = (request) => { + membrane.calls.push(request); + // A stream that fails once it has started. + const stream = new MockYieldingStream([]); + (stream as unknown as { events: unknown[] }).events.push({ type: 'error', error: new Error('overloaded') }); + return stream; + }; + const framework = await AgentFramework.create({ + storePath, + membrane: membrane.asMembrane(), + agents: [], + modules: [new EphemeralToolModule()], + syncIntervalMs: 0, + errorPolicy: { + maxRetries: 1, + onInferenceError: (_error: Error, _agent: string, attempt: number): ErrorAction => { + waiting(); + // Long enough that only stop() ends the wait. + return attempt < 1 ? { retry: true, delayMs: 60_000 } : { retry: false }; + }, + }, + }); + try { + const { agent, contextManager } = await createEphemeral(framework); + const run = runStartedEphemeral(framework, agent, contextManager, STOPPED_RUN_WATCHDOGS); + await inDelay; + const stopped = framework.stop(); + await assert.rejects(run, /refused: the framework is stopping/); + await stopped; + assert.equal(membrane.calls.length, 1, 'the retry was not compiled'); + } finally { + rmSync(tempDir, { recursive: true, force: true }); + } + }); + + it('a turn asked for once stop() has begun does not start, and its run ends at once', { timeout: 60_000 }, async () => { + // As a wake the loop dequeued before stop() ended it would be: refused + // before the turn starts. The run, whose request never reached the loop, + // ends then, not when its startup watchdog fires. + const tempDir = mkdtempSync(join(tmpdir(), 'ephemeral-late-turn-')); + // A store the framework doesn't own stays open after stop(), as the run's + // own cleanup needs it to. + const store = JsStore.openOrCreate({ path: join(tempDir, 'store') }); + let gathered = 0; + const contextModule: Module = new EphemeralToolModule(); + contextModule.gatherContext = async () => { + gathered++; + return []; + }; + const framework = await AgentFramework.create({ + store, + membrane: new MockMembrane().asMembrane(), + agents: [], + modules: [contextModule], + syncIntervalMs: 0, + }); + try { + const { agent, contextManager } = await createEphemeral(framework); + // Asked for, but the event loop never started, so its turn waits. + const run = framework.runEphemeralToCompletion(agent, contextManager, STOPPED_RUN_WATCHDOGS); + await framework.stop(); + await (framework as unknown as { startAgentStream(a: typeof agent): Promise }).startAgentStream(agent); + await assert.rejects(run, /refused: the framework is stopping/); + assert.equal(gathered, 0, 'the turn did not start'); + } finally { + store.close(); + rmSync(tempDir, { recursive: true, force: true }); + } + }); + it('resolves plain text completion and deregisters the agent', async () => { const { tempDir, storePath } = tempStorePath('ephemeral-plain-'); const membrane = new MockMembrane(); From e9689d428dc0fac5717a94643e490eb2df8249df Mon Sep 17 00:00:00 2001 From: Duncan-2410 Date: Wed, 14 Oct 2026 05:13:42 -0400 Subject: [PATCH 0197/1112] docs(deferral): the fragment no longer claims the stop() turn refusal main has since #251 #251 refuses every turn at the top of startAgentStream once stop() has closed admission and cancels a retry whose wait stop() ends, so this change no longer adds the turn refusal its fragment described. The stop() bullet now says what this change adds there: a new ephemeral run is refused, and an ephemeral run whose turn is refused because stop() closed admission, or whose retry stop() cancels, ends at once. Co-Authored-By: Claude Opus 5.5 --- changelog.d/mid-turn-window-only-held.fixed.md | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/changelog.d/mid-turn-window-only-held.fixed.md b/changelog.d/mid-turn-window-only-held.fixed.md index b622b16fa..d059c6341 100644 --- a/changelog.d/mid-turn-window-only-held.fixed.md +++ b/changelog.d/mid-turn-window-only-held.fixed.md @@ -39,14 +39,13 @@ message slot, so that reordered their shared transcript. - `stop()` now stores what is still deferred for each registered agent before the store closes, in the order it arrived; before, a memory-only - queue lost it. Once `stop()` begins, no agent turn starts: a fork's, - subagent's or retry's turn is refused before it starts, as a resident's - already was, and a new ephemeral run is refused. An ephemeral run whose - turn is refused because `stop()` has closed admission, at any of these - checks or at the ones before and after a compile, ends at once rather than - waiting for a watchdog. Since no reply can then follow a stored entry in a - stream, `stop()` stores each one even while a frame it doesn't wait for - still holds its target's turn: a turn started outside the event loop, or a + queue lost it. Once `stop()` begins, a new ephemeral run is refused, and + an ephemeral run whose turn is refused because `stop()` has closed + admission, or whose retry `stop()` cancels, ends at once rather than + waiting for a watchdog. Since no turn starts or registers a stream once + `stop()` begins, no reply can then follow a stored entry in a stream, so + `stop()` stores each one even while a frame it doesn't wait for still + holds its target's turn: a turn started outside the event loop, or a puppet call. `stop()` neither refuses nor waits for a puppet call, as before. One still running stores its pair when its tool returns. If that's before `stop()` stores the deferred entries, the pair lands first and the From 739bdbac8b290b0a1ae5c0ff985760213737cf6e Mon Sep 17 00:00:00 2001 From: Duncan-2410 Date: Wed, 14 Oct 2026 05:17:51 -0400 Subject: [PATCH 0198/1112] docs(deferral): stop() waits for a puppet call since #325; say so where this change said it didn't #325 makes stop() refuse a puppet call once it has begun and wait for one in flight, with the other operator operations, before teardown and so before this change's drain. The stop() comment and the fragment said stop() neither refuses nor waits for one, and what happened to its pair: they now say a puppet call in flight has stored its pair and run its flush by the drain, and that only one still running after that wait timed out holds its target's turn past it, its pair landing after the drained entries on the store a timed-out wait leaves open. A test's comment named a puppet call as a frame stop() doesn't wait for; it names a turn started outside the event loop. Co-Authored-By: Claude Opus 5.5 --- changelog.d/mid-turn-window-only-held.fixed.md | 15 ++++++--------- src/framework.ts | 15 +++++++-------- test/deferred-window-only.test.ts | 4 ++-- 3 files changed, 15 insertions(+), 19 deletions(-) diff --git a/changelog.d/mid-turn-window-only-held.fixed.md b/changelog.d/mid-turn-window-only-held.fixed.md index d059c6341..b82168a59 100644 --- a/changelog.d/mid-turn-window-only-held.fixed.md +++ b/changelog.d/mid-turn-window-only-held.fixed.md @@ -45,15 +45,12 @@ waiting for a watchdog. Since no turn starts or registers a stream once `stop()` begins, no reply can then follow a stored entry in a stream, so `stop()` stores each one even while a frame it doesn't wait for still - holds its target's turn: a turn started outside the event loop, or a - puppet call. `stop()` neither refuses nor waits for a puppet call, as - before. One still running stores its pair when its tool returns. If that's - before `stop()` stores the deferred entries, the pair lands first and the - puppet's own flush stores what was deferred behind it; if it's later, the - pair lands after them, and on a store `stop()` has closed that write - fails. A quiesced host keeps what is deferred in its persisted queue for - boot recovery. An entry for a target no longer registered (a disposed - fork) is dropped from a memory-only queue at exit, as before. + holds its target's turn, such as a turn started outside the event loop, or + a puppet call still running after `stop()`'s wait for operator operations + timed out; that call's pair then lands after them. A quiesced host keeps + what is deferred in its persisted queue for boot recovery. An entry for a + target no longer registered (a disposed fork) is dropped from a + memory-only queue at exit, as before. - A deferred write whose store throws goes back into the queue under its id, and so does every entry queued behind it for that agent, so none of them lands ahead of an earlier one, and a queued `tool_use` that can't be diff --git a/src/framework.ts b/src/framework.ts index 9a39160c3..157704cd4 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -2834,15 +2834,14 @@ export class AgentFramework { // queue order (residents share one message slot), each target cut at its // own first entry that can't be stored (writeDrainedInOrder). They go // past the deferral: a frame stop() doesn't wait for (a turn started - // outside the event loop, or a puppet call) can still hold its target's - // turn token, but no reply can follow an entry in any stream. + // outside the event loop) can still hold its target's turn token, but no + // reply can follow an entry in any stream. // - // stop() neither refuses nor waits for a puppet call, as before this - // drain. One still running stores its pair when its tool returns. If - // that's before this drain runs, the pair lands first, and the puppet's - // own flush stores what was deferred behind it. If it's later, the pair - // lands after these entries, and on a store closed below that write - // fails. + // A puppet call in flight is one of the operator operations stop() waited + // for above, so its pair and the flush after it have landed by now. If + // that wait timed out, the call may still be running and holding its + // target's turn: its pair lands after these entries, on the store a + // timed-out wait leaves open. // // A quiesced host defers each entry again instead, into the persisted // queue, for boot recovery. Past the deferral, a write that throws stays diff --git a/test/deferred-window-only.test.ts b/test/deferred-window-only.test.ts index e0500a338..0ceba7846 100644 --- a/test/deferred-window-only.test.ts +++ b/test/deferred-window-only.test.ts @@ -472,8 +472,8 @@ describe('mid-turn deliveries keep the window what the stream carried', () => { it("stop() stores what is deferred for an agent whose turn frame it doesn't wait for", async () => { const framework = await createFramework(); const internals = internalsOf(framework); - // A frame stop() doesn't await, such as a puppet call still running, - // holds the agent's turn token. + // A frame stop() doesn't await, such as a turn started outside the + // event loop, holds the agent's turn token. internals.activeTurnTokens.set('assistant', 1); assert.equal(internals.addMessage('Antra', text('still to land')), '', 'deferred: the turn is alive'); await framework.stop(); From 7bffdca9cbaf339ee3502f0e43c311f60e14310a Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Wed, 14 Oct 2026 05:32:13 -0400 Subject: [PATCH 0199/1112] fix(mcp): redact a username alone, JSON's escapes, and overlapping credentials Three gaps in the redaction of recorded server text: - A ws:// url whose userinfo is a username alone (a token written as the username) got no unit, since the unit was added only with a password. A username under 8 characters was then replaced nowhere. The unit is now username:password, or the username alone when there's no password, as written and decoded, anchored as before (no letter or digit just before it, its @ just after). The outer guard that the per-pair check made redundant is gone. - A token's or a credential env value's JSON form covered only an escaped slash, which JSON.stringify never writes. Its escapes for a quote, a backslash or a control character are now covered too, with and without escaped slashes. - One leftmost-first pass let a match consume the start of a string that overlapped it, leaving that string's tail in the text. The redactor now finds every match, a match beginning inside another included, and replaces each run of overlapping matches as one "redacted". Text without overlaps is redacted exactly as before, and the replacement is never scanned. Tests: ws:// beside wss:// for the userinfo unit; a username alone under both schemes, with its Basic credential; a percent-encoded userinfo whose parts are all too short for the standalone floor, as written and decoded, with the Basic credential of each; a credential with a quote and a backslash as JSON.stringify writes it; and two configurations whose tokens overlap, both replaced whole, with adjacent ones still replaced apart. The changelog fragment says each. --- .../redact-recorded-mcp-credentials.fixed.md | 2 +- src/mcpl/protocol-family.ts | 102 ++++++++++++------ test/mcp-protocol-family.test.ts | 42 ++++++++ 3 files changed, 111 insertions(+), 35 deletions(-) diff --git a/changelog.d/redact-recorded-mcp-credentials.fixed.md b/changelog.d/redact-recorded-mcp-credentials.fixed.md index 62df4f2e8..fa3bcfe38 100644 --- a/changelog.d/redact-recorded-mcp-credentials.fixed.md +++ b/changelog.d/redact-recorded-mcp-credentials.fixed.md @@ -1 +1 @@ -- Server and transport text the framework records has the server's configured credentials replaced, in both engines: connection traces, console lines, stderr traces, the ops alert (and so `logs/failures.log` and the webhook), a tool call's failure as its caller and the model get it, and the `tool:failed` trace's message and stack. The credentials are those the configuration holds: from the url, credential-like path segments and query names, credential-shaped or credential-named query values (each as written and decoded), every non-empty parameter as its whole `name=value`, and a `ws://` url's username and password, its userinfo as one unit before its `@` (with no length floor, so a short password goes with its username), and the Basic credential a WebSocket client sends for it; the static `token`, raw, with its slashes escaped as some JSON encoders write them, and as a url carries it, percent- or form-encoded with either case of hex; and a stdio server's `env` values whose names have a part ending in a credential word, singular or plural, with any digits after it (`GITHUB_TOKEN`, `API_KEYS`, `PGPASSWORD`, `DB_PASS`, `AUTHORIZATION`, `SESSION_COOKIE`, `TOKEN2`), raw, with their slashes escaped as some JSON encoders write them, and percent-encoded with either case of hex. A stdio server's command-line arguments are not derived. A string replaced on its own has at least eight characters, since a shorter one can't be told from ordinary text, and is replaced wherever it occurs. Every query parameter is also replaced as its whole `name=value`, only where no letter or digit adjoins the pair (so `v=2` leaves `dev=2` alone); a short value goes only that way. A connection's event records use a redactor built once for it, and an operation's records one built when the operation begins, so a server removed, or replaced by a restart, while the operation waits doesn't change what they replace. Text that may come from any server takes one pass over every server's strings, so a secret that begins another server's longer one doesn't cut that one short. Over HTTP, which only the modern engine speaks, a 401 or 403's response body is left out of the request error's data; where the HTTP client builds the error's message from the body, as it does for most statuses, the message is redacted like any record. A token an access provider fetches, and credentials the configuration doesn't hold, are outside this. JSON-RPC answers, and the error a host's own connect raises, pass as they are. +- Server and transport text the framework records has the server's configured credentials replaced, in both engines: connection traces, console lines, stderr traces, the ops alert (and so `logs/failures.log` and the webhook), an MCP tool call that fails without an answer, as its caller and the model get it and in the `tool:failed` trace's message and stack. The credentials are those the configuration holds: from the url, credential-like path segments and query names, credential-shaped or credential-named query values (each as written and decoded), every non-empty parameter as its whole `name=value`, and a `ws://` url's username and password, its userinfo as one unit before its `@` (the username alone when there's no password), with no length floor, so a short password goes with its username and a short username alone goes too, and the Basic credential a WebSocket client sends for it; the static `token`, raw, as JSON encoders write it (with `JSON.stringify`'s escapes, and with its slashes escaped too, as some encoders write them), and as a url carries it, percent- or form-encoded with either case of hex; and a stdio server's `env` values whose names have a part ending in a credential word, singular or plural, with any digits after it (`GITHUB_TOKEN`, `API_KEYS`, `PGPASSWORD`, `DB_PASS`, `AUTHORIZATION`, `SESSION_COOKIE`, `TOKEN2`), raw, as JSON encoders write them, and percent-encoded with either case of hex. A stdio server's command-line arguments are not derived. A string replaced on its own has at least eight characters, since a shorter one can't be told from ordinary text, and is replaced wherever it occurs. Every non-empty query parameter is also replaced as its whole `name=value`, only where no letter or digit adjoins the pair (so `v=2` leaves `dev=2` alone); a short value goes only that way. A connection's event records use a redactor built once for it. An operation's records replace the credentials of the configurations present when it began and of those present when each record is made, so a server removed, or replaced by a restart, while the operation waits is still covered, and so is one connected meanwhile. Every place a credential matches is replaced, places that overlap included, and text that may come from any server is matched against every server's strings at once, so two servers' secrets that overlap are both replaced whole. Over HTTP, which only the modern engine speaks, a 401 or 403's response body is left out of the request error's data; where the HTTP client builds the error's message from the body, as it does for most statuses, the message is redacted like any record. A token an access provider fetches, and credentials the configuration doesn't hold, are outside this. JSON-RPC answers, and the error a host's own connect raises, pass as they are. diff --git a/src/mcpl/protocol-family.ts b/src/mcpl/protocol-family.ts index fe230315f..5ba548d58 100644 --- a/src/mcpl/protocol-family.ts +++ b/src/mcpl/protocol-family.ts @@ -113,22 +113,26 @@ export function redactUrl(url: string | undefined): string { * - from the url, as written and decoded: credential-like path segments and * query names, credential-shaped or credential-named query values, every * other non-empty parameter as its whole name=value, and a ws:// url's - * username and password, its userinfo as one unit before its @ (with no - * length floor, so a short password goes with its username), and the - * Basic credential a WebSocket client sends for that userinfo; - * - the static token, raw, with its slashes escaped as some JSON encoders - * write them, and percent- or form-encoded with either case of hex; + * username and password, its userinfo as one unit before its @ (the + * username alone when there's no password), with no length floor, so a + * short password goes with its username and a short username alone goes + * too, and the Basic credential a WebSocket client sends for that userinfo; + * - the static token, raw, as JSON encoders write it (JSON.stringify's + * escapes, and with its slashes escaped too, as some encoders write them), + * and percent- or form-encoded with either case of hex; * - a stdio server's environment values whose names say they're - * credentials, raw, with their slashes JSON-escaped, and percent-encoded - * with either case of hex. + * credentials, raw, as JSON encoders write them, and percent-encoded with + * either case of hex. * A string replaced on its own has at least 8 characters, since a shorter * one can't be told from ordinary text, and it is replaced wherever it * occurs, after an escape or inside a longer token too. A query parameter is * also replaced as its whole name=value, only where no letter or digit * adjoins the pair, so v=2 leaves dev=2 alone; a short value goes only that - * way. One longest-first pass never scans its own replacement, and text - * that may come from several servers takes one pass over all their - * configurations' strings (serverCredentialPatterns). A credential the + * way. Every place one of these matches is replaced, overlapping places + * included, and the replacement is never scanned again; text that may come + * from several servers takes one pass over all their configurations' + * strings (serverCredentialPatterns), so strings of different servers that + * overlap are both replaced whole. A credential the * configuration doesn't hold, such as the token an access provider fetches * or another origin's, is outside it, and so are other encodings of the * ones it holds. JSON-RPC answers, and the error a host's own connect @@ -155,9 +159,9 @@ export interface CredentialPattern { /** * The strings {@link redactServerText} replaces for one configuration, as * patterns for {@link credentialRedactor}. Text that may come from several - * servers goes through one redactor over all their patterns, so its one - * longest-first pass takes the longest match among them all: a secret that - * begins another configuration's longer one can't cut that one short. + * servers goes through one redactor over all their patterns, so a secret + * that begins inside another configuration's, or shares its start, can't + * leave any of that one in place. */ export function serverCredentialPatterns(config: Pick): CredentialPattern[] { // Strings replaced wherever they occur (after an escape or inside a longer @@ -174,13 +178,17 @@ export function serverCredentialPatterns(config: Pick value.replace(/%[0-9A-F]{2}/g, (escape) => escape.toLowerCase()); - // A string as written, with its slashes escaped as some JSON encoders - // write them, and percent-encoded in either case of hex (a string - // encodeURIComponent refuses, such as one with an unpaired surrogate, has - // no encoded form). + // A string as written; as JSON encoders write it, with JSON.stringify's + // escapes (a quote, a backslash, a control character) and with its + // slashes escaped too, as some encoders write them; and percent-encoded in + // either case of hex (a string encodeURIComponent refuses, such as one + // with an unpaired surrogate, has no encoded form). const withEncoded = (value: string): void => { - standalone(value); - standalone(value.replace(/\//g, '\\/')); + const json = JSON.stringify(value).slice(1, -1); + for (const written of [value, json]) { + standalone(written); + standalone(written.replace(/\//g, '\\/')); + } let encoded: string; try { encoded = encodeURIComponent(value); } catch { return; /* no percent-encoded form */ } standalone(encoded); @@ -235,17 +243,18 @@ export function serverCredentialPatterns(config: Pick string { if (patterns.length === 0) return asText; + // Longest first, so at each place the longest string that matches there is + // the match. const sources = [...patterns].sort((a, b) => b.value.length - a.value.length).map(({ source }) => source); const pattern = new RegExp([...new Set(sources)].join('|'), 'g'); - return (text) => asText(text).replace(pattern, 'redacted'); + return (text) => { + const input = asText(text); + const spans: Array<[start: number, end: number]> = []; + pattern.lastIndex = 0; + for (let match = pattern.exec(input); match; match = pattern.exec(input)) { + const start = match.index; + const end = start + match[0].length; + const last = spans[spans.length - 1]; + if (last && start < last[1]) last[1] = Math.max(last[1], end); + else spans.push([start, end]); + // The next search starts one place on, not after this match, so a + // match that begins inside this one is found too. + pattern.lastIndex = start + 1; + } + let redacted = ''; + let from = 0; + for (const [start, end] of spans) { + redacted += `${input.slice(from, start)}redacted`; + from = end; + } + return redacted + input.slice(from); + }; } /** Any value as text: a value that can't be printed reads as a placeholder. */ diff --git a/test/mcp-protocol-family.test.ts b/test/mcp-protocol-family.test.ts index 5c58a79e8..afab1b238 100644 --- a/test/mcp-protocol-family.test.ts +++ b/test/mcp-protocol-family.test.ts @@ -399,6 +399,37 @@ test("recorded server text: a ws:// url's userinfo as one unit before its @, and assert.equal(redactServerText('bot:pw is not followed by its @ here', config), 'bot:pw is not followed by its @ here'); assert.equal(redactServerText('robot:pw@h', config), 'robot:pw@h'); assert.equal(redactServerText('dial wss://alice:pw@h/mcp failed', cfg({ url: 'wss://alice:pw@h/mcp' })), 'dial wss://redacted@h/mcp failed'); + // ws:// as wss://. + const plain = cfg({ url: 'ws://bot:pw@h/mcp' }); + assert.equal(redactServerText('dial ws://bot:pw@h/mcp failed', plain), 'dial ws://redacted@h/mcp failed'); + assert.equal(redactServerText('Authorization: Basic Ym90OnB3', plain), 'Authorization: Basic redacted'); + // A username alone, as a token written as the username is: the unit is the + // username, and the Basic credential is that of "name:". + for (const url of ['ws://k3y@h/mcp', 'wss://k3y@h/mcp']) { + const tokenOnly = cfg({ url }); + assert.equal(redactServerText(`dial ${url} failed`, tokenOnly), `dial ${url.replace('k3y', 'redacted')} failed`, url); + assert.equal(redactServerText(`Authorization: Basic ${Buffer.from('k3y:').toString('base64')}`, tokenOnly), 'Authorization: Basic redacted', url); + assert.equal(redactServerText('monk3y@h', tokenOnly), 'monk3y@h', url); + } + // Percent-encoded, with every part too short to replace on its own: the + // unit as written and as decoded, and the Basic credential sent for each. + const encoded = cfg({ url: 'wss://b%40t:p%40ss@h/mcp' }); + assert.equal(redactServerText('dial wss://b%40t:p%40ss@h/mcp failed', encoded), 'dial wss://redacted@h/mcp failed'); + assert.equal(redactServerText('as user b@t:p@ss@h', encoded), 'as user redacted@h'); + for (const userinfo of ['b%40t:p%40ss', 'b@t:p@ss']) { + assert.equal(redactServerText(`Authorization: Basic ${Buffer.from(userinfo).toString('base64')}`, encoded), 'Authorization: Basic redacted', userinfo); + } +}); + +test('recorded server text: a credential as JSON.stringify writes it, its quotes and backslashes escaped', () => { + const password = 'pa"ss\\w0rd/x1'; + const config = cfg({ command: 'srv', token: 'to"ken\\0123', env: { DB_PASSWORD: password } } as Partial); + for (const secret of [password, config.token!]) { + const json = JSON.stringify(secret).slice(1, -1); + for (const form of [secret, json, json.replace(/\//g, '\\/')]) { + assert.equal(redactServerText(`config {"secret":"${form}"} refused`, config), 'config {"secret":"redacted"} refused', form); + } + } }); test("one redactor over several configurations' strings takes the longest secret among them all", () => { @@ -410,6 +441,17 @@ test("one redactor over several configurations' strings takes the longest secret assert.equal(credentialRedactor([])('nothing to replace'), 'nothing to replace'); }); +test("one redactor over several configurations' strings replaces strings that overlap, both whole", () => { + // The first's end is the second's start. + const head = cfg({ id: 'a', token: 'XXXXXXXXYYYY' }); + const tail = cfg({ id: 'b', token: 'YYYYZZZZZZZZ' }); + const redact = credentialRedactor([...serverCredentialPatterns(head), ...serverCredentialPatterns(tail)]); + assert.equal(redact('auth failed for XXXXXXXXYYYYZZZZZZZZ.'), 'auth failed for redacted.'); + assert.equal(redact('XXXXXXXXYYYY and YYYYZZZZZZZZ'), 'redacted and redacted'); + // Strings next to each other, not overlapping, are each replaced, as before. + assert.equal(redact('XXXXXXXXYYYYYYYYZZZZZZZZ'), 'redactedredacted'); +}); + test('recorded server text uses longest-first, one-pass replacements and is idempotent', () => { const config = cfg({ url: 'https://h/mcp?secret=token0123456789-tail&redacted=eu', token: 'token0123456789' }); const original = 'secret=token0123456789-tail token0123456789 redacted=eu queue full'; From 96c85268a9ea1fb029962e7d52890563dd150030 Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Wed, 14 Oct 2026 05:32:13 -0400 Subject: [PATCH 0200/1112] fix(mcp): a disconnect or stop() during a restart ends it, and says the server is down A disconnect of an id a restart holds, its own or the one it moves to, already won over the restart, as a disconnect wins over a connect in flight: the restart's connect was refused or displaced after its teardown. What the restart then said was wrong, "connect it once that has finished", and nothing documented or tested it. Now: - Any failure after the restart's teardown says so first: 'MCP server "a" is down: it was disconnected for its restart, and the restart's connect failed: ' and the connect's own reason. That covers a disconnect or a stop() that began during the restart, and the new server's own failure. - The restart's own connect, meeting a disconnect of the id in flight, names it: that disconnect began during the restart (the restart's own has finished by then), and it ends the restart. - A restart's pre-check on the id it moves to words its refusal for the restart ('MCP server "a" was not restarted: ...; restart "a" once that has finished'), not for a connect. restartMcplServer's doc and the changelog now say what the restart holds and what it doesn't: a module added meanwhile can take only the tool prefix, never an id; the id and prefix it moves the server from are free once its disconnect is done; a disconnect or a stop() that begins during it ends it; and a restart whose connect fails leaves the server down, configured under the new configuration and holding its tool prefix, as any failed connect does (listed as not connected, restartable, removed by a disconnect). addModule's comment no longer names mcplPrefixCollision as create()'s check. Tests: a restart moving to an id, then to a tool prefix, another restart holds is refused before its teardown; a restart whose connect fails leaves the server listed as not connected, its prefix refused to a module, and its holds released (the id and prefix then take a working configuration); and a disconnect that begins in the restart's teardown, its gap or its connect ends it, leaving the server disconnected and the id free. --- .../mcp-standard-tool-results.changed.md | 14 +++- src/framework.ts | 53 ++++++++---- test/mcpl-runtime-lifecycle.test.ts | 80 +++++++++++++++++++ 3 files changed, 130 insertions(+), 17 deletions(-) diff --git a/changelog.d/mcp-standard-tool-results.changed.md b/changelog.d/mcp-standard-tool-results.changed.md index f17265a89..e8915081c 100644 --- a/changelog.d/mcp-standard-tool-results.changed.md +++ b/changelog.d/mcp-standard-tool-results.changed.md @@ -112,9 +112,17 @@ disconnect of the server in flight. Before, a restart disconnected first, so a refused connect left the server gone; now the server keeps running. From those checks through its connect, the restart holds the id and the - tool prefix it moves the server to, so a connect, a restart or a module - added meanwhile is refused them, naming the restart, rather than taking - them while the old server is torn down. `addModule` also refuses a name a + tool prefix it moves the server to: a connect or a restart started + meanwhile is refused them, and a module added meanwhile is refused that + prefix, each naming the restart, rather than taking them while the old + server is torn down. The id and prefix it moves the server from are free + once its disconnect is done. A disconnect of either id, or a `stop()`, + that begins during the restart ends it, as a disconnect ends a connect in + flight, and the restart's error says the server is down. A restart whose + connect fails leaves the server down, configured under the new + configuration and holding its tool prefix, as any failed connect does: + it's listed as not connected, `restartMcplServer(id)` tries it again, and + `disconnectMcplServer(id)` removes it. `addModule` also refuses a name a server's tool prefix holds, as `create()` does at startup. A disconnect releases the tool-prefix claims its own server holds. Treat a submitted configuration as immutable while the server uses it: change settings with diff --git a/src/framework.ts b/src/framework.ts index 4e23c3659..adc25250d 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -3545,9 +3545,8 @@ export class AgentFramework { */ async addModule(module: Module): Promise { // A server's tool prefix and a module's name share one namespace, as - // create() enforces at startup (mcplPrefixCollision): a module isn't - // added under a prefix a server holds, or one a restart is moving its - // server to. + // create() checks at startup: a module isn't added under a prefix a + // server holds, or one a restart is moving its server to. const server = this.mcplPrefixMap.get(module.name); if (server !== undefined) { throw new Error(`Module "${module.name}" collides with MCPL server "${server}"'s toolPrefix "${module.name}"`); @@ -16666,9 +16665,19 @@ export class AgentFramework { * unlike `reconnect: true`, which only retries after transport-level * failures. Throws if the server has no stored config and none is given. * What its connect would refuse is refused before the running server is - * torn down, and until the connect, a connect, a restart or a module added - * meanwhile is refused the id and tool prefix the restart moves the server - * to. + * torn down. Until the connect, a connect or a restart started meanwhile + * is refused the id and the tool prefix the restart moves the server to, + * and a module added meanwhile is refused that tool prefix. The id and + * prefix it moves the server from are free once its disconnect is done, as + * after any disconnect. + * + * A disconnect of either id, or a `stop()`, that begins during the restart + * ends it, as a disconnect ends a connect in flight: the restart rejects, + * saying the server is down, and its holds are released. A restart whose + * connect fails leaves the server down, configured under the new + * configuration and holding its tool prefix, as any failed connect does: + * `listMcplServers()` lists it as not connected, `restartMcplServer(id)` + * tries it again, and `disconnectMcplServer(id)` removes it. * * Treat a submitted configuration and its nested values as immutable for * as long as the server uses them. Apply changed settings with @@ -16687,7 +16696,8 @@ export class AgentFramework { // checks through its connect, the restart holds the id and the tool // prefix it moves the server to (mcpRestartClaims), so no connect, // restart or module added meanwhile takes them while the old server is - // torn down. + // torn down. A stop() or a disconnect that begins after these checks + // isn't held off: it ends the restart, as it would end a connect. if (this.mcpServerAdmissionClosed) { throw new Error(`MCP server "${id}" was not restarted: the framework is stopping or stopped`); } @@ -16713,7 +16723,7 @@ export class AgentFramework { : `MCP server "${id}" was not restarted: the restart of "${claimedBy.restarting}" holds its id; restart it once that has finished`); } if (config.id !== id) { - const taken = this.mcpServerIdTaken(config.id); + const taken = this.mcpServerIdTaken(config.id, undefined, id); if (taken) throw new Error(taken); } const collision = this.mcplPrefixCollision(config, id); @@ -16732,7 +16742,15 @@ export class AgentFramework { this.restartClaims().prefixes.set(prefix, claim); try { await this.disconnectMcplServer(id); - await this.connectMcplServerClaimed(config, claim); + try { + await this.connectMcplServerClaimed(config, claim); + } catch (error) { + // The old server is gone, whatever ended the connect: a disconnect or + // a stop() that began during the restart, or the new server's own + // failure. The caller learns that first, then the connect's reason. + const reason = error instanceof Error ? error.message : String(error); + throw new Error(`MCP server "${id}" is down: it was disconnected for its restart, and the restart's connect failed: ${reason}`, { cause: error }); + } } finally { if (this.restartClaims().ids.get(config.id) === claim) this.restartClaims().ids.delete(config.id); if (this.restartClaims().prefixes.get(prefix) === claim) this.restartClaims().prefixes.delete(prefix); @@ -16763,20 +16781,27 @@ export class AgentFramework { * disconnect in flight holds it, and so does a registered connection, * including one kept because its launch couldn't be confirmed exited, and * a restart moving its server to the id (mcpRestartClaims), except to - * that restart's own connect (`claim`). */ - private mcpServerIdTaken(id: string, claim?: McpRestartClaim): string | null { + * that restart's own connect (`claim`). That connect is still refused + * while a disconnect of the id is in flight: the restart's own disconnect + * has finished by then, so that one began during the restart, and it ends + * the restart. `restarting` words the refusal for a restart of that + * server checking the id it would move to. */ + private mcpServerIdTaken(id: string, claim?: McpRestartClaim, restarting?: string): string | null { + const refused = restarting === undefined ? '' : `MCP server "${restarting}" was not restarted: `; + const retry = restarting === undefined ? 'connect it' : `restart "${restarting}"`; const claimedBy = this.restartClaims().ids.get(id); if (claimedBy && claimedBy !== claim) { - return `MCP server "${id}" is held by the restart of "${claimedBy.restarting}"; connect it once that has finished`; + return `${refused}MCP server "${id}" is held by the restart of "${claimedBy.restarting}"; ${retry} once that has finished`; } const op = this.mcpServerOps?.get(id); if (op?.kind === 'disconnect') { - return `MCP server "${id}" is still being disconnected; connect it once that has finished`; + if (claim) return `MCP server "${id}" is being disconnected, which ends the restart of "${claim.restarting}"`; + return `${refused}MCP server "${id}" is still being disconnected; ${retry} once that has finished`; } const registered = this.modernServers().get(id) ?? this.mcplServerRegistry?.getServer(id); if (op || registered) { const idle = !op && registered && !registered.isConnected && !registered.willReconnect; - return `MCPL server "${id}" is already registered${idle ? ' (not connected and not reconnecting: disconnect it first)' : ''}`; + return `${refused}MCPL server "${id}" is already registered${idle ? ' (not connected and not reconnecting: disconnect it first)' : ''}`; } return null; } diff --git a/test/mcpl-runtime-lifecycle.test.ts b/test/mcpl-runtime-lifecycle.test.ts index 28146a928..2be6ce118 100644 --- a/test/mcpl-runtime-lifecycle.test.ts +++ b/test/mcpl-runtime-lifecycle.test.ts @@ -344,6 +344,86 @@ test('a restart that fails releases the id and the tool prefix it held', async ( }); }); +test('a restart moving to an id or a tool prefix another restart holds is refused before it tears anything down', async (t) => { + await withFramework(true, async (framework) => { + await framework.connectMcplServer(serverConfig('second') as never); + const disconnect = framework.disconnectMcplServer.bind(framework); + const refusals: string[] = []; + t.mock.method(framework, 'disconnectMcplServer', async (id: string) => { + await disconnect(id); + if (id !== 'first') return; + // Between first's disconnect and its connect, only its claims hold "moved" and "xx". + for (const attempt of [ + () => framework.restartMcplServer('second', serverConfig('moved', { toolPrefix: 'other' }) as never), + () => framework.restartMcplServer('second', serverConfig('second', { toolPrefix: 'xx' }) as never), + ]) { + await attempt().then(() => refusals.push('restarted'), (error: unknown) => refusals.push((error as Error).message)); + } + }); + await framework.restartMcplServer('first', serverConfig('moved', { toolPrefix: 'xx' }) as never); + assert.deepEqual(refusals, [ + 'MCP server "second" was not restarted: MCP server "moved" is held by the restart of "first"; restart "second" once that has finished', + 'MCPL server "second" toolPrefix "xx" is held by the restart of "first"', + ]); + assert.deepEqual(framework.listMcplServers().map(s => [s.id, s.connected]).sort(), [['moved', true], ['second', true]]); + const tools = framework.getAllTools().map(tool => tool.name); + assert.ok(tools.includes('xx--echo') && tools.includes('mcpl--second--echo'), 'each serves its own prefix'); + }); +}); + +test('a restart whose connect fails leaves the server down, configured and not connected, and releases its holds', async () => { + await withFramework(true, async (framework) => { + const exits = { id: 'renamed', command: process.execPath, args: ['-e', 'process.exit(3)'], toolPrefix: 'xx' }; + await assert.rejects( + framework.restartMcplServer('first', exits as never), + /^MCP server "first" is down: it was disconnected for its restart, and the restart's connect failed: /, + ); + // As any failed connect leaves its server: listed as not connected, holding its prefix. + assert.deepEqual(framework.listMcplServers().map(s => [s.id, s.connected]), [['renamed', false]]); + await assert.rejects( + framework.addModule(new NamedModule('xx')), + /Module "xx" collides with MCPL server "renamed"'s toolPrefix "xx"/, + ); + // The restart's holds are released, so the id and the prefix take a working configuration. + await framework.restartMcplServer('renamed', serverConfig('renamed', { toolPrefix: 'xx' }) as never); + assert.deepEqual(framework.listMcplServers().map(s => [s.id, s.connected]), [['renamed', true]]); + assert.ok(framework.getAllTools().some(tool => tool.name === 'xx--echo')); + }); +}); + +for (const when of ['teardown', 'gap', 'connect'] as const) { + test(`a disconnect that begins during a restart's ${when} ends it: the server stays disconnected and the holds are released`, async (t) => { + await withFramework(true, async (framework) => { + const disconnect = framework.disconnectMcplServer.bind(framework); + let host: Promise | undefined; + const hostDisconnect = () => { + host = disconnect('first').then(() => 'disconnected', (error: unknown) => error); + }; + t.mock.method(framework, 'disconnectMcplServer', async (id: string) => { + // The restart's own disconnect. + const own = disconnect(id); + if (when === 'teardown') hostDisconnect(); + await own; + if (when === 'gap') hostDisconnect(); + // The restart's connect takes the id within this turn, before an immediate runs. + if (when === 'connect') setImmediate(hostDisconnect); + }); + const refused = await framework.restartMcplServer('first').then(() => null, (error: unknown) => error as Error); + assert.match(refused?.message ?? 'restarted', /^MCP server "first" is down: it was disconnected for its restart, and the restart's connect failed: /); + if (when !== 'connect') { + // Its connect is refused at once, naming the disconnect. + assert.match(refused!.message, /: MCP server "first" is being disconnected, which ends the restart of "first"$/); + } + assert.equal(await host, 'disconnected', 'the disconnect completed'); + assert.deepEqual(framework.listMcplServers(), [], 'nothing is left of the server'); + t.mock.restoreAll(); + // Nothing holds the id or its prefix now. + await framework.connectMcplServer(serverConfig('first') as never); + assert.deepEqual(framework.listMcplServers().map(s => [s.id, s.connected]), [['first', true]]); + }); + }); +} + test('connectMcplServer rejects a duplicate id and a colliding prefix', async () => { await withFramework(true, async (framework) => { await assert.rejects( From 8a2d4f7b5a828d79c0dee8342c25a0c7129ae148 Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Wed, 14 Oct 2026 05:36:14 -0400 Subject: [PATCH 0201/1112] test(mcp): a stop() that begins in a restart's gap ends it, and says the server is down The restart's doc says a stop() that begins during a restart ends it, as a disconnect does. The disconnect is tested in each phase; this pins the stop(): begun between the restart's disconnect and its connect, it refuses the connect, and the restart rejects saying the server is down, with the connect's refusal after it. --- test/mcpl-runtime-lifecycle.test.ts | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/test/mcpl-runtime-lifecycle.test.ts b/test/mcpl-runtime-lifecycle.test.ts index 2be6ce118..7f5ea5041 100644 --- a/test/mcpl-runtime-lifecycle.test.ts +++ b/test/mcpl-runtime-lifecycle.test.ts @@ -424,6 +424,22 @@ for (const when of ['teardown', 'gap', 'connect'] as const) { }); } +test("a stop() that begins in a restart's gap ends it, and the restart says the server is down", async (t) => { + await withFramework(true, async (framework) => { + const disconnect = framework.disconnectMcplServer.bind(framework); + let stopping: Promise | undefined; + t.mock.method(framework, 'disconnectMcplServer', async (id: string) => { + await disconnect(id); + stopping = framework.stop(); + }); + await assert.rejects( + framework.restartMcplServer('first'), + /^MCP server "first" is down: it was disconnected for its restart, and the restart's connect failed: MCP server "first" was not connected: the framework is stopping or stopped$/, + ); + await stopping; + }); +}); + test('connectMcplServer rejects a duplicate id and a colliding prefix', async () => { await withFramework(true, async (framework) => { await assert.rejects( From 3168cd47c162f58b65061063fa67024fa692481c Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Wed, 14 Oct 2026 06:05:17 -0400 Subject: [PATCH 0202/1112] test(mcp): match the restart's refusal as assert.rejects reads it, with its Error: prefix assert.rejects tests a regular expression against the error's string form, which starts with "Error: ". The two new tests anchored theirs at the message, so they failed although the restart said what they expect. --- test/mcpl-runtime-lifecycle.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/mcpl-runtime-lifecycle.test.ts b/test/mcpl-runtime-lifecycle.test.ts index 7f5ea5041..678dbb021 100644 --- a/test/mcpl-runtime-lifecycle.test.ts +++ b/test/mcpl-runtime-lifecycle.test.ts @@ -376,7 +376,7 @@ test('a restart whose connect fails leaves the server down, configured and not c const exits = { id: 'renamed', command: process.execPath, args: ['-e', 'process.exit(3)'], toolPrefix: 'xx' }; await assert.rejects( framework.restartMcplServer('first', exits as never), - /^MCP server "first" is down: it was disconnected for its restart, and the restart's connect failed: /, + /^Error: MCP server "first" is down: it was disconnected for its restart, and the restart's connect failed: /, ); // As any failed connect leaves its server: listed as not connected, holding its prefix. assert.deepEqual(framework.listMcplServers().map(s => [s.id, s.connected]), [['renamed', false]]); @@ -434,7 +434,7 @@ test("a stop() that begins in a restart's gap ends it, and the restart says the }); await assert.rejects( framework.restartMcplServer('first'), - /^MCP server "first" is down: it was disconnected for its restart, and the restart's connect failed: MCP server "first" was not connected: the framework is stopping or stopped$/, + /^Error: MCP server "first" is down: it was disconnected for its restart, and the restart's connect failed: MCP server "first" was not connected: the framework is stopping or stopped$/, ); await stopping; }); From 406164eb44350d93bf3c75d8592ac79eb6deb087 Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Wed, 14 Oct 2026 06:10:19 -0400 Subject: [PATCH 0203/1112] fix(mcp): say which disconnect ends a restart; the test server answers what it doesn't know The restart's doc and the changelog said a disconnect of either id ends a restart. Only a disconnect of the id the restart moves the server to does, which is its own id for a restart under the same id: a disconnect of the id a move leaves only removes what that id still has, and the restart completes. They also named restartMcplServer(id) and disconnectMcplServer(id) for a server whose restart failed, where id read as the restart's parameter; the server is configured under the id it was moved to, so they now say a restart or a disconnect under that id. The lifecycle tests' stdio server never answered a request it didn't know, so each connect waited out the host's 15 s for the initial feature-set update, and a restart test took 30 s. It now answers such a request with method-not-found. An error answer and no answer reach the same handling, so what the tests check is unchanged, and the file runs in seconds. Found and measured by Beth-2389. --- .../mcp-standard-tool-results.changed.md | 15 +++++++------- src/framework.ts | 20 ++++++++++--------- test/mcpl-runtime-lifecycle.test.ts | 3 +++ 3 files changed, 22 insertions(+), 16 deletions(-) diff --git a/changelog.d/mcp-standard-tool-results.changed.md b/changelog.d/mcp-standard-tool-results.changed.md index e8915081c..b3270f64b 100644 --- a/changelog.d/mcp-standard-tool-results.changed.md +++ b/changelog.d/mcp-standard-tool-results.changed.md @@ -116,13 +116,14 @@ meanwhile is refused them, and a module added meanwhile is refused that prefix, each naming the restart, rather than taking them while the old server is torn down. The id and prefix it moves the server from are free - once its disconnect is done. A disconnect of either id, or a `stop()`, - that begins during the restart ends it, as a disconnect ends a connect in - flight, and the restart's error says the server is down. A restart whose - connect fails leaves the server down, configured under the new - configuration and holding its tool prefix, as any failed connect does: - it's listed as not connected, `restartMcplServer(id)` tries it again, and - `disconnectMcplServer(id)` removes it. `addModule` also refuses a name a + once its disconnect is done. A disconnect of the id it moves the server + to (its own, for a restart under the same id), or a `stop()`, that begins + during the restart ends it, as a disconnect ends a connect in flight, and + the restart's error says the server is down. A restart whose connect + fails leaves the server down, configured under the new configuration and + its id, and holding its tool prefix, as any failed connect does: it's + listed as not connected, and a restart or a disconnect under that id + tries it again or removes it. `addModule` also refuses a name a server's tool prefix holds, as `create()` does at startup. A disconnect releases the tool-prefix claims its own server holds. Treat a submitted configuration as immutable while the server uses it: change settings with diff --git a/src/framework.ts b/src/framework.ts index adc25250d..1f95775f9 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -16671,13 +16671,14 @@ export class AgentFramework { * prefix it moves the server from are free once its disconnect is done, as * after any disconnect. * - * A disconnect of either id, or a `stop()`, that begins during the restart - * ends it, as a disconnect ends a connect in flight: the restart rejects, - * saying the server is down, and its holds are released. A restart whose - * connect fails leaves the server down, configured under the new - * configuration and holding its tool prefix, as any failed connect does: - * `listMcplServers()` lists it as not connected, `restartMcplServer(id)` - * tries it again, and `disconnectMcplServer(id)` removes it. + * A disconnect of the id the restart moves the server to (its own id, for + * a restart under the same id), or a `stop()`, that begins during the + * restart ends it, as a disconnect ends a connect in flight: the restart + * rejects, saying the server is down, and its holds are released. A + * restart whose connect fails leaves the server down, configured under the + * new configuration and its id, and holding its tool prefix, as any failed + * connect does: `listMcplServers()` lists it as not connected, and a + * restart or a disconnect under that id tries it again or removes it. * * Treat a submitted configuration and its nested values as immutable for * as long as the server uses them. Apply changed settings with @@ -16696,8 +16697,9 @@ export class AgentFramework { // checks through its connect, the restart holds the id and the tool // prefix it moves the server to (mcpRestartClaims), so no connect, // restart or module added meanwhile takes them while the old server is - // torn down. A stop() or a disconnect that begins after these checks - // isn't held off: it ends the restart, as it would end a connect. + // torn down. A stop(), or a disconnect of the id it moves the server to, + // that begins after these checks isn't held off: it ends the restart, as + // it would end a connect. if (this.mcpServerAdmissionClosed) { throw new Error(`MCP server "${id}" was not restarted: the framework is stopping or stopped`); } diff --git a/test/mcpl-runtime-lifecycle.test.ts b/test/mcpl-runtime-lifecycle.test.ts index 678dbb021..3944e5077 100644 --- a/test/mcpl-runtime-lifecycle.test.ts +++ b/test/mcpl-runtime-lifecycle.test.ts @@ -42,6 +42,9 @@ process.stdin.on('data', (c) => { if (m.method === 'initialize') reply({ protocolVersion: '2024-11-05', capabilities: { experimental: { mcpl: { version: '0.4', pushEvents: true } } } }); else if (m.method === 'tools/list') reply({ tools: [{ name: 'echo', description: 'e', inputSchema: { type: 'object' } }] }); else if (m.method === 'tools/call') reply({ content: [{ type: 'text', text: String((m.params && m.params.arguments && m.params.arguments.text) || '') }] }); + // Any other request is answered as unknown, so the host's wait for it ends + // at once rather than at its timeout. + else if (m.id !== undefined) process.stdout.write(JSON.stringify({ jsonrpc: '2.0', id: m.id, error: { code: -32601, message: 'method not found' } }) + '\\n'); } }); `; From ae1a568dfae0fd61f95d490aae8f0858d73714f7 Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Wed, 14 Oct 2026 06:42:00 -0400 Subject: [PATCH 0204/1112] fix(mcp): stop() and a disconnect end a connect in flight at once, in either engine With #325, stop() waits for operator operations before its teardown, and a connect is one. A connect still in its handshake held that wait until its handshake's deadline, and longer when nothing bounds it: a modern one up to its request deadline, a legacy stdio one for the initialize timeout (30 s), a WebSocket one for the open timeout too, and one whose access provider never settles for good, since stop() has no budget by default. - stop() ends each connect in flight first (endConnectsInFlight): a modern connect's connection, registered before it starts, is closed, and a legacy connect's handshake is aborted. Each fails at once, its launch reaped as for any failed handshake, and its operation ends. - Each legacy connect gets its own AbortSignal (legacyConnectAborts), which McplServerRegistry.addServer(), McplServerConnection.connect(), connectChecked() and connectWithReconnect() pass to the handshake, as a reconnect's already was. An aborted connectWithReconnect() makes no retry stub. - A disconnect that takes an id over aborts the legacy connect it displaces too, rather than waiting out its handshake, so a disconnect ends a connect in flight at once in both engines. - A WebSocket dial races its access provider's wait against the signal: an aborted dial stops waiting, doesn't fall back to the static token, and ignores the provider's late answer or failure. The changelog says it, and #325's fragment now says a connect in flight is ended rather than waited for, and that a restart in flight ends too. Tests: stop() and a disconnect each end a legacy stdio connect stuck in its initialize within half a second; stop() ends a WebSocket dial still waiting for an access provider that never settles; and the modern test of a connect stuck on discover now asserts that stop() returns promptly. Without each abort, its test fails: the legacy ones wait 30 s, and the provider one never ends. --- .../mcp-standard-tool-results.changed.md | 7 ++- ...top-waits-for-operator-operations.fixed.md | 10 ++-- src/framework.ts | 47 ++++++++++++------- src/mcpl/server-connection.ts | 19 +++++--- src/mcpl/server-registry.ts | 5 +- src/mcpl/transport.ts | 18 ++++++- test/mcp-modern-framework.test.ts | 3 ++ test/mcpl-runtime-lifecycle.test.ts | 41 ++++++++++++++++ 8 files changed, 120 insertions(+), 30 deletions(-) diff --git a/changelog.d/mcp-standard-tool-results.changed.md b/changelog.d/mcp-standard-tool-results.changed.md index b3270f64b..1474d469e 100644 --- a/changelog.d/mcp-standard-tool-results.changed.md +++ b/changelog.d/mcp-standard-tool-results.changed.md @@ -87,7 +87,12 @@ (`reconnect-failed`, `permanent: true`). Reconnect attempts run one at a time, and never start before the previous child has exited. - `close()` during a reconnect handshake ends that launch before it - resolves, a WebSocket dial still opening included. + resolves, a WebSocket dial still opening included. In the framework, + `stop()` and a disconnect of the id end a first connect still in its + handshake the same way, and a WebSocket dial still waiting for its + access provider: `connect()`, `connectWithReconnect()` and + `McplServerRegistry.addServer()` take an optional `AbortSignal`, and an + aborted `connectWithReconnect()` makes no retry stub. - `McplServerRegistry` holds an id until its child is confirmed exited: `removeServer` frees it only once the close succeeds, a second `addServer` for an id still connecting is refused, and `closeAll()` diff --git a/changelog.d/stop-waits-for-operator-operations.fixed.md b/changelog.d/stop-waits-for-operator-operations.fixed.md index a6e546767..611e54765 100644 --- a/changelog.d/stop-waits-for-operator-operations.fixed.md +++ b/changelog.d/stop-waits-for-operator-operations.fixed.md @@ -2,9 +2,13 @@ refuses new ones.** A quiesce, a resume, a `host/command`, a store surgery, a branch switch, an undo or redo's config materialization, a puppet tool call, an awareness sync, or an MCPL - connect, disconnect or restart admitted before `stop()` began now - finishes before the event gate, the modules, the MCPL connections and the - store go, unless a budget runs out first (see the operators' note below). + disconnect admitted before `stop()` began now finishes before the event + gate, the modules, the MCPL connections and the store go, unless a budget + runs out first (see the operators' note below). An MCPL connect still in + flight is ended instead, in either engine: `stop()` closes it at once, + its launch reaped, rather than waiting out its handshake. A restart in + flight ends too, its connect refused once `stop()` has begun, so the + server it was restarting is down. Before, such an operation kept running through the teardown. A resume waiting on its maintenance pass or previews could go on to leave quiesce while the host stopped, so the host booted unquiesced; a puppet call diff --git a/src/framework.ts b/src/framework.ts index 27f4ecaf5..0bab16c82 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -2032,6 +2032,10 @@ export class AgentFramework { /** Connects and disconnects in flight, by server id (holdMcpServerId); * created on first use, like modernMcpConnections. */ private mcpServerOps: Map }> | undefined; + /** Each legacy connect in flight's own abort (connectHeldMcplServer): + * stop() and a disconnect of its id end it at once, its launch reaped, + * rather than waiting out its handshake. */ + private legacyConnectAborts = new Map(); /** The ids and tool prefixes restarts hold, each with the claim of the * restart holding it, from the restart's checks through its connect * (restartMcplServer): no other connect or restart takes them while the @@ -2691,16 +2695,18 @@ export class AgentFramework { return !!this.stopPromise || !!this.queue?.isClosed; } - /** stop()'s first step for a modern connect still in flight, whose - * connection is registered before it starts (connectHeldMcplServer): - * close it, so the connect fails at once and finds the stop begun. A - * legacy connect still in its handshake can't be ended early; it ends with - * its handshake and then closes itself. The close's outcome is the - * teardown's to report (closeModernMcpConnections). */ - private closeModernConnectsInFlight(): void { + /** stop()'s first step for each connect still in flight, in either engine: + * end it, so it fails at once and its operation ends, its launch reaped, + * rather than holding stop()'s wait until its handshake's deadline. A + * modern connect's connection is registered before it starts and is + * closed; a legacy one's handshake, and the access provider a WebSocket + * dial waits for, are aborted (legacyConnectAborts). A modern close's + * outcome is the teardown's to report (closeModernMcpConnections). */ + private endConnectsInFlight(): void { for (const [id, op] of this.mcpServerOps ?? []) { if (op.kind !== 'connect') continue; this.modernServers().get(id)?.close().catch(() => { /* reported by the teardown's close */ }); + this.legacyConnectAborts.get(id)?.abort(); } } @@ -2733,10 +2739,11 @@ export class AgentFramework { private async stopOwned(teardownTimeoutMs: number | undefined): Promise { // No MCP server is admitted from here on (stopHasBegun): none is launched // or dialed while the rest are being closed, so none outlives the - // framework. A modern connect still in flight is closed now, so it fails - // at once and its operator operation ends, rather than holding the wait - // for operator operations below until its handshake's deadline. - this.closeModernConnectsInFlight(); + // framework. A connect still in flight is ended now, in either engine, + // so it fails at once and its operator operation ends, rather than + // holding the wait for operator operations below until its handshake's + // deadline. + this.endConnectsInFlight(); this.pushCoalescer?.suspend(); this.flushCoalescingSnapshot(); this.running = false; @@ -16628,13 +16635,17 @@ export class AgentFramework { }); } let connection: McplServerConnection; + const abort = new AbortController(); + this.legacyConnectAborts.set(config.id, abort); try { - connection = await this.mcplServerRegistry.addServer(connectionConfig, this.mcplHostCapabilities); + connection = await this.mcplServerRegistry.addServer(connectionConfig, this.mcplHostCapabilities, abort.signal); } catch (error) { // The registry keeps a failed launch that may still be running, as the // closed connection that owns it. if (error instanceof McplUnreapedLaunchError) throw AgentFramework.unreapedConnectFailure(config.id, error); throw error; + } finally { + if (this.legacyConnectAborts.get(config.id) === abort) this.legacyConnectAborts.delete(config.id); } // stop() began while the handshake ran: its teardown has already // collected the registry, so this connection would outlive it. Or a @@ -17182,11 +17193,13 @@ export class AgentFramework { const hold = this.holdMcpServerId(id, 'disconnect'); try { // Close what the id has now, so a connect in flight ends rather than - // being waited out: a modern connect is ended by its close, and a - // legacy one already registered (setting up after its handshake) - // fails its requests. A legacy connect still handshaking ends when its - // handshake does, finding itself displaced. The registry's removal is - // the legacy close: a failed one stays registered. + // being waited out: a modern connect is ended by its close, a legacy + // one still in its handshake (or a WebSocket dial waiting for its + // access provider) by its abort, its launch reaped, and a legacy one + // already registered (setting up after its handshake) fails its + // requests and finds itself displaced. The registry's removal is the + // legacy close: a failed one stays registered. + this.legacyConnectAborts.get(id)?.abort(); const modern = this.modernServers().get(id); const closing = [modern?.close(), registry.removeServer(id)]; for (const pending of closing) pending?.catch(() => { /* its verdict is awaited below */ }); diff --git a/src/mcpl/server-connection.ts b/src/mcpl/server-connection.ts index 80efdfe3b..28b379d0a 100644 --- a/src/mcpl/server-connection.ts +++ b/src/mcpl/server-connection.ts @@ -662,20 +662,24 @@ export class McplServerConnection extends EventEmitter { static async connect( config: McplServerConfig, hostCapabilities: McplHostCapabilities, + signal?: AbortSignal, ): Promise { McplServerConnection.checkConfig(config); - return McplServerConnection.connectChecked(config, hostCapabilities); + return McplServerConnection.connectChecked(config, hostCapabilities, signal); } /** {@link connect}, for a configuration its caller has already checked - * (and warned about) once. */ + * (and warned about) once. `signal` ends the connect at once, its launch + * reaped as for any failed handshake: its caller's stop() or a + * disconnect of its id. */ private static async connectChecked( config: McplServerConfig, hostCapabilities: McplHostCapabilities, + signal?: AbortSignal, ): Promise { let established: Awaited>; try { - established = await McplServerConnection.handshake(config, hostCapabilities); + established = await McplServerConnection.handshake(config, hostCapabilities, signal); } catch (error) { if (!(error instanceof HandshakeCleanupError)) throw error; // The launch may still be running. A connection owns it, closed and @@ -880,15 +884,18 @@ export class McplServerConnection extends EventEmitter { static async connectWithReconnect( config: McplServerConfig, hostCapabilities: McplHostCapabilities, + signal?: AbortSignal, ): Promise { // Outside the retry path below: a configuration error is never retried. McplServerConnection.checkConfig(config); try { - return await McplServerConnection.connectChecked(config, hostCapabilities); + return await McplServerConnection.connectChecked(config, hostCapabilities, signal); } catch (error) { // No common protocol revision is a configuration fact, not an outage: - // a retry stub would restart the server forever for nothing. - if (!config.reconnect || error instanceof McplProtocolVersionError) { + // a retry stub would restart the server forever for nothing. A connect + // its caller ended (stop(), or a disconnect of its id) isn't retried + // either. + if (!config.reconnect || error instanceof McplProtocolVersionError || signal?.aborted) { throw error; } diff --git a/src/mcpl/server-registry.ts b/src/mcpl/server-registry.ts index 95fc6b11f..350ff9759 100644 --- a/src/mcpl/server-registry.ts +++ b/src/mcpl/server-registry.ts @@ -44,6 +44,7 @@ export class McplServerRegistry { async addServer( config: McplServerConfig, hostCapabilities: McplHostCapabilities, + signal?: AbortSignal, ): Promise { if (this.servers.has(config.id) || this.adding.has(config.id)) { throw new Error(`MCPL server "${config.id}" is already registered`); @@ -53,8 +54,8 @@ export class McplServerRegistry { let connection: McplServerConnection; try { connection = config.reconnect - ? await McplServerConnection.connectWithReconnect(config, hostCapabilities) - : await McplServerConnection.connect(config, hostCapabilities); + ? await McplServerConnection.connectWithReconnect(config, hostCapabilities, signal) + : await McplServerConnection.connect(config, hostCapabilities, signal); } catch (error) { if (error instanceof McplUnreapedLaunchError) this.register(config, error.connection); throw error; diff --git a/src/mcpl/transport.ts b/src/mcpl/transport.ts index 18375a2c9..403aa483a 100644 --- a/src/mcpl/transport.ts +++ b/src/mcpl/transport.ts @@ -317,10 +317,18 @@ export class WebSocketTransport extends McplTransport { // connect AND reconnect, so rotation happens at the transport layer and // nothing above it ever holds a credential. let token: string | null = config.token ?? null; + const abandoned = (): Error => + new Error(`MCPL server "${config.id}" WebSocket connect abandoned: the connection was closed (${redactUrl(config.url)})`); if (config.accessProvider) { + if (signal?.aborted) throw abandoned(); try { - token = (await config.accessProvider()) ?? token; + // A dial whose connection is closed meanwhile stops waiting for the + // provider; whatever the provider does later is ignored. + const provided = Promise.resolve(config.accessProvider()); + provided.catch(() => { /* a late failure after the dial was abandoned */ }); + token = (signal ? await Promise.race([provided, rejectOnAbort(signal)]) : await provided) ?? token; } catch (err) { + if (signal?.aborted) throw abandoned(); // Fall back to the static token (if any) rather than failing the dial // outright — the server will refuse a stale credential loudly anyway. // The provider's own text may quote the credential, so it isn't logged. @@ -454,6 +462,14 @@ export function isWebSocketTransport(config: McplServerConfig): boolean { * The returned transport is connected at the byte level but has NOT * performed the MCPL handshake. */ +/** A promise that rejects once `signal` aborts, and never settles otherwise. */ +function rejectOnAbort(signal: AbortSignal): Promise { + return new Promise((_resolve, reject) => { + if (signal.aborted) reject(signal.reason); + else signal.addEventListener('abort', () => reject(signal.reason), { once: true }); + }); +} + export async function openTransport(config: McplServerConfig, signal?: AbortSignal): Promise { const binding = resolveServerBinding(config); if (binding.family !== 'legacy') { diff --git a/test/mcp-modern-framework.test.ts b/test/mcp-modern-framework.test.ts index dea9c9310..bcd07f692 100644 --- a/test/mcp-modern-framework.test.ts +++ b/test/mcp-modern-framework.test.ts @@ -487,7 +487,10 @@ test('the framework owns a modern connect in flight: stop() ends it, and a secon await waitFor('the launch waiting on discover', () => lines(log).some((l) => l.includes('server/discover'))); await assert.rejects(framework.connectMcplServer(config), /already registered/); const pid = (JSON.parse(lines(log)[0]!) as { pid: number }).pid; + // stop() ends the connect in flight rather than waiting for its handshake. + const stopping = Date.now(); await framework.stop(); + assert.ok(Date.now() - stopping < 10_000, `stop() took ${Date.now() - stopping} ms, as if it waited out the connect`); const alive = (p: number) => { try { process.kill(p, 0); return true; } catch { return false; } }; assert.equal(alive(pid), false, 'the launch in flight was reaped by stop()'); await pending.catch(() => {}); diff --git a/test/mcpl-runtime-lifecycle.test.ts b/test/mcpl-runtime-lifecycle.test.ts index f6a93a08f..8026d9f51 100644 --- a/test/mcpl-runtime-lifecycle.test.ts +++ b/test/mcpl-runtime-lifecycle.test.ts @@ -452,6 +452,47 @@ test("a stop() that begins in a restart's gap ends it, and the restart says the }); }); +/** A stdio server that reads its input and never answers, so a connect waits in its handshake. */ +const SILENT = { id: 'silent', command: process.execPath, args: ['-e', 'process.stdin.resume()'] }; + +for (const ending of ['stop()', 'a disconnect'] as const) { + test(`${ending} ends a legacy connect stuck in its handshake at once`, async () => { + await withFramework(true, async (framework) => { + const connecting = framework.connectMcplServer(SILENT as never).then(() => null, (error: unknown) => error as Error); + // The child is up and its initialize sent. + await new Promise((resolve) => setTimeout(resolve, 300)); + const started = Date.now(); + if (ending === 'stop()') await framework.stop(); + else await framework.disconnectMcplServer('silent'); + const took = Date.now() - started; + assert.ok(took < 10_000, `${ending} took ${took} ms, as if it waited out the handshake`); + const refused = await connecting; + assert.match(refused?.message ?? 'connected', /MCPL server "silent" was closed while connecting/); + if (ending === 'a disconnect') assert.ok(!framework.listMcplServers().some(s => s.id === 'silent'), 'nothing is left of it'); + }); + }); +} + +test("stop() ends a WebSocket dial still waiting for its access provider", async () => { + await withFramework(true, async (framework) => { + let asked = false; + const waiting = { + id: 'dial', + url: 'ws://127.0.0.1:9/mcp', + // A provider that never settles. + accessProvider: () => { asked = true; return new Promise(() => {}); }, + }; + const connecting = framework.connectMcplServer(waiting as never).then(() => null, (error: unknown) => error as Error); + for (let i = 0; i < 50 && !asked; i++) await new Promise((resolve) => setTimeout(resolve, 10)); + assert.ok(asked, 'the dial is waiting for its provider'); + const started = Date.now(); + await framework.stop(); + assert.ok(Date.now() - started < 10_000, 'stop() did not wait for the provider'); + const refused = await connecting; + assert.match(refused?.message ?? 'connected', /MCPL server "dial" was closed while connecting/); + }); +}); + test('connectMcplServer rejects a duplicate id and a colliding prefix', async () => { await withFramework(true, async (framework) => { await assert.rejects( From 5a34a2f13fa0065ba0eceb489d97a98a5d7e2019 Mon Sep 17 00:00:00 2001 From: Ada-1017 Date: Wed, 14 Oct 2026 06:51:38 -0400 Subject: [PATCH 0205/1112] fix(mcp): an aborted legacy connect says why as a modern one does; the abort map is made on first use The full run of ae1a568d failed nine tests in three files: - The legacy connect's abort map was a field initializer, so the reaction baseline platform tests and a reconnect test, whose frameworks are built field by field, met it undefined. It's now made on first use, as the restart claims are. - A legacy connect ended by its abort rejected with the handshake's own "MCPL server "same" was closed while connecting", where a modern connect ended the same way says "MCP server "m" was not connected: it was disconnected while connecting", or that the framework stopped while it was connecting. The legacy connect now says the same, with the handshake's error as its cause, so both engines tell a host the same thing. The ownership test that pinned a disconnect waiting out the handshake now says the disconnect ends it at once, as it does; its expectations stand. --- src/framework.ts | 27 +++++++++++++++++++++------ test/mcp-server-ownership.test.ts | 2 +- test/mcpl-runtime-lifecycle.test.ts | 6 ++++-- 3 files changed, 26 insertions(+), 9 deletions(-) diff --git a/src/framework.ts b/src/framework.ts index 0bab16c82..aef76b035 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -2034,8 +2034,9 @@ export class AgentFramework { private mcpServerOps: Map }> | undefined; /** Each legacy connect in flight's own abort (connectHeldMcplServer): * stop() and a disconnect of its id end it at once, its launch reaped, - * rather than waiting out its handshake. */ - private legacyConnectAborts = new Map(); + * rather than waiting out its handshake. Created on first use, as the + * restart claims are (legacyConnectAborts()). */ + private legacyConnectAbortMap: Map | undefined; /** The ids and tool prefixes restarts hold, each with the claim of the * restart holding it, from the restart's checks through its connect * (restartMcplServer): no other connect or restart takes them while the @@ -2706,7 +2707,7 @@ export class AgentFramework { for (const [id, op] of this.mcpServerOps ?? []) { if (op.kind !== 'connect') continue; this.modernServers().get(id)?.close().catch(() => { /* reported by the teardown's close */ }); - this.legacyConnectAborts.get(id)?.abort(); + this.legacyConnectAborts().get(id)?.abort(); } } @@ -16030,6 +16031,10 @@ export class AgentFramework { return (this.mcpRestartClaims ??= { ids: new Map(), prefixes: new Map() }); } + private legacyConnectAborts(): Map { + return (this.legacyConnectAbortMap ??= new Map()); + } + /** * Hold a server id for one connect or disconnect, until `release()`. * Together with the live connections (the MCPL registry and the modern @@ -16636,16 +16641,26 @@ export class AgentFramework { } let connection: McplServerConnection; const abort = new AbortController(); - this.legacyConnectAborts.set(config.id, abort); + this.legacyConnectAborts().set(config.id, abort); try { connection = await this.mcplServerRegistry.addServer(connectionConfig, this.mcplHostCapabilities, abort.signal); } catch (error) { // The registry keeps a failed launch that may still be running, as the // closed connection that owns it. if (error instanceof McplUnreapedLaunchError) throw AgentFramework.unreapedConnectFailure(config.id, error); + // Ended by stop() or by a disconnect of its id (its abort): said as a + // modern connect ended that way says it, with the handshake's own + // error as the cause. + if (abort.signal.aborted) { + throw new Error( + `MCP server "${config.id}" was not connected: ` + + (this.stopHasBegun() ? 'the framework stopped while it was connecting' : 'it was disconnected while connecting'), + { cause: error }, + ); + } throw error; } finally { - if (this.legacyConnectAborts.get(config.id) === abort) this.legacyConnectAborts.delete(config.id); + if (this.legacyConnectAborts().get(config.id) === abort) this.legacyConnectAborts().delete(config.id); } // stop() began while the handshake ran: its teardown has already // collected the registry, so this connection would outlive it. Or a @@ -17199,7 +17214,7 @@ export class AgentFramework { // already registered (setting up after its handshake) fails its // requests and finds itself displaced. The registry's removal is the // legacy close: a failed one stays registered. - this.legacyConnectAborts.get(id)?.abort(); + this.legacyConnectAborts().get(id)?.abort(); const modern = this.modernServers().get(id); const closing = [modern?.close(), registry.removeServer(id)]; for (const pending of closing) pending?.catch(() => { /* its verdict is awaited below */ }); diff --git a/test/mcp-server-ownership.test.ts b/test/mcp-server-ownership.test.ts index 44debcc03..4ee06691c 100644 --- a/test/mcp-server-ownership.test.ts +++ b/test/mcp-server-ownership.test.ts @@ -171,7 +171,7 @@ test('a modern connect is refused while a legacy connect for the same id is stil }); }); -test('a disconnect during a legacy handshake ends that connect when the handshake finishes, and removes it whole', async () => { +test('a disconnect during a legacy handshake ends that connect at once, and removes it whole', async () => { await withFramework(async ({ framework, dir }) => { const log = join(dir, 'legacy.log'); const pending = framework.connectMcplServer(legacy('same', 'slow', log, { SLOW_INIT_MS: '600' })); diff --git a/test/mcpl-runtime-lifecycle.test.ts b/test/mcpl-runtime-lifecycle.test.ts index 8026d9f51..f81d750f0 100644 --- a/test/mcpl-runtime-lifecycle.test.ts +++ b/test/mcpl-runtime-lifecycle.test.ts @@ -467,7 +467,9 @@ for (const ending of ['stop()', 'a disconnect'] as const) { const took = Date.now() - started; assert.ok(took < 10_000, `${ending} took ${took} ms, as if it waited out the handshake`); const refused = await connecting; - assert.match(refused?.message ?? 'connected', /MCPL server "silent" was closed while connecting/); + assert.match(refused?.message ?? 'connected', ending === 'stop()' + ? /^MCP server "silent" was not connected: the framework stopped while it was connecting$/ + : /^MCP server "silent" was not connected: it was disconnected while connecting$/); if (ending === 'a disconnect') assert.ok(!framework.listMcplServers().some(s => s.id === 'silent'), 'nothing is left of it'); }); }); @@ -489,7 +491,7 @@ test("stop() ends a WebSocket dial still waiting for its access provider", async await framework.stop(); assert.ok(Date.now() - started < 10_000, 'stop() did not wait for the provider'); const refused = await connecting; - assert.match(refused?.message ?? 'connected', /MCPL server "dial" was closed while connecting/); + assert.match(refused?.message ?? 'connected', /^MCP server "dial" was not connected: the framework stopped while it was connecting$/); }); }); From 3e89b658be4ba1d91eb7dc1fd62d21b4931be517 Mon Sep 17 00:00:00 2001 From: Duncan-2410 Date: Wed, 14 Oct 2026 06:53:54 -0400 Subject: [PATCH 0206/1112] test(deferral): a requeued write is in the recovery file once; the dropped-entry line names what it holds An internal review of the final round found pieces no test pinned, and two places where a line or a comment said less than it should. - requeueDrained takes the entries it puts back out of the un-acked set. Without that, a write that fails at a flush is in the recovery file twice (once handed off, once requeued) until the batch's ack, and for as long as that ack's sync keeps failing; a crash then restores it twice and stores it twice. Removing it survived every test in the twelve files that touch the deferral. host-quiesce now fails a write at resume and the ack's sync after it, and checks that the file holds the entry that landed (waiting for its ack) and the two that went back, each once. Its removal fails it. - Boot recovery's line for a dropped entry said "no participant" (or "no id") for a value that was there but wasn't text. It now names such a value by its type ("participant of type number"), and "no " only when the field is missing. The boot-recovery test adds an entry with a numeric participant; the old line fails it. - writeDrainedInOrder's doc read as if the held later entries were logged at each flush; it's the entry that keeps failing that is. - The fragment said a held queue's later deferrals are written to the recovery file up to its size cap, and not what happens past it: the file isn't rewritten, as the host log warns, and what's deferred after that is kept in memory only. Co-Authored-By: Claude Opus 5.5 --- .../mid-turn-window-only-held.fixed.md | 4 +- src/framework.ts | 9 ++-- test/host-quiesce.test.ts | 54 +++++++++++++++++-- 3 files changed, 60 insertions(+), 7 deletions(-) diff --git a/changelog.d/mid-turn-window-only-held.fixed.md b/changelog.d/mid-turn-window-only-held.fixed.md index b82168a59..a20c5346f 100644 --- a/changelog.d/mid-turn-window-only-held.fixed.md +++ b/changelog.d/mid-turn-window-only-held.fixed.md @@ -75,7 +75,9 @@ of the queue, at once or at the resume after a quiesced boot, and logs it again, unless it's one boot drops (below). While it's held the queue stays persisted, so every later deferral is written to the recovery file, up to - the file's size cap. Only the host log names it. + the file's size cap; past it the file isn't rewritten, as the host log + warns, and what's deferred after that is kept in memory only. Only the + host log names the held write. - Boot recovery drops a recovered deferred write whose participant isn't text or whose content isn't an array of block objects, which the store could never take, and logs it by id and participant, rather than holding diff --git a/src/framework.ts b/src/framework.ts index 157704cd4..133c88c55 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -4955,9 +4955,12 @@ export class AgentFramework { && (m as { content: unknown[] }).content.every((b) => !!b && typeof b === 'object'); const malformed = data.filter((m) => !wellFormed(m)); for (const m of malformed) { + // What the entry holds there: a value that isn't text by its type. const field = (key: string): string => { const value = m && typeof m === 'object' ? (m as Record)[key] : undefined; - return typeof value === 'string' ? value : `no ${key}`; + if (typeof value === 'string') return value; + if (value === undefined) return `no ${key}`; + return `${key} ${value === null ? 'null' : `of type ${typeof value}`}`; }; console.error( `[host-mode] a recovered deferred context write is malformed and was dropped: ` + @@ -15126,8 +15129,8 @@ export class AgentFramework { * being stored alone. (A tool_use that is stored, followed by a tool_result * that can't be, stays stored without it until the tool_result lands at a * later flush. The only queued pair is puppetToolCall's defensive one, - * which no path reaches today.) An entry that keeps failing holds its - * target's later queued writes, logged by id at each flush, rather than + * which no path reaches today.) An entry that keeps failing is logged by + * id at each flush, and holds its target's later queued writes rather than * letting them land out of order. * * The hold is the queue's alone. A write that isn't deferred is stored at diff --git a/test/host-quiesce.test.ts b/test/host-quiesce.test.ts index 812ea60bb..b322016cb 100644 --- a/test/host-quiesce.test.ts +++ b/test/host-quiesce.test.ts @@ -722,6 +722,52 @@ test('resume flushes deferred writes in order: a poison write stays queued with }); }); +test('a write that fails at resume goes back into the queue once: the recovery file holds each entry once while the batch waits for its ack', async () => { + const membrane = new MockMembrane(); + await withFramework(membrane, async (framework, dir) => { + await framework.quiesce({ reason: 'refold' }); + const cm = framework.getAgent('agent')!.getContextManager(); + const add = (framework as unknown as { + addMessage(participant: string, content: unknown[]): unknown; + }).addMessage.bind(framework); + add('user', [{ type: 'text', text: 'first' }]); + add('user', [{ type: 'text', text: 'poison' }]); + add('user', [{ type: 'text', text: 'third' }]); + + const realAdd = cm.addMessage.bind(cm); + (cm as unknown as { addMessage: typeof cm.addMessage }).addMessage = ((participant, content, metadata, causedBy) => { + if ((content[0] as { text?: string }).text === 'poison') throw new Error('store write failed'); + return realAdd(participant, content, metadata, causedBy); + }) as typeof cm.addMessage; + // The batch's ack syncs the store before it rewrites the recovery file + // without what landed. While that sync fails, the batch stays in the file + // as it was handed off, beside what went back into the queue. + const store = (framework as unknown as { store: { sync(): void } }).store; + const realSync = store.sync.bind(store); + let syncFails = true; + store.sync = () => { + if (syncFails) throw new Error('sync failed'); + realSync(); + }; + try { + await framework.resume(); + } finally { + syncFails = false; + } + + const file = join(dir, 'store', 'recovery', 'deferred-writes.json'); + const pending = (JSON.parse(readFileSync(file, 'utf8')) as { + pending: Array<{ id: string; content: Array<{ text: string }> }>; + }).pending; + assert.deepEqual( + pending.map((entry) => entry.content[0]!.text), + ['first', 'poison', 'third'], + 'the entry that landed waits for its ack, and the two that went back are in the queue, each once', + ); + assert.equal(new Set(pending.map((entry) => entry.id)).size, 3, 'no id is written twice'); + }); +}); + test("resume stores several agents' writes in the order they arrived, and cuts each agent at its own stuck write, named at every flush", { timeout: 60_000 }, async () => { const dir = mkdtempSync(join(tmpdir(), 'host-quiesce-targets-')); const framework = await AgentFramework.create({ @@ -817,13 +863,15 @@ test('boot recovery drops a recovered deferred write the store could never take, await framework.stop(); // Entries no write of the framework's makes, as other code could leave - // them in the recovery file: a null block, and content that isn't blocks. + // them in the recovery file: a null block, content that isn't blocks, and + // a participant that isn't text, which the line names by its type. const file = join(storePath, 'recovery', 'deferred-writes.json'); const doc = JSON.parse(readFileSync(file, 'utf8')) as { pending: Array> }; assert.equal(doc.pending.length, 2); doc.pending.splice(1, 0, { id: 'null-block', seq: 100, participant: 'user', content: [{ type: 'text', text: 'x' }, null] }, { id: 'not-blocks', seq: 101, participant: 'user', content: 'x' }, + { id: 'number-participant', seq: 102, participant: 42, content: [{ type: 'text', text: 'x' }] }, ); writeFileSync(file, JSON.stringify(doc)); @@ -831,11 +879,11 @@ test('boot recovery drops a recovered deferred write the store could never take, framework = await AgentFramework.create(config()); console.error = error; assert.equal(framework.getHostModeStatus().deferredWrites, 2, 'the well-formed entries are restored'); - for (const entry of ['null-block (user)', 'not-blocks (user)']) { + for (const entry of ['null-block (user)', 'not-blocks (user)', 'number-participant (participant of type number)']) { assert.ok(lines.some((line) => line.includes(`malformed and was dropped: ${entry}`)), `${entry} is named:\n${lines.join('\n')}`); } const rewritten = readFileSync(file, 'utf8'); - assert.ok(!rewritten.includes('null-block') && !rewritten.includes('not-blocks'), 'the recovery file is rewritten without them'); + assert.ok(!['null-block', 'not-blocks', 'number-participant'].some((id) => rewritten.includes(id)), 'the recovery file is rewritten without them'); await framework.resume(); const texts = framework.getAgent('agent')!.getContextManager().getAllMessages().slice(before) .map((m) => (m.content[0] as { text: string }).text); From 39c4694d33f6eaab4b75d3c8552716efe2a7f09f Mon Sep 17 00:00:00 2001 From: Kestrel-2409 Date: Wed, 14 Oct 2026 06:57:42 -0400 Subject: [PATCH 0207/1112] fix(mcpl): a closed transport's held messages and pushes are kept for the next one The transport boundary dropped everything a server had sent that was still held when its transport closed. That included the world's content: a channel's incoming messages and its pushes. A bridge doesn't send those again: discord-mcpl advances and saves its forwarded watermark when it forwards a message, before the host answers, so the next process's catch-up sweep starts after it. A message held during a quiesce was lost for good when the transport dropped then. Main flushed it after the reconnect, and answered it on the new transport. - The boundary now drops only what speaks for the closed process: its requests (an inference request, a host/command), whose answers can't reach whoever asked, and its announcements (a tools/observe filter, a channel registration, a manifest change), which the next process makes for itself. channels/incoming and push/event stay held. They are admitted at the flush against the next transport's grant, like anything held. An answer to one is still never written to the next transport (sameTransport binds a responder to the transport it came on). - The set the boundary drops is built once per boundary, not once per held item. - The 'reconnect' event's doc says what a direct owner of the connection meets: the new transport starts with an empty grant and both planes closed, so the owner runs the policy exchange again and then calls ready(), as the framework's listener does. - McplRequestError's fragment says its name is McplRequestError, which is what String(error) and a stack trace begin with now. Its message is unchanged. Tests: - mcpl-reconnect-events: a channels/incoming request and a push held while the data plane is paused cross a transport drop. Both are delivered once the host establishes the next grant and opens the planes, and the request's answer isn't written to the next transport. With the boundary dropping them again, the test fails. - mcpl-request-outcome: the connection-loss case now checks that the loss itself rejected the request ("disconnected unexpectedly"), not its 10 s deadline. A new case covers close() with a request awaiting its answer: it rejects as McplRequestError 'no-response'. With close() rejecting a plain Error, it fails. Found by an internal review of this branch. Co-Authored-By: Claude Opus 5.5 --- .../mcpl-reconnect-registration.fixed.md | 10 +++-- changelog.d/mcpl-request-outcome.added.md | 4 +- src/mcpl/server-connection.ts | 36 ++++++++++------ test/mcpl-reconnect-events.test.ts | 41 +++++++++++++++++++ test/mcpl-request-outcome.test.ts | 13 +++++- 5 files changed, 86 insertions(+), 18 deletions(-) diff --git a/changelog.d/mcpl-reconnect-registration.fixed.md b/changelog.d/mcpl-reconnect-registration.fixed.md index 18460823e..cbe6e4d0a 100644 --- a/changelog.d/mcpl-reconnect-registration.fixed.md +++ b/changelog.d/mcpl-reconnect-registration.fixed.md @@ -13,10 +13,12 @@ planes opened or is announced as reconnected (`mcpl:server-reconnected`, `module:added`): its planes stay closed until the next transport's exchange settles, and an answer that arrives after its transport closed - grants nothing. What a server sent that is still held when its transport - closes, requests and notifications alike (a `tools/observe` filter, a - channel registration), is dropped rather than admitted under the next - transport's grant: the next process announces its own. An answer to a + grants nothing. What a server sent about its own process that is still + held when its transport closes (a request, a `tools/observe` filter, a + channel registration) is dropped rather than admitted under the next + transport's grant: the next process announces its own. Incoming channel + messages and pushes still held are kept and admitted under the next + grant, since the next process won't send them again. An answer to a request from a closed transport is never written to the next one. A `close()` during a reconnect's handshake now closes the new transport instead of adopting it. diff --git a/changelog.d/mcpl-request-outcome.added.md b/changelog.d/mcpl-request-outcome.added.md index 241ec226e..984e662b7 100644 --- a/changelog.d/mcpl-request-outcome.added.md +++ b/changelog.d/mcpl-request-outcome.added.md @@ -7,4 +7,6 @@ transport and no answer came back: a timeout, or the connection closed while awaiting; it may or may not have reached the server). Only `not-sent` proves the server never saw the request. Messages are unchanged, - and it is still an `Error`, so existing handling keeps working. + and it is still an `Error`, so existing handling keeps working; its `name` + is `McplRequestError`, which is what `String(error)` and a stack trace now + begin with. diff --git a/src/mcpl/server-connection.ts b/src/mcpl/server-connection.ts index 2cc4e700e..251b9cb7f 100644 --- a/src/mcpl/server-connection.ts +++ b/src/mcpl/server-connection.ts @@ -146,7 +146,11 @@ interface RequestOptions { * - `'close'` — Connection closed: `(code, signal, info: McplConnectionCloseInfo)` * - `'connect-failed'` — Initial connect failed; background retry scheduled * - `'reconnect-failed'` — A background reconnect attempt failed; next retry scheduled - * - `'reconnect'` — Background reconnect succeeded + * - `'reconnect'` — Background reconnect succeeded. The new transport + * starts with an empty grant and both planes + * closed, as at initial connect: whoever owns the + * connection runs the §5.3 policy exchange again, + * then calls ready() (the framework's listener does). */ export class McplServerConnection extends EventEmitter { /** Unique server identifier from config. */ @@ -233,16 +237,22 @@ export class McplServerConnection extends EventEmitter { this.grant = CapabilityGrant.empty(); this.policyEstablished = false; this.transportEpoch++; - // Everything the server sent that is still held in the buffer came on the - // transport that is now gone. Its requests can't be answered, and - // admitting one against the next transport's grant, or answering its id - // there, would act for a request the next server never made. Its - // notifications describe a process that is gone (a tools/observe filter, - // a channel registration): the next one announces its own. The host's - // own reports about the transport stay: its lifecycle (connect-failed, + // What the server sent that is still held came on the transport that is + // now gone. What it said about its own process is dropped: its requests + // wait on answers that can no longer reach whoever asked (an inference + // request, a host/command), and its announcements describe a process + // that is gone (a tools/observe filter, a channel registration), so the + // next one announces its own. What it reported about the world is kept: a + // channel's incoming messages and its pushes, which the next process + // won't send again (a bridge such as discord-mcpl advances its watermark + // as it forwards). They are admitted at the flush against the next + // transport's grant, like anything held, and an answer to one is never + // written to the next transport (sameTransport). The host's own reports + // about the transport stay too: its lifecycle (connect-failed, // reconnect-failed), the server's stderr, and orphaned-response audits. const held = this.bufferedEvents.length; - this.bufferedEvents = this.bufferedEvents.filter((item) => !McplServerConnection.SERVER_EVENTS.has(item.event)); + const dropped = McplServerConnection.PROCESS_EVENTS; + this.bufferedEvents = this.bufferedEvents.filter((item) => !dropped.has(item.event)); if (this.bufferedEvents.length < held) { console.error( `[mcpl] ${this.id}: dropped ${held - this.bufferedEvents.length} held message(s) from a transport that closed`, @@ -487,9 +497,11 @@ export class McplServerConnection extends EventEmitter { return typeof responder?.respond === 'function'; } - /** Events that carry a server's own messages, as routed from its transport. */ - private static get SERVER_EVENTS(): ReadonlySet { - return new Set(Object.values(McplServerConnection.METHOD_TO_EVENT)); + /** Events routed from a server's transport that speak for its process, not + * the world: dropped at a transport boundary while held. */ + private static get PROCESS_EVENTS(): ReadonlySet { + const world = new Set(['channels-incoming', 'push-event']); + return new Set(Object.values(McplServerConnection.METHOD_TO_EVENT).filter((event) => !world.has(event))); } /** Event-name form of METHOD_TO_REQUIRED_CAPABILITY, for admission-time diff --git a/test/mcpl-reconnect-events.test.ts b/test/mcpl-reconnect-events.test.ts index d2e2e098a..590de30da 100644 --- a/test/mcpl-reconnect-events.test.ts +++ b/test/mcpl-reconnect-events.test.ts @@ -248,6 +248,47 @@ describe('McplServerConnection — reconnect lifecycle events', () => { assert.deepStrictEqual(written.filter((line) => /"id":(5|7)\b/.test(line)), [], 'nothing answers the closed transport\'s requests'); }); + it('keeps an incoming message and a push held across a transport boundary, admits them under the next grant, and answers nothing there', async () => { + // What a server reported about the world isn't sent again by the next + // process: discord-mcpl advances its watermark as it forwards a message. + const grant = () => new CapabilityGrant(new Set(['channels.incoming', 'pushEvents']), []); + writeFileSync(FLAG_FILE, ''); + connection = await McplServerConnection.connect(makeConfig(), HOST_CAPS); + connection.establishGrant(grant()); + const incoming: Array<{ params: unknown; respond: (result: unknown) => void }> = []; + connection.on('channels-incoming', (params: unknown, responder: { respond: (result: unknown) => void }) => { + incoming.push({ params, respond: responder.respond }); + }); + const pushes: unknown[] = []; + connection.on('push-event', (params: unknown) => pushes.push(params)); + let reconnected = 0; + connection.on('reconnect', () => { reconnected++; }); + connection.ready(); + const serverSends = (message: Record) => + (connection as unknown as { transport: { emit(event: string, line: string): void } }).transport + .emit('line', JSON.stringify({ jsonrpc: '2.0', ...message })); + // A quiesce holds the data plane while a member's message and a push arrive. + connection.pauseDataPlane(); + serverSends({ id: 9, method: 'channels/incoming', params: { messages: [{ id: 'held-1' }] } }); + serverSends({ method: 'push/event', params: { n: 1 } }); + assert.equal(incoming.length + pushes.length, 0, 'held while paused'); + + connection.sendToolsList().catch(() => {}); + await waitFor(() => reconnected === 1, 5000, 'reconnect after crash'); + const written: string[] = []; + const transport = (connection as unknown as { transport: { writeLine(line: string): void } }).transport; + const writeLine = transport.writeLine.bind(transport); + transport.writeLine = (line: string) => { written.push(line); writeLine(line); }; + assert.equal(incoming.length + pushes.length, 0, 'still held until the planes open'); + + connection.establishGrant(grant()); + connection.ready(); + assert.deepStrictEqual(incoming.map((message) => message.params), [{ messages: [{ id: 'held-1' }] }], 'delivered under the new grant'); + assert.deepStrictEqual(pushes, [{ n: 1 }]); + incoming[0]!.respond({ ok: true }); + assert.deepStrictEqual(written.filter((line) => /"id":9\b/.test(line)), [], 'its answer is not written to the next transport'); + }); + it('drops what a transport that closed during its exchange sent, so the next transport\'s grant never admits it', async () => { writeFileSync(FLAG_FILE, ''); connection = await McplServerConnection.connect(makeConfig(), HOST_CAPS); diff --git a/test/mcpl-request-outcome.test.ts b/test/mcpl-request-outcome.test.ts index 1b0f45647..1b07bc699 100644 --- a/test/mcpl-request-outcome.test.ts +++ b/test/mcpl-request-outcome.test.ts @@ -73,7 +73,18 @@ test('a request nobody answers is no-response, and the message is unchanged', as test('a connection lost while awaiting is no-response', async () => { connection = await McplServerConnection.connect(config({ requestTimeoutMs: 10_000 }), HOST_CAPS); - await rejectsWith(connection.sendToolsCall('exit', {}), 'no-response'); + const err = await rejectsWith(connection.sendToolsCall('exit', {}), 'no-response'); + // The loss itself rejects it, not the request's own deadline. + assert.match(err.message, /disconnected unexpectedly .* while awaiting tools\/call/); +}); + +test('a connection the host closes while a request awaits its answer is no-response', async () => { + connection = await McplServerConnection.connect(config({ requestTimeoutMs: 10_000 }), HOST_CAPS); + const rejected = rejectsWith(connection.sendToolsCall('stuck', {}), 'no-response'); + await connection.close(); + const err = await rejected; + assert.match(err.message, /closed while awaiting response for tools\/call/); + connection = null; }); test('a request on a closed connection is not-sent', async () => { From 4b9fc6b4acb28b5463f35267de95aa148aa53f0d Mon Sep 17 00:00:00 2001 From: Kestrel-2409 Date: Wed, 14 Oct 2026 06:58:17 -0400 Subject: [PATCH 0208/1112] fix(awareness): a stopping host claims no marks; a failed import, a start's give-ups and a replayed release say so stop() and delivery: - A drain stops claiming once stop() begins (providerAdmissionClosed is set from its start until the next start()). Marks still due stay queued for the next start, rather than going out, or being cut off and recorded unknown, while the host shuts down. - stop() waits for running drains whatever the teardown's result. The merge made that wait depend on a successful teardown, on the view that a connection teardown couldn't close might still be serving a drain. It can't: the registry calls close() at once, and close() rejects a pending request before it awaits its transport, so a drain records that outcome and ends within microtasks. The wait stays, unconditional, as the ordering it states (no drain writes after the store closes). Removing it fails no test, since a drain has always finished by then. The journal: - load() cached its state before importing a legacy ledger, so an import that failed (an unreadable or invalid file) failed the first read, and every later read in that process answered without the import. A failed import now takes the cached state back: every read tries it again and refuses until it succeeds. Startup and the CLI fail on the first read either way; the class is exported. - An attempt the previous process left on the wire counts toward the three-attempt bound like any unanswered one. recoverAtStartup now says how many operations that brought to the bound (givenUp), and create() raises the same discord-awareness-delivery ops alert a delivery pass raises for operations it gives up on. Before, such an operation was never sent again, and nothing said so. - A refused release names what settled the batch: "it was released at