fix: what reaches a request without the context store is made well formed where it leaves for it

changelog.d/request-text-as-stored.fixed.md

@@ -0,0 +1 @@
+- What reaches a request without passing through the context store now carries a lone UTF-16 surrogate as U+FFFD, as the store keeps it (#695). That's a context injection's text and namespace, a hook's or a module's, and what a tool boundary injects into the live round: the messages it heard, with their senders' names, and its own notices. Before, every request an injection was in carried the lone half raw, and the API refuses such a request with a 400 ("The request body is not valid JSON: unexpected end of hex escape"). A namespace is still compared with channel ids as given when an injection is kept to its channel's conversation.

src/agent.ts

@@ -4,6 +4,7 @@ import { createHash } from 'node:crypto';
 import type { CacheWireReceipt, KvUnifiedRequestHooks } from './kv-unified-wire.js';
 import { ToolResultGuard } from './tool-result-guard.js';
 import { withoutTokenEstimates } from './thinking-token-stamp.js';
+import { injectionsAsStored } from './text-as-stored.js';
 import { assertBooleanSetting, holdsSettings } from './config-checks.js';
 import {
   CONTINUE_ROW,
@@ -1157,6 +1158,8 @@ export class Agent {
     if (this._state.status === 'inferring') {
       throw new Error(`Agent ${this.name} is already inferring`);
     }
+    // Unstored, so they leave for the request here (text-as-stored.ts).
+    injections = injectionsAsStored(injections);
 
     if (this._state.status === 'waiting_for_tools') {
       throw new Error(`Agent ${this.name} is waiting for tool results`);
@@ -1332,6 +1335,9 @@ export class Agent {
     compressionTools: ToolDefinition[] = availableTools,
     options: ActivationRequestOptions = {},
   ): Promise<{ request: NormalizedRequest; continuePrompt?: RequestOnlyPrompt }> {
+    // Unstored, so they leave for the request here (text-as-stored.ts), the
+    // layout hash below included.
+    injections = injectionsAsStored(injections);
     // Compression may revisit hidden tools in recorded history. Keep its full
     // definition set separate from the resident's advertised tools: the
     // autobiographical strategy must declare the same tools on its

src/framework.ts

@@ -157,6 +157,7 @@ import { bounded, renderList, renderReturn, VALUE_CHARS } from './code-execution
 import { splitProseRuns, withoutRoundsProse } from './prose-segments.js';
 import { cumulativeDelta } from './usage-accounting.js';
 import { stampThinkingTokenEstimates } from './thinking-token-stamp.js';
+import { contentAsStored, nameAsStored } from './text-as-stored.js';
 
 /** An MCPL tool call refused because its server isn't in the registry: one
  *  closed for good, one removed, or one whose connect failed without
@@ -929,7 +930,8 @@ function describeDeferredWrite(msg: { id: string; participant: string; content:
 function isInjectableMidTurn(msg: { participant: string; content: ContentBlock[] }, agentName: string): boolean {
   // Exact, as membrane's role test is: only a participant equal to the
   // request's assistantParticipant (Agent's name) renders as the assistant.
-  return msg.participant !== agentName
+  // It's the name as sent, which the boundary takes as the store keeps it.
+  return nameAsStored(msg.participant) !== agentName
     && !msg.content.some((b) => b.type === 'tool_use' || b.type === 'tool_result');
 }
 
@@ -12145,11 +12147,14 @@ export class AgentFramework {
             // 0.5.x, and from 0.6.0 declares it can't carry them, so none are
             // collected for it. Each goes as its participant and content
             // alone: a stored row's metadata is the host's record, not input
-            // for membrane.
+            // for membrane. Each goes with its name and text as the store
+            // keeps them, whether it's a heard message, rendered or not, or a
+            // notice the boundary made: this is where they leave for the
+            // request.
             currentState.stream.provideToolResults(
               agent.toolResultGuard.submissionResults(membraneResults),
               midTurnInjections.length > 0
-                ? { injectedMessages: midTurnInjections.map(({ participant, content }) => ({ participant, content })) }
+                ? { injectedMessages: midTurnInjections.map(({ participant, content }) => ({ participant: nameAsStored(participant), content: contentAsStored(content) })) }
                 : undefined,
             );
             // RFC-006 §3.3: consumption is determined at assembly, and this

src/text-as-stored.ts

@@ -0,0 +1,73 @@
+/**
+ * A request carries only text the context store would keep. Apart from the
+ * agent's own configuration, such as its name and system prompt, what a
+ * request compiles comes from the store, which writes a lone UTF-16
+ * surrogate as U+FFFD, and a provider refuses a request carrying one (#695).
+ * Two things reach a request without the store, and each is made well formed
+ * where it leaves for it: context injections, as an Agent compiles them into
+ * a request, and the copies a tool boundary injects into the live round, as
+ * they go to the stream.
+ */
+
+import type { ContentBlock } from '@animalabs/membrane';
+import type { ContextInjection } from '@animalabs/context-manager';
+
+/** A value with every string in it as the context store keeps it, through
+ *  arrays and plain objects. Keys, and objects that aren't plain, pass as
+ *  they are: content blocks are plain JSON data. Whatever is already well
+ *  formed keeps its identity. */
+function stringsAsStored<T>(value: T): T {
+  if (typeof value === 'string') return (value.isWellFormed() ? value : value.toWellFormed()) as T;
+  if (Array.isArray(value)) {
+    const out = value.map(stringsAsStored);
+    return (out.every((v, i) => v === value[i]) ? value : out) as T;
+  }
+  if (value !== null && typeof value === 'object') {
+    const proto = Object.getPrototypeOf(value);
+    if (proto !== Object.prototype && proto !== null) return value;
+    let changed = false;
+    const out: Record<string, unknown> = {};
+    for (const [k, v] of Object.entries(value)) {
+      const s = stringsAsStored(v);
+      if (s !== v) changed = true;
+      out[k] = s;
+    }
+    return (changed ? out : value) as T;
+  }
+  return value;
+}
+
+/** Content with every string in it well formed, as the context store keeps
+ *  a text block's text or an image's link. */
+export function contentAsStored(content: ContentBlock[]): ContentBlock[] {
+  return stringsAsStored(content);
+}
+
+/** A sender's name as the context store keeps it, and as a tool boundary
+ *  sends it: a lone UTF-16 surrogate becomes U+FFFD. A name that isn't a
+ *  string, from an untyped module, passes as it did: thrown here, inside the
+ *  boundary, it would keep the round's results from the stream, where it
+ *  would otherwise fail at the next compile. */
+export function nameAsStored(participant: string): string {
+  return typeof participant === 'string' ? participant.toWellFormed() : participant;
+}
+
+/** Injections with their text as the context store would keep it: each one's
+ *  content, and its namespace, which context-manager writes into a beforeUser
+ *  or afterUser injection's participant name by its text form. A namespace
+ *  that isn't a string, which MCPL §10.4 doesn't allow but the hook
+ *  orchestrator keeps when it can be read as text, is taken by that form.
+ *  They're scoped to a conversation before this, by the namespace as given,
+ *  since channel ids are compared as given. One that's already well formed
+ *  keeps its identity. */
+export function injectionsAsStored(injections: ContextInjection[] | undefined): ContextInjection[] | undefined {
+  return injections?.map((inj) => {
+    const ns: unknown = inj.namespace;
+    const named = typeof ns === 'string' || ns === undefined || ns === null ? ns : String(ns);
+    const namespace = typeof named === 'string' && !named.isWellFormed() ? named.toWellFormed() : ns;
+    const content = contentAsStored(inj.content);
+    return namespace === ns && content.every((b, i) => b === inj.content[i])
+      ? inj
+      : { ...inj, namespace: namespace as string, content };
+  });
+}

test/conversation-fork-injection-scoping.test.ts

@@ -199,6 +199,30 @@ describe('conversation fork injection scoping', () => {
     assertContains(system, 'hook retrieval context');
   });
 
+  it('scopes by a namespace as given, so one holding a lone surrogate that names another channel stays out of the fork', async () => {
+    // The namespace is made well formed for the request only after scoping:
+    // channel ids are compared as given, and well formed first it would name
+    // no channel and reach every agent.
+    membrane.pushResponse(createMockResponse([]));
+    const other = 'chanB \ud83d';
+    const framework = await makeFramework([
+      systemInjection('module other context', other),
+      systemInjection('module home context', 'chanA'),
+    ], [FORK_AGENT]);
+    installChannelRegistry(framework, ['chanA', other]);
+    bindForkHome(framework);
+    installHookInjections(framework, [systemInjection('hook other context', other)]);
+
+    framework.pushEvent(triggerEvent());
+    await framework.runUntilIdle();
+
+    assert.equal(membrane.calls.length, 1);
+    const system = membrane.calls[0]!.system;
+    assertOmits(system, 'module other context');
+    assertOmits(system, 'hook other context');
+    assertContains(system, 'module home context');
+  });
+
   it('passes module injections through for non-conversation agents', async () => {
     membrane.pushResponse(createMockResponse([]));
     const framework = await makeFramework([

test/fixtures/hook-injection-mcpl-server.mjs

@@ -0,0 +1,54 @@
+// MCPL fixture for hook-injection tests: answers context/beforeInference with
+// one injection whose text is TEXT_JSON's, at POSITION ('beforeUser', 'afterUser' or
+// 'system'), as a string when SHAPE is 'string', as a block whose type is the
+// text when SHAPE is 'unknown-type', and as a text block otherwise.
+// It lists one tool, `ready`, so a test can tell it's connected.
+import { createInterface } from 'node:readline';
+
+const position = process.env.POSITION || 'beforeUser';
+// TEXT_JSON is the text as JSON: an environment variable can't carry a lone
+// UTF-16 surrogate, which the JSON escape can.
+const text = process.env.TEXT_JSON ? JSON.parse(process.env.TEXT_JSON) : '';
+// NAMESPACE_JSON, the same way, is the injection's namespace; 'notes' without it.
+const namespace = process.env.NAMESPACE_JSON ? JSON.parse(process.env.NAMESPACE_JSON) : 'notes';
+const send = (message) => process.stdout.write(JSON.stringify(message) + '\n');
+const reply = (id, result) => send({ jsonrpc: '2.0', id, result });
+
+const rl = createInterface({ input: process.stdin });
+rl.on('close', () => process.exit(0));
+rl.on('line', (line) => {
+  let msg;
+  try { msg = JSON.parse(line); } catch { return; }
+  if (msg.method === 'initialize') {
+    reply(msg.id, {
+      protocolVersion: '2024-11-05',
+      capabilities: {
+        tools: {},
+        experimental: {
+          mcpl: {
+            version: '0.5',
+            contextHooks: { beforeInference: { inject: { [position]: true } } },
+            featureSets: {
+              notes: { description: 'injects a note', uses: [`contextHooks.beforeInference.inject.${position}`] },
+            },
+          },
+        },
+      },
+      serverInfo: { name: 'hook-injection-fixture', version: '0.0.0' },
+    });
+    return;
+  }
+  if (msg.method === 'context/beforeInference') {
+    const content = process.env.SHAPE === 'string' ? text
+      : process.env.SHAPE === 'unknown-type' ? [{ type: text }]
+        : [{ type: 'text', text }];
+    reply(msg.id, { featureSet: 'notes', contextInjections: [{ namespace, position, content }] });
+    return;
+  }
+  if (msg.method === 'tools/list') {
+    // A tool the test waits for, which says the server is connected.
+    reply(msg.id, { tools: [{ name: 'ready', description: 'Says the fixture is connected.', inputSchema: { type: 'object', properties: {} } }] });
+    return;
+  }
+  if (msg.id !== undefined && msg.id !== null && msg.method) reply(msg.id, {});
+});

test/helpers/mock-membrane.ts

@@ -211,6 +211,11 @@ export class MockMembrane {
     return this.responses[this.responseIndex++];
   }
 
+  /** The direct Agent API's call, recorded and answered as complete is. */
+  async stream(request: NormalizedRequest, _options?: unknown): Promise<NormalizedResponse> {
+    return this.complete(request);
+  }
+
   streamYielding(request: NormalizedRequest, _options?: unknown): YieldingStream {
     this.calls.push(request);
     const remaining = this.responses.slice(this.responseIndex);