From 223c9084eb6f9adedd0b1f1a87c9ec0e3d8088a1 Mon Sep 17 00:00:00 2001 From: JUN Date: Sun, 27 Sep 2026 12:10:15 +0900 Subject: [PATCH 1/3] docs(devlog): plan Codex App visualization references for routed models --- .../000_plan.md | 58 +++++++++++ .../001_app_and_external_evidence.md | 53 ++++++++++ ...2_visualization_directive_normalization.md | 98 +++++++++++++++++++ .../020_wp3_rebuild_and_live_verify.md | 25 +++++ 4 files changed, 234 insertions(+) create mode 100644 devlog/_plan/260927_directive_marker_bridge/000_plan.md create mode 100644 devlog/_plan/260927_directive_marker_bridge/001_app_and_external_evidence.md create mode 100644 devlog/_plan/260927_directive_marker_bridge/010_wp2_visualization_directive_normalization.md create mode 100644 devlog/_plan/260927_directive_marker_bridge/020_wp3_rebuild_and_live_verify.md diff --git a/devlog/_plan/260927_directive_marker_bridge/000_plan.md b/devlog/_plan/260927_directive_marker_bridge/000_plan.md new file mode 100644 index 00000000000..5f8390bf7ba --- /dev/null +++ b/devlog/_plan/260927_directive_marker_bridge/000_plan.md @@ -0,0 +1,58 @@ +# Codex App visualization references for models that cannot see private-use characters + +## Objective + +A routed model must be able to show a Codex App inline visualization. Today only models whose +provider preserves Basic Multilingual Plane private-use characters can, because the reference the +bundled Visualize skill teaches is `U+E200 visualize U+E202 {json} U+E201`. + +Observed on 2026-09-27 against the local proxy (2.68.0, `a1285fc648`): asked to list the code +points of `[A U+E200 B U+E202 C U+E201]`, `gpt-6-luna` and `xai/grok-4.7` return all six, while +`anthropic/claude-opus-5-5` and `cursor/claude-opus-5-5` return only `A B C`. OpenCodex's own +Anthropic request body still contains the three characters (checked by building the request in +process), so they are removed after the proxy. Claude therefore reads the skill template as +`visualize{"path":...}`, writes that plain text back, and the app shows it verbatim. + +## Constraints + +- Native OpenAI/ChatGPT passthrough stays byte-identical: it serializes `_rawBody`. +- Stored request history stays raw (`adapter-delivery.ts` hands `_rawBody` to `state.ts`). +- The citation filter semantics from #3150, #3843 and #6040 are unchanged. +- No new dependency, no Lab import on the request path, no file-size ratchet increase. + +## Work-phase map + +| Work-phase | Doc | Outcome | +|---|---|---| +| wp1 | this directory, 000-001 | Evidence and roadmap (docs only) | +| wp2 | [010](010_wp2_visualization_directive_normalization.md) | Parser-side normalization, tests, PR, CI, squash merge | +| wp3 | [020](020_wp3_rebuild_and_live_verify.md) | Local app rebuild from merged dev with a fresh sidecar, live Claude check | + +wp2 depends on the app evidence in [001](001_app_and_external_evidence.md); wp3 depends on wp2 being on `dev`. + +## Architect consultation (formal P) + +Architect: gpt-6-astra subagent `01a0e0ce-58fd-7fd3-8186-12e3752ec8f6` (Fermat), read-only. +Proposal D1-D14 received 2026-09-27. Main dispositions: + +| Id | Proposal | Disposition | +|---|---|---| +| D1 | New pure module `src/responses/visualization-directives.ts`, copy-on-change | Accepted | +| D2 | Hook at the parser's final `context` | Accepted; covers the replay expansion and the encrypted-agent reparse | +| D3 | Normalize conversation text only (system prompt, message strings, text parts), including fenced code | Accepted | +| D4 | Match only complete literal `U+E200visualize U+E202 … U+E201` spans | Accepted | +| D5 | Payload rules mirror the app's `f2` exactly | Accepted | +| D6 | Single template exception for `/.html` | Accepted | +| D7 | `codex-live-vis` for `type:"live"`, `mode="wide"` only for inline `mode:"wide"` | Accepted | +| D8 | Raw-body passthrough excluded | Accepted. Whether every raw-body route preserves the characters is not established; the observed failures are both on context-built adapters | +| D9 | No persistence change; history stays raw | Accepted | +| D10 | Deterministic output; Cursor checkpoint digest will differ once for affected prefixes | Accepted; the digest mismatch invalidates the old checkpoint, which is the safe direction. No dedicated Cursor test, recorded as residual | +| D11 | No output repair of bare `visualize{json}` | Accepted; revisit only if live checks show a model still emitting the bare form | +| D12 | Alternatives | Rejected: adapter hooks duplicate the rule per adapter and miss provider switches mid-thread; a request-prepare hook misses the encrypted-agent reparse and direct parser callers; changing the upstream skill text does not reach installed plugins or existing history | +| D13 | Sibling test file registered in both layout manifests | Accepted; replay-after-restart and Cursor checkpoint tests narrowed to raw-body and idempotence assertions | +| D14 | Live UI confirmation of the ASCII directive still pending | Tracked in wp3 | + +Limitations carried forward: raw-body passthrough routes are not normalized; replies already stored as bare `visualize{...}` are not repaired; a future template variant needs its own fixture; the live render of the ASCII form is confirmed only in wp3. + +Reflection: recorded in [010](010_wp2_visualization_directive_normalization.md#architect-reflection). + diff --git a/devlog/_plan/260927_directive_marker_bridge/001_app_and_external_evidence.md b/devlog/_plan/260927_directive_marker_bridge/001_app_and_external_evidence.md new file mode 100644 index 00000000000..c23fabc6e4d --- /dev/null +++ b/devlog/_plan/260927_directive_marker_bridge/001_app_and_external_evidence.md @@ -0,0 +1,53 @@ +# Evidence: how the Codex App reads visualization references + +## App bundle (primary) + +Source: `/Applications/ChatGPT.app` (bundle `com.openai.codex`, 26.924.22138), `Contents/Resources/app.asar` +extracted read-only to `.tmp/asar/app`. Paths below are inside `webview/assets/`. + +- `app-shared-36eae88777f2.js`, function `f2`: every span matching + `/U+E200visualize U+E202([^U+E201]+)U+E201/g` outside markdown code tokens is rewritten to + `::codex-inline-vis{path="<abs>" title="…" mode="wide"}` before rendering. Payload handling: + - payload starting with `{` is `JSON.parse`d (failure keeps the span); otherwise it is `{path: payload}`; + - schema `{path: string|null, title?: string, type?: "inline"|"live", mode?: "wide", wide?: boolean}`; + - `path: null` becomes `::codex-live-vis{}` for `type:"live"` and is otherwise kept; + - the span is kept when the path has a `..` segment, a double quote or CR/LF, when its basename does + not match `/^[a-z0-9]+(?:-[a-z0-9]+)*\.html$/`, or when it is not absolute and is either JSON or not a bare basename; + - attribute is `path` for an absolute path and `file` for a bare basename; + - `title` is emitted only without quotes or CR/LF; `mode="wide"` only for inline with `mode:"wide"`. +- Same file, function `Err`: a message is a visualization when it contains the private-use form OR the + literal `::codex-inline-vis`, and parsing `f2(text)` yields a `codexDirective` named + `codex-inline-vis`. The plain ASCII directive is therefore the app's canonical form. +- Same file, `sj` (absolute path): `/…` but not `//…`, `/^[A-Za-z]:[\\/]/`, `/^\\\\[^\\]+\\[^\\]+/`, `/^\/\/[^/]+\/[^/]+/`. +- `app-primary-cca0c1a58f0f.js`, `i3e`: registers `renderElement` for the inline directive and reads + its `path` or `file` attribute; `o8e` attaches it whenever `renderInlineVisualizations` is set, which + `conversation-blocks-1768e325fd75.js` (`qg`) sets for assistant message blocks. +- `app-primary-cca0c1a58f0f.js`, `a3e`: while streaming, a trailing partial `U+E200visualize U+E202` is + hidden until it closes. + +## External sources + +| Claim | Source | Status | +|---|---|---| +| OpenAI documents the private-use citation grammar (`U+E200 cite U+E202 … U+E201`) | https://developers.openai.com/api/docs/guides/citation-formatting | primary | +| Claude Code strips BMP private-use characters (U+E000-U+F8FF) from tool input and output; supplementary PUA-A survives | https://github.com/anthropics/claude-code/issues/44525 (2026-04-07) | primary report | +| Same class reported earlier for U+E0A0 | https://github.com/anthropics/claude-code/issues/31849 (2026-03-07) | primary report | +| Streamed citation markers leak into third-party output | https://community.openai.com/t/streamed-web-search-citations-leaking-citation-markers-into-text-output/1390157 (2026-08-12) | lead | +| Codex desktop instructions use plain `::name{…}` directives (`::code-comment`, `::created-thread`) | Codex App system prompt in this session; community reproduction | primary (session) | + +Luna lanes: Plato, Ampere, Avicenna (`gpt-5.6-luna`). Aside exec session `boOE2Tp2gdGCg2Do`, +report under `~/.aside/u/0/artifacts/ocx-directive-research/`. + +## Conclusion + +Converting the private-use reference into `::codex-inline-vis{…}` before the model sees it gives a model +that cannot see private-use characters an ASCII instruction it can read and repeat. The bundle parses that +form directly (`Err`, `f2`); the live render is confirmed in wp3. + + +## Session observation + +A commentary message in this thread that contained `::codex-inline-vis{…}` reached the rollout only as a +`reasoning` summary, not as an assistant `message`, so the user saw plain text. Other commentary blocks +in the same thread show the same pattern. The render test therefore uses a final answer. How routed +Claude text becomes a reasoning item is outside this unit. diff --git a/devlog/_plan/260927_directive_marker_bridge/010_wp2_visualization_directive_normalization.md b/devlog/_plan/260927_directive_marker_bridge/010_wp2_visualization_directive_normalization.md new file mode 100644 index 00000000000..0adec529e9f --- /dev/null +++ b/devlog/_plan/260927_directive_marker_bridge/010_wp2_visualization_directive_normalization.md @@ -0,0 +1,98 @@ +# wp2: normalize visualization references in model-visible text + +## Scope + +IN: `src/responses/visualization-directives.ts` (new), `src/responses/parser.ts` (one call at the +return), `tests/responses/visualization-directives.test.ts` (new), `scripts/test-layout/layout.json`, +`tests/fixtures/test-layout-expected.json`, `structure/transports/responses.md`, +`docs-site/src/content/docs/guides/codex-integration.md` (new subsection after "Routed local tools"). +OUT: adapters, raw-body passthrough, response-side repair, persistence, citation filter. + +## Files + +### NEW `src/responses/visualization-directives.ts` + +Exports: + +- `normalizeVisualizationText(text: string): string` — returns `text` unchanged (same reference) + unless it contains the exact prefix `"\uE200visualize\uE202"` (no spaces). It replaces matches of + the app's own regex `/\uE200visualize\uE202([^\uE201]+)\uE201/g` (one left-to-right pass) with + `toAsciiDirective(payload)`, or keeps the match when rejected. Spans with any other keyword never + match that regex and are copied through unchanged, as are unterminated spans. + + Intentional differences from `f2`: code tokens are normalized too (the skill's example is fenced), + and the template exception below. Using the same regex means a `visualize` span that appears after + a malformed START of another keyword is converted exactly as the app would convert it. +- `normalizeVisualizationContext(context: OcxContext): OcxContext` — copy-on-change over + `systemPrompt`, string `content`, and `type:"text"` parts of every message role; returns the same + object when nothing changed. + +`toAsciiDirective(payload)` follows [001](001_app_and_external_evidence.md) rule for rule: every valid +`type:"live"` payload becomes `::codex-live-vis{…}` with no `mode`; `wide:true` is validated and +ignored; `mode="wide"` only for inline. Template exception: a JSON payload whose `path` is exactly +`<absolute-path>/<title>.html` skips path validation and always uses the `path` attribute (the app's +`sj` would pick `file` for it); schema, type, title and mode rules still apply. Both skill templates +have exact-output assertions. + +### MODIFY `src/responses/parser.ts` + +```diff ++import { normalizeVisualizationContext } from "./visualization-directives"; +@@ return { +- context, ++ context: normalizeVisualizationContext(context), +``` + +### NEW `tests/responses/visualization-directives.test.ts` + +Cases: absolute JSON path; title; wide; live with path and with null path; bare absolute path; bare +basename (`file=`); both skill templates; spans inside a fenced block; rejected payloads kept (invalid +JSON, relative JSON path, `..`, quote, bad basename, missing path); other keywords and unterminated +spans untouched; Windows drive and UNC paths; live null path with a title; malformed optional fields +(`title` number, `type` other, `mode` other, `wide` string); a decoded compaction summary carrying the +reference; a large context with many malformed markers (linear time); idempotence; every +role and content shape through `parseRequest`, with image parts, tool-call arguments and reasoning +parts deep-equal to an unnormalized parse; a frozen input body (deep `Object.freeze`) parses without +throwing and `_rawBody` is the same object; an expanded continuation body (prior assistant output +with the private-use reference plus a new user turn) normalizes both turns; the Anthropic adapter +body contains the ASCII directive and no private-use character. + +### MODIFY layout manifests and `structure/transports/responses.md` + +Register the test under `responses`; add one paragraph to the transport doc naming the module and the +raw-body exclusion. + +### MODIFY `docs-site/src/content/docs/guides/codex-integration.md` + +New `### Inline visualizations with routed models` after `### Routed local tools`: the private-use +reference is rewritten to `::codex-inline-vis{…}` for context-built routes, raw-body passthrough routes +are unchanged, and replies already stored in the bare `visualize{…}` form stay as they are. Locale +copies gain nothing, so they do not contradict the English page. + +## Verification (PLAN-VERIFIER-REAL-01) + +| Command | Reads the target | +|---|---| +| `bun test tests/responses/visualization-directives.test.ts` | direct argument | +| `bun test tests/responses/citation-markers.test.ts tests/adapters/bridge.test.ts` | callers of the response path | +| `bun test tests/responses/responses-parser.test.ts tests/responses/responses-parser-agent-message.test.ts tests/responses/responses-parser-malformed-content.test.ts tests/responses/parser-content-audio.test.ts tests/responses/responses-state.test.ts` (run locally) | parser and replay consumers | +| `bun test tests/lab/core-lab-boundary.test.ts tests/ci-workflows/file-size-ratchet.test.ts tests/test-layout.test.ts tests/test-layout-tooling.test.ts` | Lab boundary, ratchet, layout manifests (source-reading guards `test:changed` cannot see) | +| `bun run test:changed` | import graph from `parser.ts`; local run is limited by the `~/.codex` checkout guard, the full suite is left to CI | +| `bun run typecheck`, `bun run structure:check`, `bun run privacy:scan` | whole tree | + +Activation scenarios: a rejected payload (C runs the "kept" cases and sees the private-use span unchanged); +a live null path (C sees `::codex-live-vis{}`). + +## Architect reflection + +Fermat returned MISALIGNED on the first plan revision with five gaps. Dispositions: + +1. Unknown spans opaque, nested marker fixture, exact prefix — folded above. +2. Live selection, ignored `wide`, template exception scope — folded above. +3. Opaque-field, frozen-input and replay assertions — folded above. Cursor checkpoint rejection test — + rebutted: the Cursor builder compares a digest of the context it is given, and this change only + alters that context's text, so an old checkpoint mismatches and is discarded, which is the existing + safe path; recorded as residual instead of a new Cursor fixture. +4. Existing suites and source-reading guards named explicitly — folded above. +5. Universal claims removed; alternatives' reasons and limitations recorded in 000 and 001 — folded. + diff --git a/devlog/_plan/260927_directive_marker_bridge/020_wp3_rebuild_and_live_verify.md b/devlog/_plan/260927_directive_marker_bridge/020_wp3_rebuild_and_live_verify.md new file mode 100644 index 00000000000..3ac1294ebbc --- /dev/null +++ b/devlog/_plan/260927_directive_marker_bridge/020_wp3_rebuild_and_live_verify.md @@ -0,0 +1,25 @@ +# wp3: rebuild the local app and verify with a Claude-routed model + +1. In the primary checkout, fast-forward `dev` to the merge commit. +2. Force a fresh standalone sidecar: `bun run build:standalone --target bun-darwin-arm64`. + `desktop/scripts/prepare-sidecar.ts` reuses `dist/standalone/*/ocx` when it exists, which shipped a + stale 2.61.0 proxy inside the 2.68.0 app on 2026-09-27. +3. `bun run build:gui`, then in `desktop/`: `bun run prepare-sidecar`, `bun run prepare-widget`, and + `bun run build:local` with the shared `.git/config` `core.bare` set to `false` only for the build + (Cargo reads it) and restored afterwards. +4. Extract `OpenCodex.app` from the DMG, verify `codesign --verify --deep --strict`, and check that + the bundled `ocx` contains `codex-inline-vis`. +5. Protocol check against the packaged sidecar once the user has reinstalled the app: confirm + `/healthz` reports the rebuilt version and that the running binary is the reinstalled bundle, then send a + Claude-routed request whose input carries the private-use reference and confirm the reply contains + `::codex-inline-vis{path="…"}` and no private-use character. +6. Render check in the Codex App: in this Claude-routed thread, write a real HTML fragment under the + thread's visualization directory and put `::codex-inline-vis{path="…"}` on its own line in a + **final answer** (a commentary block can be recorded as a reasoning summary, which the app does not + render as markdown directives; observed 2026-09-27). The user reports whether it renders. + Computer Use cannot inspect `com.openai.codex` (blocked by its safety policy), so rendering stays + "pending user confirmation" until that report. + +Shared `.git/config` note: Cargo (libgit2) ignores `config.worktree`, so the build flips `core.bare` for +the shortest possible window and restores it; do not run it while another task is using the checkout's +git config. From 44151d392979c505df497cd428074039cee0ad16 Mon Sep 17 00:00:00 2001 From: JUN <jun@lidgeai.com> Date: Sun, 27 Sep 2026 12:19:03 +0900 Subject: [PATCH 2/3] fix(responses): give routed models the Codex App visualization directive in ASCII --- .../000_plan.md | 1 - ...2_visualization_directive_normalization.md | 23 +- .../content/docs/guides/codex-integration.md | 13 + scripts/test-layout/layout.json | 1 + src/responses/parser.ts | 5 +- src/responses/visualization-directives.ts | 182 ++++++++++++++ structure/transports/responses-wire-shapes.md | 15 ++ tests/fixtures/test-layout-expected.json | 1 + .../visualization-directives.test.ts | 225 ++++++++++++++++++ 9 files changed, 458 insertions(+), 8 deletions(-) create mode 100644 src/responses/visualization-directives.ts create mode 100644 tests/responses/visualization-directives.test.ts diff --git a/devlog/_plan/260927_directive_marker_bridge/000_plan.md b/devlog/_plan/260927_directive_marker_bridge/000_plan.md index 5f8390bf7ba..16322b406b1 100644 --- a/devlog/_plan/260927_directive_marker_bridge/000_plan.md +++ b/devlog/_plan/260927_directive_marker_bridge/000_plan.md @@ -55,4 +55,3 @@ Proposal D1-D14 received 2026-09-27. Main dispositions: Limitations carried forward: raw-body passthrough routes are not normalized; replies already stored as bare `visualize{...}` are not repaired; a future template variant needs its own fixture; the live render of the ASCII form is confirmed only in wp3. Reflection: recorded in [010](010_wp2_visualization_directive_normalization.md#architect-reflection). - diff --git a/devlog/_plan/260927_directive_marker_bridge/010_wp2_visualization_directive_normalization.md b/devlog/_plan/260927_directive_marker_bridge/010_wp2_visualization_directive_normalization.md index 0adec529e9f..0e5550f22cb 100644 --- a/devlog/_plan/260927_directive_marker_bridge/010_wp2_visualization_directive_normalization.md +++ b/devlog/_plan/260927_directive_marker_bridge/010_wp2_visualization_directive_normalization.md @@ -15,10 +15,13 @@ OUT: adapters, raw-body passthrough, response-side repair, persistence, citation Exports: - `normalizeVisualizationText(text: string): string` — returns `text` unchanged (same reference) - unless it contains the exact prefix `"\uE200visualize\uE202"` (no spaces). It replaces matches of - the app's own regex `/\uE200visualize\uE202([^\uE201]+)\uE201/g` (one left-to-right pass) with - `toAsciiDirective(payload)`, or keeps the match when rejected. Spans with any other keyword never - match that regex and are copied through unchanged, as are unterminated spans. + unless it contains the exact prefix `"\uE200visualize\uE202"` (no spaces). It produces the same + matches as the app's regex `/\uE200visualize\uE202([^\uE201]+)\uE201/g`, but with a linear scanner: + find the next prefix with `indexOf`; the payload runs to the next END (`indexOf` from the prefix end); + an empty payload resumes the prefix search one character later; no END after a prefix means no later + match exists, so scanning stops. Each match becomes `toAsciiDirective(payload)` or stays as is. + Unterminated spans are copied through. App-regex parity supersedes D4's opaque-payload rule: a + `visualize` span inside another keyword's payload is converted, as the app would convert it. Intentional differences from `f2`: code tokens are normalized too (the skill's example is fenced), and the template exception below. Using the same regex means a `visualize` span that appears after @@ -50,7 +53,9 @@ basename (`file=`); both skill templates; spans inside a fenced block; rejected JSON, relative JSON path, `..`, quote, bad basename, missing path); other keywords and unterminated spans untouched; Windows drive and UNC paths; live null path with a title; malformed optional fields (`title` number, `type` other, `mode` other, `wide` string); a decoded compaction summary carrying the -reference; a large context with many malformed markers (linear time); idempotence; every +reference; 20,000 repeated unterminated prefixes normalized in under 200 ms; exact output for a +`visualize` span nested in a `cite` payload and for a malformed `visualize` prefix followed by a valid +one; idempotence; every role and content shape through `parseRequest`, with image parts, tool-call arguments and reasoning parts deep-equal to an unnormalized parse; a frozen input body (deep `Object.freeze`) parses without throwing and `_rawBody` is the same object; an expanded continuation body (prior assistant output @@ -87,7 +92,9 @@ a live null path (C sees `::codex-live-vis{}`). Fermat returned MISALIGNED on the first plan revision with five gaps. Dispositions: -1. Unknown spans opaque, nested marker fixture, exact prefix — folded above. +1. Exact prefix — folded. Opaque unknown spans — superseded in the second reflection by app-regex + parity (the app converts a nested span, so the model should see the same thing); nested and + malformed-prefix fixtures pin the exact output. 2. Live selection, ignored `wide`, template exception scope — folded above. 3. Opaque-field, frozen-input and replay assertions — folded above. Cursor checkpoint rejection test — rebutted: the Cursor builder compares a digest of the context it is given, and this change only @@ -96,3 +103,7 @@ Fermat returned MISALIGNED on the first plan revision with five gaps. Dispositio 4. Existing suites and source-reading guards named explicitly — folded above. 5. Universal claims removed; alternatives' reasons and limitations recorded in 000 and 001 — folded. + +Second reflection (wp2 P): MISALIGNED on regex cost (quadratic on repeated unterminated prefixes, +measured 16/63/251 ms for 2k/4k/8k) and on the opaque-span claim. Both folded above: linear scanner with +the regex's semantics, parity recorded explicitly, adversarial and nested fixtures added. diff --git a/docs-site/src/content/docs/guides/codex-integration.md b/docs-site/src/content/docs/guides/codex-integration.md index 05f44c3ba69..1b964b14529 100644 --- a/docs-site/src/content/docs/guides/codex-integration.md +++ b/docs-site/src/content/docs/guides/codex-integration.md @@ -677,6 +677,19 @@ mode unchanged. After `ocx sync` changes this metadata, restart Codex App and open a fresh task. Existing app-server processes and tasks may retain the catalog and tool plan they loaded at startup. +### Inline visualizations with routed models + +The Codex App's Visualize plugin asks the model to reply with a reference wrapped in private-use +characters (U+E200 … U+E201). Some providers remove those characters before the model sees them — +every Claude route we checked does — so the model used to answer with a bare +`visualize{"path":…}` line that Codex App printed as text. + +opencodex rewrites those references into the directive the app itself renders, +`::codex-inline-vis{path="/absolute/path/chart.html"}`, in the conversation text sent to routed +models. Any model can read and repeat that form, so the visualization renders inline. Native OpenAI +passthrough requests are forwarded unchanged. Replies that were already saved in the bare +`visualize{…}` form stay as they are; ask for the visualization again in a new reply. + ### Custom model display names A custom model can carry a human-readable **display name** that overrides the label Codex shows in diff --git a/scripts/test-layout/layout.json b/scripts/test-layout/layout.json index 2d54094a05d..aecc89fbbc0 100644 --- a/scripts/test-layout/layout.json +++ b/scripts/test-layout/layout.json @@ -433,6 +433,7 @@ "ci-structure-gate.test.ts": "ci-workflows", "ci-workflows.test.ts": "ci-workflows", "citation-markers.test.ts": "responses", + "visualization-directives.test.ts": "responses", "cl01-claude-outbound-review-regressions.test.ts": "routing", "cl01-openai-chat-review-regressions.test.ts": "routing", "cl01-review-regressions.test.ts": "routing", diff --git a/src/responses/parser.ts b/src/responses/parser.ts index 5b7aa40cfa9..236ebcd5db1 100644 --- a/src/responses/parser.ts +++ b/src/responses/parser.ts @@ -27,6 +27,7 @@ import { isObj, inputContentParts, outputTextOf, outputToToolResultContent, tool import { mapToolChoice, buildTools, customToolNamespaces } from "./parser-tools"; import { parseTextFormat } from "./parser-text-format"; import { externalTaskInputContent } from "./task-input"; +import { normalizeVisualizationContext } from "./visualization-directives"; /** * Wrap a remembered proxy-side signature as provider metadata for a replayed tool call. @@ -644,7 +645,9 @@ export function parseRequest( return { modelId: data.model, ...(data.previous_response_id ? { previousResponseId: data.previous_response_id } : {}), - context, + // Codex App visualization references in the private-use form are invisible to some models; + // hand every model the app's ASCII directive instead (visualization-directives.ts). + context: normalizeVisualizationContext(context), stream: data.stream === true, options, _rawBody: body, diff --git a/src/responses/visualization-directives.ts b/src/responses/visualization-directives.ts new file mode 100644 index 00000000000..01e64c92ca9 --- /dev/null +++ b/src/responses/visualization-directives.ts @@ -0,0 +1,182 @@ +/** + * Codex App visualization references for models that cannot see private-use characters. + * + * The bundled Visualize skill teaches the model to answer with + * + * \uE200visualize\uE202{"path":"/abs/chart.html"}\uE201 + * + * and the Codex App renders that span as an inline visualization. Some providers drop Basic + * Multilingual Plane private-use characters before the model sees them (every Claude route checked + * on 2026-09-27, direct and through Cursor), so the model reads and writes a bare + * `visualize{"path":...}` that the app shows verbatim. + * + * The app does not need the private-use form. Its own renderer rewrites each span into the plain + * directive `::codex-inline-vis{path="/abs/chart.html"}` before parsing, and it renders that + * directive when it appears directly. Rewriting the span the same way in model-visible text gives + * every model an ASCII instruction it can read and repeat. + * + * The rules mirror the app (26.924.22138, `f2` in app-shared) with two deliberate differences: code + * blocks are rewritten too, because the skill's example sits in a fenced block, and the skill's + * `<absolute-path>/<title>.html` placeholder is accepted so the model still sees the template. + * Matching follows the app's regex `/\uE200visualize\uE202([^\uE201]+)\uE201/g` exactly, scanned in + * linear time. Anything the app would keep as-is is kept as-is here. + * + * Only the parsed context is rewritten. The raw request body, which native passthrough serializes + * and which is stored for `previous_response_id` replay, is never touched. + */ +import { posix } from "node:path"; +import type { OcxContentPart, OcxContext, OcxMessage, OcxTextContent } from "../types"; + +const START = "\uE200"; +const END = "\uE201"; +const PREFIX = `${START}visualize\uE202`; +const INLINE_DIRECTIVE = "codex-inline-vis"; +const LIVE_DIRECTIVE = "codex-live-vis"; +/** The placeholder path in the Visualize skill's own template. */ +const TEMPLATE_PATH = "<absolute-path>/<title>.html"; +const HTML_BASENAME = /^[a-z0-9]+(?:-[a-z0-9]+)*\.html$/; +const PARENT_SEGMENT = /(?:^|[\\/])\.\.(?:[\\/]|$)/; +const UNSAFE_ATTRIBUTE = /["\n\r]/; + +/** The app's absolute-path test: POSIX (not `//`), drive letter, UNC with either slash. */ +function isAbsoluteAppPath(path: string): boolean { + return (path.startsWith("/") && !path.startsWith("//")) + || /^[A-Za-z]:[\\/]/.test(path) + || /^\\\\[^\\]+\\[^\\]+/.test(path) + || /^\/\/[^/]+\/[^/]+/.test(path); +} + +interface VisualizationReference { + path: string | null; + title?: string; + type?: "inline" | "live"; + mode?: "wide"; +} + +/** The app's payload schema: `path` is required but nullable; the rest are optional and typed. */ +function parseReference(value: unknown): VisualizationReference | undefined { + if (!value || typeof value !== "object" || Array.isArray(value)) return undefined; + const record = value as Record<string, unknown>; + if (!("path" in record) || (record.path !== null && typeof record.path !== "string")) return undefined; + if (record.title !== undefined && typeof record.title !== "string") return undefined; + if (record.type !== undefined && record.type !== "inline" && record.type !== "live") return undefined; + if (record.mode !== undefined && record.mode !== "wide") return undefined; + if (record.wide !== undefined && typeof record.wide !== "boolean") return undefined; + return { + path: record.path as string | null, + ...(record.title !== undefined ? { title: record.title as string } : {}), + ...(record.type !== undefined ? { type: record.type as "inline" | "live" } : {}), + ...(record.mode !== undefined ? { mode: "wide" as const } : {}), + }; +} + +/** The ASCII directive for one span payload, or undefined when the app would keep the span. */ +function toAsciiDirective(payload: string): string | undefined { + const isJson = payload.startsWith("{"); + let raw: unknown = { path: payload }; + if (isJson) { + try { + raw = JSON.parse(payload); + } catch { + return undefined; + } + } + const reference = parseReference(raw); + if (!reference) return undefined; + const live = reference.type === "live"; + const { path } = reference; + if (path === null) return live ? `::${LIVE_DIRECTIVE}{}` : undefined; + + const template = isJson && path === TEMPLATE_PATH; + if (!template) { + const basename = posix.basename(path.replaceAll("\\", "/")); + if (PARENT_SEGMENT.test(path) || UNSAFE_ATTRIBUTE.test(path) || !HTML_BASENAME.test(basename)) return undefined; + if (!isAbsoluteAppPath(path) && (isJson || path !== basename)) return undefined; + } + const attribute = template || isAbsoluteAppPath(path) ? "path" : "file"; + const title = reference.title !== undefined && !UNSAFE_ATTRIBUTE.test(reference.title) + ? ` title="${reference.title}"` + : ""; + const mode = !live && reference.mode === "wide" ? ` mode="wide"` : ""; + return `::${live ? LIVE_DIRECTIVE : INLINE_DIRECTIVE}{${attribute}="${path}"${title}${mode}}`; +} + +/** + * Rewrite every visualization span in `text` to the app's ASCII directive. + * + * Returns the same string when nothing changes. The scan finds the same matches as the app's regex: + * a prefix, one or more characters that are not END, then END. When no END follows a prefix, no + * later prefix can match either, so the scan stops instead of rescanning the tail for every prefix. + */ +export function normalizeVisualizationText(text: string): string { + let prefixAt = text.indexOf(PREFIX); + if (prefixAt === -1) return text; + let out = ""; + let copiedTo = 0; + while (prefixAt !== -1) { + const payloadStart = prefixAt + PREFIX.length; + const endAt = text.indexOf(END, payloadStart); + if (endAt === -1) break; + if (endAt === payloadStart) { + // Empty payload: the regex moves on one character and tries again. + prefixAt = text.indexOf(PREFIX, prefixAt + 1); + continue; + } + const directive = toAsciiDirective(text.slice(payloadStart, endAt)); + if (directive !== undefined) { + out += text.slice(copiedTo, prefixAt) + directive; + copiedTo = endAt + 1; + } + prefixAt = text.indexOf(PREFIX, endAt + 1); + } + return copiedTo === 0 ? text : out + text.slice(copiedTo); +} + +function normalizeParts<P extends { type: string }>(parts: P[]): P[] { + let changed = false; + const next = parts.map((part) => { + if (part.type !== "text") return part; + const text = (part as unknown as OcxTextContent).text; + if (typeof text !== "string") return part; + const normalized = normalizeVisualizationText(text); + if (normalized === text) return part; + changed = true; + return { ...part, text: normalized }; + }); + return changed ? next : parts; +} + +function normalizeContent<P extends { type: string }>(content: string | P[]): string | P[] { + return typeof content === "string" ? normalizeVisualizationText(content) : normalizeParts(content); +} + +function normalizeMessage(message: OcxMessage): OcxMessage { + if (message.role === "assistant") { + const content = normalizeParts(message.content); + return content === message.content ? message : { ...message, content }; + } + const content = normalizeContent<OcxContentPart>(message.content); + return content === message.content ? message : { ...message, content } as OcxMessage; +} + +/** + * Rewrite visualization spans in the conversation text a routed model reads: the system prompt, + * string message content, and text parts of every role. Other fields, including images, tool calls, + * reasoning parts, and tool definitions, are returned by reference. The context itself is returned + * unchanged when no text changed. + */ +export function normalizeVisualizationContext(context: OcxContext): OcxContext { + let changed = false; + const systemPrompt = context.systemPrompt?.map((text) => { + const normalized = normalizeVisualizationText(text); + if (normalized !== text) changed = true; + return normalized; + }); + const messages = context.messages.map((message) => { + const normalized = normalizeMessage(message); + if (normalized !== message) changed = true; + return normalized; + }); + if (!changed) return context; + return { ...context, ...(systemPrompt ? { systemPrompt } : {}), messages }; +} diff --git a/structure/transports/responses-wire-shapes.md b/structure/transports/responses-wire-shapes.md index 57eeb78db54..1c05e1b578a 100644 --- a/structure/transports/responses-wire-shapes.md +++ b/structure/transports/responses-wire-shapes.md @@ -533,3 +533,18 @@ summary choices remain intact. Raw display and hidden-envelope replay follow [reasoning display parity](../providers/chat-compat.md#reasoning-display-parity-hidethinkingsummary). Final-route normalization preserves visible raw reasoning when the parsed request has a validated active effort and omits summary; explicit `summary: "none"` still hides it. + +## Codex App visualization references + +The Codex App draws an inline visualization from `U+E200 visualize U+E202 {json} U+E201` in an +assistant message, and its renderer turns that span into the plain directive +`::codex-inline-vis{path="…"}` before parsing. Several providers drop private-use characters before +the model reads them (every Claude route checked), so the model saw and repeated a bare +`visualize{…}` the app printed verbatim. `src/responses/visualization-directives.ts` rewrites each +such span in the parsed context — system prompt, string content and text parts of every role — into +that ASCII directive, following the app's own payload rules, and `parseRequest` applies it to the +context it returns. `_rawBody` is not touched, so native passthrough stays byte-identical and stored +`previous_response_id` history keeps the original text. The citation filter in +`src/responses/citation-markers.ts` is separate and never removes these spans (#6040). +`tests/responses/visualization-directives.test.ts` pins the payload rules, the linear-time scan and +the parser and Anthropic request paths. diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index a365548931b..b44e92dbfec 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -269,6 +269,7 @@ "ci-structure-gate.test.ts": "ci-workflows", "ci-workflows.test.ts": "ci-workflows", "citation-markers.test.ts": "responses", + "visualization-directives.test.ts": "responses", "cl01-claude-outbound-review-regressions.test.ts": "routing", "cl01-openai-chat-review-regressions.test.ts": "routing", "cl01-review-regressions.test.ts": "routing", diff --git a/tests/responses/visualization-directives.test.ts b/tests/responses/visualization-directives.test.ts new file mode 100644 index 00000000000..33055443fcb --- /dev/null +++ b/tests/responses/visualization-directives.test.ts @@ -0,0 +1,225 @@ +import { describe, expect, test } from "bun:test"; +import { createAnthropicAdapter } from "../../src/adapters/anthropic"; +import { parseRequest } from "../../src/responses/parser"; +import { + normalizeVisualizationContext, + normalizeVisualizationText, +} from "../../src/responses/visualization-directives"; +import type { OcxContext, OcxProviderConfig } from "../../src/types"; + +/** + * Codex App visualization references (\uE200visualize\uE202{...}\uE201) are invisible to models whose + * provider drops private-use characters, so the parser hands every routed model the app's own ASCII + * directive instead. The expected strings below are what the app's renderer (26.924.22138, `f2`) + * produces for the same payloads. + */ + +const S = "\uE200"; +const P = "\uE202"; +const E = "\uE201"; +const ref = (payload: string): string => `${S}visualize${P}${payload}${E}`; +const json = (value: unknown): string => ref(JSON.stringify(value)); +const hasPrivateUse = (text: string): boolean => /[\uE000-\uF8FF]/.test(text); + +describe("visualization reference rewrite", () => { + test("an absolute path becomes the inline directive", () => { + expect(normalizeVisualizationText(`see\n${json({ path: "/Users/u/viz/chart.html" })}\nend`)) + .toBe('see\n::codex-inline-vis{path="/Users/u/viz/chart.html"}\nend'); + }); + + test("title and wide mode are carried in the app's attribute order", () => { + expect(normalizeVisualizationText(json({ mode: "wide", title: "Q3 sales", path: "/v/q3-sales.html" }))) + .toBe('::codex-inline-vis{path="/v/q3-sales.html" title="Q3 sales" mode="wide"}'); + }); + + test("a title with a quote is dropped, the directive is kept", () => { + expect(normalizeVisualizationText(json({ path: "/v/a.html", title: 'say "hi"' }))) + .toBe('::codex-inline-vis{path="/v/a.html"}'); + }); + + test("live references use the live directive and never carry mode", () => { + expect(normalizeVisualizationText(json({ type: "live", path: "/v/live-a.html", mode: "wide" }))) + .toBe('::codex-live-vis{path="/v/live-a.html"}'); + expect(normalizeVisualizationText(json({ type: "live", path: null, title: "ignored" }))).toBe("::codex-live-vis{}"); + }); + + test("wide:true is validated but does not select wide mode", () => { + expect(normalizeVisualizationText(json({ path: "/v/a.html", wide: true }))).toBe('::codex-inline-vis{path="/v/a.html"}'); + }); + + test("a bare path payload and a bare basename follow the app's path/file choice", () => { + expect(normalizeVisualizationText(ref("/v/bare-path.html"))).toBe('::codex-inline-vis{path="/v/bare-path.html"}'); + expect(normalizeVisualizationText(ref("chart.html"))).toBe('::codex-inline-vis{file="chart.html"}'); + }); + + test("Windows drive and UNC paths count as absolute", () => { + expect(normalizeVisualizationText(json({ path: "C:\\viz\\chart.html" }))) + .toBe('::codex-inline-vis{path="C:\\viz\\chart.html"}'); + expect(normalizeVisualizationText(json({ path: "\\\\host\\share\\chart.html" }))) + .toBe('::codex-inline-vis{path="\\\\host\\share\\chart.html"}'); + expect(normalizeVisualizationText(json({ path: "//host/share/chart.html" }))) + .toBe('::codex-inline-vis{path="//host/share/chart.html"}'); + }); + + test("both Visualize skill templates become readable ASCII templates", () => { + const fenced = [ + "```text", + ref('{"path":"<absolute-path>/<title>.html"}'), + "```", + "", + "```text", + ref('{"path":"<absolute-path>/<title>.html","mode":"wide"}'), + "```", + ].join("\n"); + expect(normalizeVisualizationText(fenced)).toBe([ + "```text", + '::codex-inline-vis{path="<absolute-path>/<title>.html"}', + "```", + "", + "```text", + '::codex-inline-vis{path="<absolute-path>/<title>.html" mode="wide"}', + "```", + ].join("\n")); + }); + + test("payloads the app would reject stay exactly as written", () => { + const kept = [ + ref("{not json"), + json({ path: "relative/chart.html" }), + json({ path: "/v/../chart.html" }), + json({ path: '/v/a"b.html' }), + json({ path: "/v/Chart.html" }), + json({ path: "/v/chart.htm" }), + json({ title: "no path" }), + json({ path: null }), + json({ path: "/v/a.html", title: 3 }), + json({ path: "/v/a.html", type: "embedded" }), + json({ path: "/v/a.html", mode: "tall" }), + json({ path: "/v/a.html", wide: "yes" }), + json(["/v/a.html"]), + ref("sub/chart.html"), + ]; + for (const span of kept) expect(normalizeVisualizationText(`x ${span} y`)).toBe(`x ${span} y`); + }); + + test("other keywords, empty payloads and unterminated spans are untouched", () => { + const text = `${S}cite${P}turn0search0${E} ${S}visualize${P}${E} ${S}visualize${P}{"path":"/v/a.html"}`; + expect(normalizeVisualizationText(text)).toBe(text); + }); + + test("matching follows the app's regex for nested and malformed prefixes", () => { + const nested = `${S}cite${P}turn0 ${json({ path: "/v/a.html" })}`; + expect(normalizeVisualizationText(nested)).toBe(`${S}cite${P}turn0 ::codex-inline-vis{path="/v/a.html"}`); + // An empty payload is skipped and the scan retries at the next prefix, as the regex does. + const malformed = `${S}visualize${P}${E}${json({ path: "/v/b.html" })}`; + expect(normalizeVisualizationText(malformed)).toBe(`${S}visualize${P}${E}::codex-inline-vis{path="/v/b.html"}`); + }); + + test("text without a reference is returned as the same string, and a second pass changes nothing", () => { + const plain = "no references here ${S} ${E}"; + expect(normalizeVisualizationText(plain)).toBe(plain); + const once = normalizeVisualizationText(json({ path: "/v/a.html" })); + expect(normalizeVisualizationText(once)).toBe(once); + }); + + test("many unterminated prefixes are scanned in linear time", () => { + const hostile = `${S}visualize${P}x`.repeat(20_000); + const started = performance.now(); + expect(normalizeVisualizationText(hostile)).toBe(hostile); + expect(performance.now() - started).toBeLessThan(200); + }); +}); + +describe("visualization references in the parsed request", () => { + const reference = json({ path: "/Users/u/viz/chart.html" }); + const directive = '::codex-inline-vis{path="/Users/u/viz/chart.html"}'; + + const deepFreeze = <T>(value: T): T => { + if (value && typeof value === "object") { + for (const child of Object.values(value)) deepFreeze(child); + Object.freeze(value); + } + return value; + }; + + const body = () => deepFreeze({ + model: "anthropic/claude-opus-5-5", + instructions: `Answer with ${reference} when asked.`, + input: [ + { type: "message", role: "developer", content: [{ type: "input_text", text: `dev ${reference}` }] }, + { + type: "message", + role: "user", + content: [ + { type: "input_text", text: `user ${reference}` }, + { type: "input_image", image_url: "data:image/png;base64,AAAA" }, + ], + }, + { type: "message", role: "assistant", content: [{ type: "output_text", text: `Here:\n${reference}` }] }, + { type: "function_call", call_id: "call_1", name: "exec", arguments: JSON.stringify({ note: reference }) }, + { type: "function_call_output", call_id: "call_1", output: `skill says ${reference}` }, + { type: "message", role: "user", content: "again" }, + ], + }); + + test("every conversation text reaches the model as the ASCII directive", () => { + const parsed = parseRequest(body()); + const texts = [ + ...(parsed.context.systemPrompt ?? []), + ...parsed.context.messages.flatMap((message) => { + if (typeof message.content === "string") return [message.content]; + return message.content.flatMap((part) => (part.type === "text" ? [part.text] : [])); + }), + ]; + const withDirective = texts.filter((text) => text.includes(directive)); + expect(withDirective.length).toBeGreaterThanOrEqual(5); + for (const text of texts) expect(text.includes(`${S}visualize${P}`)).toBe(false); + }); + + test("images and tool-call arguments are untouched and the raw body is the frozen input", () => { + const input = body(); + const parsed = parseRequest(input); + expect(parsed._rawBody).toBe(input); + expect(JSON.stringify(parsed._rawBody)).toContain(`${S}visualize${P}`); + const user = parsed.context.messages.find( + (message) => message.role === "user" && Array.isArray(message.content) && message.content.some((p) => p.type === "image"), + ); + expect(user && Array.isArray(user.content) ? user.content.find((p) => p.type === "image") : undefined) + .toEqual({ type: "image", imageUrl: "data:image/png;base64,AAAA" }); + const call = parsed.context.messages + .flatMap((message) => (message.role === "assistant" ? message.content : [])) + .find((part) => part.type === "toolCall"); + expect(call && "arguments" in call ? JSON.stringify(call.arguments) : "").toContain(`${S}visualize${P}`); + }); + + test("a replayed compaction summary is normalized like any other text", () => { + const summary = `earlier answer: ${reference}`; + const parsed = parseRequest({ + model: "anthropic/claude-opus-5-5", + input: [ + { type: "context_compaction", encrypted_content: "ocx1:" + Buffer.from(summary, "utf-8").toString("base64") }, + { type: "message", role: "user", content: "next" }, + ], + }); + expect(parsed.context.messages[0].content as string).toContain(directive); + }); + + test("an unaffected context is returned by reference", () => { + const context: OcxContext = { messages: [{ role: "user", content: "plain", timestamp: 0 }] }; + expect(normalizeVisualizationContext(context)).toBe(context); + }); + + test("the Anthropic request carries the directive and no private-use character in text", async () => { + const provider = { adapter: "anthropic", baseUrl: "https://api.anthropic.com", apiKey: "sk-x", authMode: "apiKey" } as unknown as OcxProviderConfig; + const parsed = parseRequest({ + model: "anthropic/claude-opus-5-5", + input: [{ type: "message", role: "user", content: `draw it: ${reference}` }], + }); + const { body: wire } = await createAnthropicAdapter(provider).buildRequest(parsed); + const serialized = typeof wire === "string" ? wire : JSON.stringify(wire); + const decoded = JSON.parse(serialized) as { messages: Array<{ content: unknown }> }; + const text = JSON.stringify(decoded.messages); + expect(text).toContain(directive.replaceAll('"', '\\"')); + expect(hasPrivateUse(text)).toBe(false); + }); +}); From 44d564b86f021e399e443eace1efc58883c69931 Mon Sep 17 00:00:00 2001 From: JUN <jun@lidgeai.com> Date: Sun, 27 Sep 2026 12:19:34 +0900 Subject: [PATCH 3/3] test(responses): use a neutral fixture path --- tests/responses/visualization-directives.test.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/responses/visualization-directives.test.ts b/tests/responses/visualization-directives.test.ts index 33055443fcb..fe5023e9d37 100644 --- a/tests/responses/visualization-directives.test.ts +++ b/tests/responses/visualization-directives.test.ts @@ -23,8 +23,8 @@ const hasPrivateUse = (text: string): boolean => /[\uE000-\uF8FF]/.test(text); describe("visualization reference rewrite", () => { test("an absolute path becomes the inline directive", () => { - expect(normalizeVisualizationText(`see\n${json({ path: "/Users/u/viz/chart.html" })}\nend`)) - .toBe('see\n::codex-inline-vis{path="/Users/u/viz/chart.html"}\nend'); + expect(normalizeVisualizationText(`see\n${json({ path: "/workspace/viz/chart.html" })}\nend`)) + .toBe('see\n::codex-inline-vis{path="/workspace/viz/chart.html"}\nend'); }); test("title and wide mode are carried in the app's attribute order", () => { @@ -131,8 +131,8 @@ describe("visualization reference rewrite", () => { }); describe("visualization references in the parsed request", () => { - const reference = json({ path: "/Users/u/viz/chart.html" }); - const directive = '::codex-inline-vis{path="/Users/u/viz/chart.html"}'; + const reference = json({ path: "/workspace/viz/chart.html" }); + const directive = '::codex-inline-vis{path="/workspace/viz/chart.html"}'; const deepFreeze = <T>(value: T): T => { if (value && typeof value === "object") {