From a40c3c7a09bae18e0a8f4f0d94f11953a8eb5489 Mon Sep 17 00:00:00 2001 From: Yuanhao Li Date: Thu, 8 Oct 2026 21:19:14 +0200 Subject: [PATCH 1/3] feat(rutis): image tool results across TypeScript, Python, dsh and rutis-agent - bridge: call_tool results and after_tool edits take yoagent-shaped content blocks (text, base64 images) instead of text; validated by content_blocks - dsh adapter: dsh image blocks (attachment references) become yoagent images via the attachments service when one is loaded, else a text placeholder - rutis-agent example: a runner's {"content": [...]} value is read as blocks (dot_picture) - yoagent.d.ts ContentBlock; README, CLAUDE.md, CHANGELOG - tests: content_blocks unit, TS/Python round trip, dsh with and without a store; dsh_test no longer reads a mid-write empty abort file Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01T7iq5hpndSiHQcnAsywKuG --- CHANGELOG.md | 7 ++ CLAUDE.md | 2 +- integrations/yoagent-rutis/README.md | 25 +++- .../examples/rutis-agent-tools/src/main.rs | 76 ++++++++++- .../plugins/dsh/dsh-tools-adapter.ts | 64 ++++++++-- .../plugins/dsh/fixture-attachments.ts | 22 ++++ .../plugins/dsh/fixture-tools.ts | 20 +++ .../yoagent-rutis/plugins/yoagent.d.ts | 27 +++- integrations/yoagent-rutis/src/languages.rs | 104 +++++++++++++-- integrations/yoagent-rutis/tests/dsh_test.rs | 58 ++++++++- .../yoagent-rutis/tests/languages_test.rs | 118 +++++++++++++++++- 11 files changed, 489 insertions(+), 34 deletions(-) create mode 100644 integrations/yoagent-rutis/plugins/dsh/fixture-attachments.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index ed76563..1179422 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ All notable changes to `yoagent` are documented here. The format loosely follows [Keep a Changelog](https://keepachangelog.com/), and the project adheres to [Semantic Versioning](https://semver.org/). +## Unreleased + +### yoagent-rutis + +- **Images in tool results, across ecosystems.** TypeScript and Python handlers' `call_tool` results and `after_tool` edits take `content` blocks in yoagent's JSON shape (`{"type": "image", "data": , "mimeType"}` next to text blocks) instead of `text`, so pictures cross the bridge both ways (`after_tool` already saw `output.content`). The dsh adapter turns dsh image blocks (references into dsh's attachment store) into yoagent images through the `attachments` service when one is loaded; the rutis-agent example reads a runner's `{"content": [...]}` value as blocks (rutis-agent results are otherwise text). Rust handlers already returned full yoagent tool results. Tests: `content_blocks` unit test, TypeScript/Python round trip in `languages_test`, dsh with and without a store in `dsh_test`. +- Test fix: `dsh_test` no longer reads the abort file mid-write (empty) as the result. + ## 0.25.0 (2026-10-08) ### Added diff --git a/CLAUDE.md b/CLAUDE.md index fa62093..150cada 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -100,7 +100,7 @@ Cloudflare Workers bindings (objects the runtime hands a Worker in `env`) as yoa ### rutis bridge (`integrations/yoagent-rutis/`, separate crate) -Its own `Cargo.toml` (not a workspace member; cargo leaves the nested package out of `cargo package` for yoagent automatically), `yoagent` by path + version, `rutis = "0.6"` (0.x caret = 0.6.x; any rutis minor bump is a yoagent-rutis minor bump — rutis types are in its API; `pub use rutis`). yoagent's core must never depend on rutis; the bridge uses only yoagent's public API. Built on `Extension` (#250): `RutisBridge::install(&root)` — **on the root**: the bridge is bound to the installing ctx's generation, so on a plugin's ctx it reads as stopped for good once that plugin reloads — provides the `Registry` service (rutis holds one binding per key, so plugins add entries; an insertion-ordered `Vec` keyed by `seq`). A plugin registers a `Handler` (closure builder, `Clone`: a name + `with_tool`/`with_tools` (per run)/`with_before_tool`/`with_after_tool`/`with_before_model`/`with_on_input`/`with_on_stop`/`with_finish` each sync or `_async` (owned arg → future of `Result<_, ExtensionError>`), `with_on_event` sync; args are plain-data `Serialize` structs `ToolCall { tool, call_id, args, user_request, latest_user_text, run }`, `Turn`, `Input`, `Stop`, each with `#[serde(flatten)] run: RunInfo { run_id, label, depth, delegated_by, parent_run_id }`; decisions are yoagent's `ToolDecision`/`TurnDecision`/`InputDecision`/`StopDecision`, `after_tool` edits yoagent's `ToolOutput`). Behind it the crate-private `HandlerImpl` trait (`hooks() -> Hooks`, `static_tools`, one async method per hook, `events(run) -> Option>`), which language handlers implement too. `Registry::register(ctx, handler)` / `PluginCtxExt::register_handler` ties the entry to `ctx.effect_named` on the plugin's fiber (removed on dispose/restart/update/dependency eviction); handler names and **static** tool names unique across plugins (`CordisError::ServiceExists`, warned with the holder, no retry); each entry has a `Gate` = the plugin generation's `cancellation_token()` (cancelled at the start of an unload) + a `removed` token. `bridge.extension()` → `RutisExtension` (`impl Extension`, name "rutis"; host-set `.required()`, `.filters_tool_output()`, `.rechecks_modified_calls()`, `.require_policy()`, `.with_{policy,input,turn}_timeout(Option)`, `.with_timeout`): `start_run` snapshots available entries (`Registry::snapshot`) into a `RunState` (`impl RunHooks`). Every handler call goes through `extension::call` (gate checked → `Missed::Unavailable`; `catch_unwind`; `select!` against the gate going (in flight → unavailable); per-hook timeout → `Missed::Failed`); yoagent's own dispatcher already races cancel. Combination, registration order: `tools` static then per-run, dedup earlier wins, each wrapped in `LiveTool` (call after unload → "no longer available", in flight → "plugin unloaded during the call"); `on_input` first `Reject`, any miss rejects; `before_model` notes joined `\n`, first `Stop` ends, a miss is skipped (required → `Fail`); `before_tool` `Deny` wins / `Modify` feeds next / any miss (incl. unavailable) denies; `after_tool` chained, a failure → `Err` (yoagent withholds; required fails the run), an unavailable handler → the bridge withholds itself and returns `Ok` (an unload never fails a run); `on_stop` `Continue` messages joined, miss skipped/required fails; `on_event` sync per handler (panic → that handler off for the run; never unwinds, so publishing and other handlers go on); `finish` flushes event sinks, then calls all handlers concurrently. A required `tools`/`on_event` failure is never raised as a panic (yoagent would switch the whole extension's `on_event` off — bus publishing and every other handler): it is recorded in `RunState::failure` (tool calls denied / output withheld while pending, handed to yoagent through `RunHooks::take_failure` (required only), which the loop polls at every boundary incl. the run's end; only one on `AgentEnd` is just logged). Static tools of a handler already unavailable at `tools` are not offered. Timeouts: policy 60 s (`before_tool`, `after_tool`, `on_stop`), input 30 s (`on_input`), turn 5 s (`before_model`, `tools`, `finish`). `require_policy()` = a run starting with no `before_tool` handler denies every call (covers the reload window; no input counterpart). `Host::is_closed` (the generation token captured at **install** — a disposed or restarted root reads as stopped for good, since the registry went with that generation — + the ctx's current token + root fiber state) at `start_run` → deny every tool call / reject input / no notes, and per `before_tool` call (a shutdown mid-run). Events: `on_event` publishes every event with rutis `emit` as `AgentEventEmitted { run_id, label, depth, event }` (no listener → no task; one queue per bus; skipped when closed). Removed in #250: the four adapters, `attach`/`AgentRutisExt`, `ToolRegistry`, the `event_sender` tee, the waterfall/serial chains and their workarounds. `PluginCtxExt` (sealed: `register_handler`, `provide_tool` (= a `tool:` handler), `on_agent_event`) + `AgentPlugin::new(handler)` (named after the handler, injects `Registry`). Published with yoagent 0.25 (yoagent-rutis 0.1.0, requires yoagent 0.25). Tests share `tests/common` (`setup()`, `Setup(name, closure)`, `GreeterFactory`, `plugin`/`handler`/`deny_all`/`run_error`); policy edges drive `RutisExtension::start_run` + `RunHooks` directly with `RunContext::new` / `ToolCallRequest::new`. **TypeScript / Python plugins** (`languages.rs`, features `node`/`python`/`websocket` → optional `rutis-bridge = "0.7"`, `default-features = false`; the default build is Rust-only; `rutis-loader` 0.7 only a dev-dependency): `install` also provides `dyn HostDispatch` under `host_key("yoagent")` (`methods` = `{register: sync}`). `register(name, handler, options?)`: the handler is a live object (`Reference::is_object`: JS objects with functions, Python instances; hooks probed with a sync `get` on the plugin's call chain — `undefined`/`null`/Python `AttributeError` = absent, a function = present, anything else or another error refuses the registration — called with `call_method_async`, so `this` works) or a `Value::Record` of function refs (a Python dict; a `None`/`null` entry = absent, as on an object); `tools` needs `call_tool`; `options.events` (AgentEvent `type` tags; unknown ones warned) is required with `on_event` and only with it; returns a `Value::callback` disposer (cancels the registration's lifetime token = its `Gate` generation, removes the entry); `session::caller()` (the runtime's `Connection`) is watched with `closed()`, so a crashed or exited runtime's handlers are removed. `RemoteHandler` implements `HandlerImpl`: every hook is one `call` with JSON args (`after_tool(call, output)` two), results parsed strictly (`null` = default; `{deny}`/`{args}`, string/`{note}`/`{stop}`, `{reject}`, `{continue}`/`{fail}`, `{text,details,is_error}` with typed fields (`{}` = empty text, like `after_tool`'s edit), `{args}` must be an object; anything else is an error, so it fails closed); `RemoteTool` runs `call_tool` (no timeout), raced against the tool's cancel; `on_event` is delivered by one task per run and handler (in order, the extension's turn timeout per event, a bounded queue of 1024, filtered by `event_type` before serializing); the first failure (error, timeout, full queue, a panicking delivery — each delivery runs inside `catch_unwind` — or a delivery task found gone: `TrySendError::Closed` on `send`, or a `flush` whose send or ack fails) is kept in the sink: the task logs it (`warn!`) and drops the rest, `send` stops queueing, and `RunState` reads it via `EventSink::failure` — right after each `send` in `on_event` and in `check_sinks` at every decision point (`pending`/`take_failure`) — never as a panic; `EventSink::flush` (an ack through the queue) before `finish`; yoagent sends `AgentEnd` after `finish`, so the queue stays open until the hooks drop. Cancel handle: `Target::call` turns the first argument (when an object) into a `Value::Record` of its fields as `Value::Data` plus `signal: Value::Signal` (`with_signal`, `SIGNAL_FIELD`) — rutis-bridge 0.7 allows a signal nested in a call argument; the JS runtime decodes it as a real `AbortSignal`, Python as `rutis.peer.Signal` (`.cancelled`, `await .wait()`) — so it is aborted whenever the host drops the call future (run cancel, hook timeout, plugin unload; never after completion), and fixed-signature Python methods still accept the call (a positional signal would break them). `on_event` uses `call_plain` (no handle: events are data). Python's `json.dumps(call)` now refuses the argument (documented). Tested in `languages_test` (`JS_CANCEL` / `PY_CANCEL` fixtures: run cancel and policy timeout abort, completed `*_quick` calls never do). **dsh adapter** (`plugins/dsh/`, own `package.json` + lock pinning `@arcships/rutis*` 0.7.0, dsh 0.2.0-rc.2 incl. `dsh-settings` (dsh-free-search imports an error class from it), `@deepseek-ai/cordis` 4.0.4, `dsh-free-search` 0.8.1; separate from `plugins/package.json` to keep that install small): `dsh-tools-adapter.ts` (a rutis `definePlugin`, injects `tools`, `systemPrompt`, `yoagent`; `tools` = `schemas()` (config allowlist `tools`), `call_tool` = `execute({callId: "yoagent:", name, arguments, signal: call.signal})`, `isError` → `{text, is_error: true}`, `before_model` = `systemPrompt.assemble()` minus `harness:identity` / `deployment:persona-{prefix,suffix}`, each section rendered alone with `renderPrompt` (unset variables → skipped), joined under `[Guidance from dsh plugins]`, capped at `maxNoteChars` 2000); `fixture-tools.ts` (a plain Cordis plugin: `defineTool` echo / fail / slow-until-abort writing `config.abortFile`, and a `fixture:guidance` section); `tests/dsh_test.rs` (`required-features = ["node"]`, skips without `plugins/dsh/node_modules` unless `YOAGENT_RUTIS_REQUIRE_RUNTIMES=1`); `examples/dsh_tools.rs` (feature `node`; rows `dsh-web`, `dsh-system-prompt`, `dsh-tools`, `dsh-free-search` (bing, en-US), the adapter; scripted `platform_search` hits the real web; `--live` DeepSeek via `DEEPSEEK_API_KEY` / `~/.dskey`; self-checks). **rutis-agent example** `examples/rutis-agent-tools/` (own `[workspace]`, `/target` git-ignored, excluded from the package, not auto-discovered, not in CI): rutis-agent from git (`arcships/rutis` tag v0.7.0 — crates.io's 0.2.0 is on rutis 0.2) + `[patch.crates-io] rutis` at the same tag, so one `rutis` (verified with `cargo tree -d`); a `RutisAgentTools` plugin (injects `tools_key()` + `Registry`) registers a `Handler` whose per-run `with_tools` maps `ToolRegistry::schemas()` (`aimux_core::options::Tool::Function`) to tools calling `execute(&tool_call(..), &ctx.cancel)` (`ok: false` → `ToolError::Failed`, cancelled → `Cancelled`); shows `replace_text`, a hot-added `word_count`, and (scripted) a cancelled `slow` tool; `--live` DeepSeek. `ToolSpec` `description`/`parameters` `null` = missing. `plugins/`: `package.json` + lock (`@arcships/rutis`, `@arcships/rutis-runtime` 0.7.0), `requirements.txt` (`rutis==0.7.0`), `yoagent.d.ts` (the handler shape), `ts/example.ts`, `python/yoagent_example.py`; `node_modules`/`.venv` git-ignored and excluded from the package. `tests/languages_test.rs` (`required-features = ["node", "python"]`, unix): real runtimes via rutis-loader rows, a `probe` host service, fixtures written to a temp dir (JS imports `@arcships/rutis` by `file://` URL); a missing Node 24 / `plugins/node_modules` / Python with rutis 0.7 (`YOAGENT_RUTIS_PYTHON` or `plugins/.venv/bin/python`) prints `SKIPPED:` and passes, or fails with `YOAGENT_RUTIS_REQUIRE_RUNTIMES=1`. Example `language_plugins` (same features). CI jobs `rutis-bridge` (fmt/clippy/doc/test/example, `--manifest-path`), `rutis-bridge-msrv` (1.86, yoke-derive pin; also checks the language features) and `rutis-bridge-languages` (Linux, Node 24, Python 3.12: `npm ci` in `plugins/` and `plugins/dsh/`, clippy per feature, docs, tests with the runtimes required, the `language_plugins` example; `dsh_tools` is not run in CI — it needs the network). A PR whose base is not main gets no CI (the workflow triggers on main/release). Run its checks with `--manifest-path integrations/yoagent-rutis/Cargo.toml`. +Its own `Cargo.toml` (not a workspace member; cargo leaves the nested package out of `cargo package` for yoagent automatically), `yoagent` by path + version, `rutis = "0.6"` (0.x caret = 0.6.x; any rutis minor bump is a yoagent-rutis minor bump — rutis types are in its API; `pub use rutis`). yoagent's core must never depend on rutis; the bridge uses only yoagent's public API. Built on `Extension` (#250): `RutisBridge::install(&root)` — **on the root**: the bridge is bound to the installing ctx's generation, so on a plugin's ctx it reads as stopped for good once that plugin reloads — provides the `Registry` service (rutis holds one binding per key, so plugins add entries; an insertion-ordered `Vec` keyed by `seq`). A plugin registers a `Handler` (closure builder, `Clone`: a name + `with_tool`/`with_tools` (per run)/`with_before_tool`/`with_after_tool`/`with_before_model`/`with_on_input`/`with_on_stop`/`with_finish` each sync or `_async` (owned arg → future of `Result<_, ExtensionError>`), `with_on_event` sync; args are plain-data `Serialize` structs `ToolCall { tool, call_id, args, user_request, latest_user_text, run }`, `Turn`, `Input`, `Stop`, each with `#[serde(flatten)] run: RunInfo { run_id, label, depth, delegated_by, parent_run_id }`; decisions are yoagent's `ToolDecision`/`TurnDecision`/`InputDecision`/`StopDecision`, `after_tool` edits yoagent's `ToolOutput`). Behind it the crate-private `HandlerImpl` trait (`hooks() -> Hooks`, `static_tools`, one async method per hook, `events(run) -> Option>`), which language handlers implement too. `Registry::register(ctx, handler)` / `PluginCtxExt::register_handler` ties the entry to `ctx.effect_named` on the plugin's fiber (removed on dispose/restart/update/dependency eviction); handler names and **static** tool names unique across plugins (`CordisError::ServiceExists`, warned with the holder, no retry); each entry has a `Gate` = the plugin generation's `cancellation_token()` (cancelled at the start of an unload) + a `removed` token. `bridge.extension()` → `RutisExtension` (`impl Extension`, name "rutis"; host-set `.required()`, `.filters_tool_output()`, `.rechecks_modified_calls()`, `.require_policy()`, `.with_{policy,input,turn}_timeout(Option)`, `.with_timeout`): `start_run` snapshots available entries (`Registry::snapshot`) into a `RunState` (`impl RunHooks`). Every handler call goes through `extension::call` (gate checked → `Missed::Unavailable`; `catch_unwind`; `select!` against the gate going (in flight → unavailable); per-hook timeout → `Missed::Failed`); yoagent's own dispatcher already races cancel. Combination, registration order: `tools` static then per-run, dedup earlier wins, each wrapped in `LiveTool` (call after unload → "no longer available", in flight → "plugin unloaded during the call"); `on_input` first `Reject`, any miss rejects; `before_model` notes joined `\n`, first `Stop` ends, a miss is skipped (required → `Fail`); `before_tool` `Deny` wins / `Modify` feeds next / any miss (incl. unavailable) denies; `after_tool` chained, a failure → `Err` (yoagent withholds; required fails the run), an unavailable handler → the bridge withholds itself and returns `Ok` (an unload never fails a run); `on_stop` `Continue` messages joined, miss skipped/required fails; `on_event` sync per handler (panic → that handler off for the run; never unwinds, so publishing and other handlers go on); `finish` flushes event sinks, then calls all handlers concurrently. A required `tools`/`on_event` failure is never raised as a panic (yoagent would switch the whole extension's `on_event` off — bus publishing and every other handler): it is recorded in `RunState::failure` (tool calls denied / output withheld while pending, handed to yoagent through `RunHooks::take_failure` (required only), which the loop polls at every boundary incl. the run's end; only one on `AgentEnd` is just logged). Static tools of a handler already unavailable at `tools` are not offered. Timeouts: policy 60 s (`before_tool`, `after_tool`, `on_stop`), input 30 s (`on_input`), turn 5 s (`before_model`, `tools`, `finish`). `require_policy()` = a run starting with no `before_tool` handler denies every call (covers the reload window; no input counterpart). `Host::is_closed` (the generation token captured at **install** — a disposed or restarted root reads as stopped for good, since the registry went with that generation — + the ctx's current token + root fiber state) at `start_run` → deny every tool call / reject input / no notes, and per `before_tool` call (a shutdown mid-run). Events: `on_event` publishes every event with rutis `emit` as `AgentEventEmitted { run_id, label, depth, event }` (no listener → no task; one queue per bus; skipped when closed). Removed in #250: the four adapters, `attach`/`AgentRutisExt`, `ToolRegistry`, the `event_sender` tee, the waterfall/serial chains and their workarounds. `PluginCtxExt` (sealed: `register_handler`, `provide_tool` (= a `tool:` handler), `on_agent_event`) + `AgentPlugin::new(handler)` (named after the handler, injects `Registry`). Published with yoagent 0.25 (yoagent-rutis 0.1.0, requires yoagent 0.25). Tests share `tests/common` (`setup()`, `Setup(name, closure)`, `GreeterFactory`, `plugin`/`handler`/`deny_all`/`run_error`); policy edges drive `RutisExtension::start_run` + `RunHooks` directly with `RunContext::new` / `ToolCallRequest::new`. **TypeScript / Python plugins** (`languages.rs`, features `node`/`python`/`websocket` → optional `rutis-bridge = "0.7"`, `default-features = false`; the default build is Rust-only; `rutis-loader` 0.7 only a dev-dependency): `install` also provides `dyn HostDispatch` under `host_key("yoagent")` (`methods` = `{register: sync}`). `register(name, handler, options?)`: the handler is a live object (`Reference::is_object`: JS objects with functions, Python instances; hooks probed with a sync `get` on the plugin's call chain — `undefined`/`null`/Python `AttributeError` = absent, a function = present, anything else or another error refuses the registration — called with `call_method_async`, so `this` works) or a `Value::Record` of function refs (a Python dict; a `None`/`null` entry = absent, as on an object); `tools` needs `call_tool`; `options.events` (AgentEvent `type` tags; unknown ones warned) is required with `on_event` and only with it; returns a `Value::callback` disposer (cancels the registration's lifetime token = its `Gate` generation, removes the entry); `session::caller()` (the runtime's `Connection`) is watched with `closed()`, so a crashed or exited runtime's handlers are removed. `RemoteHandler` implements `HandlerImpl`: every hook is one `call` with JSON args (`after_tool(call, output)` two), results parsed strictly (`null` = default; `{deny}`/`{args}`, string/`{note}`/`{stop}`, `{reject}`, `{continue}`/`{fail}`, `{text,details,is_error}` with typed fields (`{}` = empty text, like `after_tool`'s edit) — or `content` blocks instead of `text` (never both), parsed by `content_blocks` (yoagent's JSON shape: `{type:"text",text}` / `{type:"image",data (base64 charset, non-empty),mimeType (image/*)}`, nothing else) — which `after_tool`'s edit also takes (`content` replaces every block; `after_tool` sees `output.content`); an `is_error` result's message is its text blocks joined, `{args}` must be an object; anything else is an error, so it fails closed); `RemoteTool` runs `call_tool` (no timeout), raced against the tool's cancel; `on_event` is delivered by one task per run and handler (in order, the extension's turn timeout per event, a bounded queue of 1024, filtered by `event_type` before serializing); the first failure (error, timeout, full queue, a panicking delivery — each delivery runs inside `catch_unwind` — or a delivery task found gone: `TrySendError::Closed` on `send`, or a `flush` whose send or ack fails) is kept in the sink: the task logs it (`warn!`) and drops the rest, `send` stops queueing, and `RunState` reads it via `EventSink::failure` — right after each `send` in `on_event` and in `check_sinks` at every decision point (`pending`/`take_failure`) — never as a panic; `EventSink::flush` (an ack through the queue) before `finish`; yoagent sends `AgentEnd` after `finish`, so the queue stays open until the hooks drop. Cancel handle: `Target::call` turns the first argument (when an object) into a `Value::Record` of its fields as `Value::Data` plus `signal: Value::Signal` (`with_signal`, `SIGNAL_FIELD`) — rutis-bridge 0.7 allows a signal nested in a call argument; the JS runtime decodes it as a real `AbortSignal`, Python as `rutis.peer.Signal` (`.cancelled`, `await .wait()`) — so it is aborted whenever the host drops the call future (run cancel, hook timeout, plugin unload; never after completion), and fixed-signature Python methods still accept the call (a positional signal would break them). `on_event` uses `call_plain` (no handle: events are data). Python's `json.dumps(call)` now refuses the argument (documented). Tested in `languages_test` (`JS_CANCEL` / `PY_CANCEL` fixtures: run cancel and policy timeout abort, completed `*_quick` calls never do). **dsh adapter** (`plugins/dsh/`, own `package.json` + lock pinning `@arcships/rutis*` 0.7.0, dsh 0.2.0-rc.2 incl. `dsh-settings` (dsh-free-search imports an error class from it), `@deepseek-ai/cordis` 4.0.4, `dsh-free-search` 0.8.1; separate from `plugins/package.json` to keep that install small): `dsh-tools-adapter.ts` (a rutis `definePlugin`, injects `tools`, `systemPrompt`, `yoagent`; `tools` = `schemas()` (config allowlist `tools`), `call_tool` = `execute({callId: "yoagent:", name, arguments, signal: call.signal})`, `isError` → `{text, is_error: true}`, text blocks kept, an image block (`{attachment: ImageAttachmentRef}`) read with the `attachments` service (`readImage(ref, signal)` → base64 + `ref.mediaType`) looked up **per image** via `ctx.use` in a try (never injected — the adapter runs without a store; none or a failed read → `[image …: not available here]` text), other blocks named, `before_model` = `systemPrompt.assemble()` minus `harness:identity` / `deployment:persona-{prefix,suffix}`, each section rendered alone with `renderPrompt` (unset variables → skipped), joined under `[Guidance from dsh plugins]`, capped at `maxNoteChars` 2000); `fixture-tools.ts` (a plain Cordis plugin: `defineTool` echo / fail / slow-until-abort writing `config.abortFile`, and a `fixture:guidance` section); `tests/dsh_test.rs` (`required-features = ["node"]`, skips without `plugins/dsh/node_modules` unless `YOAGENT_RUTIS_REQUIRE_RUNTIMES=1`); `examples/dsh_tools.rs` (feature `node`; rows `dsh-web`, `dsh-system-prompt`, `dsh-tools`, `dsh-free-search` (bing, en-US), the adapter; scripted `platform_search` hits the real web; `--live` DeepSeek via `DEEPSEEK_API_KEY` / `~/.dskey`; self-checks). **rutis-agent example** `examples/rutis-agent-tools/` (own `[workspace]`, `/target` git-ignored, excluded from the package, not auto-discovered, not in CI): rutis-agent from git (`arcships/rutis` tag v0.7.0 — crates.io's 0.2.0 is on rutis 0.2) + `[patch.crates-io] rutis` at the same tag, so one `rutis` (verified with `cargo tree -d`); a `RutisAgentTools` plugin (injects `tools_key()` + `Registry`) registers a `Handler` whose per-run `with_tools` maps `ToolRegistry::schemas()` (`aimux_core::options::Tool::Function`) to tools calling `execute(&tool_call(..), &ctx.cancel)` (`ok: false` → `ToolError::Failed`, cancelled → `Cancelled`; an output that parses wholly as `{"content": [text/image blocks]}` → those yoagent blocks — the adapter's image convention, rutis-agent itself has text results only — shown by `dot_picture`); shows `replace_text`, a hot-added `word_count`, and (scripted) a cancelled `slow` tool; `--live` DeepSeek. `ToolSpec` `description`/`parameters` `null` = missing. `plugins/`: `package.json` + lock (`@arcships/rutis`, `@arcships/rutis-runtime` 0.7.0), `requirements.txt` (`rutis==0.7.0`), `yoagent.d.ts` (the handler shape), `ts/example.ts`, `python/yoagent_example.py`; `node_modules`/`.venv` git-ignored and excluded from the package. `tests/languages_test.rs` (`required-features = ["node", "python"]`, unix): real runtimes via rutis-loader rows, a `probe` host service, fixtures written to a temp dir (JS imports `@arcships/rutis` by `file://` URL); a missing Node 24 / `plugins/node_modules` / Python with rutis 0.7 (`YOAGENT_RUTIS_PYTHON` or `plugins/.venv/bin/python`) prints `SKIPPED:` and passes, or fails with `YOAGENT_RUTIS_REQUIRE_RUNTIMES=1`. Example `language_plugins` (same features). CI jobs `rutis-bridge` (fmt/clippy/doc/test/example, `--manifest-path`), `rutis-bridge-msrv` (1.86, yoke-derive pin; also checks the language features) and `rutis-bridge-languages` (Linux, Node 24, Python 3.12: `npm ci` in `plugins/` and `plugins/dsh/`, clippy per feature, docs, tests with the runtimes required, the `language_plugins` example; `dsh_tools` is not run in CI — it needs the network). A PR whose base is not main gets no CI (the workflow triggers on main/release). Run its checks with `--manifest-path integrations/yoagent-rutis/Cargo.toml`. ### OpenAPI Integration (`openapi/`, feature-gated) diff --git a/integrations/yoagent-rutis/README.md b/integrations/yoagent-rutis/README.md index 975b21b..4414fdd 100644 --- a/integrations/yoagent-rutis/README.md +++ b/integrations/yoagent-rutis/README.md @@ -209,6 +209,21 @@ def apply(ctx, config): return await res.text() } ``` +- **Images, both ways.** A tool result and an `after_tool` edit carry + `content` blocks in yoagent's JSON shape (pi's and MCP's too) instead of + `text` — `{"type": "text", "text"}` and + `{"type": "image", "data": , "mimeType": "image/…"}` — and + `after_tool` sees the output's blocks as `output.content`, so a handler + can return pictures, and keep, add or drop them when it edits a result. + Text and blocks are exclusive in one answer; anything else in `content` + (another block type, a non-`image/*` type, data that is not base64) fails + the answer. + + ```ts + async call_tool(call) { + return { content: [{ type: 'text', text: 'the chart' }, { type: 'image', data: png.toString('base64'), mimeType: 'image/png' }] } + } + ``` - **The bridge never loads plugins**: the host does, typically with [rutis-loader](https://crates.io/crates/rutis-loader) rows, and must share `yoagent` in the loader's catalog (`catalog.register_shared("yoagent")` or @@ -265,7 +280,7 @@ offers every tool in dsh's tool registry (the `tools` service of | Hook | What the adapter does | |---|---| | `tools` | `tools.schemas()` → name, description, parameters (config `tools`: an allowlist) | -| `call_tool` | `tools.execute({callId, name, arguments, signal})` with the bridge's cancel handle as dsh's `signal`: cancelling the run aborts the dsh call. `isError` → an error tool result; text blocks joined | +| `call_tool` | `tools.execute({callId, name, arguments, signal})` with the bridge's cancel handle as dsh's `signal`: cancelling the run aborts the dsh call. `isError` → an error tool result. Text blocks stay text; an image block — a reference into dsh's attachment store — becomes a yoagent image, its bytes read with the `attachments` service when one is loaded (looked up per image, so the adapter also runs without it; then, or when a read fails, the image is a text placeholder); other blocks are named | | `before_model` | the system-prompt sections dsh plugins added (the harness identity and persona slots left out, sections whose variables are unset skipped), as one note, capped at `maxNoteChars` (2000) | Load, as rows of one Node runtime whose `package.json` is `plugins/dsh/`'s: @@ -286,7 +301,8 @@ cargo run --features node --example dsh_tools -- --live # DeepSeek: DEEPSEEK_AP the adapter, runs one search, and checks that a dsh tool answered (and, scripted, that free-search's prompt section reached the model). `tests/dsh_test.rs` covers the adapter offline with a fixture dsh plugin -(`plugins/dsh/fixture-tools.ts`). +(`plugins/dsh/fixture-tools.ts`) and, for images, a stand-in attachment +store (`plugins/dsh/fixture-attachments.ts`). ### rutis-agent tools @@ -297,6 +313,11 @@ Rust rutis plugin that maps every `ToolDef` to a yoagent tool per run (results as text, failures as `ToolError`s, yoagent's cancel token passed as rutis-agent's), shown with rutis-agent's `replace_text` and a tool registered into the registry while the host runs, which the next run offers. +rutis-agent's results are text (a runner's JSON value is serialized), so +images use a convention of this adapter: a runner returning +`{"content": [blocks]}` in yoagent's block shape gives yoagent those blocks, +images included (`dot_picture` in the example); rutis-agent's own agent +still sees the JSON text. ```sh cargo run --manifest-path examples/rutis-agent-tools/Cargo.toml [-- --live] diff --git a/integrations/yoagent-rutis/examples/rutis-agent-tools/src/main.rs b/integrations/yoagent-rutis/examples/rutis-agent-tools/src/main.rs index ce1272d..bf06112 100644 --- a/integrations/yoagent-rutis/examples/rutis-agent-tools/src/main.rs +++ b/integrations/yoagent-rutis/examples/rutis-agent-tools/src/main.rs @@ -11,6 +11,12 @@ //! - its schema (name, description, parameters) is the tool's; //! - a result is text, a failure (rutis-agent's `ok: false`, its //! `error: ...` text) is a real `ToolError::Failed`; +//! - images: rutis-agent's results are text (a runner's JSON value is +//! serialized). By this adapter's convention a runner that returns +//! `{"content": [blocks]}` — yoagent's text and image blocks, +//! `{"type": "image", "data": , "mimeType": "image/…"}` — gives +//! yoagent those blocks, images included (rutis-agent's own agent still +//! sees the JSON text); //! - yoagent's cancel token *is* the token rutis-agent's `execute` watches, //! so cancelling the run stops the runner. The tool returns //! `ToolError::Cancelled`, but the transcript shows yoagent's "Tool result @@ -25,7 +31,9 @@ //! 2. while the host runs, a `word_count` tool is registered into //! rutis-agent's `ToolRegistry` — it appears on the agent's next run, with //! no change to the agent or the adapter (hot add); -//! 3. (scripted only) a slow tool, added the same way, is cancelled with +//! 3. (scripted only) a `dot_picture` tool, added the same way, returns a +//! picture: it reaches yoagent as an image; +//! 4. (scripted only) a slow tool, added the same way, is cancelled with //! `Agent::abort()`: its runner never finishes. //! //! The model is scripted by default; `--live` asks DeepSeek instead @@ -163,12 +171,37 @@ impl AgentTool for RegistryTool { return Err(ToolError::Failed(out.output)); } Ok(ToolResult { - content: vec![Content::Text { text: out.output }], + content: content_of(&out.output) + .unwrap_or_else(|| vec![Content::Text { text: out.output }]), details: Value::Null, }) } } +/// A runner's `{"content": [blocks]}` value (as rutis-agent serialized it), +/// read back as yoagent content: only text and image blocks, and only when +/// the whole value is that shape — any other output stays text. +fn content_of(output: &str) -> Option> { + if !output.starts_with('{') { + return None; + } + let mut value: serde_json::Map = serde_json::from_str(output).ok()?; + if value.len() != 1 { + return None; + } + let blocks: Vec = serde_json::from_value(value.remove("content")?).ok()?; + blocks + .iter() + .all(|b| match b { + Content::Text { .. } => true, + Content::Image { data, mime_type } => { + !data.is_empty() && mime_type.starts_with("image/") + } + _ => false, + }) + .then_some(blocks) +} + // ── The host ──────────────────────────────────────────────────── /// A tool registered into rutis-agent's registry while the host runs. @@ -192,6 +225,26 @@ fn word_count_tool() -> ToolDef { ) } +/// A 1×1 PNG, base64. +const DOT_PNG: &str = + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGMAAQAABQABDQottAAAAABJRU5ErkJggg=="; + +/// A tool returning a picture: text and an image, in the content-block +/// convention this adapter reads. +fn dot_picture_tool() -> ToolDef { + ToolDef::new( + "dot_picture", + "Shows a picture of a dot.", + json!({"type": "object", "properties": {}}), + |_| async move { + Ok(json!({"content": [ + {"type": "text", "text": "a dot"}, + {"type": "image", "data": DOT_PNG, "mimeType": "image/png"}, + ]})) + }, + ) +} + /// A tool that takes a minute unless cancelled; `finished` says whether its /// runner ever completed. fn slow_tool(finished: Arc) -> ToolDef { @@ -348,7 +401,10 @@ async fn main() -> Result<(), BoxError> { // Run 2: the tool added while the host ran. call("word_count", json!({"path": path})), MockResponse::Text("Counted.".into()), - // Run 3: cancelled mid-call. + // Run 3: a picture. + call("dot_picture", json!({})), + MockResponse::Text("Seen.".into()), + // Run 4: cancelled mid-call. call("slow", json!({})), MockResponse::Text("never sent".into()), ]); @@ -415,7 +471,19 @@ async fn main() -> Result<(), BoxError> { format!("word_count was not hot-added between the runs: {offered:?}"), )?; - // 3. Cancel: yoagent's token stops rutis-agent's runner. + // 3. Images: a runner's content blocks reach yoagent as an image. + registry.register(dot_picture_tool()); + let before = agent.messages().len(); + run(&mut agent, "Show me the dot.").await?; + let image = agent.messages()[before..].iter().any(|m| { + matches!(m, AgentMessage::Llm(Message::ToolResult { tool_name, content, .. }) + if tool_name == "dot_picture" + && content.iter().any(|c| matches!(c, Content::Image { data, .. } if data == DOT_PNG))) + }); + println!(" dot_picture returned an image: {image}"); + check(image, "dot_picture's image did not reach yoagent")?; + + // 4. Cancel: yoagent's token stops rutis-agent's runner. let finished = Arc::new(AtomicBool::new(false)); registry.register(slow_tool(finished.clone())); let mut events = agent.prompt("Take your time.").await; diff --git a/integrations/yoagent-rutis/plugins/dsh/dsh-tools-adapter.ts b/integrations/yoagent-rutis/plugins/dsh/dsh-tools-adapter.ts index efe4cfd..64571a1 100644 --- a/integrations/yoagent-rutis/plugins/dsh/dsh-tools-adapter.ts +++ b/integrations/yoagent-rutis/plugins/dsh/dsh-tools-adapter.ts @@ -20,14 +20,30 @@ // call_tool `tools.execute({callId, name, arguments, signal})`, with the // bridge's cancel handle as dsh's `signal`: cancelling the // yoagent run aborts the dsh call. `isError` → an error tool -// result; text blocks are joined, other blocks named. +// result. Text blocks stay text; an image block (a reference +// into dsh's attachment store) becomes a yoagent image, read +// with the `attachments` service when one is loaded (looked up +// per image, not injected: the adapter runs without it); with +// none, or when a read fails, the image is a text placeholder. +// Other blocks are named. // before_model the sections dsh plugins added to `systemPrompt` (the // harness identity and persona slots left out), rendered and // capped, as a note on the request's latest user turn. import { definePlugin } from '@arcships/rutis' import { renderPrompt } from '@deepseek-ai/dsh-system-prompt' -import type { Cancellable, ToolCall, ToolResult, ToolSpec, Yoagent } from '../yoagent.d.ts' +import type { Cancellable, ContentBlock, ToolCall, ToolResult, ToolSpec, Yoagent } from '../yoagent.d.ts' + +/** The slice of `@deepseek-ai/dsh-attachment`'s `AttachmentStore` this adapter uses. */ +interface DshAttachments { + readImage( + ref: { attachmentId: string; mediaType: string }, + signal?: AbortSignal, + ): Promise<{ ref: { mediaType: string }; data: Uint8Array }> +} + +/** A dsh content block as a tool result carries it. */ +type DshBlock = { type: string; text?: string; attachment?: { attachmentId: string; mediaType: string; name?: string } } /** The slice of `@deepseek-ai/dsh-tools`' `ToolRuntime` this adapter uses. */ interface DshTools { @@ -39,7 +55,7 @@ interface DshTools { signal: AbortSignal }): Promise<{ isError: boolean - content?: { type: string; text?: string }[] + content?: DshBlock[] error?: { message?: string; info?: { code?: string } } }> } @@ -87,8 +103,34 @@ export default definePlugin({ const prompt = ctx.use('systemPrompt') const yoagent = ctx.use('yoagent') const only = config?.tools ? new Set(config.tools) : undefined + /** dsh's attachment store, when one is loaded: optional, so looked up per use rather than injected. */ + const attachments = (): DshAttachments | undefined => { + try { + return ctx.use('attachments') + } catch { + return undefined + } + } const maxNote = config?.maxNoteChars ?? 2000 + /** An image reference as a yoagent image: its bytes from dsh's attachment store. */ + const image = async ( + ref: { attachmentId: string; mediaType: string; name?: string }, + signal: AbortSignal, + ): Promise => { + const label = `[image${ref.name ? ` ${ref.name}` : ''}: not available here]` + const store = attachments() + if (!store) return { type: 'text', text: label } + try { + const stored = await store.readImage(ref, signal) + return { type: 'image', data: Buffer.from(stored.data).toString('base64'), mimeType: stored.ref.mediaType } + } catch (error) { + if (signal.aborted) throw error + console.warn(`[dsh] image ${ref.attachmentId} could not be read: ${error}`) + return { type: 'text', text: label } + } + } + ctx.effect( yoagent.register(config?.name ?? 'dsh-tools', { async tools(): Promise { @@ -109,13 +151,21 @@ export default definePlugin({ arguments: call.args, signal: call.signal, }) - const text = (out.content ?? []) - .map((block) => (block.type === 'text' ? (block.text ?? '') : `[${block.type} block]`)) - .join('\n') + const content: ContentBlock[] = [] + for (const block of out.content ?? []) { + if (block.type === 'text') { + content.push({ type: 'text', text: block.text ?? '' }) + } else if (block.type === 'image' && block.attachment) { + content.push(await image(block.attachment, call.signal)) + } else { + content.push({ type: 'text', text: `[${block.type} block]` }) + } + } if (out.isError) { + const text = content.flatMap((b) => (b.type === 'text' ? [b.text] : [])).join('\n') return { text: text || out.error?.message || 'the dsh tool failed', is_error: true } } - return { text } + return { content } }, async before_model(turn: Cancellable) { diff --git a/integrations/yoagent-rutis/plugins/dsh/fixture-attachments.ts b/integrations/yoagent-rutis/plugins/dsh/fixture-attachments.ts new file mode 100644 index 0000000..8508b10 --- /dev/null +++ b/integrations/yoagent-rutis/plugins/dsh/fixture-attachments.ts @@ -0,0 +1,22 @@ +// A test fixture: a stand-in for dsh's `attachments` service +// (`@deepseek-ai/dsh-attachment`'s `AttachmentStore`), holding one image — +// a 1×1 PNG under the id `fixture-dot`. Only `readImage` is implemented. +// Not for production use. + +import { definePlugin } from '@arcships/rutis' + +const PNG = Buffer.from( + 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGMAAQAABQABDQottAAAAABJRU5ErkJggg==', + 'base64', +) + +export default definePlugin({ + apply(ctx) { + ctx.provide('attachments', { + async readImage(ref: { attachmentId: string }) { + if (ref.attachmentId !== 'fixture-dot') throw new Error(`no attachment ${ref.attachmentId}`) + return { ref, data: new Uint8Array(PNG) } + }, + }) + }, +}) diff --git a/integrations/yoagent-rutis/plugins/dsh/fixture-tools.ts b/integrations/yoagent-rutis/plugins/dsh/fixture-tools.ts index 60be07c..fb5323e 100644 --- a/integrations/yoagent-rutis/plugins/dsh/fixture-tools.ts +++ b/integrations/yoagent-rutis/plugins/dsh/fixture-tools.ts @@ -5,6 +5,8 @@ // // fixture_echo {text} → "echo: " // fixture_fail {why} → throws: dsh reports an `isError` result +// fixture_dot {} → a text block and an image block referring to the +// attachment `fixture-dot` (see fixture-attachments.ts) // fixture_slow {} → writes "started" to `config.abortFile`, waits for // `exec.signal` to abort (60 s at most), then // writes how it ended there @@ -49,6 +51,24 @@ export function apply(ctx: any, config: Config | undefined) { throw new Error(`fixture failure: ${args.why}`) }, }), + defineTool({ + name: 'fixture_dot', + description: 'Shows a picture of a dot.', + parameters: {}, + output: { + schema: TEXT, + render: () => [ + { type: 'text' as const, text: 'a dot' }, + { + type: 'image' as const, + attachment: { attachmentId: 'fixture-dot', mediaType: 'image/png', bytes: 70, width: 1, height: 1 }, + }, + ], + }, + async execute() { + return 'a dot' + }, + }), defineTool({ name: 'fixture_slow', description: 'Waits until it is cancelled.', diff --git a/integrations/yoagent-rutis/plugins/yoagent.d.ts b/integrations/yoagent-rutis/plugins/yoagent.d.ts index a2c0feb..4fb20bb 100644 --- a/integrations/yoagent-rutis/plugins/yoagent.d.ts +++ b/integrations/yoagent-rutis/plugins/yoagent.d.ts @@ -152,17 +152,30 @@ export interface ToolCall extends RunInfo { /** Allow (nothing), deny with a reason the model sees, or rewrite the arguments. */ export type ToolVerdict = void | null | { deny: string } | { args: Record } +/** + * A content block, in yoagent's JSON shape (also pi's and MCP's): text, or + * an image as base64 `data` with an `image/*` `mimeType`. + */ +export type ContentBlock = { type: 'text'; text: string } | { type: 'image'; data: string; mimeType: string } + export interface ToolOutput { /** The text blocks, joined. */ text: string /** yoagent's content blocks (text and images). */ - content: unknown[] + content: ContentBlock[] details: unknown is_error: boolean } -/** Keep the output (nothing), or replace parts of it: `text` replaces every content block with one text block. */ -export type OutputEdit = void | null | { text?: string; details?: unknown; is_error?: boolean } +/** + * Keep the output (nothing), or replace parts of it: `text` replaces every + * content block with one text block, `content` with the given blocks (so + * images can be kept, added or dropped). Not both. + */ +export type OutputEdit = + | void + | null + | { text?: string; content?: ContentBlock[]; details?: unknown; is_error?: boolean } export interface Turn extends RunInfo { model: string @@ -213,9 +226,13 @@ export interface ToolSpec { /** * The tool's text, or the text plus details (a missing `text` — `{}` too — - * is an empty text); `is_error` (or a throw) fails the call. + * is an empty text), or `content` blocks instead of `text` (images included; + * not both); `is_error` (or a throw) fails the call, with the text blocks as + * its message. */ -export type ToolResult = string | { text?: string; details?: unknown; is_error?: boolean } +export type ToolResult = + | string + | { text?: string; content?: ContentBlock[]; details?: unknown; is_error?: boolean } export type AgentEventType = | 'agentStart' diff --git a/integrations/yoagent-rutis/src/languages.rs b/integrations/yoagent-rutis/src/languages.rs index 4687853..daba871 100644 --- a/integrations/yoagent-rutis/src/languages.rs +++ b/integrations/yoagent-rutis/src/languages.rs @@ -512,7 +512,8 @@ impl HandlerImpl for RemoteHandler { }; if edit .keys() - .any(|k| !["text", "details", "is_error"].contains(&k.as_str())) + .any(|k| !["text", "content", "details", "is_error"].contains(&k.as_str())) + || (edit.contains_key("text") && edit.contains_key("content")) { return Err(unexpected("after_tool", &value)); } @@ -524,6 +525,10 @@ impl HandlerImpl for RemoteHandler { text: text.to_string(), }]; } + if let Some(content) = edit.get("content") { + output.result.content = + content_blocks(content).ok_or_else(|| unexpected("after_tool", &value))?; + } if let Some(details) = edit.get("details") { output.result.details = details.clone(); } @@ -895,18 +900,26 @@ impl AgentTool for RemoteTool { .map_err(|e| ToolError::Failed(e.to_string()))?; let unexpected = || ToolError::Failed(format!("`call_tool` returned an unexpected value: {value}")); - let (text, details, is_error) = match &value { - Json::String(text) => (text.clone(), Json::Null, false), + let (content, details, is_error) = match &value { + Json::String(text) => ( + vec![Content::Text { text: text.clone() }], + Json::Null, + false, + ), // `{}` is an empty text, as `after_tool`'s edit reads it. Json::Object(fields) if fields .keys() - .all(|k| ["text", "details", "is_error"].contains(&k.as_str())) => + .all(|k| ["text", "content", "details", "is_error"].contains(&k.as_str())) + && !(fields.contains_key("text") && fields.contains_key("content")) => { - let text = match fields.get("text") { - None => String::new(), - Some(Json::String(text)) => text.clone(), - Some(_) => return Err(unexpected()), + let content = match (fields.get("text"), fields.get("content")) { + (Some(Json::String(text)), _) => vec![Content::Text { text: text.clone() }], + (Some(_), _) => return Err(unexpected()), + (None, Some(blocks)) => content_blocks(blocks).ok_or_else(unexpected)?, + (None, None) => vec![Content::Text { + text: String::new(), + }], }; let is_error = match fields.get("is_error") { None => false, @@ -914,20 +927,59 @@ impl AgentTool for RemoteTool { Some(_) => return Err(unexpected()), }; let details = fields.get("details").cloned().unwrap_or(Json::Null); - (text, details, is_error) + (content, details, is_error) } _ => return Err(unexpected()), }; if is_error { + let text = content + .iter() + .filter_map(|c| match c { + Content::Text { text } => Some(text.as_str()), + _ => None, + }) + .collect::>() + .join("\n"); return Err(ToolError::Failed(text)); } - Ok(ToolResult { - content: vec![Content::Text { text }], - details, - }) + Ok(ToolResult { content, details }) } } +/// A handler's content blocks — `{"type": "text", "text"}` and +/// `{"type": "image", "data", "mimeType"}`, yoagent's own JSON shape (and +/// pi's, and MCP's) — as yoagent content. `None` for anything else: not an +/// array, an unknown block type, an extra or missing field, an image whose +/// `mimeType` is not `image/*` or whose `data` is empty or not base64. +fn content_blocks(blocks: &Json) -> Option> { + blocks + .as_array()? + .iter() + .map(|block| { + let fields = block.as_object()?; + let field = |name: &str| fields.get(name)?.as_str(); + match field("type")? { + "text" if fields.len() == 2 => Some(Content::Text { + text: field("text")?.to_string(), + }), + "image" if fields.len() == 3 => { + let data = field("data")?; + let mime_type = field("mimeType")?; + let base64 = !data.is_empty() + && data + .bytes() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'+' | b'/' | b'=')); + (base64 && mime_type.starts_with("image/")).then(|| Content::Image { + data: data.to_string(), + mime_type: mime_type.to_string(), + }) + } + _ => None, + } + }) + .collect() +} + #[cfg(test)] mod tests { use super::*; @@ -937,6 +989,32 @@ mod tests { type Failed = Arc>>; + #[test] + fn content_blocks_take_text_and_images_in_yoagents_shape() { + let blocks = content_blocks(&json!([ + {"type": "text", "text": "see"}, + {"type": "image", "data": "iVBORw0KGgo=", "mimeType": "image/png"}, + ])) + .unwrap(); + assert!(matches!(&blocks[0], Content::Text { text } if text == "see")); + assert!(matches!(&blocks[1], Content::Image { data, mime_type } + if data == "iVBORw0KGgo=" && mime_type == "image/png")); + assert!(content_blocks(&json!([])).unwrap().is_empty()); + for bad in [ + json!("text"), + json!([{"type": "text"}]), + json!([{"type": "text", "text": 1}]), + json!([{"type": "text", "text": "a", "extra": 1}]), + json!([{"type": "image", "data": "", "mimeType": "image/png"}]), + json!([{"type": "image", "data": "not base64!", "mimeType": "image/png"}]), + json!([{"type": "image", "data": "AAAA", "mimeType": "text/plain"}]), + json!([{"type": "image", "data": "AAAA"}]), + json!([{"type": "file", "data": "AAAA", "mimeType": "image/png"}]), + ] { + assert!(content_blocks(&bad).is_none(), "{bad}"); + } + } + /// How the test handler's event delivery goes wrong. #[derive(Clone, Copy)] enum Breaks { diff --git a/integrations/yoagent-rutis/tests/dsh_test.rs b/integrations/yoagent-rutis/tests/dsh_test.rs index 51b676f..5c27fad 100644 --- a/integrations/yoagent-rutis/tests/dsh_test.rs +++ b/integrations/yoagent-rutis/tests/dsh_test.rs @@ -25,6 +25,7 @@ use rutis_loader::{ }; use serde_json::{json, Value}; use yoagent::provider::mock::{MockResponse, MockToolCall}; +use yoagent::{AgentMessage, Content, Message}; use yoagent_rutis::RutisBridge; fn dsh_dir() -> PathBuf { @@ -234,7 +235,62 @@ async fn dsh_tools_reach_a_yoagent_agent_through_the_adapter() { .await .expect("the cancelled run ends"); agent.finish().await; - let ended = wait_file(&abort_file, |t| t != "started").await; + // Not empty: writeFileSync truncates before it writes. + let ended = wait_file(&abort_file, |t| t != "started" && !t.is_empty()).await; assert_eq!(ended, "aborted: AbortError"); host.root.shutdown().await.unwrap(); } + +#[tokio::test(flavor = "multi_thread")] +async fn dsh_images_arrive_as_images_when_an_attachment_store_is_loaded() { + let Some(runtime) = node_runtime() else { + return; + }; + for with_store in [true, false] { + let host = host(&runtime).await; + let mut rows = vec![ + json!({ "id": "system-prompt", "name": "@deepseek-ai/dsh-system-prompt" }), + json!({ "id": "tools", "name": "@deepseek-ai/dsh-tools" }), + json!({ "id": "fixture", "name": dsh_dir().join("fixture-tools.ts") }), + json!({ "id": "adapter", "name": dsh_dir().join("dsh-tools-adapter.ts") }), + ]; + if with_store { + // Loaded after the adapter: it is looked up when an image is read. + rows.push( + json!({ "id": "attachments", "name": dsh_dir().join("fixture-attachments.ts") }), + ); + } + host.load(Value::Array(rows)).await; + let (agent, _) = agent(vec![call("fixture_dot", json!({})), text("done")]); + let mut agent = agent.with_extension(host.bridge.extension()); + run(&mut agent, "go").await; + let content = agent + .messages() + .iter() + .find_map(|m| match m { + AgentMessage::Llm(Message::ToolResult { + tool_name, content, .. + }) if tool_name == "fixture_dot" => Some(content.clone()), + _ => None, + }) + .expect("fixture_dot ran"); + assert!( + matches!(&content[0], Content::Text { text } if text == "a dot"), + "{content:?}" + ); + if with_store { + // The attachment's bytes, read from the store, as a yoagent image. + assert!( + matches!(&content[1], Content::Image { data, mime_type } + if mime_type == "image/png" && data.starts_with("iVBORw0KGgo")), + "{content:?}" + ); + } else { + assert!( + matches!(&content[1], Content::Text { text } if text.contains("not available")), + "{content:?}" + ); + } + host.root.shutdown().await.unwrap(); + } +} diff --git a/integrations/yoagent-rutis/tests/languages_test.rs b/integrations/yoagent-rutis/tests/languages_test.rs index b392bc6..ea579bc 100644 --- a/integrations/yoagent-rutis/tests/languages_test.rs +++ b/integrations/yoagent-rutis/tests/languages_test.rs @@ -30,7 +30,9 @@ use rutis_loader::{ use serde_json::{json, Value}; use yoagent::extension::RunContext; use yoagent::provider::mock::{MockResponse, MockToolCall}; -use yoagent::{AgentEvent, Extension, ToolCallRequest, ToolDecision}; +use yoagent::{ + AgentEvent, AgentMessage, Content, Extension, Message, ToolCallRequest, ToolDecision, +}; use yoagent_rutis::RutisBridge; fn plugins_dir() -> PathBuf { @@ -414,6 +416,54 @@ export default definePlugin({ }) "#; +/// A 1×1 PNG, base64. +const PNG_B64: &str = + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGMAAQAABQABDQottAAAAABJRU5ErkJggg=="; + +/// Image tool results, both ways: a tool returning text and an image, and an +/// `after_tool` edit replacing the content with blocks (images kept). +const JS_IMAGES: &str = r#" +import { definePlugin } from 'RUTIS' +export default definePlugin({ + inject: ['yoagent'], + apply(ctx) { + const yoagent = ctx.use('yoagent') + ctx.effect(yoagent.register('js-images', { + async tools() { return [{ name: 'js_image', description: 'returns a picture' }] }, + async call_tool() { + return { content: [{ type: 'text', text: 'a dot' }, { type: 'image', data: 'PNG_B64', mimeType: 'image/png' }] } + }, + async after_tool(call, output) { + // Annotates the result, keeping its blocks (the image included). + if (call.tool === 'js_image') return { content: [...output.content, { type: 'text', text: 'checked by js' }] } + }, + })) + }, +}) +"#; + +const PY_IMAGES: &str = r#" +inject = ["yoagent"] + + +def apply(ctx, config): + yoagent = ctx.use("yoagent") + + async def tools(run): + return [{"name": "py_image", "description": "returns a picture"}] + + async def call_tool(call): + return {"content": [{"type": "text", "text": "a dot"}, {"type": "image", "data": "PNG_B64", "mimeType": "image/png"}], "details": {"px": 1}} + + async def after_tool(call, output): + # Drops the text, keeps only the images. + if call["tool"] == "py_image": + return {"content": [b for b in output["content"] if b["type"] == "image"]} + return None + + ctx.effect(yoagent.register("py-images", {"tools": tools, "call_tool": call_tool, "after_tool": after_tool})) +"#; + /// A dict of functions, in Python, and a tool that kills its runtime. const PY_HOOKS: &str = r#" import os @@ -595,6 +645,7 @@ struct Fixtures { js: PathBuf, strict: PathBuf, cancel: PathBuf, + images: PathBuf, py: PathBuf, _dir: tempfile::TempDir, } @@ -612,14 +663,28 @@ fn fixtures() -> Fixtures { std::fs::write(&strict, JS_STRICT.replace("RUTIS", &rutis)).unwrap(); let cancel = dir.path().join("cancel.mjs"); std::fs::write(&cancel, JS_CANCEL.replace("RUTIS", &rutis)).unwrap(); + let images = dir.path().join("images.mjs"); + std::fs::write( + &images, + JS_IMAGES + .replace("RUTIS", &rutis) + .replace("PNG_B64", PNG_B64), + ) + .unwrap(); let py = dir.path().join("py"); std::fs::create_dir_all(&py).unwrap(); std::fs::write(py.join("py_hooks.py"), PY_HOOKS).unwrap(); std::fs::write(py.join("py_cancel.py"), PY_CANCEL).unwrap(); + std::fs::write( + py.join("py_images.py"), + PY_IMAGES.replace("PNG_B64", PNG_B64), + ) + .unwrap(); Fixtures { js, strict, cancel, + images, py, _dir: dir, } @@ -1243,3 +1308,54 @@ async fn a_python_hook_sees_its_signal_set_when_the_run_is_cancelled() { ); host.root.shutdown().await.unwrap(); } + +#[tokio::test(flavor = "multi_thread")] +async fn image_tool_results_cross_both_ways_in_typescript_and_python() { + let (Some(node), Some(python)) = (node_runtime(), python()) else { + return; + }; + let fx = fixtures(); + let host = host(Some(&node), Some((&fx.py, &python))).await; + host.load(vec![ + ts_row("images", &fx.images, json!({})), + py_row("py-images", "py_images", json!({})), + ]) + .await; + host.until_handlers(&["js-images", "py-images"]).await; + + let (agent, _) = agent(vec![ + call("js_image", json!({})), + call("py_image", json!({})), + text("done"), + ]); + let mut agent = agent.with_extension(host.bridge.extension()); + run(&mut agent, "go").await; + let results: Vec<(String, Vec)> = agent + .messages() + .iter() + .filter_map(|m| match m { + AgentMessage::Llm(Message::ToolResult { + tool_name, content, .. + }) => Some((tool_name.clone(), content.clone())), + _ => None, + }) + .collect(); + let image = |c: &Content| matches!(c, Content::Image { data, mime_type } if data == PNG_B64 && mime_type == "image/png"); + // js_image: text + image from call_tool, and JS's edit appended a note + // keeping both. + let (_, js) = &results[0]; + assert_eq!(js.len(), 3, "{js:?}"); + assert!( + matches!(&js[0], Content::Text { text } if text == "a dot"), + "{js:?}" + ); + assert!(image(&js[1]), "{js:?}"); + assert!( + matches!(&js[2], Content::Text { text } if text == "checked by js"), + "{js:?}" + ); + // py_image: text + image from call_tool, and Python's edit kept only the image. + let (_, py) = &results[1]; + assert!(py.len() == 1 && image(&py[0]), "{py:?}"); + host.root.shutdown().await.unwrap(); +} From d5a9dbf5b8389421175f15cf5207979225401ab0 Mon Sep 17 00:00:00 2001 From: Yuanhao Li Date: Thu, 8 Oct 2026 21:27:42 +0200 Subject: [PATCH 2/3] =?UTF-8?q?fix(rutis):=20image=20review=20=E2=80=94=20?= =?UTF-8?q?strict=20base64,=20provider=20image=20types,=20size=20limits?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - content_blocks decodes base64 (STANDARD), takes png/jpeg/gif/webp only, and refuses images over 10 MB (websocket frame limit): an image a provider refuses would sit in history and fail every later request - dsh adapter: dsh's offloaded images and images over 3.75 MB (under Anthropic's 5 MB base64) stay text; an error result's images are not read - rutis-agent example: blocks only with at least one image - docs: limits, picking fields in after_tool edits, text join Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01T7iq5hpndSiHQcnAsywKuG --- CLAUDE.md | 2 +- integrations/yoagent-rutis/Cargo.toml | 2 + integrations/yoagent-rutis/README.md | 18 ++++---- .../examples/rutis-agent-tools/src/main.rs | 19 ++++----- .../plugins/dsh/dsh-tools-adapter.ts | 41 ++++++++++++++----- .../plugins/dsh/fixture-tools.ts | 16 +++++++- .../yoagent-rutis/plugins/yoagent.d.ts | 17 +++++--- integrations/yoagent-rutis/src/languages.rs | 40 ++++++++++++++---- integrations/yoagent-rutis/tests/dsh_test.rs | 9 ++++ 9 files changed, 118 insertions(+), 46 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 150cada..160fcbb 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -100,7 +100,7 @@ Cloudflare Workers bindings (objects the runtime hands a Worker in `env`) as yoa ### rutis bridge (`integrations/yoagent-rutis/`, separate crate) -Its own `Cargo.toml` (not a workspace member; cargo leaves the nested package out of `cargo package` for yoagent automatically), `yoagent` by path + version, `rutis = "0.6"` (0.x caret = 0.6.x; any rutis minor bump is a yoagent-rutis minor bump — rutis types are in its API; `pub use rutis`). yoagent's core must never depend on rutis; the bridge uses only yoagent's public API. Built on `Extension` (#250): `RutisBridge::install(&root)` — **on the root**: the bridge is bound to the installing ctx's generation, so on a plugin's ctx it reads as stopped for good once that plugin reloads — provides the `Registry` service (rutis holds one binding per key, so plugins add entries; an insertion-ordered `Vec` keyed by `seq`). A plugin registers a `Handler` (closure builder, `Clone`: a name + `with_tool`/`with_tools` (per run)/`with_before_tool`/`with_after_tool`/`with_before_model`/`with_on_input`/`with_on_stop`/`with_finish` each sync or `_async` (owned arg → future of `Result<_, ExtensionError>`), `with_on_event` sync; args are plain-data `Serialize` structs `ToolCall { tool, call_id, args, user_request, latest_user_text, run }`, `Turn`, `Input`, `Stop`, each with `#[serde(flatten)] run: RunInfo { run_id, label, depth, delegated_by, parent_run_id }`; decisions are yoagent's `ToolDecision`/`TurnDecision`/`InputDecision`/`StopDecision`, `after_tool` edits yoagent's `ToolOutput`). Behind it the crate-private `HandlerImpl` trait (`hooks() -> Hooks`, `static_tools`, one async method per hook, `events(run) -> Option>`), which language handlers implement too. `Registry::register(ctx, handler)` / `PluginCtxExt::register_handler` ties the entry to `ctx.effect_named` on the plugin's fiber (removed on dispose/restart/update/dependency eviction); handler names and **static** tool names unique across plugins (`CordisError::ServiceExists`, warned with the holder, no retry); each entry has a `Gate` = the plugin generation's `cancellation_token()` (cancelled at the start of an unload) + a `removed` token. `bridge.extension()` → `RutisExtension` (`impl Extension`, name "rutis"; host-set `.required()`, `.filters_tool_output()`, `.rechecks_modified_calls()`, `.require_policy()`, `.with_{policy,input,turn}_timeout(Option)`, `.with_timeout`): `start_run` snapshots available entries (`Registry::snapshot`) into a `RunState` (`impl RunHooks`). Every handler call goes through `extension::call` (gate checked → `Missed::Unavailable`; `catch_unwind`; `select!` against the gate going (in flight → unavailable); per-hook timeout → `Missed::Failed`); yoagent's own dispatcher already races cancel. Combination, registration order: `tools` static then per-run, dedup earlier wins, each wrapped in `LiveTool` (call after unload → "no longer available", in flight → "plugin unloaded during the call"); `on_input` first `Reject`, any miss rejects; `before_model` notes joined `\n`, first `Stop` ends, a miss is skipped (required → `Fail`); `before_tool` `Deny` wins / `Modify` feeds next / any miss (incl. unavailable) denies; `after_tool` chained, a failure → `Err` (yoagent withholds; required fails the run), an unavailable handler → the bridge withholds itself and returns `Ok` (an unload never fails a run); `on_stop` `Continue` messages joined, miss skipped/required fails; `on_event` sync per handler (panic → that handler off for the run; never unwinds, so publishing and other handlers go on); `finish` flushes event sinks, then calls all handlers concurrently. A required `tools`/`on_event` failure is never raised as a panic (yoagent would switch the whole extension's `on_event` off — bus publishing and every other handler): it is recorded in `RunState::failure` (tool calls denied / output withheld while pending, handed to yoagent through `RunHooks::take_failure` (required only), which the loop polls at every boundary incl. the run's end; only one on `AgentEnd` is just logged). Static tools of a handler already unavailable at `tools` are not offered. Timeouts: policy 60 s (`before_tool`, `after_tool`, `on_stop`), input 30 s (`on_input`), turn 5 s (`before_model`, `tools`, `finish`). `require_policy()` = a run starting with no `before_tool` handler denies every call (covers the reload window; no input counterpart). `Host::is_closed` (the generation token captured at **install** — a disposed or restarted root reads as stopped for good, since the registry went with that generation — + the ctx's current token + root fiber state) at `start_run` → deny every tool call / reject input / no notes, and per `before_tool` call (a shutdown mid-run). Events: `on_event` publishes every event with rutis `emit` as `AgentEventEmitted { run_id, label, depth, event }` (no listener → no task; one queue per bus; skipped when closed). Removed in #250: the four adapters, `attach`/`AgentRutisExt`, `ToolRegistry`, the `event_sender` tee, the waterfall/serial chains and their workarounds. `PluginCtxExt` (sealed: `register_handler`, `provide_tool` (= a `tool:` handler), `on_agent_event`) + `AgentPlugin::new(handler)` (named after the handler, injects `Registry`). Published with yoagent 0.25 (yoagent-rutis 0.1.0, requires yoagent 0.25). Tests share `tests/common` (`setup()`, `Setup(name, closure)`, `GreeterFactory`, `plugin`/`handler`/`deny_all`/`run_error`); policy edges drive `RutisExtension::start_run` + `RunHooks` directly with `RunContext::new` / `ToolCallRequest::new`. **TypeScript / Python plugins** (`languages.rs`, features `node`/`python`/`websocket` → optional `rutis-bridge = "0.7"`, `default-features = false`; the default build is Rust-only; `rutis-loader` 0.7 only a dev-dependency): `install` also provides `dyn HostDispatch` under `host_key("yoagent")` (`methods` = `{register: sync}`). `register(name, handler, options?)`: the handler is a live object (`Reference::is_object`: JS objects with functions, Python instances; hooks probed with a sync `get` on the plugin's call chain — `undefined`/`null`/Python `AttributeError` = absent, a function = present, anything else or another error refuses the registration — called with `call_method_async`, so `this` works) or a `Value::Record` of function refs (a Python dict; a `None`/`null` entry = absent, as on an object); `tools` needs `call_tool`; `options.events` (AgentEvent `type` tags; unknown ones warned) is required with `on_event` and only with it; returns a `Value::callback` disposer (cancels the registration's lifetime token = its `Gate` generation, removes the entry); `session::caller()` (the runtime's `Connection`) is watched with `closed()`, so a crashed or exited runtime's handlers are removed. `RemoteHandler` implements `HandlerImpl`: every hook is one `call` with JSON args (`after_tool(call, output)` two), results parsed strictly (`null` = default; `{deny}`/`{args}`, string/`{note}`/`{stop}`, `{reject}`, `{continue}`/`{fail}`, `{text,details,is_error}` with typed fields (`{}` = empty text, like `after_tool`'s edit) — or `content` blocks instead of `text` (never both), parsed by `content_blocks` (yoagent's JSON shape: `{type:"text",text}` / `{type:"image",data (base64 charset, non-empty),mimeType (image/*)}`, nothing else) — which `after_tool`'s edit also takes (`content` replaces every block; `after_tool` sees `output.content`); an `is_error` result's message is its text blocks joined, `{args}` must be an object; anything else is an error, so it fails closed); `RemoteTool` runs `call_tool` (no timeout), raced against the tool's cancel; `on_event` is delivered by one task per run and handler (in order, the extension's turn timeout per event, a bounded queue of 1024, filtered by `event_type` before serializing); the first failure (error, timeout, full queue, a panicking delivery — each delivery runs inside `catch_unwind` — or a delivery task found gone: `TrySendError::Closed` on `send`, or a `flush` whose send or ack fails) is kept in the sink: the task logs it (`warn!`) and drops the rest, `send` stops queueing, and `RunState` reads it via `EventSink::failure` — right after each `send` in `on_event` and in `check_sinks` at every decision point (`pending`/`take_failure`) — never as a panic; `EventSink::flush` (an ack through the queue) before `finish`; yoagent sends `AgentEnd` after `finish`, so the queue stays open until the hooks drop. Cancel handle: `Target::call` turns the first argument (when an object) into a `Value::Record` of its fields as `Value::Data` plus `signal: Value::Signal` (`with_signal`, `SIGNAL_FIELD`) — rutis-bridge 0.7 allows a signal nested in a call argument; the JS runtime decodes it as a real `AbortSignal`, Python as `rutis.peer.Signal` (`.cancelled`, `await .wait()`) — so it is aborted whenever the host drops the call future (run cancel, hook timeout, plugin unload; never after completion), and fixed-signature Python methods still accept the call (a positional signal would break them). `on_event` uses `call_plain` (no handle: events are data). Python's `json.dumps(call)` now refuses the argument (documented). Tested in `languages_test` (`JS_CANCEL` / `PY_CANCEL` fixtures: run cancel and policy timeout abort, completed `*_quick` calls never do). **dsh adapter** (`plugins/dsh/`, own `package.json` + lock pinning `@arcships/rutis*` 0.7.0, dsh 0.2.0-rc.2 incl. `dsh-settings` (dsh-free-search imports an error class from it), `@deepseek-ai/cordis` 4.0.4, `dsh-free-search` 0.8.1; separate from `plugins/package.json` to keep that install small): `dsh-tools-adapter.ts` (a rutis `definePlugin`, injects `tools`, `systemPrompt`, `yoagent`; `tools` = `schemas()` (config allowlist `tools`), `call_tool` = `execute({callId: "yoagent:", name, arguments, signal: call.signal})`, `isError` → `{text, is_error: true}`, text blocks kept, an image block (`{attachment: ImageAttachmentRef}`) read with the `attachments` service (`readImage(ref, signal)` → base64 + `ref.mediaType`) looked up **per image** via `ctx.use` in a try (never injected — the adapter runs without a store; none or a failed read → `[image …: not available here]` text), other blocks named, `before_model` = `systemPrompt.assemble()` minus `harness:identity` / `deployment:persona-{prefix,suffix}`, each section rendered alone with `renderPrompt` (unset variables → skipped), joined under `[Guidance from dsh plugins]`, capped at `maxNoteChars` 2000); `fixture-tools.ts` (a plain Cordis plugin: `defineTool` echo / fail / slow-until-abort writing `config.abortFile`, and a `fixture:guidance` section); `tests/dsh_test.rs` (`required-features = ["node"]`, skips without `plugins/dsh/node_modules` unless `YOAGENT_RUTIS_REQUIRE_RUNTIMES=1`); `examples/dsh_tools.rs` (feature `node`; rows `dsh-web`, `dsh-system-prompt`, `dsh-tools`, `dsh-free-search` (bing, en-US), the adapter; scripted `platform_search` hits the real web; `--live` DeepSeek via `DEEPSEEK_API_KEY` / `~/.dskey`; self-checks). **rutis-agent example** `examples/rutis-agent-tools/` (own `[workspace]`, `/target` git-ignored, excluded from the package, not auto-discovered, not in CI): rutis-agent from git (`arcships/rutis` tag v0.7.0 — crates.io's 0.2.0 is on rutis 0.2) + `[patch.crates-io] rutis` at the same tag, so one `rutis` (verified with `cargo tree -d`); a `RutisAgentTools` plugin (injects `tools_key()` + `Registry`) registers a `Handler` whose per-run `with_tools` maps `ToolRegistry::schemas()` (`aimux_core::options::Tool::Function`) to tools calling `execute(&tool_call(..), &ctx.cancel)` (`ok: false` → `ToolError::Failed`, cancelled → `Cancelled`; an output that parses wholly as `{"content": [text/image blocks]}` → those yoagent blocks — the adapter's image convention, rutis-agent itself has text results only — shown by `dot_picture`); shows `replace_text`, a hot-added `word_count`, and (scripted) a cancelled `slow` tool; `--live` DeepSeek. `ToolSpec` `description`/`parameters` `null` = missing. `plugins/`: `package.json` + lock (`@arcships/rutis`, `@arcships/rutis-runtime` 0.7.0), `requirements.txt` (`rutis==0.7.0`), `yoagent.d.ts` (the handler shape), `ts/example.ts`, `python/yoagent_example.py`; `node_modules`/`.venv` git-ignored and excluded from the package. `tests/languages_test.rs` (`required-features = ["node", "python"]`, unix): real runtimes via rutis-loader rows, a `probe` host service, fixtures written to a temp dir (JS imports `@arcships/rutis` by `file://` URL); a missing Node 24 / `plugins/node_modules` / Python with rutis 0.7 (`YOAGENT_RUTIS_PYTHON` or `plugins/.venv/bin/python`) prints `SKIPPED:` and passes, or fails with `YOAGENT_RUTIS_REQUIRE_RUNTIMES=1`. Example `language_plugins` (same features). CI jobs `rutis-bridge` (fmt/clippy/doc/test/example, `--manifest-path`), `rutis-bridge-msrv` (1.86, yoke-derive pin; also checks the language features) and `rutis-bridge-languages` (Linux, Node 24, Python 3.12: `npm ci` in `plugins/` and `plugins/dsh/`, clippy per feature, docs, tests with the runtimes required, the `language_plugins` example; `dsh_tools` is not run in CI — it needs the network). A PR whose base is not main gets no CI (the workflow triggers on main/release). Run its checks with `--manifest-path integrations/yoagent-rutis/Cargo.toml`. +Its own `Cargo.toml` (not a workspace member; cargo leaves the nested package out of `cargo package` for yoagent automatically), `yoagent` by path + version, `rutis = "0.6"` (0.x caret = 0.6.x; any rutis minor bump is a yoagent-rutis minor bump — rutis types are in its API; `pub use rutis`). yoagent's core must never depend on rutis; the bridge uses only yoagent's public API. Built on `Extension` (#250): `RutisBridge::install(&root)` — **on the root**: the bridge is bound to the installing ctx's generation, so on a plugin's ctx it reads as stopped for good once that plugin reloads — provides the `Registry` service (rutis holds one binding per key, so plugins add entries; an insertion-ordered `Vec` keyed by `seq`). A plugin registers a `Handler` (closure builder, `Clone`: a name + `with_tool`/`with_tools` (per run)/`with_before_tool`/`with_after_tool`/`with_before_model`/`with_on_input`/`with_on_stop`/`with_finish` each sync or `_async` (owned arg → future of `Result<_, ExtensionError>`), `with_on_event` sync; args are plain-data `Serialize` structs `ToolCall { tool, call_id, args, user_request, latest_user_text, run }`, `Turn`, `Input`, `Stop`, each with `#[serde(flatten)] run: RunInfo { run_id, label, depth, delegated_by, parent_run_id }`; decisions are yoagent's `ToolDecision`/`TurnDecision`/`InputDecision`/`StopDecision`, `after_tool` edits yoagent's `ToolOutput`). Behind it the crate-private `HandlerImpl` trait (`hooks() -> Hooks`, `static_tools`, one async method per hook, `events(run) -> Option>`), which language handlers implement too. `Registry::register(ctx, handler)` / `PluginCtxExt::register_handler` ties the entry to `ctx.effect_named` on the plugin's fiber (removed on dispose/restart/update/dependency eviction); handler names and **static** tool names unique across plugins (`CordisError::ServiceExists`, warned with the holder, no retry); each entry has a `Gate` = the plugin generation's `cancellation_token()` (cancelled at the start of an unload) + a `removed` token. `bridge.extension()` → `RutisExtension` (`impl Extension`, name "rutis"; host-set `.required()`, `.filters_tool_output()`, `.rechecks_modified_calls()`, `.require_policy()`, `.with_{policy,input,turn}_timeout(Option)`, `.with_timeout`): `start_run` snapshots available entries (`Registry::snapshot`) into a `RunState` (`impl RunHooks`). Every handler call goes through `extension::call` (gate checked → `Missed::Unavailable`; `catch_unwind`; `select!` against the gate going (in flight → unavailable); per-hook timeout → `Missed::Failed`); yoagent's own dispatcher already races cancel. Combination, registration order: `tools` static then per-run, dedup earlier wins, each wrapped in `LiveTool` (call after unload → "no longer available", in flight → "plugin unloaded during the call"); `on_input` first `Reject`, any miss rejects; `before_model` notes joined `\n`, first `Stop` ends, a miss is skipped (required → `Fail`); `before_tool` `Deny` wins / `Modify` feeds next / any miss (incl. unavailable) denies; `after_tool` chained, a failure → `Err` (yoagent withholds; required fails the run), an unavailable handler → the bridge withholds itself and returns `Ok` (an unload never fails a run); `on_stop` `Continue` messages joined, miss skipped/required fails; `on_event` sync per handler (panic → that handler off for the run; never unwinds, so publishing and other handlers go on); `finish` flushes event sinks, then calls all handlers concurrently. A required `tools`/`on_event` failure is never raised as a panic (yoagent would switch the whole extension's `on_event` off — bus publishing and every other handler): it is recorded in `RunState::failure` (tool calls denied / output withheld while pending, handed to yoagent through `RunHooks::take_failure` (required only), which the loop polls at every boundary incl. the run's end; only one on `AgentEnd` is just logged). Static tools of a handler already unavailable at `tools` are not offered. Timeouts: policy 60 s (`before_tool`, `after_tool`, `on_stop`), input 30 s (`on_input`), turn 5 s (`before_model`, `tools`, `finish`). `require_policy()` = a run starting with no `before_tool` handler denies every call (covers the reload window; no input counterpart). `Host::is_closed` (the generation token captured at **install** — a disposed or restarted root reads as stopped for good, since the registry went with that generation — + the ctx's current token + root fiber state) at `start_run` → deny every tool call / reject input / no notes, and per `before_tool` call (a shutdown mid-run). Events: `on_event` publishes every event with rutis `emit` as `AgentEventEmitted { run_id, label, depth, event }` (no listener → no task; one queue per bus; skipped when closed). Removed in #250: the four adapters, `attach`/`AgentRutisExt`, `ToolRegistry`, the `event_sender` tee, the waterfall/serial chains and their workarounds. `PluginCtxExt` (sealed: `register_handler`, `provide_tool` (= a `tool:` handler), `on_agent_event`) + `AgentPlugin::new(handler)` (named after the handler, injects `Registry`). Published with yoagent 0.25 (yoagent-rutis 0.1.0, requires yoagent 0.25). Tests share `tests/common` (`setup()`, `Setup(name, closure)`, `GreeterFactory`, `plugin`/`handler`/`deny_all`/`run_error`); policy edges drive `RutisExtension::start_run` + `RunHooks` directly with `RunContext::new` / `ToolCallRequest::new`. **TypeScript / Python plugins** (`languages.rs`, features `node`/`python`/`websocket` → optional `rutis-bridge = "0.7"`, `default-features = false`; the default build is Rust-only; `rutis-loader` 0.7 only a dev-dependency): `install` also provides `dyn HostDispatch` under `host_key("yoagent")` (`methods` = `{register: sync}`). `register(name, handler, options?)`: the handler is a live object (`Reference::is_object`: JS objects with functions, Python instances; hooks probed with a sync `get` on the plugin's call chain — `undefined`/`null`/Python `AttributeError` = absent, a function = present, anything else or another error refuses the registration — called with `call_method_async`, so `this` works) or a `Value::Record` of function refs (a Python dict; a `None`/`null` entry = absent, as on an object); `tools` needs `call_tool`; `options.events` (AgentEvent `type` tags; unknown ones warned) is required with `on_event` and only with it; returns a `Value::callback` disposer (cancels the registration's lifetime token = its `Gate` generation, removes the entry); `session::caller()` (the runtime's `Connection`) is watched with `closed()`, so a crashed or exited runtime's handlers are removed. `RemoteHandler` implements `HandlerImpl`: every hook is one `call` with JSON args (`after_tool(call, output)` two), results parsed strictly (`null` = default; `{deny}`/`{args}`, string/`{note}`/`{stop}`, `{reject}`, `{continue}`/`{fail}`, `{text,details,is_error}` with typed fields (`{}` = empty text, like `after_tool`'s edit) — or `content` blocks instead of `text` (never both), parsed by `content_blocks` (yoagent's JSON shape: `{type:"text",text}` / `{type:"image",data,mimeType}` — data decoded with `base64` STANDARD, 1 B to `MAX_IMAGE_BYTES` 10 MB (websocket frame limit 16 MiB), mimeType in `IMAGE_TYPES` png/jpeg/gif/webp; nothing else) — which `after_tool`'s edit also takes (`content` replaces every block; `after_tool` sees `output.content`); an `is_error` result's message is its text blocks joined, `{args}` must be an object; anything else is an error, so it fails closed); `RemoteTool` runs `call_tool` (no timeout), raced against the tool's cancel; `on_event` is delivered by one task per run and handler (in order, the extension's turn timeout per event, a bounded queue of 1024, filtered by `event_type` before serializing); the first failure (error, timeout, full queue, a panicking delivery — each delivery runs inside `catch_unwind` — or a delivery task found gone: `TrySendError::Closed` on `send`, or a `flush` whose send or ack fails) is kept in the sink: the task logs it (`warn!`) and drops the rest, `send` stops queueing, and `RunState` reads it via `EventSink::failure` — right after each `send` in `on_event` and in `check_sinks` at every decision point (`pending`/`take_failure`) — never as a panic; `EventSink::flush` (an ack through the queue) before `finish`; yoagent sends `AgentEnd` after `finish`, so the queue stays open until the hooks drop. Cancel handle: `Target::call` turns the first argument (when an object) into a `Value::Record` of its fields as `Value::Data` plus `signal: Value::Signal` (`with_signal`, `SIGNAL_FIELD`) — rutis-bridge 0.7 allows a signal nested in a call argument; the JS runtime decodes it as a real `AbortSignal`, Python as `rutis.peer.Signal` (`.cancelled`, `await .wait()`) — so it is aborted whenever the host drops the call future (run cancel, hook timeout, plugin unload; never after completion), and fixed-signature Python methods still accept the call (a positional signal would break them). `on_event` uses `call_plain` (no handle: events are data). Python's `json.dumps(call)` now refuses the argument (documented). Tested in `languages_test` (`JS_CANCEL` / `PY_CANCEL` fixtures: run cancel and policy timeout abort, completed `*_quick` calls never do). **dsh adapter** (`plugins/dsh/`, own `package.json` + lock pinning `@arcships/rutis*` 0.7.0, dsh 0.2.0-rc.2 incl. `dsh-settings` (dsh-free-search imports an error class from it), `@deepseek-ai/cordis` 4.0.4, `dsh-free-search` 0.8.1; separate from `plugins/package.json` to keep that install small): `dsh-tools-adapter.ts` (a rutis `definePlugin`, injects `tools`, `systemPrompt`, `yoagent`; `tools` = `schemas()` (config allowlist `tools`), `call_tool` = `execute({callId: "yoagent:", name, arguments, signal: call.signal})`, `isError` → `{text, is_error: true}`, text blocks kept, an image block (`{attachment: ImageAttachmentRef}`) read with the `attachments` service (`readImage(ref, signal)` → base64 + `ref.mediaType`) looked up **per image** via `ctx.use` in a try (never injected — the adapter runs without a store; none or a failed read → `[image …: not available here]` text; dsh's `offloaded` → `[…: offloaded]`; over `MAX_IMAGE_BYTES` 3.75 MB (ref `bytes` or read size) → `[…: too large to send]`; an `isError` result's images are not read), other blocks named, `before_model` = `systemPrompt.assemble()` minus `harness:identity` / `deployment:persona-{prefix,suffix}`, each section rendered alone with `renderPrompt` (unset variables → skipped), joined under `[Guidance from dsh plugins]`, capped at `maxNoteChars` 2000); `fixture-tools.ts` (a plain Cordis plugin: `defineTool` echo / fail / slow-until-abort writing `config.abortFile`, and a `fixture:guidance` section); `tests/dsh_test.rs` (`required-features = ["node"]`, skips without `plugins/dsh/node_modules` unless `YOAGENT_RUTIS_REQUIRE_RUNTIMES=1`); `examples/dsh_tools.rs` (feature `node`; rows `dsh-web`, `dsh-system-prompt`, `dsh-tools`, `dsh-free-search` (bing, en-US), the adapter; scripted `platform_search` hits the real web; `--live` DeepSeek via `DEEPSEEK_API_KEY` / `~/.dskey`; self-checks). **rutis-agent example** `examples/rutis-agent-tools/` (own `[workspace]`, `/target` git-ignored, excluded from the package, not auto-discovered, not in CI): rutis-agent from git (`arcships/rutis` tag v0.7.0 — crates.io's 0.2.0 is on rutis 0.2) + `[patch.crates-io] rutis` at the same tag, so one `rutis` (verified with `cargo tree -d`); a `RutisAgentTools` plugin (injects `tools_key()` + `Registry`) registers a `Handler` whose per-run `with_tools` maps `ToolRegistry::schemas()` (`aimux_core::options::Tool::Function`) to tools calling `execute(&tool_call(..), &ctx.cancel)` (`ok: false` → `ToolError::Failed`, cancelled → `Cancelled`; an output that parses wholly as `{"content": [text/image blocks]}` with at least one image → those yoagent blocks — the adapter's image convention, rutis-agent itself has text results only — shown by `dot_picture`); shows `replace_text`, a hot-added `word_count`, and (scripted) a cancelled `slow` tool; `--live` DeepSeek. `ToolSpec` `description`/`parameters` `null` = missing. `plugins/`: `package.json` + lock (`@arcships/rutis`, `@arcships/rutis-runtime` 0.7.0), `requirements.txt` (`rutis==0.7.0`), `yoagent.d.ts` (the handler shape), `ts/example.ts`, `python/yoagent_example.py`; `node_modules`/`.venv` git-ignored and excluded from the package. `tests/languages_test.rs` (`required-features = ["node", "python"]`, unix): real runtimes via rutis-loader rows, a `probe` host service, fixtures written to a temp dir (JS imports `@arcships/rutis` by `file://` URL); a missing Node 24 / `plugins/node_modules` / Python with rutis 0.7 (`YOAGENT_RUTIS_PYTHON` or `plugins/.venv/bin/python`) prints `SKIPPED:` and passes, or fails with `YOAGENT_RUTIS_REQUIRE_RUNTIMES=1`. Example `language_plugins` (same features). CI jobs `rutis-bridge` (fmt/clippy/doc/test/example, `--manifest-path`), `rutis-bridge-msrv` (1.86, yoke-derive pin; also checks the language features) and `rutis-bridge-languages` (Linux, Node 24, Python 3.12: `npm ci` in `plugins/` and `plugins/dsh/`, clippy per feature, docs, tests with the runtimes required, the `language_plugins` example; `dsh_tools` is not run in CI — it needs the network). A PR whose base is not main gets no CI (the workflow triggers on main/release). Run its checks with `--manifest-path integrations/yoagent-rutis/Cargo.toml`. ### OpenAPI Integration (`openapi/`, feature-gated) diff --git a/integrations/yoagent-rutis/Cargo.toml b/integrations/yoagent-rutis/Cargo.toml index 4583c35..63e211f 100644 --- a/integrations/yoagent-rutis/Cargo.toml +++ b/integrations/yoagent-rutis/Cargo.toml @@ -34,6 +34,8 @@ rutis = "0.6" # rutis-bridge minor bump is a yoagent-rutis minor bump too. rutis-bridge = { version = "0.7", optional = true, default-features = false } async-trait = "0.1" +# Decodes plugin images to check them (yoagent itself uses 0.22). +base64 = "0.22" futures = "0.3" serde = { version = "1", features = ["derive"] } serde_json = "1" diff --git a/integrations/yoagent-rutis/README.md b/integrations/yoagent-rutis/README.md index 4414fdd..18bb4cd 100644 --- a/integrations/yoagent-rutis/README.md +++ b/integrations/yoagent-rutis/README.md @@ -215,9 +215,12 @@ def apply(ctx, config): `{"type": "image", "data": , "mimeType": "image/…"}` — and `after_tool` sees the output's blocks as `output.content`, so a handler can return pictures, and keep, add or drop them when it edits a result. - Text and blocks are exclusive in one answer; anything else in `content` - (another block type, a non-`image/*` type, data that is not base64) fails - the answer. + Text and blocks are exclusive in one answer (so return picked fields, + not the `output` you were given). An image must be standard base64 of at + most 10 MB, typed `image/png`, `image/jpeg`, `image/gif` or `image/webp` + (what every provider takes); anything else in `content` fails the answer. + Stay under your provider's own limit too (Anthropic: 5 MB base64): an + image it refuses sits in the history and fails every later request. ```ts async call_tool(call) { @@ -280,7 +283,7 @@ offers every tool in dsh's tool registry (the `tools` service of | Hook | What the adapter does | |---|---| | `tools` | `tools.schemas()` → name, description, parameters (config `tools`: an allowlist) | -| `call_tool` | `tools.execute({callId, name, arguments, signal})` with the bridge's cancel handle as dsh's `signal`: cancelling the run aborts the dsh call. `isError` → an error tool result. Text blocks stay text; an image block — a reference into dsh's attachment store — becomes a yoagent image, its bytes read with the `attachments` service when one is loaded (looked up per image, so the adapter also runs without it; then, or when a read fails, the image is a text placeholder); other blocks are named | +| `call_tool` | `tools.execute({callId, name, arguments, signal})` with the bridge's cancel handle as dsh's `signal`: cancelling the run aborts the dsh call. `isError` → an error tool result. Text blocks stay text; an image block — a reference into dsh's attachment store — becomes a yoagent image, its bytes read with the `attachments` service when one is loaded (looked up per image, so the adapter also runs without it). Without one, when a read fails, when dsh marked the image `offloaded`, or over 3.75 MB (under Anthropic's 5 MB once base64), the image is a text placeholder; an error result's images are not read; other blocks are named | | `before_model` | the system-prompt sections dsh plugins added (the harness identity and persona slots left out, sections whose variables are unset skipped), as one note, capped at `maxNoteChars` (2000) | Load, as rows of one Node runtime whose `package.json` is `plugins/dsh/`'s: @@ -315,9 +318,10 @@ rutis-agent's), shown with rutis-agent's `replace_text` and a tool registered into the registry while the host runs, which the next run offers. rutis-agent's results are text (a runner's JSON value is serialized), so images use a convention of this adapter: a runner returning -`{"content": [blocks]}` in yoagent's block shape gives yoagent those blocks, -images included (`dot_picture` in the example); rutis-agent's own agent -still sees the JSON text. +`{"content": [blocks]}` in yoagent's block shape, with at least one image, +gives yoagent those blocks (`dot_picture` in the example); rutis-agent's +own agent still sees the JSON text. A tool that prints exactly such JSON +(an image included) as its text would be read as blocks too. ```sh cargo run --manifest-path examples/rutis-agent-tools/Cargo.toml [-- --live] diff --git a/integrations/yoagent-rutis/examples/rutis-agent-tools/src/main.rs b/integrations/yoagent-rutis/examples/rutis-agent-tools/src/main.rs index bf06112..baaf84b 100644 --- a/integrations/yoagent-rutis/examples/rutis-agent-tools/src/main.rs +++ b/integrations/yoagent-rutis/examples/rutis-agent-tools/src/main.rs @@ -179,8 +179,9 @@ impl AgentTool for RegistryTool { } /// A runner's `{"content": [blocks]}` value (as rutis-agent serialized it), -/// read back as yoagent content: only text and image blocks, and only when -/// the whole value is that shape — any other output stays text. +/// read back as yoagent content: only text and image blocks, only when the +/// whole value is that shape, and only with at least one image — so a tool +/// that prints such JSON as text (`cat result.json`) mostly stays text. fn content_of(output: &str) -> Option> { if !output.starts_with('{') { return None; @@ -190,16 +191,12 @@ fn content_of(output: &str) -> Option> { return None; } let blocks: Vec = serde_json::from_value(value.remove("content")?).ok()?; - blocks + let image = |b: &Content| matches!(b, Content::Image { data, mime_type } if !data.is_empty() && mime_type.starts_with("image/")); + let valid = blocks .iter() - .all(|b| match b { - Content::Text { .. } => true, - Content::Image { data, mime_type } => { - !data.is_empty() && mime_type.starts_with("image/") - } - _ => false, - }) - .then_some(blocks) + .all(|b| matches!(b, Content::Text { .. }) || image(b)) + && blocks.iter().any(image); + valid.then_some(blocks) } // ── The host ──────────────────────────────────────────────────── diff --git a/integrations/yoagent-rutis/plugins/dsh/dsh-tools-adapter.ts b/integrations/yoagent-rutis/plugins/dsh/dsh-tools-adapter.ts index 64571a1..294078e 100644 --- a/integrations/yoagent-rutis/plugins/dsh/dsh-tools-adapter.ts +++ b/integrations/yoagent-rutis/plugins/dsh/dsh-tools-adapter.ts @@ -23,9 +23,12 @@ // result. Text blocks stay text; an image block (a reference // into dsh's attachment store) becomes a yoagent image, read // with the `attachments` service when one is loaded (looked up -// per image, not injected: the adapter runs without it); with -// none, or when a read fails, the image is a text placeholder. -// Other blocks are named. +// per image, not injected: the adapter runs without it). With +// none, when a read fails, when dsh marked the image +// `offloaded`, or when it is over MAX_IMAGE_BYTES (provider-safe: +// Anthropic takes 5 MB base64), the image is a text +// placeholder. Other blocks are named. An error result's +// images are not read. // before_model the sections dsh plugins added to `systemPrompt` (the // harness identity and persona slots left out), rendered and // capped, as a note on the request's latest user turn. @@ -43,7 +46,15 @@ interface DshAttachments { } /** A dsh content block as a tool result carries it. */ -type DshBlock = { type: string; text?: string; attachment?: { attachmentId: string; mediaType: string; name?: string } } +type DshBlock = { + type: string + text?: string + attachment?: { attachmentId: string; mediaType: string; name?: string; bytes?: number } + offloaded?: true +} + +/** Largest image sent as an image: under Anthropic's 5 MB once base64-encoded. */ +const MAX_IMAGE_BYTES = 3_750_000 /** The slice of `@deepseek-ai/dsh-tools`' `ToolRuntime` this adapter uses. */ interface DshTools { @@ -115,14 +126,20 @@ export default definePlugin({ /** An image reference as a yoagent image: its bytes from dsh's attachment store. */ const image = async ( - ref: { attachmentId: string; mediaType: string; name?: string }, + ref: { attachmentId: string; mediaType: string; name?: string; bytes?: number }, + offloaded: boolean, signal: AbortSignal, ): Promise => { - const label = `[image${ref.name ? ` ${ref.name}` : ''}: not available here]` + const named = `image${ref.name ? ` ${ref.name}` : ''}` + const label = `[${named}: not available here]` + // dsh decided this image goes out as text; and an oversize one would fail the request. + if (offloaded) return { type: 'text', text: `[${named}: offloaded]` } + if ((ref.bytes ?? 0) > MAX_IMAGE_BYTES) return { type: 'text', text: `[${named}: too large to send]` } const store = attachments() if (!store) return { type: 'text', text: label } try { const stored = await store.readImage(ref, signal) + if (stored.data.byteLength > MAX_IMAGE_BYTES) return { type: 'text', text: `[${named}: too large to send]` } return { type: 'image', data: Buffer.from(stored.data).toString('base64'), mimeType: stored.ref.mediaType } } catch (error) { if (signal.aborted) throw error @@ -151,20 +168,22 @@ export default definePlugin({ arguments: call.args, signal: call.signal, }) + if (out.isError) { + const text = (out.content ?? []) + .map((block) => (block.type === 'text' ? (block.text ?? '') : `[${block.type} block]`)) + .join('\n') + return { text: text || out.error?.message || 'the dsh tool failed', is_error: true } + } const content: ContentBlock[] = [] for (const block of out.content ?? []) { if (block.type === 'text') { content.push({ type: 'text', text: block.text ?? '' }) } else if (block.type === 'image' && block.attachment) { - content.push(await image(block.attachment, call.signal)) + content.push(await image(block.attachment, block.offloaded === true, call.signal)) } else { content.push({ type: 'text', text: `[${block.type} block]` }) } } - if (out.isError) { - const text = content.flatMap((b) => (b.type === 'text' ? [b.text] : [])).join('\n') - return { text: text || out.error?.message || 'the dsh tool failed', is_error: true } - } return { content } }, diff --git a/integrations/yoagent-rutis/plugins/dsh/fixture-tools.ts b/integrations/yoagent-rutis/plugins/dsh/fixture-tools.ts index fb5323e..7dfe039 100644 --- a/integrations/yoagent-rutis/plugins/dsh/fixture-tools.ts +++ b/integrations/yoagent-rutis/plugins/dsh/fixture-tools.ts @@ -5,8 +5,9 @@ // // fixture_echo {text} → "echo: " // fixture_fail {why} → throws: dsh reports an `isError` result -// fixture_dot {} → a text block and an image block referring to the -// attachment `fixture-dot` (see fixture-attachments.ts) +// fixture_dot {} → a text block, an image block referring to the +// attachment `fixture-dot` (see fixture-attachments.ts), +// an offloaded one and an oversize one // fixture_slow {} → writes "started" to `config.abortFile`, waits for // `exec.signal` to abort (60 s at most), then // writes how it ended there @@ -63,6 +64,17 @@ export function apply(ctx: any, config: Config | undefined) { type: 'image' as const, attachment: { attachmentId: 'fixture-dot', mediaType: 'image/png', bytes: 70, width: 1, height: 1 }, }, + // dsh decided to send this one as text. + { + type: 'image' as const, + attachment: { attachmentId: 'fixture-dot', mediaType: 'image/png', bytes: 70, width: 1, height: 1, name: 'offloaded.png' }, + offloaded: true as const, + }, + // Over the adapter's provider-safe limit. + { + type: 'image' as const, + attachment: { attachmentId: 'fixture-huge', mediaType: 'image/png', bytes: 9_000_000, width: 9000, height: 9000, name: 'huge.png' }, + }, ], }, async execute() { diff --git a/integrations/yoagent-rutis/plugins/yoagent.d.ts b/integrations/yoagent-rutis/plugins/yoagent.d.ts index 4fb20bb..8816dcc 100644 --- a/integrations/yoagent-rutis/plugins/yoagent.d.ts +++ b/integrations/yoagent-rutis/plugins/yoagent.d.ts @@ -153,13 +153,17 @@ export interface ToolCall extends RunInfo { export type ToolVerdict = void | null | { deny: string } | { args: Record } /** - * A content block, in yoagent's JSON shape (also pi's and MCP's): text, or - * an image as base64 `data` with an `image/*` `mimeType`. + * A content block, in yoagent's JSON shape (also pi's, and MCP's bare + * blocks): text, or an image as standard base64 `data` (at most 10 MB + * decoded) with a `mimeType` of `image/png`, `image/jpeg`, `image/gif` or + * `image/webp` — the types every provider takes. Stay under your provider's + * own limit too (Anthropic: 5 MB base64): an image it refuses is in the + * history, and fails every later request. */ export type ContentBlock = { type: 'text'; text: string } | { type: 'image'; data: string; mimeType: string } export interface ToolOutput { - /** The text blocks, joined. */ + /** The text blocks, concatenated with no separator. */ text: string /** yoagent's content blocks (text and images). */ content: ContentBlock[] @@ -170,7 +174,8 @@ export interface ToolOutput { /** * Keep the output (nothing), or replace parts of it: `text` replaces every * content block with one text block, `content` with the given blocks (so - * images can be kept, added or dropped). Not both. + * images can be kept, added or dropped). Not both — so do not return the + * `output` you were given (`{...output, text}` has both): pick the fields. */ export type OutputEdit = | void @@ -227,8 +232,8 @@ export interface ToolSpec { /** * The tool's text, or the text plus details (a missing `text` — `{}` too — * is an empty text), or `content` blocks instead of `text` (images included; - * not both); `is_error` (or a throw) fails the call, with the text blocks as - * its message. + * not both); `is_error` (or a throw) fails the call, with the text blocks, + * joined by newlines, as its message. */ export type ToolResult = | string diff --git a/integrations/yoagent-rutis/src/languages.rs b/integrations/yoagent-rutis/src/languages.rs index daba871..c590035 100644 --- a/integrations/yoagent-rutis/src/languages.rs +++ b/integrations/yoagent-rutis/src/languages.rs @@ -946,12 +946,23 @@ impl AgentTool for RemoteTool { } } +/// Image types every provider takes; others (svg, bmp, tiff, ...) are refused +/// rather than put into history, where they would fail every later request. +const IMAGE_TYPES: [&str; 4] = ["image/png", "image/jpeg", "image/gif", "image/webp"]; + +/// Largest image a plugin may hand over, decoded. Keeps one answer under the +/// websocket transport's frame limit (16 MiB) once base64-encoded; providers +/// may take less (Anthropic: 5 MB) — plugins should stay under theirs. +const MAX_IMAGE_BYTES: usize = 10 * 1024 * 1024; + /// A handler's content blocks — `{"type": "text", "text"}` and /// `{"type": "image", "data", "mimeType"}`, yoagent's own JSON shape (and -/// pi's, and MCP's) — as yoagent content. `None` for anything else: not an -/// array, an unknown block type, an extra or missing field, an image whose -/// `mimeType` is not `image/*` or whose `data` is empty or not base64. +/// pi's, and MCP's bare blocks) — as yoagent content. `None` for anything +/// else: not an array, an unknown block type, an extra or missing field, an +/// image whose `mimeType` is not one of IMAGE_TYPES or whose `data` is not +/// standard base64 of 1 to MAX_IMAGE_BYTES bytes. fn content_blocks(blocks: &Json) -> Option> { + use base64::Engine as _; blocks .as_array()? .iter() @@ -965,11 +976,15 @@ fn content_blocks(blocks: &Json) -> Option> { "image" if fields.len() == 3 => { let data = field("data")?; let mime_type = field("mimeType")?; - let base64 = !data.is_empty() - && data - .bytes() - .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'+' | b'/' | b'=')); - (base64 && mime_type.starts_with("image/")).then(|| Content::Image { + // A decoded length bound before decoding: 3 bytes per 4 characters. + let fits = data.len() / 4 * 3 <= MAX_IMAGE_BYTES + 2; + let bytes = fits + .then(|| base64::engine::general_purpose::STANDARD.decode(data).ok()) + .flatten()?; + let valid = !bytes.is_empty() + && bytes.len() <= MAX_IMAGE_BYTES + && IMAGE_TYPES.contains(&mime_type); + valid.then(|| Content::Image { data: data.to_string(), mime_type: mime_type.to_string(), }) @@ -1008,6 +1023,15 @@ mod tests { json!([{"type": "image", "data": "", "mimeType": "image/png"}]), json!([{"type": "image", "data": "not base64!", "mimeType": "image/png"}]), json!([{"type": "image", "data": "AAAA", "mimeType": "text/plain"}]), + // Mis-padded, or padding in the middle: not base64. + json!([{"type": "image", "data": "AAAAA", "mimeType": "image/png"}]), + json!([{"type": "image", "data": "A=AA", "mimeType": "image/png"}]), + json!([{"type": "image", "data": "=", "mimeType": "image/png"}]), + // A type providers do not take. + json!([{"type": "image", "data": "AAAA", "mimeType": "image/svg+xml"}]), + json!([{"type": "image", "data": "AAAA", "mimeType": "image/"}]), + // Over the size limit. + json!([{"type": "image", "data": "A".repeat((MAX_IMAGE_BYTES + 3) / 3 * 4), "mimeType": "image/png"}]), json!([{"type": "image", "data": "AAAA"}]), json!([{"type": "file", "data": "AAAA", "mimeType": "image/png"}]), ] { diff --git a/integrations/yoagent-rutis/tests/dsh_test.rs b/integrations/yoagent-rutis/tests/dsh_test.rs index 5c27fad..0ded5bd 100644 --- a/integrations/yoagent-rutis/tests/dsh_test.rs +++ b/integrations/yoagent-rutis/tests/dsh_test.rs @@ -278,6 +278,15 @@ async fn dsh_images_arrive_as_images_when_an_attachment_store_is_loaded() { matches!(&content[0], Content::Text { text } if text == "a dot"), "{content:?}" ); + // dsh's offloaded image and the oversize one stay text, store or not. + assert!( + matches!(&content[2], Content::Text { text } if text.contains("offloaded.png: offloaded")), + "{content:?}" + ); + assert!( + matches!(&content[3], Content::Text { text } if text.contains("huge.png: too large")), + "{content:?}" + ); if with_store { // The attachment's bytes, read from the store, as a yoagent image. assert!( From 49cb4ef82c5ed39b35bc4002f63bbfba96e26200 Mon Sep 17 00:00:00 2001 From: Yuanhao Li Date: Thu, 8 Oct 2026 21:31:12 +0200 Subject: [PATCH 3/3] fix(rutis): an after_tool edit may keep an image yoagent let in An image identical to one in the output the edit was given passes as it is: yoagent's read_file takes bmp up to 20 MB, MCP tools other types, and keeping them must not withhold the result. New or changed images are checked as before. Tests: kept bmp passes, changed data does not; the size pre-check's own rejection. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01T7iq5hpndSiHQcnAsywKuG --- CLAUDE.md | 2 +- integrations/yoagent-rutis/README.md | 5 +- .../yoagent-rutis/plugins/yoagent.d.ts | 4 +- integrations/yoagent-rutis/src/languages.rs | 60 ++++++++++++++----- 4 files changed, 54 insertions(+), 17 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 160fcbb..cfd9083 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -100,7 +100,7 @@ Cloudflare Workers bindings (objects the runtime hands a Worker in `env`) as yoa ### rutis bridge (`integrations/yoagent-rutis/`, separate crate) -Its own `Cargo.toml` (not a workspace member; cargo leaves the nested package out of `cargo package` for yoagent automatically), `yoagent` by path + version, `rutis = "0.6"` (0.x caret = 0.6.x; any rutis minor bump is a yoagent-rutis minor bump — rutis types are in its API; `pub use rutis`). yoagent's core must never depend on rutis; the bridge uses only yoagent's public API. Built on `Extension` (#250): `RutisBridge::install(&root)` — **on the root**: the bridge is bound to the installing ctx's generation, so on a plugin's ctx it reads as stopped for good once that plugin reloads — provides the `Registry` service (rutis holds one binding per key, so plugins add entries; an insertion-ordered `Vec` keyed by `seq`). A plugin registers a `Handler` (closure builder, `Clone`: a name + `with_tool`/`with_tools` (per run)/`with_before_tool`/`with_after_tool`/`with_before_model`/`with_on_input`/`with_on_stop`/`with_finish` each sync or `_async` (owned arg → future of `Result<_, ExtensionError>`), `with_on_event` sync; args are plain-data `Serialize` structs `ToolCall { tool, call_id, args, user_request, latest_user_text, run }`, `Turn`, `Input`, `Stop`, each with `#[serde(flatten)] run: RunInfo { run_id, label, depth, delegated_by, parent_run_id }`; decisions are yoagent's `ToolDecision`/`TurnDecision`/`InputDecision`/`StopDecision`, `after_tool` edits yoagent's `ToolOutput`). Behind it the crate-private `HandlerImpl` trait (`hooks() -> Hooks`, `static_tools`, one async method per hook, `events(run) -> Option>`), which language handlers implement too. `Registry::register(ctx, handler)` / `PluginCtxExt::register_handler` ties the entry to `ctx.effect_named` on the plugin's fiber (removed on dispose/restart/update/dependency eviction); handler names and **static** tool names unique across plugins (`CordisError::ServiceExists`, warned with the holder, no retry); each entry has a `Gate` = the plugin generation's `cancellation_token()` (cancelled at the start of an unload) + a `removed` token. `bridge.extension()` → `RutisExtension` (`impl Extension`, name "rutis"; host-set `.required()`, `.filters_tool_output()`, `.rechecks_modified_calls()`, `.require_policy()`, `.with_{policy,input,turn}_timeout(Option)`, `.with_timeout`): `start_run` snapshots available entries (`Registry::snapshot`) into a `RunState` (`impl RunHooks`). Every handler call goes through `extension::call` (gate checked → `Missed::Unavailable`; `catch_unwind`; `select!` against the gate going (in flight → unavailable); per-hook timeout → `Missed::Failed`); yoagent's own dispatcher already races cancel. Combination, registration order: `tools` static then per-run, dedup earlier wins, each wrapped in `LiveTool` (call after unload → "no longer available", in flight → "plugin unloaded during the call"); `on_input` first `Reject`, any miss rejects; `before_model` notes joined `\n`, first `Stop` ends, a miss is skipped (required → `Fail`); `before_tool` `Deny` wins / `Modify` feeds next / any miss (incl. unavailable) denies; `after_tool` chained, a failure → `Err` (yoagent withholds; required fails the run), an unavailable handler → the bridge withholds itself and returns `Ok` (an unload never fails a run); `on_stop` `Continue` messages joined, miss skipped/required fails; `on_event` sync per handler (panic → that handler off for the run; never unwinds, so publishing and other handlers go on); `finish` flushes event sinks, then calls all handlers concurrently. A required `tools`/`on_event` failure is never raised as a panic (yoagent would switch the whole extension's `on_event` off — bus publishing and every other handler): it is recorded in `RunState::failure` (tool calls denied / output withheld while pending, handed to yoagent through `RunHooks::take_failure` (required only), which the loop polls at every boundary incl. the run's end; only one on `AgentEnd` is just logged). Static tools of a handler already unavailable at `tools` are not offered. Timeouts: policy 60 s (`before_tool`, `after_tool`, `on_stop`), input 30 s (`on_input`), turn 5 s (`before_model`, `tools`, `finish`). `require_policy()` = a run starting with no `before_tool` handler denies every call (covers the reload window; no input counterpart). `Host::is_closed` (the generation token captured at **install** — a disposed or restarted root reads as stopped for good, since the registry went with that generation — + the ctx's current token + root fiber state) at `start_run` → deny every tool call / reject input / no notes, and per `before_tool` call (a shutdown mid-run). Events: `on_event` publishes every event with rutis `emit` as `AgentEventEmitted { run_id, label, depth, event }` (no listener → no task; one queue per bus; skipped when closed). Removed in #250: the four adapters, `attach`/`AgentRutisExt`, `ToolRegistry`, the `event_sender` tee, the waterfall/serial chains and their workarounds. `PluginCtxExt` (sealed: `register_handler`, `provide_tool` (= a `tool:` handler), `on_agent_event`) + `AgentPlugin::new(handler)` (named after the handler, injects `Registry`). Published with yoagent 0.25 (yoagent-rutis 0.1.0, requires yoagent 0.25). Tests share `tests/common` (`setup()`, `Setup(name, closure)`, `GreeterFactory`, `plugin`/`handler`/`deny_all`/`run_error`); policy edges drive `RutisExtension::start_run` + `RunHooks` directly with `RunContext::new` / `ToolCallRequest::new`. **TypeScript / Python plugins** (`languages.rs`, features `node`/`python`/`websocket` → optional `rutis-bridge = "0.7"`, `default-features = false`; the default build is Rust-only; `rutis-loader` 0.7 only a dev-dependency): `install` also provides `dyn HostDispatch` under `host_key("yoagent")` (`methods` = `{register: sync}`). `register(name, handler, options?)`: the handler is a live object (`Reference::is_object`: JS objects with functions, Python instances; hooks probed with a sync `get` on the plugin's call chain — `undefined`/`null`/Python `AttributeError` = absent, a function = present, anything else or another error refuses the registration — called with `call_method_async`, so `this` works) or a `Value::Record` of function refs (a Python dict; a `None`/`null` entry = absent, as on an object); `tools` needs `call_tool`; `options.events` (AgentEvent `type` tags; unknown ones warned) is required with `on_event` and only with it; returns a `Value::callback` disposer (cancels the registration's lifetime token = its `Gate` generation, removes the entry); `session::caller()` (the runtime's `Connection`) is watched with `closed()`, so a crashed or exited runtime's handlers are removed. `RemoteHandler` implements `HandlerImpl`: every hook is one `call` with JSON args (`after_tool(call, output)` two), results parsed strictly (`null` = default; `{deny}`/`{args}`, string/`{note}`/`{stop}`, `{reject}`, `{continue}`/`{fail}`, `{text,details,is_error}` with typed fields (`{}` = empty text, like `after_tool`'s edit) — or `content` blocks instead of `text` (never both), parsed by `content_blocks` (yoagent's JSON shape: `{type:"text",text}` / `{type:"image",data,mimeType}` — data decoded with `base64` STANDARD, 1 B to `MAX_IMAGE_BYTES` 10 MB (websocket frame limit 16 MiB), mimeType in `IMAGE_TYPES` png/jpeg/gif/webp; nothing else) — which `after_tool`'s edit also takes (`content` replaces every block; `after_tool` sees `output.content`); an `is_error` result's message is its text blocks joined, `{args}` must be an object; anything else is an error, so it fails closed); `RemoteTool` runs `call_tool` (no timeout), raced against the tool's cancel; `on_event` is delivered by one task per run and handler (in order, the extension's turn timeout per event, a bounded queue of 1024, filtered by `event_type` before serializing); the first failure (error, timeout, full queue, a panicking delivery — each delivery runs inside `catch_unwind` — or a delivery task found gone: `TrySendError::Closed` on `send`, or a `flush` whose send or ack fails) is kept in the sink: the task logs it (`warn!`) and drops the rest, `send` stops queueing, and `RunState` reads it via `EventSink::failure` — right after each `send` in `on_event` and in `check_sinks` at every decision point (`pending`/`take_failure`) — never as a panic; `EventSink::flush` (an ack through the queue) before `finish`; yoagent sends `AgentEnd` after `finish`, so the queue stays open until the hooks drop. Cancel handle: `Target::call` turns the first argument (when an object) into a `Value::Record` of its fields as `Value::Data` plus `signal: Value::Signal` (`with_signal`, `SIGNAL_FIELD`) — rutis-bridge 0.7 allows a signal nested in a call argument; the JS runtime decodes it as a real `AbortSignal`, Python as `rutis.peer.Signal` (`.cancelled`, `await .wait()`) — so it is aborted whenever the host drops the call future (run cancel, hook timeout, plugin unload; never after completion), and fixed-signature Python methods still accept the call (a positional signal would break them). `on_event` uses `call_plain` (no handle: events are data). Python's `json.dumps(call)` now refuses the argument (documented). Tested in `languages_test` (`JS_CANCEL` / `PY_CANCEL` fixtures: run cancel and policy timeout abort, completed `*_quick` calls never do). **dsh adapter** (`plugins/dsh/`, own `package.json` + lock pinning `@arcships/rutis*` 0.7.0, dsh 0.2.0-rc.2 incl. `dsh-settings` (dsh-free-search imports an error class from it), `@deepseek-ai/cordis` 4.0.4, `dsh-free-search` 0.8.1; separate from `plugins/package.json` to keep that install small): `dsh-tools-adapter.ts` (a rutis `definePlugin`, injects `tools`, `systemPrompt`, `yoagent`; `tools` = `schemas()` (config allowlist `tools`), `call_tool` = `execute({callId: "yoagent:", name, arguments, signal: call.signal})`, `isError` → `{text, is_error: true}`, text blocks kept, an image block (`{attachment: ImageAttachmentRef}`) read with the `attachments` service (`readImage(ref, signal)` → base64 + `ref.mediaType`) looked up **per image** via `ctx.use` in a try (never injected — the adapter runs without a store; none or a failed read → `[image …: not available here]` text; dsh's `offloaded` → `[…: offloaded]`; over `MAX_IMAGE_BYTES` 3.75 MB (ref `bytes` or read size) → `[…: too large to send]`; an `isError` result's images are not read), other blocks named, `before_model` = `systemPrompt.assemble()` minus `harness:identity` / `deployment:persona-{prefix,suffix}`, each section rendered alone with `renderPrompt` (unset variables → skipped), joined under `[Guidance from dsh plugins]`, capped at `maxNoteChars` 2000); `fixture-tools.ts` (a plain Cordis plugin: `defineTool` echo / fail / slow-until-abort writing `config.abortFile`, and a `fixture:guidance` section); `tests/dsh_test.rs` (`required-features = ["node"]`, skips without `plugins/dsh/node_modules` unless `YOAGENT_RUTIS_REQUIRE_RUNTIMES=1`); `examples/dsh_tools.rs` (feature `node`; rows `dsh-web`, `dsh-system-prompt`, `dsh-tools`, `dsh-free-search` (bing, en-US), the adapter; scripted `platform_search` hits the real web; `--live` DeepSeek via `DEEPSEEK_API_KEY` / `~/.dskey`; self-checks). **rutis-agent example** `examples/rutis-agent-tools/` (own `[workspace]`, `/target` git-ignored, excluded from the package, not auto-discovered, not in CI): rutis-agent from git (`arcships/rutis` tag v0.7.0 — crates.io's 0.2.0 is on rutis 0.2) + `[patch.crates-io] rutis` at the same tag, so one `rutis` (verified with `cargo tree -d`); a `RutisAgentTools` plugin (injects `tools_key()` + `Registry`) registers a `Handler` whose per-run `with_tools` maps `ToolRegistry::schemas()` (`aimux_core::options::Tool::Function`) to tools calling `execute(&tool_call(..), &ctx.cancel)` (`ok: false` → `ToolError::Failed`, cancelled → `Cancelled`; an output that parses wholly as `{"content": [text/image blocks]}` with at least one image → those yoagent blocks — the adapter's image convention, rutis-agent itself has text results only — shown by `dot_picture`); shows `replace_text`, a hot-added `word_count`, and (scripted) a cancelled `slow` tool; `--live` DeepSeek. `ToolSpec` `description`/`parameters` `null` = missing. `plugins/`: `package.json` + lock (`@arcships/rutis`, `@arcships/rutis-runtime` 0.7.0), `requirements.txt` (`rutis==0.7.0`), `yoagent.d.ts` (the handler shape), `ts/example.ts`, `python/yoagent_example.py`; `node_modules`/`.venv` git-ignored and excluded from the package. `tests/languages_test.rs` (`required-features = ["node", "python"]`, unix): real runtimes via rutis-loader rows, a `probe` host service, fixtures written to a temp dir (JS imports `@arcships/rutis` by `file://` URL); a missing Node 24 / `plugins/node_modules` / Python with rutis 0.7 (`YOAGENT_RUTIS_PYTHON` or `plugins/.venv/bin/python`) prints `SKIPPED:` and passes, or fails with `YOAGENT_RUTIS_REQUIRE_RUNTIMES=1`. Example `language_plugins` (same features). CI jobs `rutis-bridge` (fmt/clippy/doc/test/example, `--manifest-path`), `rutis-bridge-msrv` (1.86, yoke-derive pin; also checks the language features) and `rutis-bridge-languages` (Linux, Node 24, Python 3.12: `npm ci` in `plugins/` and `plugins/dsh/`, clippy per feature, docs, tests with the runtimes required, the `language_plugins` example; `dsh_tools` is not run in CI — it needs the network). A PR whose base is not main gets no CI (the workflow triggers on main/release). Run its checks with `--manifest-path integrations/yoagent-rutis/Cargo.toml`. +Its own `Cargo.toml` (not a workspace member; cargo leaves the nested package out of `cargo package` for yoagent automatically), `yoagent` by path + version, `rutis = "0.6"` (0.x caret = 0.6.x; any rutis minor bump is a yoagent-rutis minor bump — rutis types are in its API; `pub use rutis`). yoagent's core must never depend on rutis; the bridge uses only yoagent's public API. Built on `Extension` (#250): `RutisBridge::install(&root)` — **on the root**: the bridge is bound to the installing ctx's generation, so on a plugin's ctx it reads as stopped for good once that plugin reloads — provides the `Registry` service (rutis holds one binding per key, so plugins add entries; an insertion-ordered `Vec` keyed by `seq`). A plugin registers a `Handler` (closure builder, `Clone`: a name + `with_tool`/`with_tools` (per run)/`with_before_tool`/`with_after_tool`/`with_before_model`/`with_on_input`/`with_on_stop`/`with_finish` each sync or `_async` (owned arg → future of `Result<_, ExtensionError>`), `with_on_event` sync; args are plain-data `Serialize` structs `ToolCall { tool, call_id, args, user_request, latest_user_text, run }`, `Turn`, `Input`, `Stop`, each with `#[serde(flatten)] run: RunInfo { run_id, label, depth, delegated_by, parent_run_id }`; decisions are yoagent's `ToolDecision`/`TurnDecision`/`InputDecision`/`StopDecision`, `after_tool` edits yoagent's `ToolOutput`). Behind it the crate-private `HandlerImpl` trait (`hooks() -> Hooks`, `static_tools`, one async method per hook, `events(run) -> Option>`), which language handlers implement too. `Registry::register(ctx, handler)` / `PluginCtxExt::register_handler` ties the entry to `ctx.effect_named` on the plugin's fiber (removed on dispose/restart/update/dependency eviction); handler names and **static** tool names unique across plugins (`CordisError::ServiceExists`, warned with the holder, no retry); each entry has a `Gate` = the plugin generation's `cancellation_token()` (cancelled at the start of an unload) + a `removed` token. `bridge.extension()` → `RutisExtension` (`impl Extension`, name "rutis"; host-set `.required()`, `.filters_tool_output()`, `.rechecks_modified_calls()`, `.require_policy()`, `.with_{policy,input,turn}_timeout(Option)`, `.with_timeout`): `start_run` snapshots available entries (`Registry::snapshot`) into a `RunState` (`impl RunHooks`). Every handler call goes through `extension::call` (gate checked → `Missed::Unavailable`; `catch_unwind`; `select!` against the gate going (in flight → unavailable); per-hook timeout → `Missed::Failed`); yoagent's own dispatcher already races cancel. Combination, registration order: `tools` static then per-run, dedup earlier wins, each wrapped in `LiveTool` (call after unload → "no longer available", in flight → "plugin unloaded during the call"); `on_input` first `Reject`, any miss rejects; `before_model` notes joined `\n`, first `Stop` ends, a miss is skipped (required → `Fail`); `before_tool` `Deny` wins / `Modify` feeds next / any miss (incl. unavailable) denies; `after_tool` chained, a failure → `Err` (yoagent withholds; required fails the run), an unavailable handler → the bridge withholds itself and returns `Ok` (an unload never fails a run); `on_stop` `Continue` messages joined, miss skipped/required fails; `on_event` sync per handler (panic → that handler off for the run; never unwinds, so publishing and other handlers go on); `finish` flushes event sinks, then calls all handlers concurrently. A required `tools`/`on_event` failure is never raised as a panic (yoagent would switch the whole extension's `on_event` off — bus publishing and every other handler): it is recorded in `RunState::failure` (tool calls denied / output withheld while pending, handed to yoagent through `RunHooks::take_failure` (required only), which the loop polls at every boundary incl. the run's end; only one on `AgentEnd` is just logged). Static tools of a handler already unavailable at `tools` are not offered. Timeouts: policy 60 s (`before_tool`, `after_tool`, `on_stop`), input 30 s (`on_input`), turn 5 s (`before_model`, `tools`, `finish`). `require_policy()` = a run starting with no `before_tool` handler denies every call (covers the reload window; no input counterpart). `Host::is_closed` (the generation token captured at **install** — a disposed or restarted root reads as stopped for good, since the registry went with that generation — + the ctx's current token + root fiber state) at `start_run` → deny every tool call / reject input / no notes, and per `before_tool` call (a shutdown mid-run). Events: `on_event` publishes every event with rutis `emit` as `AgentEventEmitted { run_id, label, depth, event }` (no listener → no task; one queue per bus; skipped when closed). Removed in #250: the four adapters, `attach`/`AgentRutisExt`, `ToolRegistry`, the `event_sender` tee, the waterfall/serial chains and their workarounds. `PluginCtxExt` (sealed: `register_handler`, `provide_tool` (= a `tool:` handler), `on_agent_event`) + `AgentPlugin::new(handler)` (named after the handler, injects `Registry`). Published with yoagent 0.25 (yoagent-rutis 0.1.0, requires yoagent 0.25). Tests share `tests/common` (`setup()`, `Setup(name, closure)`, `GreeterFactory`, `plugin`/`handler`/`deny_all`/`run_error`); policy edges drive `RutisExtension::start_run` + `RunHooks` directly with `RunContext::new` / `ToolCallRequest::new`. **TypeScript / Python plugins** (`languages.rs`, features `node`/`python`/`websocket` → optional `rutis-bridge = "0.7"`, `default-features = false`; the default build is Rust-only; `rutis-loader` 0.7 only a dev-dependency): `install` also provides `dyn HostDispatch` under `host_key("yoagent")` (`methods` = `{register: sync}`). `register(name, handler, options?)`: the handler is a live object (`Reference::is_object`: JS objects with functions, Python instances; hooks probed with a sync `get` on the plugin's call chain — `undefined`/`null`/Python `AttributeError` = absent, a function = present, anything else or another error refuses the registration — called with `call_method_async`, so `this` works) or a `Value::Record` of function refs (a Python dict; a `None`/`null` entry = absent, as on an object); `tools` needs `call_tool`; `options.events` (AgentEvent `type` tags; unknown ones warned) is required with `on_event` and only with it; returns a `Value::callback` disposer (cancels the registration's lifetime token = its `Gate` generation, removes the entry); `session::caller()` (the runtime's `Connection`) is watched with `closed()`, so a crashed or exited runtime's handlers are removed. `RemoteHandler` implements `HandlerImpl`: every hook is one `call` with JSON args (`after_tool(call, output)` two), results parsed strictly (`null` = default; `{deny}`/`{args}`, string/`{note}`/`{stop}`, `{reject}`, `{continue}`/`{fail}`, `{text,details,is_error}` with typed fields (`{}` = empty text, like `after_tool`'s edit) — or `content` blocks instead of `text` (never both), parsed by `content_blocks` (yoagent's JSON shape: `{type:"text",text}` / `{type:"image",data,mimeType}` — data decoded with `base64` STANDARD, 1 B to `MAX_IMAGE_BYTES` 10 MB (websocket frame limit 16 MiB), mimeType in `IMAGE_TYPES` png/jpeg/gif/webp; nothing else; in `after_tool` an image identical to one in the given output (`kept`) passes unchecked) — which `after_tool`'s edit also takes (`content` replaces every block; `after_tool` sees `output.content`); an `is_error` result's message is its text blocks joined, `{args}` must be an object; anything else is an error, so it fails closed); `RemoteTool` runs `call_tool` (no timeout), raced against the tool's cancel; `on_event` is delivered by one task per run and handler (in order, the extension's turn timeout per event, a bounded queue of 1024, filtered by `event_type` before serializing); the first failure (error, timeout, full queue, a panicking delivery — each delivery runs inside `catch_unwind` — or a delivery task found gone: `TrySendError::Closed` on `send`, or a `flush` whose send or ack fails) is kept in the sink: the task logs it (`warn!`) and drops the rest, `send` stops queueing, and `RunState` reads it via `EventSink::failure` — right after each `send` in `on_event` and in `check_sinks` at every decision point (`pending`/`take_failure`) — never as a panic; `EventSink::flush` (an ack through the queue) before `finish`; yoagent sends `AgentEnd` after `finish`, so the queue stays open until the hooks drop. Cancel handle: `Target::call` turns the first argument (when an object) into a `Value::Record` of its fields as `Value::Data` plus `signal: Value::Signal` (`with_signal`, `SIGNAL_FIELD`) — rutis-bridge 0.7 allows a signal nested in a call argument; the JS runtime decodes it as a real `AbortSignal`, Python as `rutis.peer.Signal` (`.cancelled`, `await .wait()`) — so it is aborted whenever the host drops the call future (run cancel, hook timeout, plugin unload; never after completion), and fixed-signature Python methods still accept the call (a positional signal would break them). `on_event` uses `call_plain` (no handle: events are data). Python's `json.dumps(call)` now refuses the argument (documented). Tested in `languages_test` (`JS_CANCEL` / `PY_CANCEL` fixtures: run cancel and policy timeout abort, completed `*_quick` calls never do). **dsh adapter** (`plugins/dsh/`, own `package.json` + lock pinning `@arcships/rutis*` 0.7.0, dsh 0.2.0-rc.2 incl. `dsh-settings` (dsh-free-search imports an error class from it), `@deepseek-ai/cordis` 4.0.4, `dsh-free-search` 0.8.1; separate from `plugins/package.json` to keep that install small): `dsh-tools-adapter.ts` (a rutis `definePlugin`, injects `tools`, `systemPrompt`, `yoagent`; `tools` = `schemas()` (config allowlist `tools`), `call_tool` = `execute({callId: "yoagent:", name, arguments, signal: call.signal})`, `isError` → `{text, is_error: true}`, text blocks kept, an image block (`{attachment: ImageAttachmentRef}`) read with the `attachments` service (`readImage(ref, signal)` → base64 + `ref.mediaType`) looked up **per image** via `ctx.use` in a try (never injected — the adapter runs without a store; none or a failed read → `[image …: not available here]` text; dsh's `offloaded` → `[…: offloaded]`; over `MAX_IMAGE_BYTES` 3.75 MB (ref `bytes` or read size) → `[…: too large to send]`; an `isError` result's images are not read), other blocks named, `before_model` = `systemPrompt.assemble()` minus `harness:identity` / `deployment:persona-{prefix,suffix}`, each section rendered alone with `renderPrompt` (unset variables → skipped), joined under `[Guidance from dsh plugins]`, capped at `maxNoteChars` 2000); `fixture-tools.ts` (a plain Cordis plugin: `defineTool` echo / fail / slow-until-abort writing `config.abortFile`, and a `fixture:guidance` section); `tests/dsh_test.rs` (`required-features = ["node"]`, skips without `plugins/dsh/node_modules` unless `YOAGENT_RUTIS_REQUIRE_RUNTIMES=1`); `examples/dsh_tools.rs` (feature `node`; rows `dsh-web`, `dsh-system-prompt`, `dsh-tools`, `dsh-free-search` (bing, en-US), the adapter; scripted `platform_search` hits the real web; `--live` DeepSeek via `DEEPSEEK_API_KEY` / `~/.dskey`; self-checks). **rutis-agent example** `examples/rutis-agent-tools/` (own `[workspace]`, `/target` git-ignored, excluded from the package, not auto-discovered, not in CI): rutis-agent from git (`arcships/rutis` tag v0.7.0 — crates.io's 0.2.0 is on rutis 0.2) + `[patch.crates-io] rutis` at the same tag, so one `rutis` (verified with `cargo tree -d`); a `RutisAgentTools` plugin (injects `tools_key()` + `Registry`) registers a `Handler` whose per-run `with_tools` maps `ToolRegistry::schemas()` (`aimux_core::options::Tool::Function`) to tools calling `execute(&tool_call(..), &ctx.cancel)` (`ok: false` → `ToolError::Failed`, cancelled → `Cancelled`; an output that parses wholly as `{"content": [text/image blocks]}` with at least one image → those yoagent blocks — the adapter's image convention, rutis-agent itself has text results only — shown by `dot_picture`); shows `replace_text`, a hot-added `word_count`, and (scripted) a cancelled `slow` tool; `--live` DeepSeek. `ToolSpec` `description`/`parameters` `null` = missing. `plugins/`: `package.json` + lock (`@arcships/rutis`, `@arcships/rutis-runtime` 0.7.0), `requirements.txt` (`rutis==0.7.0`), `yoagent.d.ts` (the handler shape), `ts/example.ts`, `python/yoagent_example.py`; `node_modules`/`.venv` git-ignored and excluded from the package. `tests/languages_test.rs` (`required-features = ["node", "python"]`, unix): real runtimes via rutis-loader rows, a `probe` host service, fixtures written to a temp dir (JS imports `@arcships/rutis` by `file://` URL); a missing Node 24 / `plugins/node_modules` / Python with rutis 0.7 (`YOAGENT_RUTIS_PYTHON` or `plugins/.venv/bin/python`) prints `SKIPPED:` and passes, or fails with `YOAGENT_RUTIS_REQUIRE_RUNTIMES=1`. Example `language_plugins` (same features). CI jobs `rutis-bridge` (fmt/clippy/doc/test/example, `--manifest-path`), `rutis-bridge-msrv` (1.86, yoke-derive pin; also checks the language features) and `rutis-bridge-languages` (Linux, Node 24, Python 3.12: `npm ci` in `plugins/` and `plugins/dsh/`, clippy per feature, docs, tests with the runtimes required, the `language_plugins` example; `dsh_tools` is not run in CI — it needs the network). A PR whose base is not main gets no CI (the workflow triggers on main/release). Run its checks with `--manifest-path integrations/yoagent-rutis/Cargo.toml`. ### OpenAPI Integration (`openapi/`, feature-gated) diff --git a/integrations/yoagent-rutis/README.md b/integrations/yoagent-rutis/README.md index 18bb4cd..9e40160 100644 --- a/integrations/yoagent-rutis/README.md +++ b/integrations/yoagent-rutis/README.md @@ -218,7 +218,10 @@ def apply(ctx, config): Text and blocks are exclusive in one answer (so return picked fields, not the `output` you were given). An image must be standard base64 of at most 10 MB, typed `image/png`, `image/jpeg`, `image/gif` or `image/webp` - (what every provider takes); anything else in `content` fails the answer. + (what every provider takes); anything else in `content` fails the answer + — except, in an `after_tool` edit, an image identical to one in + `output.content`: keeping what yoagent let in (`read_file` takes bmp, up + to 20 MB) never fails the edit. Stay under your provider's own limit too (Anthropic: 5 MB base64): an image it refuses sits in the history and fails every later request. diff --git a/integrations/yoagent-rutis/plugins/yoagent.d.ts b/integrations/yoagent-rutis/plugins/yoagent.d.ts index 8816dcc..466ef9f 100644 --- a/integrations/yoagent-rutis/plugins/yoagent.d.ts +++ b/integrations/yoagent-rutis/plugins/yoagent.d.ts @@ -174,7 +174,9 @@ export interface ToolOutput { /** * Keep the output (nothing), or replace parts of it: `text` replaces every * content block with one text block, `content` with the given blocks (so - * images can be kept, added or dropped). Not both — so do not return the + * images can be kept, added or dropped; a kept image — identical to one in + * `output.content` — passes as it is, a new one must meet ContentBlock's + * rules). Not both — so do not return the * `output` you were given (`{...output, text}` has both): pick the fields. */ export type OutputEdit = diff --git a/integrations/yoagent-rutis/src/languages.rs b/integrations/yoagent-rutis/src/languages.rs index c590035..c714bf4 100644 --- a/integrations/yoagent-rutis/src/languages.rs +++ b/integrations/yoagent-rutis/src/languages.rs @@ -526,8 +526,8 @@ impl HandlerImpl for RemoteHandler { }]; } if let Some(content) = edit.get("content") { - output.result.content = - content_blocks(content).ok_or_else(|| unexpected("after_tool", &value))?; + output.result.content = content_blocks(content, &output.result.content) + .ok_or_else(|| unexpected("after_tool", &value))?; } if let Some(details) = edit.get("details") { output.result.details = details.clone(); @@ -916,7 +916,7 @@ impl AgentTool for RemoteTool { let content = match (fields.get("text"), fields.get("content")) { (Some(Json::String(text)), _) => vec![Content::Text { text: text.clone() }], (Some(_), _) => return Err(unexpected()), - (None, Some(blocks)) => content_blocks(blocks).ok_or_else(unexpected)?, + (None, Some(blocks)) => content_blocks(blocks, &[]).ok_or_else(unexpected)?, (None, None) => vec![Content::Text { text: String::new(), }], @@ -960,8 +960,11 @@ const MAX_IMAGE_BYTES: usize = 10 * 1024 * 1024; /// pi's, and MCP's bare blocks) — as yoagent content. `None` for anything /// else: not an array, an unknown block type, an extra or missing field, an /// image whose `mimeType` is not one of IMAGE_TYPES or whose `data` is not -/// standard base64 of 1 to MAX_IMAGE_BYTES bytes. -fn content_blocks(blocks: &Json) -> Option> { +/// standard base64 of 1 to MAX_IMAGE_BYTES bytes. An image identical to one +/// in `kept` (the output an `after_tool` edit was given) passes as it is: +/// yoagent let it in (`read_file` takes bmp, up to 20 MB), and keeping it +/// must not fail the edit. +fn content_blocks(blocks: &Json, kept: &[Content]) -> Option> { use base64::Engine as _; blocks .as_array()? @@ -976,6 +979,15 @@ fn content_blocks(blocks: &Json) -> Option> { "image" if fields.len() == 3 => { let data = field("data")?; let mime_type = field("mimeType")?; + let unchanged = kept.iter().any(|c| { + matches!(c, Content::Image { data: d, mime_type: m } if d == data && m == mime_type) + }); + if unchanged { + return Some(Content::Image { + data: data.to_string(), + mime_type: mime_type.to_string(), + }); + } // A decoded length bound before decoding: 3 bytes per 4 characters. let fits = data.len() / 4 * 3 <= MAX_IMAGE_BYTES + 2; let bytes = fits @@ -1006,15 +1018,18 @@ mod tests { #[test] fn content_blocks_take_text_and_images_in_yoagents_shape() { - let blocks = content_blocks(&json!([ - {"type": "text", "text": "see"}, - {"type": "image", "data": "iVBORw0KGgo=", "mimeType": "image/png"}, - ])) + let blocks = content_blocks( + &json!([ + {"type": "text", "text": "see"}, + {"type": "image", "data": "iVBORw0KGgo=", "mimeType": "image/png"}, + ]), + &[], + ) .unwrap(); assert!(matches!(&blocks[0], Content::Text { text } if text == "see")); assert!(matches!(&blocks[1], Content::Image { data, mime_type } if data == "iVBORw0KGgo=" && mime_type == "image/png")); - assert!(content_blocks(&json!([])).unwrap().is_empty()); + assert!(content_blocks(&json!([]), &[]).unwrap().is_empty()); for bad in [ json!("text"), json!([{"type": "text"}]), @@ -1023,6 +1038,8 @@ mod tests { json!([{"type": "image", "data": "", "mimeType": "image/png"}]), json!([{"type": "image", "data": "not base64!", "mimeType": "image/png"}]), json!([{"type": "image", "data": "AAAA", "mimeType": "text/plain"}]), + json!([{"type": "image", "data": "AAAA"}]), + json!([{"type": "file", "data": "AAAA", "mimeType": "image/png"}]), // Mis-padded, or padding in the middle: not base64. json!([{"type": "image", "data": "AAAAA", "mimeType": "image/png"}]), json!([{"type": "image", "data": "A=AA", "mimeType": "image/png"}]), @@ -1030,15 +1047,30 @@ mod tests { // A type providers do not take. json!([{"type": "image", "data": "AAAA", "mimeType": "image/svg+xml"}]), json!([{"type": "image", "data": "AAAA", "mimeType": "image/"}]), - // Over the size limit. + // Over the size limit: refused after decoding, and before it. json!([{"type": "image", "data": "A".repeat((MAX_IMAGE_BYTES + 3) / 3 * 4), "mimeType": "image/png"}]), - json!([{"type": "image", "data": "AAAA"}]), - json!([{"type": "file", "data": "AAAA", "mimeType": "image/png"}]), + json!([{"type": "image", "data": "A".repeat(MAX_IMAGE_BYTES / 3 * 4 + 8), "mimeType": "image/png"}]), ] { - assert!(content_blocks(&bad).is_none(), "{bad}"); + assert!(content_blocks(&bad, &[]).is_none(), "{bad}"); } } + #[test] + fn an_after_tool_edit_may_keep_an_image_yoagent_let_in() { + // read_file returns bmp; MCP tools other types: keeping one is no new image. + let kept = [Content::Image { + data: "Qk0=".into(), + mime_type: "image/bmp".into(), + }]; + let same = json!([{"type": "image", "data": "Qk0=", "mimeType": "image/bmp"}]); + assert!(content_blocks(&same, &[]).is_none()); + let blocks = content_blocks(&same, &kept).unwrap(); + assert!(matches!(&blocks[0], Content::Image { mime_type, .. } if mime_type == "image/bmp")); + // Changed data is a new image, checked as one. + let changed = json!([{"type": "image", "data": "Qk1=", "mimeType": "image/bmp"}]); + assert!(content_blocks(&changed, &kept).is_none()); + } + /// How the test handler's event delivery goes wrong. #[derive(Clone, Copy)] enum Breaks {