From 3238f503a93546725cc22adff7d8190e5b85bf70 Mon Sep 17 00:00:00 2001 From: Mihai Chirculescu Date: Tue, 29 Sep 2026 23:39:26 +0300 Subject: [PATCH 01/20] feat(delegate): drive the public @uipath/delegate-stdio host Replace the first-party delegate_host.mjs wrapper around @uipath/delegate-sdk with the published @uipath/delegate-stdio host (^1.202.1), which pulls in the SDK and interop runtime itself. - Speak the host's protocol: `event` frames, terminal `result` / `error`, `destroyed`; read toolArgs / toolResult / toolStatus, isStepStart deltas, Anthropic-convention `usage`, and `turnUsages` as the authoritative call count. - Export auth into the host's environment under the names it reads (ORG_SLUG -> ORG_LOGICAL_NAME, TENANT_SLUG -> TENANT_NAME) instead of an `auth` init option; DELEGATE_ENV becomes the `env` option. - Send enableSkills explicitly (the host defaults it off). - Rename DELEGATE_SDK_PATH / DELEGATE_SDK_NODE_MODULES to DELEGATE_STDIO_PATH / DELEGATE_STDIO_NODE_MODULES across code, docs, CI. - Keep max_turns client-side: the host's maxSteps does not stop a turn. - Live-test gate now honours DELEGATE_AUTH_TOKEN. Verified live against alpha: all delegate live tests (saved login and env-token paths) and tasks/delegate/fizzbuzz_delegate.yaml pass. Co-Authored-By: Claude Opus 5.5 (1M context) --- .claude/notes/agents.md | 131 +++---- .env.example | 18 +- .github/workflows/pr-checks.yml | 10 +- docs/USER_GUIDE.md | 2 +- docs/agents/DELEGATE.md | 56 +-- docs/agents/HARNESS_PARITY.md | 10 +- pyproject.toml | 2 +- src/coder_eval/agents/__init__.py | 2 +- .../agents/delegate/delegate_host.mjs | 175 --------- src/coder_eval/agents/delegate/package.json | 4 +- src/coder_eval/agents/delegate_agent.py | 344 +++++++++--------- src/coder_eval/models/agent_config.py | 10 +- tasks/delegate/hello_date_delegate.yaml | 2 +- tests/test_delegate_agent.py | 312 +++++++++------- tests/test_delegate_agent_live.py | 25 +- tests/test_timing_identity_contract.py | 8 +- 16 files changed, 478 insertions(+), 633 deletions(-) delete mode 100644 src/coder_eval/agents/delegate/delegate_host.mjs diff --git a/.claude/notes/agents.md b/.claude/notes/agents.md index b36815af..eea6f3f3 100644 --- a/.claude/notes/agents.md +++ b/.claude/notes/agents.md @@ -744,88 +744,74 @@ release adding an unclassified field fails loudly instead of silently passing th ## Delegate agent `DelegateAgent` drives UiPath Autopilot's Delegate agent — reasoning in the UiPath backend, -tools executing locally through the SDK's bundled interop process. Unlike every other CLI-driven -agent in this file, there is no vendor-provided stdio protocol host to spawn: `@uipath/delegate-stdio`, -the internal package a UiPath-only sibling plugin (`coder_eval_uipath`) drives, is not public. Only -`@uipath/delegate-sdk` (a programmatic library exposing a `DelegateAgent` class with -`initialize()`/`onEvent()`/`sendMessage()`/`destroy()`) and `@uipath/delegate-cli` (a terminal wrapper -around it) are. So this agent ships its own first-party Node host, `agents/delegate/delegate_host.mjs`, -which wraps the SDK class in a newline-JSON stdio protocol this framework designed, not one it had to -reverse-engineer — verified live against a real `@uipath/delegate-sdk@0.1.12` install with no token, -confirming both the plain-public-install story and that the SDK's own event vocabulary -(`session_start`/`thinking`/`message`/`tool_call`/`tool_result`/`error`/`done`/`step`) substantially -matches the internal sibling's protocol. - -**No multi-generation transcript splitting.** The internal sibling reconstructs one `AssistantMessage` -per backend round-trip from an `isStepStart` flag and a `turnUsages` array on the host's terminal -message. Neither is present on `DelegateAgent.onEvent`'s payloads (confirmed absent by grepping the -installed SDK bundle's string literals), so there is no reliable per-round-trip boundary signal at this -API layer. `DelegateAgent` builds exactly ONE `AssistantMessage` per `communicate()` call instead — a -deliberate simplification, not an oversight; richer segmentation can be added once a real boundary -signal is confirmed against a live backend. `timing.close_window` still opens that one window from the -turn's own start (there is nothing to tile from), so the head and tail both measure ~0. - -**Deferred, not ported speculatively.** The internal sibling's module docstring documents several -hard-won failure-signature-specific recoveries: a Cloudflare WAF-block-page rewrite, an SSE-connect- -timeout rewrite, session-conflict fresh-host recovery, and first-response stall-timeout+resend. None -are ported here — porting a marker tuned to the internal sibling's own observed failures risks matching -nothing (or the wrong thing) against the public SDK/backend's actual error surface. A crash still ends -the turn correctly as a retryable `AgentCrashError`; it is just not specially diagnosed. Port these once -the same failures are actually observed running this agent for real. - -**The process handle must be cleared on every path that leaves the host dead or dying** — EOF, a host -`fatal` message, a timeout (both the top-of-loop pre-check AND a timeout elapsing while blocked inside +tools executing locally through the SDK's bundled interop process. It spawns the host that the +public `@uipath/delegate-stdio` npm package ships (`dist/delegate_stdio.mjs`); that package depends +on `@uipath/delegate-sdk` and the platform's `@uipath/delegate-runtime-*` interop binaries, so one +`npm install` is the whole prerequisite. The package README is the wire-protocol SSOT; every frame +shape the adapter reads (`event` wrapper, `toolArgs`/`toolResult`/`toolStatus`, `isStepStart`, +`result.usage`/`turnUsages`/`model`, top-level `error`, `destroyed`) was confirmed against a live +`1.202.1` transcript before the port, and `tests/test_delegate_agent.py`'s frame builders replay +those shapes. + +An earlier version of this agent wrapped `@uipath/delegate-sdk` in a first-party host +(`delegate_host.mjs`) because `delegate-stdio` was not yet public. That host and its protocol are +gone; git history has them. + +**Auth goes through the host's environment, not the init options.** The host reads `AUTH_TOKEN` / +`TENANT_ID` / `ORG_ID` and, for an `env`-derived backend URL, `ORG_LOGICAL_NAME` / `TENANT_NAME` +from its own environment. coder_eval keeps its user-facing names (`DELEGATE_`-namespaced first, then +`AUTH_TOKEN` / `TENANT_ID` / `ORG_ID` / `ORG_SLUG` / `TENANT_SLUG`) and `_host_auth_env` re-exports +them under the host's names. Confirmed live with no saved login: with the slugs the turn runs; without +them init fails with `env="alpha" requires org/tenant slugs`. `DELEGATE_ENV` becomes the `env` init +option; `DELEGATE_BACKEND_URL` becomes `backendUrl`, which the host ranks above `env`. + +**`max_turns` stays client-side.** The host accepts `maxSteps` on `send`, but it does not stop the +turn: live, `maxSteps: 2` ran 7 steps and only reported `maxStepsReached: true` in the `result`. +Forwarding it would suggest a cap that does not exist, so the adapter keeps counting calls from the +event stream and abandons the host when call N+1 opens. Once the `result` arrives, `len(turnUsages)` +(one entry per backend round-trip, confirmed equal to `assistantStepCount`) replaces the running +estimate as `num_turns`. + +**`enableSkills` must be sent explicitly.** The host's default is `false`, so a `bundledSkillsPath` +alone loads nothing; the adapter sets `enableSkills` to whether a plugin resolved. + +**No multi-generation transcript splitting (yet).** `DelegateAgent` builds exactly ONE +`AssistantMessage` per `communicate()` call. The host now carries what a per-round-trip split needs — +`isStepStart` on `message` events and per-round-trip `turnUsages` on `result` — but the adapter only +uses `isStepStart` to merge streamed deltas into one text block. `timing.close_window` opens that one +window from the turn's own start, so the head and tail both measure ~0. + +**Deferred, not ported speculatively.** The out-of-tree `delegate-sdk` adapter carries +failure-signature-specific recoveries: a Cloudflare WAF-block-page rewrite, an SSE-connect-timeout +rewrite, session-conflict fresh-host recovery, first-response stall-timeout+resend, and an S2S +token-file refresher. None are ported here. A crash still ends the turn correctly as a retryable +`AgentCrashError`; it is just not specially diagnosed. Port each once its failure is observed here. + +**The process handle must be cleared on every path that leaves the host dead or dying** — EOF, an +`error` frame, a timeout (both the top-of-loop pre-check AND a timeout elapsing while blocked inside `asyncio.wait_for`), cooperative stop, and `max_turns` exhaustion. A real bug shipped once during this agent's own development: a timeout elapsing mid-read fell through to the generic crash path instead of `TurnTimeoutError`, and left the process handle set, so the NEXT `communicate()` call reused a host with a `"send"` still nominally in flight instead of respawning — risking a stale response being consumed as the new turn's. `tests/test_delegate_agent.py`'s `test_timeout_elapsing_mid_read_still_raises_turn_timeout_error` -pins the fix. +pins the fix. The host survives a failed `send` (it answers the next `destroy` with `destroyed`), but +the adapter still does not reuse it, so no late event from the failed turn can leak into the retry. **Skills mapping is deliberately NOT the shared `agents/_skills.py` resolver.** That resolver enumerates individual skill directories for a repeated `--skill `-style CLI argument (OpenCode/Pi's shape). The Delegate SDK's `bundledSkillsPath` wants exactly ONE parent directory whose children are skill -folders — a genuinely different shape — so `_resolve_bundled_skills_path` mirrors the internal sibling's -own mapping (`/skills`, first plugin wins) instead of force-fitting the shared helper. +folders — a genuinely different shape — so `_resolve_bundled_skills_path` maps `/skills`, +first plugin wins, instead of force-fitting the shared helper. **Registered unconditionally**, exactly like `codex`/`antigravity` — there is no conditional-registration gate keyed on an optional-dependency extra anywhere in this codebase; `opencode`/`pi` declare an EMPTY extra purely as packaging-metadata documentation, and `delegate` follows that same shape (no new pip -package — Node/`@uipath/delegate-sdk` is the real prerequisite, resolved lazily in `start()`). - -**`environment` resolution needs org/tenant SLUGS, not just IDs — confirmed by reading the installed -SDK's own bundle, not by guessing from its error message.** When `DELEGATE_BACKEND_URL` is absent and -`DELEGATE_ENV` (-> `environment` init option) is used instead, the SDK resolves the backend URL from -`organizationName`/`orgLogicalName` and `tenantName` fields on the `auth` object passed to -`initialize()` — NOT from `ORG_SLUG`/`TENANT_SLUG` read off `process.env` directly, despite that being -exactly what the SDK's own thrown error suggests ("set ORG_SLUG and TENANT_SLUG env vars alongside -ORG_ID/TENANT_ID"). That advice describes the separate `@uipath/delegate-cli`/`delegate-stdio` wrapper's -own env-var-driven bootstrapping, not the `DelegateAgent` class itself, which we drive directly. So -`_build_init_options` forwards `ORG_SLUG`/`TENANT_SLUG` (when set) into `auth.organizationName`/ -`auth.tenantName` itself — confirmed against `node_modules/@uipath/delegate-sdk/dist/index.mjs`'s own -`yie()` resolver function live in CI (`pyright`/tests can't catch this class of bug; it only surfaces -against a real backend). `tests/test_delegate_agent.py::TestStart::test_org_and_tenant_slug_forwarded_into_auth` -pins it. Passing `DELEGATE_BACKEND_URL` directly instead of `DELEGATE_ENV` skips this whole path. - -**Token usage comes from two getters called after `sendMessage()` resolves, not from any event or the -resolved value itself — confirmed by reading the installed SDK's bundle, not by guessing.** A first -pass guessed at a flat `usage` dict carried on the resolved `sendMessage()` value or on a forwarded -event, tried several plausible snake_case/camelCase bucket-name spellings, and silently returned zero -tokens every turn against a real backend (`tests/test_delegate_agent_live.py::test_delegate_live_token_usage_populated` -failed live: `record.crashed is False`, real text/tool output, `token_usage=None`). Reading -`node_modules/@uipath/delegate-sdk/dist/index.mjs` directly settled it: `sendMessage()` always resolves -to a plain string (never an object), and no event this host forwards via `agent.onEvent()` ever carries -a `usage` field — the SDK's own internal event vocabulary has no `"usage"` member (that name IS used -internally, but only inside the SDK's own Zustand store reducer that updates its "Token usage" UI -panel, never re-emitted through the public `onEvent` bus). The only way to reach it is -`DelegateAgent.getLastTurnUsage(sessionId?)` (defaults to the just-used session), so -`delegate_host.mjs`'s `handleSend` calls it — and `getSessionId()` — right after `sendMessage()` -resolves, and attaches both to the `send_ok` message itself. That getter's shape, read straight off the -SDK's own `setUsage` store action, is `{promptTokens, completionTokens, promptTokensCached, -cacheCreationTokens, turnTokenUnits, contextBreakdown}` — `promptTokens` is the TOTAL input token count -(OpenAI-style, cached + uncached), `promptTokensCached` the cache-READ subset of it, so -`_parse_usage` computes `uncached_input_tokens = promptTokens - promptTokensCached`. This is now -CONFIRMED, not a guess, so `_parse_usage`'s docstring no longer marks it `# UNVERIFIED`. +package — Node/`@uipath/delegate-stdio` is the real prerequisite, resolved lazily in `start()`). + +**Windows path length.** The interop binary sits deep under `node_modules/@uipath/delegate-runtime-win32-x64/`; +an install root long enough to push that path past 260 characters makes the SDK's spawn fail with +`ENOENT` even though the file exists. Seen live from a scratch directory; the docs tell users to +install into a short path. ### Delegate agent pricing @@ -859,10 +845,9 @@ introduced or should silently "fix" by picking one number over the other. `AgentKind.DELEGATE` is excluded from `tests/test_agent_golden_master.py`'s `_NO_GOLDEN_COVERAGE` allowlist rather than given fixture scenarios: a golden snapshot pins a byte-identical `TurnRecord` -for a scripted event stream, and every event field name below the confirmed `type`/`content`/ -`toolName`/`sessionId`/`model`/`usage` set is a guess (see this file's own "Delegate agent" section -above and `delegate_agent.py`'s module docstring). Snapshotting a guess would make it look verified. -Add real scenarios once the fields are confirmed against a live backend. +for a scripted event stream. The frame shapes are now confirmed against a live `delegate-stdio` +transcript (see "Delegate agent" above), so this exemption can be lifted by recording real scenarios; +until then `tests/test_delegate_agent.py` replays those shapes directly. This does NOT extend to `tests/test_timing_identity_contract.py::test_delegate_buckets_tile_the_turn`, which IS a real, unexempted case: the ms-exact four-bucket identity depends only on this agent's own diff --git a/.env.example b/.env.example index 58128023..238c5a35 100644 --- a/.env.example +++ b/.env.example @@ -89,24 +89,24 @@ LOG_TO_FILE=false # Set to true to enable file logging # CODER_EVAL_REMEDIATE_HOME_PLUGINS=0 # Delegate agent settings (UiPath Autopilot's Delegate agent, requires Node.js -# on PATH plus `npm install @uipath/delegate-sdk` — a genuinely public package, +# on PATH plus `npm install @uipath/delegate-stdio` — a public package, # no token needed). See docs/agents/DELEGATE.md. # -# DELEGATE_SDK_NODE_MODULES / DELEGATE_SDK_PATH — override the auto-located +# DELEGATE_STDIO_NODE_MODULES / DELEGATE_STDIO_PATH — override the auto-located # install (ancestor-walk from cwd, plus home); set at most one. -# DELEGATE_SDK_NODE_MODULES="/path/to/install/root" -# DELEGATE_SDK_PATH="/path/to/node_modules/@uipath/delegate-sdk/dist/index.mjs" +# DELEGATE_STDIO_NODE_MODULES="/path/to/install/root" +# DELEGATE_STDIO_PATH="/path/to/node_modules/@uipath/delegate-stdio/dist/delegate_stdio.mjs" # -# DELEGATE_ENV — cloud environment slug (alpha/staging/production/localhost), -# used to derive the backend URL from the auth record's org/tenant slugs. +# DELEGATE_ENV — cloud environment slug (alpha/staging/production), +# used to derive the backend URL from the org/tenant slugs. # DELEGATE_ENV="alpha" # # DELEGATE_BACKEND_URL — pin the full agent-service URL directly instead # (wins over DELEGATE_ENV when both are set). # DELEGATE_BACKEND_URL="https://cloud.uipath.com///delegate_" # -# Auth: either the token set below, or a prior `delegate-sdk`/`delegate-cli` -# login that wrote ~/.aria/sdk-auth.json (read by the Node host itself, not +# Auth: either the token set below, or a prior `delegate-cli login` +# that wrote ~/.aria/sdk-auth.json (read by the Node host itself, not # by coder_eval). Each var also accepts a DELEGATE_-namespaced spelling # (DELEGATE_AUTH_TOKEN, DELEGATE_TENANT_ID, ...), checked first -- prefer that # spelling in a shared environment, since the bare names collide with what @@ -116,7 +116,7 @@ LOG_TO_FILE=false # Set to true to enable file logging # ORG_ID="..." # With DELEGATE_ENV (not DELEGATE_BACKEND_URL) also set these two -- the # human-readable org/tenant names, not the GUIDs above -- or init fails with -# "--env alpha needs org/tenant slugs". See docs/agents/DELEGATE.md. +# 'env="alpha" requires org/tenant slugs'. See docs/agents/DELEGATE.md. # ORG_SLUG="..." # TENANT_SLUG="..." diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index 1690fba9..124d1f57 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -1175,8 +1175,8 @@ jobs: retention-days: 7 delegate-live-tests: - # Live Delegate e2e: spawns the real Node host (agents/delegate/delegate_host.mjs) - # wrapping the public @uipath/delegate-sdk against a real UiPath backend. Runs on + # Live Delegate e2e: spawns the real Node host from the public + # @uipath/delegate-stdio package against a real UiPath backend. Runs on # every PR/push like the other live-test jobs (codex-live-tests, byoa-live-tests) # now that the DELEGATE_ROPC_* secrets are provisioned; the presence-check gate # below skips cleanly ONLY on a fork PR (no secrets available there) and fails @@ -1260,7 +1260,7 @@ jobs: if: steps.gate.outputs.present == 'true' run: uv sync --frozen --extra dev - - name: Install @uipath/delegate-sdk (public npm, no token needed) + - name: Install @uipath/delegate-stdio (public npm, no token needed) if: steps.gate.outputs.present == 'true' working-directory: src/coder_eval/agents/delegate # --safe-chain-skip-minimum-package-age (not a growing exclusion-list entry @@ -1372,7 +1372,7 @@ jobs: - name: Run fizzbuzz delegate task (assert PASS) if: steps.gate.outputs.present == 'true' env: - DELEGATE_SDK_NODE_MODULES: ${{ github.workspace }}/src/coder_eval/agents/delegate + DELEGATE_STDIO_NODE_MODULES: ${{ github.workspace }}/src/coder_eval/agents/delegate DELEGATE_ENV: ${{ github.event.inputs.delegate_uipath_env || 'alpha' }} AUTH_TOKEN: ${{ steps.ropc.outputs.access_token }} TENANT_ID: ${{ secrets.DELEGATE_TENANT_ID }} @@ -1405,7 +1405,7 @@ jobs: - name: Run Delegate live unit tests (assert PASS) if: steps.gate.outputs.present == 'true' env: - DELEGATE_SDK_NODE_MODULES: ${{ github.workspace }}/src/coder_eval/agents/delegate + DELEGATE_STDIO_NODE_MODULES: ${{ github.workspace }}/src/coder_eval/agents/delegate DELEGATE_ENV: ${{ github.event.inputs.delegate_uipath_env || 'alpha' }} AUTH_TOKEN: ${{ steps.ropc.outputs.access_token }} TENANT_ID: ${{ secrets.DELEGATE_TENANT_ID }} diff --git a/docs/USER_GUIDE.md b/docs/USER_GUIDE.md index dda76b66..69d9aa6b 100644 --- a/docs/USER_GUIDE.md +++ b/docs/USER_GUIDE.md @@ -429,7 +429,7 @@ Set these in `.env` (copy from `.env.example`). | `BEDROCK_SMALL_MODEL` | No | Cross-region Bedrock small/fast model ID | | `CODEX_API_KEY` / `CODEX_BASE_URL` / `CODEX_MODEL` / `CODEX_API_VERSION` | For Codex | Codex agent auth & endpoint routing — see [Codex Agent Guide](agents/CODEX.md#endpoint-routing). | | `GEMINI_API_KEY` / `ANTIGRAVITY_MODEL` | For Antigravity | Antigravity (Gemini) agent auth & model — see [Antigravity Agent Guide](agents/ANTIGRAVITY.md#setup). | -| `DELEGATE_ENV` / `DELEGATE_BACKEND_URL` / `AUTH_TOKEN` / `TENANT_ID` / `ORG_ID` / `ORG_SLUG` / `TENANT_SLUG` / `DELEGATE_SDK_NODE_MODULES` / `DELEGATE_SDK_PATH` | For Delegate | Delegate agent backend routing, auth & SDK install location — see [Delegate Agent Guide](agents/DELEGATE.md#setup). | +| `DELEGATE_ENV` / `DELEGATE_BACKEND_URL` / `AUTH_TOKEN` / `TENANT_ID` / `ORG_ID` / `ORG_SLUG` / `TENANT_SLUG` / `DELEGATE_STDIO_NODE_MODULES` / `DELEGATE_STDIO_PATH` | For Delegate | Delegate agent backend routing, auth & host install location — see [Delegate Agent Guide](agents/DELEGATE.md#setup). | | `UIPATH_PLUGIN_MARKETPLACE_DIR` | No | Conventional base directory for local plugins. Not special-cased by the framework: **any** `$VAR` / `${VAR}` referenced in a plugin `path` is expanded from the environment, so a plugin path like `$UIPATH_PLUGIN_MARKETPLACE_DIR/my-plugin` resolves against this variable. (An undefined variable in a plugin path logs a warning.) | | `PLUGIN_TOOLS_DIR` | No | A *separate* mechanism from the above: the canonical `node_modules/@uipath` directory used to pin UiPath CLI plugin discovery (not path substitution). When unset, the sandbox auto-derives it from the resolved `uip` binary. | | `CODER_EVAL_REMEDIATE_HOME_PLUGINS` | No | **DESTRUCTIVE.** Truthy deletes `$HOME/node_modules/@uipath` at sandbox setup to clear sibling-task pollution on dedicated eval hosts. Off by default; do **not** enable on developer workstations. | diff --git a/docs/agents/DELEGATE.md b/docs/agents/DELEGATE.md index ae097f7c..4fa7d244 100644 --- a/docs/agents/DELEGATE.md +++ b/docs/agents/DELEGATE.md @@ -1,7 +1,7 @@ --- description: >- Run UiPath Autopilot's Delegate agent as the agent under evaluation in Coder - Eval — installing the public @uipath/delegate-sdk, authentication, task + Eval — installing the public @uipath/delegate-stdio host, authentication, task configuration, and how Delegate telemetry maps to sandboxed, weighted scoring. --- @@ -11,41 +11,45 @@ description: >- Coder Eval can run UiPath Autopilot's **Delegate agent** as the agent under evaluation. Its reasoning runs in the UiPath backend, but its tools (shell, file, Office, PDF) execute locally through the SDK's bundled interop process — so file-based success criteria work as usual. `DelegateAgent` plugs into the same sandbox, scoring, and telemetry pipeline as every other agent: set `agent.type: delegate` in a task and the rest of the framework works unchanged. -Under the hood, this agent spawns a small first-party Node host, `agents/delegate/delegate_host.mjs`, that wraps the public `@uipath/delegate-sdk` package's `DelegateAgent` class in a newline-JSON stdio protocol. That host exists because `@uipath/delegate-stdio` — a ready-made protocol host UiPath's internal-only tooling drives — is not a public package; only `@uipath/delegate-sdk` (a programmatic library) and `@uipath/delegate-cli` (a terminal wrapper around it) are. +Under the hood, this agent spawns the host that the public [`@uipath/delegate-stdio`](https://www.npmjs.com/package/@uipath/delegate-stdio) package ships (`dist/delegate_stdio.mjs`). The host runs the Delegate agent as a subprocess that speaks newline-delimited JSON over stdio. `@uipath/delegate-sdk` and your platform's `@uipath/delegate-runtime-*` interop binaries are dependencies of that package, so one install is the complete install. ## Setup -### 1. Install Node and the Delegate SDK +### 1. Install Node and the Delegate host 1. **Node.js** on your `PATH`. -2. **`@uipath/delegate-sdk`**, a genuinely public npm package (verified: no token, no custom registry) — install it plain: +2. **`@uipath/delegate-stdio`**, a public npm package (no token, no custom registry). Install it plain: ```bash -npm install @uipath/delegate-sdk +npm install @uipath/delegate-stdio ``` -**You usually don't need to set anything about where.** When neither `DELEGATE_SDK_NODE_MODULES` nor `DELEGATE_SDK_PATH` is set, coder_eval auto-locates the install by walking up from the current directory through its ancestors (and home) — the same way Node resolves modules. Running the install inside `src/coder_eval/agents/delegate/` (this agent's own directory, which ships a `package.json` naming the dependency) is a convenient default location. +**You usually don't need to set anything about where.** When neither `DELEGATE_STDIO_NODE_MODULES` nor `DELEGATE_STDIO_PATH` is set, coder_eval auto-locates the install by walking up from the current directory through its ancestors (and home) — the same way Node resolves modules. Running the install inside `src/coder_eval/agents/delegate/` (this agent's own directory, which ships a `package.json` naming the dependency) is a convenient default location. To override the auto-search, set **one** of: | Variable | Purpose | |---|---| -| `DELEGATE_SDK_NODE_MODULES` | Install root that holds `node_modules/@uipath/...`. | -| `DELEGATE_SDK_PATH` | Absolute path straight to `dist/index.mjs`. | +| `DELEGATE_STDIO_NODE_MODULES` | Install root that holds `node_modules/@uipath/...`. | +| `DELEGATE_STDIO_PATH` | Absolute path straight to `@uipath/delegate-stdio/dist/delegate_stdio.mjs`. | + +On Windows, install into a short path. The interop binary sits deep inside `node_modules`, and a path longer than 260 characters makes its spawn fail with `ENOENT`. ### 2. Choose a backend | Variable | Purpose | |---|---| -| `DELEGATE_ENV` | Cloud environment slug (`alpha` / `staging` / `production` / `localhost`). Derives the backend URL from the auth record's org/tenant slugs. | +| `DELEGATE_ENV` | Cloud environment slug (`alpha` / `staging` / `production`), sent as the host's `env` init option. The host derives the backend URL from the org/tenant slugs. | | `DELEGATE_BACKEND_URL` | Pin the full agent-service URL directly (e.g. `https://cloud.uipath.com///delegate_`, or `http://localhost:5002` for a local backend). Wins over `DELEGATE_ENV` when both are set. | ### 3. Authenticate Either: -- **Environment token** — `AUTH_TOKEN`, `TENANT_ID`, `ORG_ID` env vars (each also accepts a `DELEGATE_`-namespaced spelling — `DELEGATE_AUTH_TOKEN`, `DELEGATE_TENANT_ID`, `DELEGATE_ORG_ID` — checked first, since the bare names collide with what other tooling like npm/Vault/Terraform commonly exports). Confirmed live: when `DELEGATE_ENV` (rather than `DELEGATE_BACKEND_URL`) resolves the backend, also set `ORG_SLUG` and `TENANT_SLUG` (or `DELEGATE_ORG_SLUG`/`DELEGATE_TENANT_SLUG`; the human-readable org/tenant names, not the GUIDs above) — the SDK's `environment` resolution reads `organizationName`/`tenantName` off the `auth` object passed to `initialize()`, not off `process.env` directly (its own error message's "set ORG_SLUG and TENANT_SLUG env vars" advice describes the separate `delegate-cli` wrapper's behavior, not this SDK class), so this agent forwards them into `auth.organizationName`/`auth.tenantName` itself. Without them, init fails with `--env alpha needs org/tenant slugs`. Skip both by setting `DELEGATE_BACKEND_URL` directly instead. -- **Saved login** — a prior `delegate-sdk` `runLoginFlow` / `delegate-cli login` that wrote `~/.aria/sdk-auth.json`. The Node host reads this itself (via the SDK's own `loadAndRefreshAuth()`) when no `AUTH_TOKEN` is supplied — this agent does not parse that file in Python, so auth-freshness logic lives in exactly one place. This path already carries the slugs, so `ORG_SLUG`/`TENANT_SLUG` aren't needed. +- **Environment token** — `AUTH_TOKEN`, `TENANT_ID`, `ORG_ID` env vars. Each also accepts a `DELEGATE_`-namespaced spelling (`DELEGATE_AUTH_TOKEN`, `DELEGATE_TENANT_ID`, `DELEGATE_ORG_ID`), which is checked first, because the bare names collide with what other tooling (npm, Vault, Terraform) commonly exports. When `DELEGATE_ENV` (not `DELEGATE_BACKEND_URL`) resolves the backend, also set `ORG_SLUG` and `TENANT_SLUG` (or `DELEGATE_ORG_SLUG` / `DELEGATE_TENANT_SLUG`). These are the human-readable org/tenant names, not the GUIDs. coder_eval exports all five values into the host's environment under the names the host reads (`AUTH_TOKEN`, `TENANT_ID`, `ORG_ID`, `ORG_LOGICAL_NAME`, `TENANT_NAME`). Without the slugs, init fails with `env="alpha" requires org/tenant slugs`. To skip the slugs, set `DELEGATE_BACKEND_URL` instead. +- **Saved login** — a prior `npx @uipath/delegate-cli login --env ` that wrote `~/.aria/sdk-auth.json`. The host reads and refreshes this file itself when no `AUTH_TOKEN` is supplied. This agent does not parse that file in Python, so auth-freshness logic lives in exactly one place. The saved login already carries the slugs, so `ORG_SLUG` / `TENANT_SLUG` are not needed. + +The host also reads its own advanced variables directly, such as `DELEGATE_AUTH_TOKEN_FILE` (a token file that an external process keeps fresh), `INTEROP_URL` and `DELEGATE_STDIO_VERBOSE=1` (trace every frame to stderr). See the [package README](https://www.npmjs.com/package/@uipath/delegate-stdio). ## Usage @@ -81,7 +85,7 @@ success_criteria: ### Skills -A `plugins:` entry with `type: local` and a `path` is mounted as `/skills` (the Delegate SDK's `bundledSkillsPath`, which expects one directory whose direct children are skill folders). If more than one plugin is configured, the first wins and a warning is logged. +A `plugins:` entry with `type: local` and a `path` is mounted as `/skills` (the Delegate SDK's `bundledSkillsPath`, which expects one directory whose direct children are skill folders), and turns on `enableSkills`. With no plugin, skills stay off. If more than one plugin is configured, the first wins and a warning is logged. ## Architecture @@ -90,15 +94,15 @@ A `plugins:` entry with `type: local` and a `path` is mounted as `/skills` ``` Agent (ABC) └── DelegateAgent - ├── Node host subprocess (agents/delegate/delegate_host.mjs) + ├── Node host subprocess (@uipath/delegate-stdio's dist/delegate_stdio.mjs) │ └── @uipath/delegate-sdk's DelegateAgent class └── Streaming telemetry (commands, token usage, agent text) ``` ### Key Methods -- **`start(working_directory)`** — resolve the Node/SDK install, spawn the host, send the `init` command. -- **`communicate(user_input, timeout, stream_callback)`** — send one `"send"` command and drain the host's forwarded events until `send_ok`/`send_error`/`fatal` or EOF. +- **`start(working_directory)`** — resolve the host install, spawn the host, send the `init` command and wait for `init_ok`. +- **`communicate(user_input, timeout, stream_callback)`** — send one `"send"` command and drain the host's `event` frames until `result`, `error` or EOF. - **`stop()`** — send `destroy`, wait bounded, then SIGKILL. - **`kill()` / `kill_sync()`** — force-terminate the host subprocess (the latter safe to call from the orchestrator's watchdog thread). @@ -108,8 +112,9 @@ Each turn returns a `TurnRecord` with: - `agent_output` — the SDK's final response, falling back to accumulated streamed text. - `commands` — one `CommandTelemetry` per `tool_call`/`tool_result` pair. - `messages` — exactly **one** `AssistantMessage` per turn (see Known Limitations). -- `token_usage` — best-effort parse of the SDK's `usage` payload (see Known Limitations). -- `model_used` — the SDK-reported model, falling back to the pinned `agent.model`. +- `token_usage` — the `result` frame's `usage` (Anthropic convention: `input_tokens` excludes cache reads and writes). +- `num_turns` — the length of the `result` frame's `turnUsages` (one entry per backend round-trip). +- `model_used` — the `result` frame's `model`, falling back to the pinned `agent.model`. ## Implementation Details @@ -119,7 +124,7 @@ A wall-clock deadline (`timeout`) is enforced both between reads (a top-of-loop ### Error Recovery -On any crash (host death, a `send_error`/`fatal` protocol message, or an unexpected exception), the agent: +On any crash (host death, an `error` frame during a turn, or an unexpected exception), the agent: 1. Sets `pending_turn` to a `crashed=True` TurnRecord with captured telemetry. 2. Raises `AgentCrashError` (retryable) or `AgentConfigError` (non-retryable — missing Node/SDK install, or an SDK init rejection such as a missing `backendUrl`). 3. The orchestrator reads `pending_turn` and calls `discard_pending_turn()` to roll back state. @@ -132,13 +137,13 @@ A dead host is always detected and its handle cleared, so a retried `communicate ### Non-JSON stdout tolerance -Importing `@uipath/delegate-sdk` can itself write a non-JSON diagnostic line to stdout (confirmed live) before this agent's own host script produces any output. The Python-side reader skips a line that fails to parse as JSON rather than treating it as a protocol violation, mirroring every other CLI-driven agent's tolerance for interleaved non-JSON notices. +The host writes some non-JSON diagnostic lines to stdout (confirmed live), for example `[backendUrl] Module loaded ...` and `[DelegateAgent] Using model: ...`. The Python-side reader skips a line that fails to parse as JSON rather than treating it as a protocol violation, mirroring every other CLI-driven agent's tolerance for interleaved non-JSON notices. ## Differences from Claude Code Agent | Feature | Claude Code | Delegate | |---------|------------|----------| -| **SDK Type** | Subprocess (CLI via JSON generator) | Subprocess (a first-party Node host wrapping a programmatic SDK class) | +| **SDK Type** | Subprocess (CLI via JSON generator) | Subprocess (the `@uipath/delegate-stdio` Node host) | | **Reasoning location** | Local process | UiPath backend | | **Tool execution** | Local | Local, through the SDK's bundled interop process | | **System prompt** | `system_prompt` appended to the default prompt | No SDK equivalent — warned about, not enforced | @@ -151,9 +156,9 @@ Run-limit semantics per harness: [Run-Limit Parity](HARNESS_PARITY.md). ## Known Limitations -1. **No multi-generation transcript splitting.** The SDK's event stream carries no round-trip boundary signal (no `isStepStart`/`turnUsages` equivalent), so this agent builds one `AssistantMessage` per `communicate()` call rather than one per backend round-trip. -2. **Token-bucket field names are best-effort.** The SDK confirms an event-level `usage` field exists, but not its exact internal bucket names; several plausible spellings are tried and unrecognized shapes fall back to zero rather than raising. -3. **No WAF-block-page rewrite, SSE-connect-timeout rewrite, session-conflict fresh-host recovery, or first-response stall-timeout+resend.** These are hard-won failure-signature-specific recoveries UiPath's internal tooling has needed against its own CI; they are not ported here until the same failures are observed against the public SDK/backend from this agent. A crash still ends the turn correctly as a retryable `AgentCrashError` — it is just not specially diagnosed. +1. **No multi-generation transcript splitting.** This agent builds one `AssistantMessage` per `communicate()` call, not one per backend round-trip. The host does send the signals a split needs (`isStepStart` on `message` events and per-round-trip `turnUsages` on `result`), but this agent does not use them yet. +2. **`max_turns` is enforced by coder_eval, not by the host.** The host's `maxSteps` option on `send` does not stop the turn (confirmed live: `maxSteps: 2` ran 7 steps and only reported `maxStepsReached: true`), so this agent does not send it. See [Run-Limit Parity](HARNESS_PARITY.md). +3. **No WAF-block-page rewrite, SSE-connect-timeout rewrite, session-conflict fresh-host recovery, or first-response stall-timeout+resend.** These are recoveries for specific failure signatures. They are not ported until the same failures are observed from this agent. A crash still ends the turn correctly as a retryable `AgentCrashError` — it is just not specially diagnosed. 4. **No `sdk_options` passthrough.** Unlike Claude Code, there is no allowlisted escape hatch for arbitrary SDK fields — only `effort`, `project_id`, `session_id`, and `enable_computer_use` are exposed as typed config fields. 5. **`enable_computer_use: true` requires local permissions and is unavailable on Linux.** macOS needs Accessibility + Screen Recording grants; the SDK throws unconditionally on Linux when this is enabled. @@ -171,7 +176,7 @@ Registration/pricing tests: uv run pytest tests/test_delegate_agent_registration.py ``` -Live integration tests (drive a real `@uipath/delegate-sdk` install against a real backend; skipped unless credentials are configured): +Live integration tests (drive a real `@uipath/delegate-stdio` install against a real backend; skipped unless credentials are configured): ```bash uv run pytest -m live tests/test_delegate_agent_live.py @@ -187,6 +192,7 @@ are provisioned. ## References -- SDK package: [`@uipath/delegate-sdk`](https://www.npmjs.com/package/@uipath/delegate-sdk) +- Host package: [`@uipath/delegate-stdio`](https://www.npmjs.com/package/@uipath/delegate-stdio) — its README is the wire-protocol reference +- SDK package (installed by the host package): [`@uipath/delegate-sdk`](https://www.npmjs.com/package/@uipath/delegate-sdk) - CLI package (not required by this agent, but shares the same auth file): [`@uipath/delegate-cli`](https://www.npmjs.com/package/@uipath/delegate-cli) - Task examples: `tasks/delegate/*.yaml` diff --git a/docs/agents/HARNESS_PARITY.md b/docs/agents/HARNESS_PARITY.md index 86fb07bb..73ffc19f 100644 --- a/docs/agents/HARNESS_PARITY.md +++ b/docs/agents/HARNESS_PARITY.md @@ -500,10 +500,10 @@ needed to drive it. - **This page's `delegate` column is a DIFFERENT agent** from the one described in the bullet immediately below. `delegate` (this repo, `AgentKind.DELEGATE`) - drives the now-public `@uipath/delegate-sdk` through a first-party Node host; + drives the public `@uipath/delegate-stdio` Node host; `delegate-sdk` (the bullet below) is an older, UiPath-internal-only agent in the separate `coder_eval_uipath` plugin, driving the non-public - `@uipath/delegate-stdio` package. They are not the same code and this page + `@uipath/delegate-stdio` package. They are not the same adapter code and this page does not claim their timing behavior matches. - **Delegate (`delegate-sdk`, out of tree)** records `duration_ms` but no execution bounds, so its tool calls cannot be placed on a timeline. Its @@ -563,8 +563,10 @@ Each harness finds the call boundary in its own stream (the table above): step, so a rise in that total closes the call, and the next call opens with a MODEL step at a new `step_index`. - **OpenCode** and **Pi** stream one `step_start` or `turn_start` per call. -- **Delegate**'s SDK has no round-trip marker, and a tool-only reply streams only - its tool call, with no text before it. So the next call opens when every tool the +- **Delegate**'s event stream marks a round-trip only on a text reply + (`isStepStart`), and a tool-only reply streams only its tool call, with no text + before it. The host's own `maxSteps` option does not stop the turn, so the + adapter does not send it and enforces the cap itself. So the next call opens when every tool the previous call announced has returned, since those results go back to the model, and the cap fires before call N+1 can run anything. A reply that announces several tools before their results counts once. If a tool never returns, the diff --git a/pyproject.toml b/pyproject.toml index 007c0390..014b74d3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -85,7 +85,7 @@ dev = [ # tasks that invoke `uv run uipath eval ...`) # - the Delegate agent's home in the packaging metadata (`agent.type: delegate`). # Like `opencode`/`pi` below, its real prerequisite is a Node package -# (`npm install @uipath/delegate-sdk`, genuinely public — no token needed), +# (`npm install @uipath/delegate-stdio`, public — no token needed), # so nothing new is added to THIS list; it just shares this extra's name. # Without this extra, the framework still installs and runs; UiPath-dependent # code paths fail at dispatch with a clear hint pointing back here. diff --git a/src/coder_eval/agents/__init__.py b/src/coder_eval/agents/__init__.py index 337bb9ff..00ebac53 100644 --- a/src/coder_eval/agents/__init__.py +++ b/src/coder_eval/agents/__init__.py @@ -22,7 +22,7 @@ def register_builtins(registry: type[AgentRegistry]) -> None: ensure those modules are imported, which the package import already did. ``delegate`` registers unconditionally, exactly like every sibling agent - (``codex``/``antigravity``): its Node/``@uipath/delegate-sdk`` prerequisite + (``codex``/``antigravity``): its Node/``@uipath/delegate-stdio`` prerequisite is resolved lazily in ``start()``, which raises a clear ``AgentConfigError`` if it is missing. There is no conditional-registration gate — that would be a second, ad-hoc dispatch mechanism the registry pattern already replaces. diff --git a/src/coder_eval/agents/delegate/delegate_host.mjs b/src/coder_eval/agents/delegate/delegate_host.mjs deleted file mode 100644 index 0999faad..00000000 --- a/src/coder_eval/agents/delegate/delegate_host.mjs +++ /dev/null @@ -1,175 +0,0 @@ -// delegate_host.mjs — a stdio JSON-Lines host wrapping @uipath/delegate-sdk's -// DelegateAgent class, standing in for the internal (non-public) -// @uipath/delegate-stdio package the framework's UiPath-only sibling used to -// drive. This host is first-party to coder_eval (there is no vendor-provided -// equivalent to point at once @uipath/delegate-sdk is the only public piece), -// so IT defines the wire protocol below rather than reverse-engineering one. -// -// See src/coder_eval/agents/delegate_agent.py's module docstring for the -// Python-side consumer of this exact protocol, and docs/agents/DELEGATE.md -// for the end-to-end setup story. -// -// Usage: node delegate_host.mjs -// -// Protocol (newline-delimited JSON, UTF-8, one object per line): -// -// stdin (coder_eval -> host): -// {"cmd": "init", "options": {...}} -- forwarded verbatim to DelegateAgent.initialize() -// {"cmd": "send", "prompt": str, "sessionId": str|null} -// {"cmd": "destroy"} -// -// stdout (host -> coder_eval): -// {"type": "init_ok"} -// {"type": "init_error", "message": str} -// {"type": "send_ok", "result": , -// "usage": , -// "sessionId": } -// -- usage/sessionId are read from these two getters AFTER sendMessage() -// resolves, not from the resolved value itself (a plain string) or from -// any forwarded event: CONFIRMED (reading the installed SDK's bundled -// source) that no event this host forwards ever carries a `usage` field. -// {"type": "send_error", "message": str} -// {"type": "destroy_error", "message": str} -// {"type": "protocol_error", "message": str} -- malformed/unknown stdin command; host keeps running -// {"type": "fatal", "message": str} -- unrecoverable; the process exits non-zero right after -// {"type": , ...event} -- every DelegateAgent.onEvent() callback, -// forwarded with ALL of its own enumerable fields spread onto the line, `type` -// included. This host does no filtering or renaming: the SDK's own event -// vocabulary (session_start / thinking / message / tool_call / tool_result / -// error / done / step / ... at last check) and per-event field names are -// whatever the installed @uipath/delegate-sdk version emits. A schema change -// on a future SDK release surfaces to the Python side as an unrecognized -// `type` string or missing field, never as a silently dropped event. -// -// CONFIRMED LIVE (installing @uipath/delegate-sdk@0.1.12 and driving this host -// against it directly): importing the SDK can itself write a non-JSON diagnostic -// line to STDOUT before any of this host's own output (observed: -// "[backendUrl] Module loaded - VITE_USE_CLOUD_URL: ..."), not stderr, so the -// Python reader must skip a non-JSON stdout line rather than treat it as a -// protocol violation -- mirroring every other agent's established resilience -// pattern for interleaved non-JSON CLI notices (see opencode_agent.py's -// `_handle_line`). This host does not attempt to suppress the SDK's own stdout -// writes (fragile across SDK versions); the tolerance lives on the read side. -"use strict"; - -import { createInterface } from "node:readline"; -import { pathToFileURL } from "node:url"; - -const sdkEntryPath = process.argv[2]; -if (!sdkEntryPath) { - process.stderr.write("delegate_host.mjs: missing required argv[1] (path to delegate-sdk's dist/index.mjs)\n"); - process.exit(2); -} - -function writeLine(obj, callback) { - process.stdout.write(JSON.stringify(obj) + "\n", callback); -} - -let agent = null; -let initialized = false; - -async function handleInit(msg) { - const { DelegateAgent } = await import(pathToFileURL(sdkEntryPath).href); - agent = new DelegateAgent(); - agent.onEvent((event) => { - try { - writeLine({ ...event }); - } catch (err) { - // A non-JSON-serializable event field (e.g. a circular reference) must not - // kill the host silently -- surface it as fatal so the Python side sees a - // clear crash instead of hanging on a line that will never arrive. Exits, - // like every other `fatal` site: the Python side's crash-handling treats - // ALL `fatal` messages as "the host is exiting", so this one must too. - writeLine({ type: "fatal", message: `event serialization failed: ${err}` }, () => process.exit(1)); - } - }); - await agent.initialize(msg.options || {}); - initialized = true; - writeLine({ type: "init_ok" }); -} - -async function handleSend(msg) { - if (!initialized || !agent) { - writeLine({ type: "send_error", message: "not initialized -- send {\"cmd\":\"init\"} first" }); - return; - } - const result = await agent.sendMessage(msg.prompt, msg.sessionId || undefined); - // CONFIRMED (reading @uipath/delegate-sdk@0.1.12's bundled dist/index.mjs): - // sendMessage() resolves to a plain string (the final response text) -- - // never an object -- and no event this host forwards via agent.onEvent() - // ever carries a `usage` field. The SDK's own per-turn token accounting is - // internal state, reachable only through these two getters, called here - // once the turn (and the backend's own internal "usage" store update) has - // settled. Guarded with typeof, not called unconditionally: an older/newer - // SDK build that drops either method must degrade to "no usage this turn", - // never crash the host. - const usage = typeof agent.getLastTurnUsage === "function" ? agent.getLastTurnUsage() : null; - const sessionId = typeof agent.getSessionId === "function" ? agent.getSessionId() : null; - writeLine({ type: "send_ok", result: result ?? null, usage: usage ?? null, sessionId: sessionId ?? null }); -} - -async function handleDestroy() { - if (agent) { - await agent.destroy(); - } - process.exit(0); -} - -const rl = createInterface({ input: process.stdin, crlfDelay: Infinity }); - -rl.on("line", (line) => { - const trimmed = line.trim(); - if (!trimmed) return; - - let msg; - try { - msg = JSON.parse(trimmed); - } catch (err) { - writeLine({ type: "protocol_error", message: `invalid JSON on stdin: ${err}` }); - return; - } - - const cmd = msg && msg.cmd; - if (cmd !== "init" && cmd !== "send" && cmd !== "destroy") { - writeLine({ type: "protocol_error", message: `unknown cmd: ${cmd}` }); - return; - } - - const dispatch = cmd === "init" ? handleInit(msg) : cmd === "send" ? handleSend(msg) : handleDestroy(); - const errorKind = { init: "init_error", send: "send_error", destroy: "destroy_error" }[cmd]; - dispatch.catch((err) => { - writeLine({ type: errorKind, message: String((err && err.message) || err) }); - // destroy is the teardown path: a failed agent.destroy() must still end - // the process, or coder_eval's bounded wait burns its full timeout before - // falling back to SIGKILL on every task. - if (cmd === "destroy") { - process.exit(1); - } - }); -}); - -// coder_eval closing stdin (its own exit, or a SIGKILL) must end this process -// too, or the host -- and the interop child it may have spawned -- outlives it. -rl.on("close", () => { - process.exit(0); -}); - -// Both handlers below write a `fatal` line before exiting so the Python side's -// read loop never hangs waiting for a line that will never arrive -- the -// original failure mode this design exists to avoid (see -// .claude/notes/agents.md § Delegate agent). The exit happens from the write's -// own callback, not the next statement: stdout is a pipe here, so -// process.stdout.write() is asynchronous, and process.exit() called before it -// flushes can drop this exact line -- degrading the Python side's crash -// message to the generic "closed its output stream unexpectedly". -process.on("unhandledRejection", (err) => { - writeLine({ type: "fatal", message: `unhandled rejection: ${(err && err.message) || err}` }, () => - process.exit(1) - ); -}); - -process.on("uncaughtException", (err) => { - writeLine({ type: "fatal", message: `uncaught exception: ${(err && err.message) || err}` }, () => - process.exit(1) - ); -}); diff --git a/src/coder_eval/agents/delegate/package.json b/src/coder_eval/agents/delegate/package.json index eca0995f..e4d9276c 100644 --- a/src/coder_eval/agents/delegate/package.json +++ b/src/coder_eval/agents/delegate/package.json @@ -1,9 +1,9 @@ { "name": "coder-eval-delegate-host", "private": true, - "description": "npm install target for the delegate agent's Node prerequisite (@uipath/delegate-sdk). Installing here, or in any ancestor directory, lets coder_eval's ancestor-walk resolver find it with zero configuration.", + "description": "npm install target for the delegate agent's Node prerequisite (@uipath/delegate-stdio, which pulls in @uipath/delegate-sdk). Installing here, or in any ancestor directory, lets coder_eval's ancestor-walk resolver find it with zero configuration.", "type": "module", "dependencies": { - "@uipath/delegate-sdk": "^0.1.12" + "@uipath/delegate-stdio": "^1.202.1" } } diff --git a/src/coder_eval/agents/delegate_agent.py b/src/coder_eval/agents/delegate_agent.py index 9d71738d..09d52874 100644 --- a/src/coder_eval/agents/delegate_agent.py +++ b/src/coder_eval/agents/delegate_agent.py @@ -2,18 +2,24 @@ The reasoning runs in the UiPath backend; tools (shell, file, Office, PDF) execute locally through the SDK's bundled interop process, so file-based success criteria -work as usual. This agent spawns a first-party Node host this framework ships, -``agents/delegate/delegate_host.mjs``, wrapping the public ``@uipath/delegate-sdk`` -package's ``DelegateAgent`` class in a newline-JSON stdio protocol — see that -file's header for the exact wire format. +work as usual. This agent spawns the host that the public ``@uipath/delegate-stdio`` +npm package ships (``dist/delegate_stdio.mjs``). That host runs the Delegate agent +as a subprocess speaking newline-delimited JSON over stdio. ``@uipath/delegate-sdk`` +is a dependency of that package, so installing it is the complete install. + +Wire protocol (one JSON object per line; the package README is the SSOT):: + + stdin stdout + {"cmd":"init","options":{...}} -> {"type":"init_ok"} + {"cmd":"send","prompt":.., {"type":"event","event":{...}} (zero or more) + "sessionId":..} -> {"type":"result","response":..,"sessionId":.., + "usage":{..},"turnUsages":[..],"model":..} + {"cmd":"destroy"} -> {"type":"destroyed"} + {"type":"error","message":..} (any command) Prerequisites are documented in ``docs/agents/DELEGATE.md`` and enforced with a clear ``AgentConfigError`` at ``start()`` (Node.js, ``npm install -@uipath/delegate-sdk``, UiPath auth). Deliberate scope reductions versus the -UiPath-internal sibling agent's more hardened adapter (no multi-generation -transcript splitting, no WAF/SSE/session-conflict/stall-resend recovery) and -every remaining ``# UNVERIFIED`` spot's rationale live in one place, not -scattered: +@uipath/delegate-stdio``, UiPath auth). Rationale: .claude/notes/agents.md § Delegate agent """ @@ -78,8 +84,17 @@ # --- Host resolution --------------------------------------------------------- -_HOST_SCRIPT = Path(__file__).parent / "delegate" / "delegate_host.mjs" -_SDK_ENTRY_REL_PATH = Path("node_modules") / "@uipath" / "delegate-sdk" / "dist" / "index.mjs" +_HOST_PACKAGE = "@uipath/delegate-stdio" +_HOST_BUNDLE_REL_PATH = Path("node_modules") / "@uipath" / "delegate-stdio" / "dist" / "delegate_stdio.mjs" + +_HOST_AUTH_ENV: tuple[tuple[str, str], ...] = ( + ("AUTH_TOKEN", "AUTH_TOKEN"), + ("TENANT_ID", "TENANT_ID"), + ("ORG_ID", "ORG_ID"), + ("ORG_LOGICAL_NAME", "ORG_SLUG"), + ("TENANT_NAME", "TENANT_SLUG"), +) +"""``(name the host reads, name coder_eval reads via _env)`` for each auth var.""" _UNSUPPORTED_CONFIG_FIELDS: tuple[str, ...] = ( "allowed_tools", @@ -99,28 +114,31 @@ indefinitely, unlike every turn-scoped read, which is deadline-bounded.""" _SIGKILL: signal.Signals = getattr(signal, "SIGKILL", signal.SIGTERM) -# The SDK event types this host forwards verbatim that carry model-turn content. -# `session_start` / `step` / `done` are recognized-but-informational; anything -# else is logged and ignored rather than silently dropped. +# Event types (inside the host's `event` frames) that carry model-turn content. +# `session_start` / `done` are informational; anything else is logged and ignored. _TEXT_EVENT_TYPES = frozenset({"thinking", "message"}) -def _tool_id(msg: dict[str, Any]) -> str: - return str(msg.get("toolId") or msg.get("id") or msg.get("callId") or "") +def _tool_id(event: dict[str, Any]) -> str: + return str(event.get("toolId") or "") def _env(bare_name: str) -> str | None: """Read a ``DELEGATE_``-namespaced auth var, falling back to the bare name. - coder_eval controls these names (it forwards the values into the SDK's - ``auth`` object; the SDK never reads process.env itself), so the bare - spellings (``AUTH_TOKEN``, ``TENANT_ID``, ...) collide with names other - tooling (npm, Vault, Terraform) commonly exports. The namespaced spelling - is checked first; the bare one stays for delegate-cli compatibility. + The bare spellings (``AUTH_TOKEN``, ``TENANT_ID``, ...) collide with names + other tooling (npm, Vault, Terraform) commonly exports, so the namespaced + spelling is checked first. ``_host_auth_env`` re-exports the value under + the name the host itself reads. """ return os.environ.get(f"DELEGATE_{bare_name}") or os.environ.get(bare_name) +def _host_auth_env() -> dict[str, str]: + """The auth vars to set in the host's environment, keyed by the host's names.""" + return {host_name: value for host_name, ours in _HOST_AUTH_ENV if (value := _env(ours))} + + def _candidate_install_roots() -> list[Path]: """Ancestor-walk search roots: cwd, its ancestors, and home. @@ -136,11 +154,11 @@ def _candidate_install_roots() -> list[Path]: return roots -def _resolve_sdk_entry() -> Path: - """Locate the installed ``@uipath/delegate-sdk``'s ``dist/index.mjs``. +def _resolve_host_bundle() -> Path: + """Locate the installed ``@uipath/delegate-stdio``'s ``dist/delegate_stdio.mjs``. - Resolution order: ``DELEGATE_SDK_PATH`` (explicit file path) -> - ``DELEGATE_SDK_NODE_MODULES`` (explicit install root, probed exactly) -> + Resolution order: ``DELEGATE_STDIO_PATH`` (explicit file path) -> + ``DELEGATE_STDIO_NODE_MODULES`` (explicit install root, probed exactly) -> ancestor walk from cwd (plus home), so an ``npm install`` anywhere in that chain — including this module's own ``agents/delegate/`` directory, which ships a ``package.json`` naming the dependency — is found automatically. @@ -148,40 +166,40 @@ def _resolve_sdk_entry() -> Path: Raises: AgentConfigError: no install found anywhere searched. """ - explicit = os.environ.get("DELEGATE_SDK_PATH") + explicit = os.environ.get("DELEGATE_STDIO_PATH") if explicit: path = Path(explicit).expanduser().resolve() if not path.is_file(): raise AgentConfigError( - f"DELEGATE_SDK_PATH={path} does not point to a file. Point it at " - + "@uipath/delegate-sdk's dist/index.mjs." + f"DELEGATE_STDIO_PATH={path} does not point to a file. Point it at " + + f"{_HOST_PACKAGE}'s dist/delegate_stdio.mjs." ) return path - root_override = os.environ.get("DELEGATE_SDK_NODE_MODULES") + root_override = os.environ.get("DELEGATE_STDIO_NODE_MODULES") if root_override: - path = (Path(root_override).expanduser().resolve() / _SDK_ENTRY_REL_PATH).resolve() + path = (Path(root_override).expanduser().resolve() / _HOST_BUNDLE_REL_PATH).resolve() if not path.is_file(): raise AgentConfigError( - f"DELEGATE_SDK_NODE_MODULES={root_override}: @uipath/delegate-sdk not found at {path}. " - + "Run `npm install @uipath/delegate-sdk` there, or set DELEGATE_SDK_PATH directly." + f"DELEGATE_STDIO_NODE_MODULES={root_override}: {_HOST_PACKAGE} not found at {path}. " + + f"Run `npm install {_HOST_PACKAGE}` there, or set DELEGATE_STDIO_PATH directly." ) return path searched: list[Path] = [] for root in _candidate_install_roots(): - candidate = (root / _SDK_ENTRY_REL_PATH).resolve() + candidate = (root / _HOST_BUNDLE_REL_PATH).resolve() searched.append(candidate) if candidate.is_file(): return candidate searched_block = "\n ".join(str(p) for p in searched) raise AgentConfigError( - "@uipath/delegate-sdk not found. Searched the cwd, its ancestors, and home:\n" + f"{_HOST_PACKAGE} not found. Searched the cwd, its ancestors, and home:\n" + f" {searched_block}\n" - + "Run `npm install @uipath/delegate-sdk` (plain, public install — no token needed), " - + "e.g. in this framework's own agents/delegate/ directory, or set DELEGATE_SDK_NODE_MODULES " - + "to the install root, or DELEGATE_SDK_PATH to the dist/index.mjs file directly. " + + f"Run `npm install {_HOST_PACKAGE}` (plain, public install — no token needed), " + + "e.g. in this framework's own agents/delegate/ directory, or set DELEGATE_STDIO_NODE_MODULES " + + "to the install root, or DELEGATE_STDIO_PATH to the dist/delegate_stdio.mjs file directly. " + "See docs/agents/DELEGATE.md." ) @@ -210,23 +228,13 @@ def _resolve_bundled_skills_path(plugins: list[dict[str, Any]] | None) -> str | def _parse_usage(raw: Any) -> TokenUsage | None: - """Parse the SDK's per-turn usage payload (from ``getLastTurnUsage()``) into ``TokenUsage``. - - CONFIRMED (reading the installed ``@uipath/delegate-sdk@0.1.12``'s bundled - ``dist/index.mjs``): no event this host forwards ever carries a ``usage`` - field -- the SDK's per-turn token accounting lives only in its internal - store, reachable through ``DelegateAgent.getLastTurnUsage()``, which - ``delegate_host.mjs`` calls after ``sendMessage()`` resolves and attaches - to the ``send_ok`` message as ``usage``. That getter's shape, from the - SDK's own ``setUsage`` store action: ``{promptTokens, completionTokens, - promptTokensCached, cacheCreationTokens, turnTokenUnits, - contextBreakdown}``. ``promptTokens`` is the TOTAL input token count - (cached + uncached, OpenAI-style); ``promptTokensCached`` is the - cache-READ subset of it, so ``uncached = promptTokens - promptTokensCached``. - Falls back to 0 for anything absent (e.g. before the backend's first - internal usage report), mirroring the project's "warn on drift, never - raise" contract -- a future SDK release renaming one of these fields - degrades to zero tokens for that bucket, not a crash. + """Parse the host's ``result.usage`` payload into ``TokenUsage``. + + The host follows the Anthropic convention: ``input_tokens`` excludes cache + reads and writes, which arrive separately as ``cache_read_input_tokens`` / + ``cache_creation_input_tokens``. An absent or invalid bucket reads as 0, and + a payload with no recognised bucket returns ``None`` with a warning — a + renamed field degrades to zero tokens, never a crash. """ if not isinstance(raw, dict): return None @@ -237,28 +245,25 @@ def _int(key: str) -> int: return 0 return value if isinstance(value, int) and value >= 0 else 0 - prompt_total = _int("promptTokens") - prompt_cached = _int("promptTokensCached") - output_tokens = _int("completionTokens") - cache_creation = _int("cacheCreationTokens") - if prompt_total == 0 and output_tokens == 0 and prompt_cached == 0 and cache_creation == 0: + usage = TokenUsage( + uncached_input_tokens=_int("input_tokens"), + output_tokens=_int("output_tokens"), + cache_creation_input_tokens=_int("cache_creation_input_tokens"), + cache_read_input_tokens=_int("cache_read_input_tokens"), + ) + if usage.is_empty(): if raw: logger.warning("delegate: usage payload matched none of the known bucket spellings: %r", sorted(raw)) return None - return TokenUsage( - uncached_input_tokens=max(prompt_total - prompt_cached, 0), - output_tokens=output_tokens, - cache_creation_input_tokens=cache_creation, - cache_read_input_tokens=prompt_cached, - ) + return usage class _TurnState: """Per-``communicate()`` accumulator. - One ``AssistantMessage`` per turn (see module docstring's "No - multi-generation transcript splitting"), so this is far smaller than the - per-round-trip segment machinery a richer transcript would need. + One ``AssistantMessage`` per turn (see .claude/notes/agents.md § Delegate + agent), so this is far smaller than the per-round-trip segment machinery a + richer transcript would need. """ def __init__(self, *, iteration: int, user_input: str, model: str | None) -> None: @@ -298,8 +303,7 @@ def agent_output(self) -> str: class DelegateAgent(Agent[DelegateAgentConfig]): """Drives UiPath Autopilot's Delegate agent through a persistent Node host subprocess. - See module docstring for prerequisites and the deliberate scope reductions - versus the UiPath-only sibling plugin's more hardened adapter. + See the module docstring for prerequisites and the wire protocol. """ # The host streams one event per model/tool step, checked after each. @@ -329,7 +333,7 @@ def __init__( self._session_id: str | None = None self._state = AgentState.WORKING - self._sdk_entry: Path | None = None + self._host_bundle: Path | None = None self._process: asyncio.subprocess.Process | None = None self._stdout_task: asyncio.Task[None] | None = None self._stderr_task: asyncio.Task[None] | None = None @@ -349,9 +353,9 @@ async def start( if shutil.which("node") is None: raise AgentConfigError( "Node.js was not found on PATH. Install it (https://nodejs.org/), " - + "then `npm install @uipath/delegate-sdk`. See docs/agents/DELEGATE.md." + + f"then `npm install {_HOST_PACKAGE}`. See docs/agents/DELEGATE.md." ) - self._sdk_entry = _resolve_sdk_entry() + self._host_bundle = _resolve_host_bundle() ignored = [f for f in _UNSUPPORTED_CONFIG_FIELDS if getattr(self.config, f, None)] if ignored: @@ -370,7 +374,7 @@ async def start( await self._spawn_and_init() async def _spawn_and_init(self) -> None: - assert self._sdk_entry is not None, "start() must resolve the SDK entry before spawning" + assert self._host_bundle is not None, "start() must resolve the host bundle before spawning" env = dict(os.environ) if self._env_path_prepend: env["PATH"] = os.pathsep.join([*self._env_path_prepend, env.get("PATH", "")]) @@ -378,15 +382,15 @@ async def _spawn_and_init(self) -> None: env["PLUGIN_TOOLS_DIR"] = self._plugin_tools_dir # Respect an operator's own telemetry choice; only default it off. env.setdefault("DELEGATE_TELEMETRY_DISABLED", "1") + env.update(_host_auth_env()) await self._cancel_drain_tasks() self._stdout_queue = asyncio.Queue() self._stderr_lines.clear() self._process = await asyncio.create_subprocess_exec( "node", - str(_HOST_SCRIPT), - str(self._sdk_entry), - cwd=str(_HOST_SCRIPT.parent), + str(self._host_bundle), + cwd=self.working_directory, stdin=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, @@ -405,7 +409,7 @@ async def _spawn_and_init(self) -> None: init_options = self._build_init_options() await self._send_command({"cmd": "init", "options": init_options}) try: - ack = await asyncio.wait_for(self._read_until(("init_ok", "init_error")), timeout=_INIT_TIMEOUT_SEC) + ack = await asyncio.wait_for(self._read_until(("init_ok", "error")), timeout=_INIT_TIMEOUT_SEC) except TimeoutError as exc: await self._force_kill_host() self._process = None @@ -413,7 +417,9 @@ async def _spawn_and_init(self) -> None: f"Delegate SDK init did not respond within {_INIT_TIMEOUT_SEC:.0f}s " + "(hung auth refresh or backend connect?)." ) from exc - if ack.get("type") == "init_error": + if ack.get("type") == "error": + await self._force_kill_host() + self._process = None raise AgentConfigError(f"Delegate SDK init failed: {ack.get('message', 'unknown error')}") def _build_init_options(self) -> dict[str, Any]: @@ -436,29 +442,10 @@ def _build_init_options(self) -> dict[str, Any]: options["backendUrl"] = backend_url environment = os.environ.get("DELEGATE_ENV") if environment: - options["environment"] = environment - auth_token = _env("AUTH_TOKEN") - if auth_token: - auth: dict[str, Any] = { - "accessToken": auth_token, - "tenantId": _env("TENANT_ID"), - "organizationId": _env("ORG_ID"), - } - # CONFIRMED LIVE: when `environment` (rather than `backendUrl`) is set, - # the SDK resolves the backend URL from THIS auth object's - # organizationName/tenantName fields, not from ORG_SLUG/TENANT_SLUG - # process.env directly (that pairing is documented only in the SDK's - # own error message, aimed at delegate-cli's env-var-driven wrapper — - # the SDK class we drive here never reads those two vars itself). - org_slug = _env("ORG_SLUG") - tenant_slug = _env("TENANT_SLUG") - if org_slug: - auth["organizationName"] = org_slug - if tenant_slug: - auth["tenantName"] = tenant_slug - options["auth"] = auth + options["env"] = environment # list[LocalPluginConfig] is not list[dict[str, Any]] under list invariance. skills_path = _resolve_bundled_skills_path(self.config.plugins) # type: ignore[arg-type] + options["enableSkills"] = skills_path is not None if skills_path: options["bundledSkillsPath"] = skills_path return options @@ -583,7 +570,7 @@ async def communicate( if self._process is None or self._process.returncode is not None: # A prior cooperative stop or crash left no live host: respawn # fresh rather than fail fast. Session-conflict-specific recovery - # is deferred (see module docstring); this simpler policy covers + # is deferred (see .claude/notes/agents.md); this simpler policy covers # both "the previous turn stopped cleanly" and "it crashed". try: await self._spawn_and_init() @@ -645,30 +632,22 @@ def emit(event: StreamEvent) -> None: ) mtype = msg.get("type") - if mtype == "send_ok": - self._handle_send_ok(msg, state) + if mtype == "result": + self._handle_result(msg, state) break - if mtype == "send_error": - # Reuse of a live host after a recoverable `send_error` is - # not attempted: force-kill so the next communicate() always - # respawns onto a fresh queue rather than risk consuming a - # stale onEvent callback the SDK fires after this rejection. + if mtype == "error": + # The host may survive a failed send, but it is not reused: + # force-kill so the next communicate() respawns onto a fresh + # queue rather than risk consuming a stale event from this turn. await self._abandon_host_and_crash( state, collector, emit, f"Delegate send failed: {msg.get('message', 'unknown error')}" ) - if mtype == "fatal": - # The host exits right after writing this line (every - # `fatal` site calls process.exit) -- force-kill is then a - # no-op, but stays here to match the EOF branch exactly - # rather than trust that invariant from this side too. - await self._abandon_host_and_crash( - state, collector, emit, f"Delegate host crashed: {msg.get('message', 'unknown error')}" - ) - if mtype in ("protocol_error", "destroy_error"): - logger.warning("delegate: host reported %s: %s", mtype, msg.get("message")) + event = msg.get("event") + if mtype != "event" or not isinstance(event, dict): + logger.debug("delegate: ignoring host message %r", mtype) continue - self._handle_event(msg, state, emit) + self._handle_event(event, state, emit) if max_turns is not None and state.api_calls > max_turns: state.max_turns_exhausted = True @@ -711,63 +690,63 @@ def finalize_on_cancel( self._crash_turn(state, collector, emit, f"Delegate turn failed: {e!s}", cause=e) raise # unreachable — _crash_turn is NoReturn - def _handle_event(self, msg: dict[str, Any], state: _TurnState, emit: Callable[[StreamEvent], None]) -> None: - """Dispatch one forwarded SDK event. Never raises on unrecognized shape.""" - event_type = msg.get("type") - session_id = msg.get("sessionId") + def _handle_event(self, event: dict[str, Any], state: _TurnState, emit: Callable[[StreamEvent], None]) -> None: + """Dispatch one SDK event unwrapped from an ``event`` frame. Never raises on unrecognized shape.""" + event_type = event.get("type") + session_id = event.get("sessionId") if isinstance(session_id, str) and session_id: self._session_id = session_id - model = msg.get("model") - if isinstance(model, str) and model: - state.model_used = model - usage = _parse_usage(msg.get("usage")) - if usage is not None: - state.usage = usage if state.api_calls == 0 and (event_type in _TEXT_EVENT_TYPES or event_type == "tool_call"): state.api_calls = 1 elif state.results_incomplete and ( - event_type in _TEXT_EVENT_TYPES or (event_type == "tool_call" and _tool_id(msg) not in state.open_tools) + event_type in _TEXT_EVENT_TYPES or (event_type == "tool_call" and _tool_id(event) not in state.open_tools) ): self._close_open_tools(state, emit) state.api_calls += 1 state.results_incomplete = False - if event_type in _TEXT_EVENT_TYPES: - text = msg.get("content") + if event_type == "message": + text = event.get("content") if isinstance(text, str) and text: - state.text_parts.append(text) - if event_type == "message": - state.message_events += 1 - state.content_blocks.append( - ContentBlock(block_type="text", sequence=len(state.content_blocks), text=text) - ) - emit(TextChunkEvent(task_id=self.task_id, turn_id=state.turn_id, text=text)) - else: - state.content_blocks.append( - ContentBlock(block_type="thinking", sequence=len(state.content_blocks), thinking=text) - ) + self._append_message_text(state, text, starts_step=event.get("isStepStart") is not False) + emit(TextChunkEvent(task_id=self.task_id, turn_id=state.turn_id, text=text)) + elif event_type == "thinking": + text = event.get("content") + if isinstance(text, str) and text: + state.content_blocks.append( + ContentBlock(block_type="thinking", sequence=len(state.content_blocks), thinking=text) + ) elif event_type == "tool_call": - self._handle_tool_call(msg, state, emit) + self._handle_tool_call(event, state, emit) elif event_type == "tool_result": - self._handle_tool_result(msg, state, emit) + self._handle_tool_result(event, state, emit) state.results_incomplete = bool(state.open_tools) if not state.open_tools: state.api_calls += 1 elif event_type == "error": - message = msg.get("message") or msg.get("content") or "unknown error" - state.error_message = str(message) + state.error_message = str(event.get("error") or "unknown error") logger.warning("delegate: SDK reported an error event: %s", state.error_message) - elif event_type in ("session_start", "step", "done"): + elif event_type in ("session_start", "done"): pass # informational; no telemetry to record else: logger.debug("delegate: unrecognized event type %r", event_type) - def _handle_tool_call(self, msg: dict[str, Any], state: _TurnState, emit: Callable[[StreamEvent], None]) -> None: - # UNVERIFIED: exact id-field spelling. - tool_id = _tool_id(msg) or str(uuid.uuid4()) - tool_name = str(msg.get("toolName") or msg.get("tool") or "unknown") - parameters = msg.get("input") + @staticmethod + def _append_message_text(state: _TurnState, text: str, *, starts_step: bool) -> None: + """Record assistant text; a streamed delta (``isStepStart: false``) extends the open text block.""" + state.text_parts.append(text) + last = state.content_blocks[-1] if state.content_blocks else None + if not starts_step and last is not None and last.block_type == "text": + last.text = (last.text or "") + text + return + state.message_events += 1 + state.content_blocks.append(ContentBlock(block_type="text", sequence=len(state.content_blocks), text=text)) + + def _handle_tool_call(self, event: dict[str, Any], state: _TurnState, emit: Callable[[StreamEvent], None]) -> None: + tool_id = _tool_id(event) or str(uuid.uuid4()) + tool_name = str(event.get("toolName") or "unknown") + parameters = event.get("toolArgs") parameters = parameters if isinstance(parameters, dict) else {} state.sequence += 1 telemetry = CommandTelemetry( @@ -786,8 +765,10 @@ def _handle_tool_call(self, msg: dict[str, Any], state: _TurnState, emit: Callab ) emit(ToolStartEvent(task_id=self.task_id, turn_id=state.turn_id, tool=telemetry)) - def _handle_tool_result(self, msg: dict[str, Any], state: _TurnState, emit: Callable[[StreamEvent], None]) -> None: - tool_id = _tool_id(msg) + def _handle_tool_result( + self, event: dict[str, Any], state: _TurnState, emit: Callable[[StreamEvent], None] + ) -> None: + tool_id = _tool_id(event) telemetry = state.open_tools.pop(tool_id, None) if telemetry is None: # A result with no matching open call (id mismatch or unknown shape). @@ -801,19 +782,13 @@ def _handle_tool_result(self, msg: dict[str, Any], state: _TurnState, emit: Call timestamp=datetime.now(), sequence_number=state.sequence, ) - error_text = msg.get("error") - output = msg.get("output") if "output" in msg else msg.get("content") + output = event.get("toolResult") + if isinstance(output, dict) and isinstance(output.get("content"), str): + output = output["content"] completed = datetime.now() telemetry.execution_completed_at = completed if telemetry.execution_started_at is not None: telemetry.duration_ms = (completed - telemetry.execution_started_at).total_seconds() * 1000 - if error_text: - status = ToolEndStatus.ERROR - telemetry.result_status = "error" - telemetry.error_message = str(error_text) - else: - status = ToolEndStatus.OK - telemetry.result_status = "success" if isinstance(output, str): telemetry.result_summary = output elif output is not None: @@ -822,24 +797,36 @@ def _handle_tool_result(self, msg: dict[str, Any], state: _TurnState, emit: Call telemetry.result_summary = json.dumps(output) else: telemetry.result_summary = None + if event.get("toolStatus") == "failed": + status = ToolEndStatus.ERROR + telemetry.result_status = "error" + telemetry.error_message = telemetry.result_summary or "tool failed" + else: + status = ToolEndStatus.OK + telemetry.result_status = "success" emit(ToolEndEvent(task_id=self.task_id, turn_id=state.turn_id, tool=telemetry, status=status)) - def _handle_send_ok(self, msg: dict[str, Any], state: _TurnState) -> None: - # CONFIRMED (reading the installed SDK's bundled source): - # sendMessage()'s resolved value is always a plain string, never an - # object -- `usage`/`sessionId` are NOT nested under it. delegate_host.mjs - # instead reads them off `getLastTurnUsage()`/`getSessionId()` after - # sendMessage() resolves and attaches them to this message's own - # top level (see delegate_host.mjs's wire-protocol header comment). - result = msg.get("result") - if isinstance(result, str): - state.final_response = result + def _handle_result(self, msg: dict[str, Any], state: _TurnState) -> None: + """Fold the host's terminal ``result`` frame into the turn state. + + ``turnUsages`` has one entry per backend round-trip, so its length is the + authoritative call count and replaces the running estimate. + """ + response = msg.get("response") + if isinstance(response, str): + state.final_response = response session_id = msg.get("sessionId") if isinstance(session_id, str) and session_id: self._session_id = session_id + model = msg.get("model") + if isinstance(model, str) and model: + state.model_used = model usage = _parse_usage(msg.get("usage")) if usage is not None: state.usage = usage + turn_usages = msg.get("turnUsages") + if isinstance(turn_usages, list) and turn_usages: + state.api_calls = len(turn_usages) def _close_open_tools(self, state: _TurnState, emit: Callable[[StreamEvent], None]) -> None: for tool_id, telemetry in list(state.open_tools.items()): @@ -880,9 +867,9 @@ def _finalize_turn( messages = [] if state.content_blocks: completed = datetime.now() - # Single generation window for the whole turn (see module docstring's - # "No multi-generation transcript splitting"): tiles from the turn's - # own start, since there is no prior emission to tile from. + # Single generation window for the whole turn (see .claude/notes/agents.md + # § Delegate agent): tiles from the turn's own start, since there is no + # prior emission to tile from. started, generation_ms = close_window(mark=state.started_dt, now=completed) messages.append( AssistantMessage( @@ -1014,8 +1001,7 @@ async def _read_until(self, accepted_types: tuple[str, ...]) -> dict[str, Any]: raise AgentCrashError(f"Delegate host exited before responding. stderr tail:\n{tail}") if msg.get("type") in accepted_types: return msg - if msg.get("type") in ("protocol_error", "fatal"): - logger.warning("delegate: host reported %s during init: %s", msg.get("type"), msg.get("message")) + logger.debug("delegate: ignoring host message %r during init", msg.get("type")) async def _drain_stdout(self, stream: asyncio.StreamReader) -> None: try: diff --git a/src/coder_eval/models/agent_config.py b/src/coder_eval/models/agent_config.py index a9aef571..acf19584 100644 --- a/src/coder_eval/models/agent_config.py +++ b/src/coder_eval/models/agent_config.py @@ -396,11 +396,11 @@ class PiAgentConfig(BaseAgentConfig): class DelegateAgentConfig(BaseAgentConfig): """Delegate agent configuration (UiPath Autopilot's Delegate agent). - Drives a Node host subprocess this framework ships (``agents/delegate/delegate_host.mjs``) - that wraps the public ``@uipath/delegate-sdk`` npm package's ``DelegateAgent`` class in a - stdio JSON-Lines protocol. The SDK's reasoning runs in the UiPath backend; its tools - (shell, file, Office, PDF) execute locally through a bundled interop process, so file-based - success criteria work as usual. See ``docs/agents/DELEGATE.md``. + Drives the Node host that the public ``@uipath/delegate-stdio`` npm package ships, which + runs the ``@uipath/delegate-sdk`` agent behind a stdio JSON-Lines protocol. The SDK's + reasoning runs in the UiPath backend; its tools (shell, file, Office, PDF) execute locally + through a bundled interop process, so file-based success criteria work as usual. See + ``docs/agents/DELEGATE.md``. ``allowed_tools`` / ``disallowed_tools`` / ``system_prompt`` / ``system_prompt_file`` have no Delegate SDK equivalent and are warned about, not enforced, at ``start()``. diff --git a/tasks/delegate/hello_date_delegate.yaml b/tasks/delegate/hello_date_delegate.yaml index a299d5bb..c7b9c153 100644 --- a/tasks/delegate/hello_date_delegate.yaml +++ b/tasks/delegate/hello_date_delegate.yaml @@ -2,7 +2,7 @@ task_id: "hello_date_delegate_smoke_test" description: "Smoke-test the Delegate agent harness: create and run a small Python script." initial_prompt: "Create a Python file named app.py in the current working directory that prints 'Hello, Delegate!' on one line, and today's date in YYYY-MM-DD format on the next line. Use the datetime module. Then run the script with: python app.py" # No `smoke-pass`: that tag routes a task into the CI E2E bucket, which has no -# UiPath auth or @uipath/delegate-sdk install. Run this task locally (or in a +# UiPath auth or @uipath/delegate-stdio install. Run this task locally (or in a # job that provisions both) instead. tags: [smoke, basic, pure-python, delegate] diff --git a/tests/test_delegate_agent.py b/tests/test_delegate_agent.py index faa8f29b..202c2f09 100644 --- a/tests/test_delegate_agent.py +++ b/tests/test_delegate_agent.py @@ -11,12 +11,13 @@ import asyncio import json import os +from pathlib import Path from typing import Any import pytest from coder_eval.agents import delegate_agent as agent_module -from coder_eval.agents.delegate_agent import DelegateAgent, _resolve_sdk_entry +from coder_eval.agents.delegate_agent import DelegateAgent, _resolve_host_bundle from coder_eval.agents.registry import AgentRegistry, create_agent from coder_eval.errors import AgentConfigError, AgentCrashError, TurnTimeoutError from coder_eval.models import AgentKind, DelegateAgentConfig @@ -27,6 +28,29 @@ def _line(obj: dict[str, Any]) -> bytes: return (json.dumps(obj) + "\n").encode("utf-8") +def _ev(**event: Any) -> bytes: + """One ``event`` frame wrapping an SDK event, as ``delegate-stdio`` writes it.""" + return _line({"type": "event", "event": event}) + + +def _result(**fields: Any) -> bytes: + return _line({"type": "result", **fields}) + + +def _tool_call(tool_id: str, name: str = "shell", **args: Any) -> bytes: + return _ev(type="tool_call", toolId=tool_id, toolName=name, toolArgs=args, toolStatus="pending") + + +def _tool_result(tool_id: str, content: str = "ok", *, status: str = "completed") -> bytes: + return _ev( + type="tool_result", + toolId=tool_id, + toolName="shell", + toolResult={"responseType": "success", "content": content}, + toolStatus=status, + ) + + class _FakeStreamReader: def __init__(self, lines: list[bytes], *, hang_after: bool = False) -> None: self._lines = list(lines) @@ -68,6 +92,8 @@ def __init__( self.returncode: int | None = None self.pid = 4242 self._killed = False + self.spawn_args: tuple[Any, ...] = () + self.spawn_kwargs: dict[str, Any] = {} async def wait(self) -> int: while self.returncode is None: @@ -96,11 +122,13 @@ def _install( proc = _FakeProcess(stdout_lines, stderr_lines, hang_after=hang_after) async def fake_exec(*args: Any, **kwargs: Any) -> _FakeProcess: + proc.spawn_args = args + proc.spawn_kwargs = kwargs return proc monkeypatch.setattr(asyncio, "create_subprocess_exec", fake_exec) monkeypatch.setattr("shutil.which", lambda _name: "/usr/local/bin/node") - monkeypatch.setattr(agent_module, "_resolve_sdk_entry", lambda: agent_module._HOST_SCRIPT) + monkeypatch.setattr(agent_module, "_resolve_host_bundle", lambda: Path("/opt/delegate_stdio.mjs")) # raising=False: os.killpg does not exist on Windows, where the sweep is a # no-op -- the stub must still install so the fixture works on every platform. monkeypatch.setattr(os, "killpg", lambda pgid, sig: None, raising=False) @@ -122,42 +150,42 @@ async def _started_agent( return agent, proc -class TestResolveSdkEntry: +class TestResolveHostBundle: def test_explicit_path_env_var(self, tmp_path, monkeypatch): - entry = tmp_path / "index.mjs" - entry.write_text("export const DelegateAgent = class {};") - monkeypatch.setenv("DELEGATE_SDK_PATH", str(entry)) - assert _resolve_sdk_entry() == entry.resolve() + entry = tmp_path / "delegate_stdio.mjs" + entry.write_text("#!/usr/bin/env node") + monkeypatch.setenv("DELEGATE_STDIO_PATH", str(entry)) + assert _resolve_host_bundle() == entry.resolve() def test_explicit_path_missing_file_raises(self, tmp_path, monkeypatch): - monkeypatch.setenv("DELEGATE_SDK_PATH", str(tmp_path / "nope.mjs")) + monkeypatch.setenv("DELEGATE_STDIO_PATH", str(tmp_path / "nope.mjs")) with pytest.raises(AgentConfigError, match="does not point to a file"): - _resolve_sdk_entry() + _resolve_host_bundle() def test_node_modules_override(self, tmp_path, monkeypatch): - monkeypatch.delenv("DELEGATE_SDK_PATH", raising=False) - monkeypatch.setenv("DELEGATE_SDK_NODE_MODULES", str(tmp_path)) - entry = tmp_path / "node_modules" / "@uipath" / "delegate-sdk" / "dist" / "index.mjs" + monkeypatch.delenv("DELEGATE_STDIO_PATH", raising=False) + monkeypatch.setenv("DELEGATE_STDIO_NODE_MODULES", str(tmp_path)) + entry = tmp_path / "node_modules" / "@uipath" / "delegate-stdio" / "dist" / "delegate_stdio.mjs" entry.parent.mkdir(parents=True) - entry.write_text("export const DelegateAgent = class {};") - assert _resolve_sdk_entry() == entry.resolve() + entry.write_text("#!/usr/bin/env node") + assert _resolve_host_bundle() == entry.resolve() def test_node_modules_override_missing_raises(self, tmp_path, monkeypatch): - monkeypatch.delenv("DELEGATE_SDK_PATH", raising=False) - monkeypatch.setenv("DELEGATE_SDK_NODE_MODULES", str(tmp_path)) + monkeypatch.delenv("DELEGATE_STDIO_PATH", raising=False) + monkeypatch.setenv("DELEGATE_STDIO_NODE_MODULES", str(tmp_path)) with pytest.raises(AgentConfigError, match="not found"): - _resolve_sdk_entry() + _resolve_host_bundle() def test_no_config_and_not_found_raises_with_search_list(self, tmp_path, monkeypatch): - monkeypatch.delenv("DELEGATE_SDK_PATH", raising=False) - monkeypatch.delenv("DELEGATE_SDK_NODE_MODULES", raising=False) + monkeypatch.delenv("DELEGATE_STDIO_PATH", raising=False) + monkeypatch.delenv("DELEGATE_STDIO_NODE_MODULES", raising=False) monkeypatch.chdir(tmp_path) monkeypatch.setattr(os, "getcwd", lambda: str(tmp_path)) # A real npm install under the developer's actual home directory must # not make this test flaky -- pin every search root to tmp_path. monkeypatch.setattr(agent_module, "_candidate_install_roots", lambda: [tmp_path]) with pytest.raises(AgentConfigError, match="Searched the cwd"): - _resolve_sdk_entry() + _resolve_host_bundle() class TestStart: @@ -173,12 +201,14 @@ async def test_init_ok_completes_start(self, patch_exec, tmp_path): sent = proc.stdin.written[0] assert sent["cmd"] == "init" assert sent["options"]["workingDirectory"] == str(tmp_path) + assert proc.spawn_args == ("node", str(Path("/opt/delegate_stdio.mjs"))) async def test_init_error_raises_agent_config_error(self, patch_exec, tmp_path): - patch_exec([_line({"type": "init_error", "message": "backendUrl is required"})]) + patch_exec([_line({"type": "error", "message": "backendUrl is required", "stack": "Error: ..."})]) agent = DelegateAgent(_config()) with pytest.raises(AgentConfigError, match="backendUrl is required"): await agent.start(str(tmp_path)) + assert agent._process is None async def test_eof_during_init_raises_crash(self, patch_exec, tmp_path): patch_exec([]) # EOF immediately @@ -209,76 +239,109 @@ async def test_unsupported_fields_warn(self, patch_exec, tmp_path, caplog): await _started_agent(patch_exec, [], tmp_path, system_prompt="be nice") assert any("has no Delegate SDK equivalent" in r.message for r in caplog.records) - async def test_auth_token_forwards_ids_but_not_slugs_by_default(self, patch_exec, tmp_path, monkeypatch): - monkeypatch.setenv("AUTH_TOKEN", "tok-1") - monkeypatch.setenv("TENANT_ID", "tenant-guid") - monkeypatch.setenv("ORG_ID", "org-guid") - monkeypatch.delenv("ORG_SLUG", raising=False) - monkeypatch.delenv("TENANT_SLUG", raising=False) + async def test_delegate_env_becomes_the_env_option(self, patch_exec, tmp_path, monkeypatch): + monkeypatch.setenv("DELEGATE_ENV", "alpha") _agent, proc = await _started_agent(patch_exec, [], tmp_path) - auth = proc.stdin.written[0]["options"]["auth"] - assert auth == {"accessToken": "tok-1", "tenantId": "tenant-guid", "organizationId": "org-guid"} - - async def test_org_and_tenant_slug_forwarded_into_auth(self, patch_exec, tmp_path, monkeypatch): - """Regression test: the SDK's `environment` resolution reads - organizationName/tenantName off the `auth` object it's given, NOT - ORG_SLUG/TENANT_SLUG from process.env directly (confirmed live against - the installed @uipath/delegate-sdk) -- so these two env vars must be - translated into `auth` fields here, or `environment: "alpha"` fails - init with "--env alpha needs org/tenant slugs" even with a valid - AUTH_TOKEN/TENANT_ID/ORG_ID triple. - """ - monkeypatch.setenv("AUTH_TOKEN", "tok-1") + assert proc.stdin.written[0]["options"]["env"] == "alpha" + + async def test_skills_enabled_only_with_a_plugin(self, patch_exec, tmp_path): + _agent, proc = await _started_agent(patch_exec, [], tmp_path) + assert proc.stdin.written[0]["options"]["enableSkills"] is False + + plugin_dir = tmp_path / "plugin" + plugin_dir.mkdir() + _agent, proc = await _started_agent( + patch_exec, [], tmp_path, plugins=[{"type": "local", "path": str(plugin_dir)}] + ) + options = proc.stdin.written[0]["options"] + assert options["enableSkills"] is True + assert options["bundledSkillsPath"] == str(plugin_dir / "skills") + + async def test_auth_is_exported_under_the_hosts_own_env_names(self, patch_exec, tmp_path, monkeypatch): + """The host reads auth from its environment, so nothing auth-shaped goes into init options.""" + monkeypatch.setenv("DELEGATE_AUTH_TOKEN", "tok-1") + monkeypatch.setenv("AUTH_TOKEN", "stray-npm-token") monkeypatch.setenv("TENANT_ID", "tenant-guid") monkeypatch.setenv("ORG_ID", "org-guid") monkeypatch.setenv("ORG_SLUG", "my-org") - monkeypatch.setenv("TENANT_SLUG", "my-tenant") + monkeypatch.setenv("DELEGATE_TENANT_SLUG", "my-tenant") _agent, proc = await _started_agent(patch_exec, [], tmp_path) - auth = proc.stdin.written[0]["options"]["auth"] - assert auth == { - "accessToken": "tok-1", - "tenantId": "tenant-guid", - "organizationId": "org-guid", - "organizationName": "my-org", - "tenantName": "my-tenant", - } + env = proc.spawn_kwargs["env"] + assert env["AUTH_TOKEN"] == "tok-1" + assert env["TENANT_ID"] == "tenant-guid" + assert env["ORG_ID"] == "org-guid" + assert env["ORG_LOGICAL_NAME"] == "my-org" + assert env["TENANT_NAME"] == "my-tenant" + assert "auth" not in proc.stdin.written[0]["options"] class TestCommunicate: async def test_happy_path_text_and_tool(self, patch_exec, tmp_path): events = [ - _line({"type": "thinking", "content": "let me think"}), - _line({"type": "message", "content": "here is my answer"}), - _line({"type": "tool_call", "toolName": "Bash", "toolId": "tool-1", "input": {"command": "ls"}}), - _line({"type": "tool_result", "toolId": "tool-1", "output": "file.txt"}), - _line({"type": "send_ok", "result": "here is my answer", "sessionId": "sess-1"}), + _ev(type="session_start", sessionId="sess-1"), + _ev(type="thinking", content="let me think"), + _tool_call("tool-1", "ExecutePowershellCommand", command="ls"), + _tool_result("tool-1", "file.txt"), + _ev(type="thinking", content=""), + _ev(type="message", content="here is my answer", isStepStart=True), + _ev(type="done", sessionId="sess-1"), + _result(response="here is my answer", sessionId="sess-2", model="virtuoso-1-5"), ] agent, proc = await _started_agent(patch_exec, events, tmp_path) record = await agent.communicate("do something") assert record.agent_output == "here is my answer" + assert record.model_used == "virtuoso-1-5" assert not record.crashed assert len(record.commands) == 1 - assert record.commands[0].tool_name == "Bash" - assert record.commands[0].result_status == "success" + command = record.commands[0] + assert command.tool_name == "ExecutePowershellCommand" + assert command.parameters == {"command": "ls"} + assert command.result_status == "success" + assert command.result_summary == "file.txt" sent = proc.stdin.written[1] assert sent == {"cmd": "send", "prompt": "do something", "sessionId": None} - # Session id from send_ok is remembered for the NEXT turn. - assert agent._session_id == "sess-1" + # The result's session id is remembered for the NEXT turn. + assert agent._session_id == "sess-2" - async def test_send_error_raises_crash(self, patch_exec, tmp_path): - events = [_line({"type": "send_error", "message": "network blip"})] + async def test_streamed_message_deltas_form_one_text_block(self, patch_exec, tmp_path): + events = [ + _ev(type="message", content="P", isStepStart=True), + _ev(type="message", content="ONG", isStepStart=False), + _result(response="PONG"), + ] agent, _ = await _started_agent(patch_exec, events, tmp_path) - with pytest.raises(AgentCrashError, match="network blip"): - await agent.communicate("hi") - assert agent.pending_turn is not None - assert agent.pending_turn.crashed is True + record = await agent.communicate("hi") + (message,) = record.messages + text_blocks = [b for b in message.content_blocks if b.block_type == "text"] + assert [b.text for b in text_blocks] == ["PONG"] + assert record.assistant_turn_count == 1 - async def test_fatal_raises_crash(self, patch_exec, tmp_path): - events = [_line({"type": "fatal", "message": "uncaught exception: boom"})] + async def test_turn_usages_length_is_the_call_count(self, patch_exec, tmp_path): + usage = {"input_tokens": 1, "output_tokens": 1} + events = [ + _tool_call("a"), + _tool_result("a"), + _ev(type="message", content="done", isStepStart=True), + _result(response="done", usage=usage, turnUsages=[usage, usage, usage]), + ] agent, _ = await _started_agent(patch_exec, events, tmp_path) - with pytest.raises(AgentCrashError): + record = await agent.communicate("hi") + assert record.num_turns == 3 + + async def test_send_error_raises_crash(self, patch_exec, tmp_path): + events = [ + _ev(type="error", error="There was a problem with your request."), + _ev(type="done", sessionId="s"), + _line({"type": "error", "message": "Delegate backend error: HTTP 422", "stack": "Error: ..."}), + ] + agent, proc = await _started_agent(patch_exec, events, tmp_path) + with pytest.raises(AgentCrashError, match="HTTP 422"): await agent.communicate("hi") + assert agent.pending_turn is not None + assert agent.pending_turn.crashed is True + assert proc._killed + assert agent._process is None async def test_eof_mid_turn_raises_crash(self, patch_exec, tmp_path): agent, _ = await _started_agent(patch_exec, [], tmp_path) @@ -288,7 +351,7 @@ async def test_eof_mid_turn_raises_crash(self, patch_exec, tmp_path): async def test_non_json_stdout_line_is_skipped_not_fatal(self, patch_exec, tmp_path): events = [ b"[backendUrl] Module loaded - VITE_USE_CLOUD_URL: undefined\n", - _line({"type": "send_ok", "result": "done"}), + _result(response="done"), ] agent, _ = await _started_agent(patch_exec, events, tmp_path) record = await agent.communicate("hi") @@ -296,9 +359,9 @@ async def test_non_json_stdout_line_is_skipped_not_fatal(self, patch_exec, tmp_p async def test_tool_error_marks_error_status(self, patch_exec, tmp_path): events = [ - _line({"type": "tool_call", "toolName": "Bash", "toolId": "t1", "input": {}}), - _line({"type": "tool_result", "toolId": "t1", "error": "command not found"}), - _line({"type": "send_ok", "result": "done"}), + _tool_call("t1"), + _tool_result("t1", "command not found", status="failed"), + _result(response="done"), ] agent, _ = await _started_agent(patch_exec, events, tmp_path) record = await agent.communicate("hi") @@ -306,18 +369,15 @@ async def test_tool_error_marks_error_status(self, patch_exec, tmp_path): assert record.commands[0].error_message == "command not found" async def test_orphaned_tool_call_is_force_closed(self, patch_exec, tmp_path): - events = [ - _line({"type": "tool_call", "toolName": "Bash", "toolId": "t1", "input": {}}), - _line({"type": "send_ok", "result": "done"}), - ] + events = [_tool_call("t1"), _result(response="done")] agent, _ = await _started_agent(patch_exec, events, tmp_path) record = await agent.communicate("hi") assert record.commands[0].result_status == "unknown" async def test_cooperative_stop_ends_cleanly(self, patch_exec, tmp_path): events = [ - _line({"type": "message", "content": "partial"}), - _line({"type": "message", "content": "more"}), + _ev(type="message", content="partial", isStepStart=True), + _ev(type="message", content="more", isStepStart=False), ] agent, proc = await _started_agent(patch_exec, events, tmp_path) calls = {"n": 0} @@ -332,18 +392,18 @@ def should_stop() -> bool: assert agent._process is None # dropped so the next turn respawns async def test_cooperative_stop_then_next_turn_respawns_and_completes(self, patch_exec, tmp_path): - events = [_line({"type": "message", "content": "partial"})] + events = [_ev(type="message", content="partial", isStepStart=True)] agent, _proc = await _started_agent(patch_exec, events, tmp_path) await agent.communicate("hi", should_stop=lambda: True) - patch_exec([_line({"type": "init_ok"}), _line({"type": "send_ok", "result": "resumed cleanly"})]) + patch_exec([_line({"type": "init_ok"}), _result(response="resumed cleanly")]) record = await agent.communicate("hi again") assert record.agent_output == "resumed cleanly" assert record.crashed is False async def test_should_stop_not_polled_means_full_completion(self, patch_exec, tmp_path): - events = [_line({"type": "send_ok", "result": "done"})] + events = [_result(response="done")] agent, _ = await _started_agent(patch_exec, events, tmp_path) record = await agent.communicate("hi") assert record.agent_output == "done" @@ -376,18 +436,14 @@ async def test_timeout_elapsing_mid_read_still_raises_turn_timeout_error(self, p # A fresh host is spawned for the NEXT turn -- no cross-wiring with # the abandoned "send" from the timed-out one. - patch_exec([_line({"type": "init_ok"}), _line({"type": "send_ok", "result": "clean turn"})]) + patch_exec([_line({"type": "init_ok"}), _result(response="clean turn")]) record = await agent.communicate("hi again") assert record.agent_output == "clean turn" @staticmethod def _round_trip(n: int) -> list[bytes]: """One backend round-trip: an empty reply, then its tool call and result.""" - return [ - _line({"type": "message", "content": ""}), - _line({"type": "tool_call", "toolId": f"t{n}", "toolName": "shell", "input": {}}), - _line({"type": "tool_result", "toolId": f"t{n}", "output": "ok"}), - ] + return [_ev(type="message", content="", isStepStart=True), _tool_call(f"t{n}"), _tool_result(f"t{n}")] async def test_max_turns_exhausted(self, patch_exec, tmp_path): events = [*self._round_trip(0), *self._round_trip(1), *self._round_trip(2)] @@ -401,10 +457,10 @@ async def test_max_turns_counts_round_trips_not_text_chunks(self, patch_exec, tm """The SDK streams a reply as several message events; they are one round-trip.""" events = [ *self._round_trip(0), - _line({"type": "thinking", "content": "wrap up"}), - _line({"type": "message", "content": "all "}), - _line({"type": "message", "content": "done"}), - _line({"type": "send_ok", "result": "all done"}), + _ev(type="thinking", content="wrap up"), + _ev(type="message", content="all ", isStepStart=True), + _ev(type="message", content="done", isStepStart=False), + _result(response="all done"), ] agent, _proc = await _started_agent(patch_exec, events, tmp_path) record = await agent.communicate("hi", max_turns=2) @@ -414,12 +470,8 @@ async def test_max_turns_counts_round_trips_not_text_chunks(self, patch_exec, tm async def test_max_turns_stops_a_tool_only_reply_before_its_tool_runs(self, patch_exec, tmp_path): """A tool-only reply streams no text event, only its tool call.""" events = [ - _line({"type": "message", "content": "on it"}), - *[ - _line({"type": kind, "toolId": f"t{n}", "toolName": "shell", "output": "ok"}) - for n in range(3) - for kind in ("tool_call", "tool_result") - ], + _ev(type="message", content="on it", isStepStart=True), + *[frame for n in range(3) for frame in (_tool_call(f"t{n}"), _tool_result(f"t{n}"))], ] agent, _proc = await _started_agent(patch_exec, events, tmp_path) record = await agent.communicate("hi", max_turns=2) @@ -429,13 +481,13 @@ async def test_max_turns_stops_a_tool_only_reply_before_its_tool_runs(self, patc async def test_max_turns_counts_a_batched_reply_once(self, patch_exec, tmp_path): events = [ - _line({"type": "message", "content": ""}), - _line({"type": "tool_call", "toolId": "a", "toolName": "shell"}), - _line({"type": "tool_call", "toolId": "b", "toolName": "shell"}), - _line({"type": "tool_result", "toolId": "a", "output": "ok"}), - _line({"type": "tool_result", "toolId": "b", "output": "ok"}), - _line({"type": "message", "content": "done"}), - _line({"type": "send_ok", "result": "done"}), + _ev(type="message", content="", isStepStart=True), + _tool_call("a"), + _tool_call("b"), + _tool_result("a"), + _tool_result("b"), + _ev(type="message", content="done", isStepStart=True), + _result(response="done"), ] agent, _proc = await _started_agent(patch_exec, events, tmp_path) record = await agent.communicate("hi", max_turns=2) @@ -445,14 +497,10 @@ async def test_max_turns_counts_a_batched_reply_once(self, patch_exec, tmp_path) async def test_max_turns_still_counts_after_a_tool_never_returns(self, patch_exec, tmp_path): """Tool b never returns, so the next call opens on its first new tool call.""" events = [ - _line({"type": "tool_call", "toolId": "a", "toolName": "shell"}), - _line({"type": "tool_call", "toolId": "b", "toolName": "shell"}), - _line({"type": "tool_result", "toolId": "a", "output": "ok"}), - *[ - _line({"type": kind, "toolId": f"t{n}", "toolName": "shell", "output": "ok"}) - for n in range(3) - for kind in ("tool_call", "tool_result") - ], + _tool_call("a"), + _tool_call("b"), + _tool_result("a"), + *[frame for n in range(3) for frame in (_tool_call(f"t{n}"), _tool_result(f"t{n}"))], ] agent, _proc = await _started_agent(patch_exec, events, tmp_path) record = await agent.communicate("hi", max_turns=2) @@ -472,12 +520,12 @@ async def test_respawns_after_crash_on_next_turn(self, patch_exec, tmp_path): await agent.discard_pending_turn() # A fresh host is spawned for the retry. - patch_exec([_line({"type": "init_ok"}), _line({"type": "send_ok", "result": "recovered"})]) + patch_exec([_line({"type": "init_ok"}), _result(response="recovered")]) record = await agent.communicate("hi again") assert record.agent_output == "recovered" async def test_emits_exactly_one_start_and_end_event(self, patch_exec, tmp_path): - events = [_line({"type": "send_ok", "result": "done"})] + events = [_result(response="done")] agent, _ = await _started_agent(patch_exec, events, tmp_path) seen: list[Any] = [] @@ -495,10 +543,15 @@ def on_event(self, event: Any) -> None: @pytest.mark.parametrize( ("usage_payload", "expected_uncached_input", "expected_output", "expected_cache_read", "expected_cache_write"), [ - ({"promptTokens": 10, "completionTokens": 5}, 10, 5, 0, 0), - ({"promptTokens": 10, "completionTokens": 5, "promptTokensCached": 4}, 6, 5, 4, 0), + ({"input_tokens": 10, "output_tokens": 5}, 10, 5, 0, 0), + ({"input_tokens": 6, "output_tokens": 5, "cache_read_input_tokens": 4}, 6, 5, 4, 0), ( - {"promptTokens": 10, "completionTokens": 5, "promptTokensCached": 4, "cacheCreationTokens": 3}, + { + "input_tokens": 6, + "output_tokens": 5, + "cache_read_input_tokens": 4, + "cache_creation_input_tokens": 3, + }, 6, 5, 4, @@ -516,7 +569,7 @@ async def test_usage_bucket_spellings_populate_token_usage( expected_cache_read, expected_cache_write, ): - events = [_line({"type": "send_ok", "result": "done", "usage": usage_payload})] + events = [_result(response="done", usage=usage_payload)] agent, _ = await _started_agent(patch_exec, events, tmp_path) record = await agent.communicate("hi") assert record.token_usage is not None @@ -526,7 +579,7 @@ async def test_usage_bucket_spellings_populate_token_usage( assert record.token_usage.cache_creation_input_tokens == expected_cache_write async def test_usage_all_zero_is_none_and_warns(self, patch_exec, tmp_path, caplog): - events = [_line({"type": "send_ok", "result": "done", "usage": {"weird_bucket": 3}})] + events = [_result(response="done", usage={"weird_bucket": 3})] agent, _ = await _started_agent(patch_exec, events, tmp_path) with caplog.at_level("WARNING"): record = await agent.communicate("hi") @@ -534,15 +587,7 @@ async def test_usage_all_zero_is_none_and_warns(self, patch_exec, tmp_path, capl assert any("usage payload matched none" in r.message for r in caplog.records) async def test_cost_wired_into_record_for_configured_model(self, patch_exec, tmp_path): - events = [ - _line( - { - "type": "send_ok", - "result": "done", - "usage": {"promptTokens": 1_000_000, "completionTokens": 1_000_000}, - } - ) - ] + events = [_result(response="done", usage={"input_tokens": 1_000_000, "output_tokens": 1_000_000})] agent, _ = await _started_agent(patch_exec, events, tmp_path, model="virtuoso-1-5") record = await agent.communicate("hi") assert record.model_used == "virtuoso-1-5" @@ -550,7 +595,7 @@ async def test_cost_wired_into_record_for_configured_model(self, patch_exec, tmp assert record.token_usage.total_cost_usd == pytest.approx(0.95 + 4.0) async def test_send_error_delivers_end_events_to_stream_callback(self, patch_exec, tmp_path): - events = [_line({"type": "send_error", "message": "network blip"})] + events = [_line({"type": "error", "message": "network blip"})] agent, _ = await _started_agent(patch_exec, events, tmp_path) seen: list[Any] = [] @@ -642,10 +687,7 @@ def test_create_agent_dispatches(self): assert isinstance(agent, DelegateAgent) -def test_host_script_ships_with_the_package(): - """Every test above patches the fixture to return ``_HOST_SCRIPT`` without - ever checking the file exists on disk -- a build-config change that dropped - it from the wheel would surface only as a runtime MODULE_NOT_FOUND inside a - live run, never here. - """ - assert agent_module._HOST_SCRIPT.is_file() +def test_install_target_names_the_host_package(): + """``agents/delegate/package.json`` is the documented ``npm install`` target the resolver finds.""" + package_json = Path(agent_module.__file__).parent / "delegate" / "package.json" + assert agent_module._HOST_PACKAGE in json.loads(package_json.read_text())["dependencies"] diff --git a/tests/test_delegate_agent_live.py b/tests/test_delegate_agent_live.py index 257cee55..c3c10cca 100644 --- a/tests/test_delegate_agent_live.py +++ b/tests/test_delegate_agent_live.py @@ -1,14 +1,14 @@ """Live integration tests for DelegateAgent. Hit a real Delegate backend through a real Node subprocess; skipped by default, -only run with ``pytest -m live``. Needs Node + a real ``@uipath/delegate-sdk`` +only run with ``pytest -m live``. Needs Node + a real ``@uipath/delegate-stdio`` install, UiPath auth (``AUTH_TOKEN``/``TENANT_ID``/``ORG_ID`` or a saved login), a backend (``DELEGATE_ENV``/``DELEGATE_BACKEND_URL``), and optionally ``DELEGATE_MODEL``. -Also the mechanism for empirically confirming this agent's ``# UNVERIFIED`` -field-name guesses (see ``.claude/notes/agents.md`` § Delegate agent) against a -real payload — a failure here is the first place to check them. +A failure here while the unit tests pass usually means the host's frame shapes +moved: compare a real ``delegate-stdio`` transcript against ``tests/test_delegate_agent.py``'s +``_ev`` / ``_result`` / ``_tool_call`` / ``_tool_result`` builders. """ import os @@ -17,7 +17,7 @@ import pytest -from coder_eval.agents.delegate_agent import DelegateAgent, _resolve_sdk_entry +from coder_eval.agents.delegate_agent import DelegateAgent, _env, _resolve_host_bundle from coder_eval.errors import AgentConfigError from coder_eval.models import AgentKind, parse_agent_config @@ -29,15 +29,15 @@ def _have_prerequisites() -> bool: if shutil.which("node") is None: return False try: - _resolve_sdk_entry() + _resolve_host_bundle() except AgentConfigError: return False - has_auth = bool(os.getenv("AUTH_TOKEN")) or (Path.home() / ".aria" / "sdk-auth.json").is_file() + has_auth = bool(_env("AUTH_TOKEN")) or (Path.home() / ".aria" / "sdk-auth.json").is_file() has_backend = bool(os.getenv("DELEGATE_ENV") or os.getenv("DELEGATE_BACKEND_URL")) return has_auth and has_backend -_skip_reason = "Live Delegate tests need Node + @uipath/delegate-sdk + UiPath auth + a backend" +_skip_reason = "Live Delegate tests need Node + @uipath/delegate-stdio + UiPath auth + a backend" pytestmark = [_live, pytest.mark.skipif(not _have_prerequisites(), reason=_skip_reason)] @@ -83,6 +83,7 @@ async def test_delegate_live_runs_shell_command_captured_as_telemetry(tmp_path): assert record.crashed is False assert record.commands, "expected at least one command in telemetry" + assert any("coder-eval-live" in (c.result_summary or "") for c in record.commands) @_live @@ -123,12 +124,7 @@ async def test_delegate_live_two_turns_reuse_session(tmp_path): @_live async def test_delegate_live_token_usage_populated(tmp_path): - """Token usage is captured from the SDK and attached to the TurnRecord. - - If this fails while the turn otherwise completes, the `_parse_usage` - UNVERIFIED bucket-name guesses in delegate_agent.py are the first thing - to check against the real payload. - """ + """Token usage and the call count are read off the host's ``result`` frame.""" agent = _make_agent() await agent.start(str(tmp_path)) try: @@ -139,3 +135,4 @@ async def test_delegate_live_token_usage_populated(tmp_path): assert record.crashed is False assert record.token_usage is not None assert record.token_usage.output_tokens > 0 + assert record.num_turns is not None and record.num_turns >= 1 diff --git a/tests/test_timing_identity_contract.py b/tests/test_timing_identity_contract.py index 6ee23b9e..1ffbda20 100644 --- a/tests/test_timing_identity_contract.py +++ b/tests/test_timing_identity_contract.py @@ -263,11 +263,13 @@ def emit(e: Any) -> None: state = _TurnState(iteration=1, user_input="go", model="m") _SteppedDatetime.at_ms = 700 - agent._handle_tool_call({"type": "tool_call", "toolId": "c1", "toolName": "bash", "input": {}}, state, emit) + agent._handle_tool_call({"type": "tool_call", "toolId": "c1", "toolName": "bash", "toolArgs": {}}, state, emit) _SteppedDatetime.at_ms = 1200 - agent._handle_tool_result({"type": "tool_result", "toolId": "c1", "output": "ok"}, state, emit) + agent._handle_tool_result( + {"type": "tool_result", "toolId": "c1", "toolResult": {"content": "ok"}, "toolStatus": "completed"}, state, emit + ) _SteppedDatetime.at_ms = 1800 - agent._handle_event({"type": "message", "content": "done"}, state, emit) + agent._handle_event({"type": "message", "content": "done", "isStepStart": True}, state, emit) _SteppedDatetime.at_ms = 2000 # last flush: the single window closes here agent._finalize_turn(state, AgentEndStatus.COMPLETED, emit) From e5336742c4eecdcca2fcb6b241b960e4a65a18ae Mon Sep 17 00:00:00 2001 From: Mihai Chirculescu Date: Tue, 29 Sep 2026 23:46:13 +0300 Subject: [PATCH 02/20] fix(delegate): record an interrupted tool as an error, not a success The SDK's tool_result carries toolStatus "interrupted" for a tool that did not complete. Only "failed" was treated as an error, so an interrupted tool was recorded with result_status "success". Co-Authored-By: Claude Opus 5.5 (1M context) --- src/coder_eval/agents/delegate_agent.py | 4 +++- tests/test_delegate_agent.py | 5 +++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/coder_eval/agents/delegate_agent.py b/src/coder_eval/agents/delegate_agent.py index 09d52874..db6b1af4 100644 --- a/src/coder_eval/agents/delegate_agent.py +++ b/src/coder_eval/agents/delegate_agent.py @@ -117,6 +117,8 @@ # Event types (inside the host's `event` frames) that carry model-turn content. # `session_start` / `done` are informational; anything else is logged and ignored. _TEXT_EVENT_TYPES = frozenset({"thinking", "message"}) +_FAILED_TOOL_STATUSES = frozenset({"failed", "interrupted"}) +"""``toolStatus`` values the SDK's ``tool_result`` carries for a tool that did not complete.""" def _tool_id(event: dict[str, Any]) -> str: @@ -797,7 +799,7 @@ def _handle_tool_result( telemetry.result_summary = json.dumps(output) else: telemetry.result_summary = None - if event.get("toolStatus") == "failed": + if event.get("toolStatus") in _FAILED_TOOL_STATUSES: status = ToolEndStatus.ERROR telemetry.result_status = "error" telemetry.error_message = telemetry.result_summary or "tool failed" diff --git a/tests/test_delegate_agent.py b/tests/test_delegate_agent.py index 202c2f09..f3692991 100644 --- a/tests/test_delegate_agent.py +++ b/tests/test_delegate_agent.py @@ -357,10 +357,11 @@ async def test_non_json_stdout_line_is_skipped_not_fatal(self, patch_exec, tmp_p record = await agent.communicate("hi") assert record.agent_output == "done" - async def test_tool_error_marks_error_status(self, patch_exec, tmp_path): + @pytest.mark.parametrize("status", ["failed", "interrupted"]) + async def test_tool_error_marks_error_status(self, patch_exec, tmp_path, status): events = [ _tool_call("t1"), - _tool_result("t1", "command not found", status="failed"), + _tool_result("t1", "command not found", status=status), _result(response="done"), ] agent, _ = await _started_agent(patch_exec, events, tmp_path) From 7e23755c3790f7d524ff5ba0453c2f284c7af7e1 Mon Sep 17 00:00:00 2001 From: Mihai Chirculescu Date: Tue, 29 Sep 2026 23:50:34 +0300 Subject: [PATCH 03/20] fix(delegate): categorize WAF blocks, SSE connect timeouts and session conflicts Port three failure signatures from the out-of-tree delegate-sdk adapter, which drove this same delegate-stdio host against this same backend: - A Cloudflare WAF block page (a 403 for shell-like text in the request body) is rewritten to a "content filter" reason, so it is not retried: the same payload is blocked again. - An SSE connect timeout is rewritten to a "connection" reason with "timeout" defanged, so it is retried as AGENT_API_ERROR instead of ending the task as a non-retryable AGENT_TIMEOUT. - A session conflict ("A reply is already being generated") drops the remembered session id, so the retry starts a new conversation instead of conflicting again. A rewritten reason omits the host stderr tail, because a "timeout" in the tail would undo the categorization; the tail is logged instead. Co-Authored-By: Claude Opus 5.5 (1M context) --- .claude/notes/agents.md | 35 +++++++++-- docs/agents/DELEGATE.md | 2 +- src/coder_eval/agents/delegate_agent.py | 82 ++++++++++++++++++++++--- tests/test_delegate_agent.py | 50 +++++++++++++++ 4 files changed, 154 insertions(+), 15 deletions(-) diff --git a/.claude/notes/agents.md b/.claude/notes/agents.md index eea6f3f3..778fdcd8 100644 --- a/.claude/notes/agents.md +++ b/.claude/notes/agents.md @@ -781,11 +781,36 @@ alone loads nothing; the adapter sets `enableSkills` to whether a plugin resolve uses `isStepStart` to merge streamed deltas into one text block. `timing.close_window` opens that one window from the turn's own start, so the head and tail both measure ~0. -**Deferred, not ported speculatively.** The out-of-tree `delegate-sdk` adapter carries -failure-signature-specific recoveries: a Cloudflare WAF-block-page rewrite, an SSE-connect-timeout -rewrite, session-conflict fresh-host recovery, first-response stall-timeout+resend, and an S2S -token-file refresher. None are ported here. A crash still ends the turn correctly as a retryable -`AgentCrashError`; it is just not specially diagnosed. Port each once its failure is observed here. +**Known host failures are recategorized.** The out-of-tree `delegate-sdk` adapter drove this same +`delegate-stdio` host against this same backend, so three of its failure signatures are ported, in +`_describe_host_error` / `_crash_on_host_error`. Each routes a host `error` frame to the category +`errors/categorization.py` should give it: + +- **Cloudflare WAF block page** (`Continue with UiPath Platform`, "not available in + your country"). A 403 from a managed WAF rule that matched shell-like text in the request body — + for example a skill doc's `python -c "...open(...,'w')..."` echoed back by a file read. It is not a + geo or auth block. The same payload is blocked again on retry, so the reason is rewritten to carry + "content filter" (`AGENT_INVALID_OUTPUT`, not retried). Two markers, because the host truncates its + message near 50 KB and the country sentence sits after ~48 KB of inline font CSS; the `` is + in the first ~300 bytes. +- **SSE connect timeout.** The request got no response headers inside the SDK's connect watchdog + (30 s) on each of its three internal attempts. The raw "timeout" would route to `AGENT_TIMEOUT`, + which is not retried, but no headers means the backend never started the turn — a transient + availability window. The rewrite carries "connection" (`AGENT_API_ERROR`, retried with backoff) and + defangs "timeout". +- **Session conflict** ("A reply is already being generated"). A backend 409 for a send into a + conversation whose previous generation still runs. A retry into that conversation can only + conflict again, so the remembered `_session_id` is dropped and the retry starts a new one. The host + is replaced either way. A `session_id` pinned in config still reaches the new host as an init + option, so a pinned run does not recover from this. + +A rewritten reason omits the stderr tail — the tail logs the SDK's own watchdog lines, and one +"timeout" in it would re-route the crash to `AGENT_TIMEOUT` (the `timeout` rule runs before both +"content filter" and "connection"). The tail is logged at WARNING instead. + +Not ported: the first-response stall-timeout+resend (opt-in, and a stall already ends as a turn +timeout) and the S2S token-file refresher (it depends on a token-file seam this host has not been +confirmed to read). Port either once its failure is observed here. **The process handle must be cleared on every path that leaves the host dead or dying** — EOF, an `error` frame, a timeout (both the top-of-loop pre-check AND a timeout elapsing while blocked inside diff --git a/docs/agents/DELEGATE.md b/docs/agents/DELEGATE.md index 4fa7d244..f6eafe9d 100644 --- a/docs/agents/DELEGATE.md +++ b/docs/agents/DELEGATE.md @@ -158,7 +158,7 @@ Run-limit semantics per harness: [Run-Limit Parity](HARNESS_PARITY.md). 1. **No multi-generation transcript splitting.** This agent builds one `AssistantMessage` per `communicate()` call, not one per backend round-trip. The host does send the signals a split needs (`isStepStart` on `message` events and per-round-trip `turnUsages` on `result`), but this agent does not use them yet. 2. **`max_turns` is enforced by coder_eval, not by the host.** The host's `maxSteps` option on `send` does not stop the turn (confirmed live: `maxSteps: 2` ran 7 steps and only reported `maxStepsReached: true`), so this agent does not send it. See [Run-Limit Parity](HARNESS_PARITY.md). -3. **No WAF-block-page rewrite, SSE-connect-timeout rewrite, session-conflict fresh-host recovery, or first-response stall-timeout+resend.** These are recoveries for specific failure signatures. They are not ported until the same failures are observed from this agent. A crash still ends the turn correctly as a retryable `AgentCrashError` — it is just not specially diagnosed. +3. **Only three backend failures get a specific diagnosis.** A Cloudflare WAF block page is reported as a content-filter failure and is not retried: the same prompt or tool result is blocked again. An SSE connect timeout is reported as a connection failure and is retried. A session conflict ("A reply is already being generated") is retried in a new conversation. There is no first-response stall detection: a stalled turn ends at its turn timeout. Every other crash ends the turn as a retryable `AgentCrashError`. 4. **No `sdk_options` passthrough.** Unlike Claude Code, there is no allowlisted escape hatch for arbitrary SDK fields — only `effort`, `project_id`, `session_id`, and `enable_computer_use` are exposed as typed config fields. 5. **`enable_computer_use: true` requires local permissions and is unavailable on Linux.** macOS needs Accessibility + Screen Recording grants; the SDK throws unconditionally on Linux when this is enabled. diff --git a/src/coder_eval/agents/delegate_agent.py b/src/coder_eval/agents/delegate_agent.py index db6b1af4..bbae41bb 100644 --- a/src/coder_eval/agents/delegate_agent.py +++ b/src/coder_eval/agents/delegate_agent.py @@ -31,6 +31,7 @@ import json import logging import os +import re import shutil import signal import time @@ -260,6 +261,48 @@ def _int(key: str) -> int: return usage +_WAF_BLOCK_PAGE_MARKERS = ("continue with uipath platform", "not available in your country") +"""Fingerprints of UiPath's Cloudflare block page, served as a 403 when a WAF managed +rule matches shell-like text in the REQUEST BODY. It is not a geo or auth block.""" + +_SESSION_CONFLICT_MARKER = "already being generated" +"""The backend's 409 for a send into a conversation whose previous generation still runs.""" + +_SSE_CONNECT_TIMEOUT_MARKER = "sse connect timeout" +"""The SDK's error once its SSE connect watchdog has failed all of its internal retries.""" + + +def _describe_host_error(message: str) -> str | None: + """A correctly categorized crash reason for a known host failure, or ``None``. + + The raw message would mis-route under ``errors/categorization.py``: a WAF + block reads as geo/auth but is deterministic per payload (stamped "content + filter", non-retryable), and an SSE connect failure says "timeout" but is a + transient backend window (stamped "connection", retryable, "timeout" defanged). + A session conflict keeps its raw message; the caller handles it. + + Rationale: .claude/notes/agents.md § Delegate agent + """ + lowered = message.lower() + if any(marker in lowered for marker in _WAF_BLOCK_PAGE_MARKERS): + prefix = message.split("<", 1)[0].strip().rstrip(":").strip() + return ( + "Delegate backend request blocked by the Cloudflare WAF content filter in front of the UiPath " + + "backend (the generic 'not available in your country' 403 page, not a geo or auth problem): " + + "shell-like text in the prompt or a tool result matched a managed rule, and the same payload " + + f"would be blocked again on retry. [{prefix}]" + ) + if _SSE_CONNECT_TIMEOUT_MARKER in lowered and _SESSION_CONFLICT_MARKER not in lowered: + original = message[lowered.find(_SSE_CONNECT_TIMEOUT_MARKER) :] + defanged = re.sub("timeout", "time-out", original, flags=re.IGNORECASE) + return ( + "Delegate backend connection failure: the turn's request got no response headers within the " + + "SDK's SSE connect watchdog, on every internal attempt. This is a transient backend " + + f"availability window, not a task-budget breach. [{defanged}]" + ) + return None + + class _TurnState: """Per-``communicate()`` accumulator. @@ -571,9 +614,7 @@ async def communicate( if self._process is None or self._process.returncode is not None: # A prior cooperative stop or crash left no live host: respawn - # fresh rather than fail fast. Session-conflict-specific recovery - # is deferred (see .claude/notes/agents.md); this simpler policy covers - # both "the previous turn stopped cleanly" and "it crashed". + # fresh rather than fail fast. try: await self._spawn_and_init() except AgentConfigError as exc: @@ -638,12 +679,7 @@ def emit(event: StreamEvent) -> None: self._handle_result(msg, state) break if mtype == "error": - # The host may survive a failed send, but it is not reused: - # force-kill so the next communicate() respawns onto a fresh - # queue rather than risk consuming a stale event from this turn. - await self._abandon_host_and_crash( - state, collector, emit, f"Delegate send failed: {msg.get('message', 'unknown error')}" - ) + await self._crash_on_host_error(state, collector, emit, str(msg.get("message", "unknown error"))) event = msg.get("event") if mtype != "event" or not isinstance(event, dict): logger.debug("delegate: ignoring host message %r", mtype) @@ -945,6 +981,34 @@ async def _abandon_host_and_crash( self._process = None self._crash_turn(state, collector, emit, f"{message}. stderr tail:\n{self._stderr_tail()}") + async def _crash_on_host_error( + self, + state: _TurnState, + collector: EventCollector, + emit: Callable[[StreamEvent], None], + message: str, + ) -> NoReturn: + """Crash the turn on a host ``error`` frame; the host is never reused. + + A rewritten reason goes out without the stderr tail, which can itself + say "timeout" and so undo the rewrite's categorization; the tail is + logged instead. A session conflict also drops the remembered session id, + because a retry into the same conversation can only conflict again. + """ + reason = _describe_host_error(message) + if reason is None: + if _SESSION_CONFLICT_MARKER in message.lower(): + logger.warning( + "delegate: session %s is still generating a reply; the retry starts a new conversation", + self._session_id, + ) + self._session_id = None + await self._abandon_host_and_crash(state, collector, emit, f"Delegate send failed: {message}") + logger.warning("delegate: %s stderr tail:\n%s", reason, self._stderr_tail()) + await self._force_kill_host() + self._process = None + self._crash_turn(state, collector, emit, reason) + def _crash_turn( self, state: _TurnState, diff --git a/tests/test_delegate_agent.py b/tests/test_delegate_agent.py index f3692991..40dad543 100644 --- a/tests/test_delegate_agent.py +++ b/tests/test_delegate_agent.py @@ -20,6 +20,8 @@ from coder_eval.agents.delegate_agent import DelegateAgent, _resolve_host_bundle from coder_eval.agents.registry import AgentRegistry, create_agent from coder_eval.errors import AgentConfigError, AgentCrashError, TurnTimeoutError +from coder_eval.errors.categories import ErrorCategory +from coder_eval.errors.categorization import categorize_error from coder_eval.models import AgentKind, DelegateAgentConfig from coder_eval.streaming.events import AgentEndEvent, AgentEndStatus, AgentStartEvent @@ -343,6 +345,54 @@ async def test_send_error_raises_crash(self, patch_exec, tmp_path): assert proc._killed assert agent._process is None + @pytest.mark.parametrize( + ("host_message", "category"), + [ + ( + "Delegate backend error: HTTP 403: <!DOCTYPE html><title>Continue with UiPath Platform", + ErrorCategory.AGENT_INVALID_OUTPUT, + ), + ("Delegate backend error: SSE connect timeout after 30s", ErrorCategory.AGENT_API_ERROR), + ], + ids=["waf-block", "sse-connect-timeout"], + ) + async def test_known_host_errors_are_recategorized(self, patch_exec, tmp_path, host_message, category): + # The stderr tail says "timeout"; a rewritten reason must not carry it. + proc = patch_exec( + [_line({"type": "init_ok"}), _line({"type": "error", "message": host_message})], + [b"[agenticApi] SSE connect timeout, retrying\n"], + ) + agent = DelegateAgent(_config(), task_id="t1") + await agent.start(str(tmp_path)) + await asyncio.sleep(0) + assert agent._stderr_lines + with pytest.raises(AgentCrashError) as excinfo: + await agent.communicate("hi") + assert categorize_error(excinfo.value, {"component": "agent"}) is category + assert "<" not in str(excinfo.value) + assert proc._killed + assert agent._process is None + + async def test_session_conflict_drops_the_session_id(self, patch_exec, tmp_path): + events = [ + _line( + { + "type": "error", + "message": "HTTP 409: A reply is already being generated for this conversation.", + } + ) + ] + agent, _ = await _started_agent(patch_exec, events, tmp_path) + agent._session_id = "wedged" + with pytest.raises(AgentCrashError, match="already being generated"): + await agent.communicate("hi") + await agent.discard_pending_turn() + + proc = patch_exec([_line({"type": "init_ok"}), _result(response="recovered")]) + record = await agent.communicate("hi again") + assert record.agent_output == "recovered" + assert proc.stdin.written[1]["sessionId"] is None + async def test_eof_mid_turn_raises_crash(self, patch_exec, tmp_path): agent, _ = await _started_agent(patch_exec, [], tmp_path) with pytest.raises(AgentCrashError, match="closed its output stream"): From dc9c6943606bd5dcaae0e88dad92f1f205da2fd1 Mon Sep 17 00:00:00 2001 From: Mihai Chirculescu Date: Tue, 29 Sep 2026 23:55:42 +0300 Subject: [PATCH 04/20] fix(delegate): keep the LLMGW_* secret out of agent shells when a token file is set The agent's shell tools inherit the host env, so the gateway S2S client secret there is readable by the code under test. The host refreshes its token from that pair only when no token file is configured; with DELEGATE_AUTH_TOKEN_FILE (or the older AUTH_TOKEN_FILE) set, the file wins and the pair is unused. Remove LLMGW_CLIENT_ID / LLMGW_CLIENT_SECRET / LLMGW_URL from the host env in that case only, mirroring the host's own lookup, so a long run without a token file still refreshes its token. Co-Authored-By: Claude Opus 5.5 (1M context) --- .claude/notes/agents.md | 13 +++++++++++-- docs/agents/DELEGATE.md | 2 ++ src/coder_eval/agents/delegate_agent.py | 23 ++++++++++++++++++++++ tests/test_delegate_agent.py | 26 +++++++++++++++++++++++++ 4 files changed, 62 insertions(+), 2 deletions(-) diff --git a/.claude/notes/agents.md b/.claude/notes/agents.md index 778fdcd8..58757dbf 100644 --- a/.claude/notes/agents.md +++ b/.claude/notes/agents.md @@ -808,9 +808,18 @@ A rewritten reason omits the stderr tail — the tail logs the SDK's own watchdo "timeout" in it would re-route the crash to `AGENT_TIMEOUT` (the `timeout` rule runs before both "content filter" and "connection"). The tail is logged at WARNING instead. +**`LLMGW_*` leaves the host env only when a token file is configured.** The host's shells inherit +its env, so the gateway client secret there is readable by the code under test. But the host's own +`selectTokenSource` uses that S2S pair as its refresh source whenever no token file is set, so +stripping it unconditionally would end token refresh an hour into a long run. With a token file +the file wins and the pair is dead weight, so `_strip_redundant_gateway_creds` removes it — mirroring +the host's lookup exactly (`DELEGATE_AUTH_TOKEN_FILE ?? AUTH_TOKEN_FILE`, split on the path +delimiter, blank entries ignored; an empty `DELEGATE_AUTH_TOKEN_FILE` shadows the legacy name). +Removing it in every case needs the out-of-tree adapter's S2S token-file refresher, which mints +adapter-side and hands the host a file. + Not ported: the first-response stall-timeout+resend (opt-in, and a stall already ends as a turn -timeout) and the S2S token-file refresher (it depends on a token-file seam this host has not been -confirmed to read). Port either once its failure is observed here. +timeout) and that S2S token-file refresher. Port either once its need is observed here. **The process handle must be cleared on every path that leaves the host dead or dying** — EOF, an `error` frame, a timeout (both the top-of-loop pre-check AND a timeout elapsing while blocked inside diff --git a/docs/agents/DELEGATE.md b/docs/agents/DELEGATE.md index f6eafe9d..df045d0b 100644 --- a/docs/agents/DELEGATE.md +++ b/docs/agents/DELEGATE.md @@ -51,6 +51,8 @@ Either: The host also reads its own advanced variables directly, such as `DELEGATE_AUTH_TOKEN_FILE` (a token file that an external process keeps fresh), `INTEROP_URL` and `DELEGATE_STDIO_VERBOSE=1` (trace every frame to stderr). See the [package README](https://www.npmjs.com/package/@uipath/delegate-stdio). +For runs longer than the token's lifetime (about one hour), the host refreshes the token itself. It uses the token file when `DELEGATE_AUTH_TOKEN_FILE` (or the older `AUTH_TOKEN_FILE`) is set. If no token file is set, it uses the `LLMGW_CLIENT_ID` / `LLMGW_CLIENT_SECRET` / `LLMGW_URL` S2S pair. The agent's shell tools inherit the host's environment, so when a token file is set, coder_eval removes the `LLMGW_*` variables from that environment: the host does not need them, and the code under test cannot read the client secret. When no token file is set, the variables stay, because the host needs them to refresh the token. + ## Usage ### Command Line diff --git a/src/coder_eval/agents/delegate_agent.py b/src/coder_eval/agents/delegate_agent.py index bbae41bb..d6d96f14 100644 --- a/src/coder_eval/agents/delegate_agent.py +++ b/src/coder_eval/agents/delegate_agent.py @@ -142,6 +142,27 @@ def _host_auth_env() -> dict[str, str]: return {host_name: value for host_name, ours in _HOST_AUTH_ENV if (value := _env(ours))} +_GATEWAY_S2S_ENV_VARS = ("LLMGW_CLIENT_ID", "LLMGW_CLIENT_SECRET", "LLMGW_URL") + + +def _strip_redundant_gateway_creds(env: dict[str, str]) -> tuple[str, ...]: + """Remove the ``LLMGW_*`` S2S pair from ``env`` when the host would not use it. + + The agent's shell tools inherit the host env, so a live client secret there is + exposed to the code under test. The host refreshes its token from that pair + only when no token file is configured; a token file wins. This mirrors the + host's own lookup (``DELEGATE_AUTH_TOKEN_FILE``, else ``AUTH_TOKEN_FILE``, + split on ``os.pathsep``) and keeps the pair when it is the refresh source. + Returns the names removed. + """ + raw = env.get("DELEGATE_AUTH_TOKEN_FILE") + if raw is None: + raw = env.get("AUTH_TOKEN_FILE", "") + if not any(entry.strip() for entry in raw.split(os.pathsep)): + return () + return tuple(name for name in _GATEWAY_S2S_ENV_VARS if env.pop(name, None) is not None) + + def _candidate_install_roots() -> list[Path]: """Ancestor-walk search roots: cwd, its ancestors, and home. @@ -428,6 +449,8 @@ async def _spawn_and_init(self) -> None: # Respect an operator's own telemetry choice; only default it off. env.setdefault("DELEGATE_TELEMETRY_DISABLED", "1") env.update(_host_auth_env()) + if stripped := _strip_redundant_gateway_creds(env): + logger.debug("delegate: a token file is configured; removed %s from the host env", ", ".join(stripped)) await self._cancel_drain_tasks() self._stdout_queue = asyncio.Queue() diff --git a/tests/test_delegate_agent.py b/tests/test_delegate_agent.py index 40dad543..de85cadf 100644 --- a/tests/test_delegate_agent.py +++ b/tests/test_delegate_agent.py @@ -276,6 +276,32 @@ async def test_auth_is_exported_under_the_hosts_own_env_names(self, patch_exec, assert env["TENANT_NAME"] == "my-tenant" assert "auth" not in proc.stdin.written[0]["options"] + @pytest.mark.parametrize( + ("token_file_env", "stripped"), + [ + ({}, False), + ({"DELEGATE_AUTH_TOKEN_FILE": "/run/token"}, True), + ({"AUTH_TOKEN_FILE": "/run/token"}, True), + ({"DELEGATE_AUTH_TOKEN_FILE": f" {os.pathsep} "}, False), + ({"DELEGATE_AUTH_TOKEN_FILE": "", "AUTH_TOKEN_FILE": "/run/token"}, False), + ], + ids=["no-token-file", "token-file", "legacy-token-file", "blank-entries", "empty-shadows-legacy"], + ) + async def test_gateway_creds_leave_the_host_env_only_when_a_token_file_wins( + self, patch_exec, tmp_path, monkeypatch, token_file_env, stripped + ): + """The host refreshes from LLMGW_* only without a token file, so only then must they stay.""" + for name in ("DELEGATE_AUTH_TOKEN_FILE", "AUTH_TOKEN_FILE"): + monkeypatch.delenv(name, raising=False) + for name, value in token_file_env.items(): + monkeypatch.setenv(name, value) + for name in ("LLMGW_CLIENT_ID", "LLMGW_CLIENT_SECRET", "LLMGW_URL"): + monkeypatch.setenv(name, "value") + _agent, proc = await _started_agent(patch_exec, [], tmp_path) + env = proc.spawn_kwargs["env"] + for name in ("LLMGW_CLIENT_ID", "LLMGW_CLIENT_SECRET", "LLMGW_URL"): + assert (name in env) is not stripped + class TestCommunicate: async def test_happy_path_text_and_tool(self, patch_exec, tmp_path): From 780a4fbb749e08c3b94c7a620343accf0e8778fa Mon Sep 17 00:00:00 2001 From: Mihai Chirculescu Date: Wed, 30 Sep 2026 00:06:35 +0300 Subject: [PATCH 05/20] feat(delegate)!: read the delegate-stdio host's own env var names; route effort through sdk_options BREAKING CHANGE: the Delegate agent now uses the names that the @uipath/delegate-stdio host reads itself, and passes its environment to the host unchanged. DELEGATE_ENV -> DELEGATE_SDK_ENV, DELEGATE_BACKEND_URL -> BACKEND_URL, ORG_SLUG -> ORG_LOGICAL_NAME, TENANT_SLUG -> TENANT_NAME. The DELEGATE_-prefixed auth spellings (DELEGATE_AUTH_TOKEN, ...) are no longer read; set AUTH_TOKEN / TENANT_ID / ORG_ID. The Delegate-only `effort` field is replaced by `sdk_options.effort`, the key Claude Code already uses, so `-D agent.sdk_options.effort=high` now works for delegate tasks and the reports' Effort row shows it. Co-Authored-By: Claude Opus 5.5 (1M context) --- .claude/notes/agents.md | 15 ++++--- .env.example | 23 +++++------ .github/workflows/pr-checks.yml | 19 ++++----- docs/USER_GUIDE.md | 2 +- docs/agents/DELEGATE.md | 16 ++++---- src/coder_eval/agents/delegate_agent.py | 50 +++++------------------ src/coder_eval/models/agent_config.py | 26 +++++++++--- src/coder_eval/orchestration/overrides.py | 22 ++++++++-- tests/test_delegate_agent.py | 48 +++++++++++++++------- tests/test_delegate_agent_config.py | 14 +++++-- tests/test_delegate_agent_live.py | 8 ++-- tests/test_merge_characterization.py | 2 +- tests/test_overrides_engine.py | 9 +++- 13 files changed, 142 insertions(+), 112 deletions(-) diff --git a/.claude/notes/agents.md b/.claude/notes/agents.md index 58757dbf..83c8110a 100644 --- a/.claude/notes/agents.md +++ b/.claude/notes/agents.md @@ -759,11 +759,16 @@ gone; git history has them. **Auth goes through the host's environment, not the init options.** The host reads `AUTH_TOKEN` / `TENANT_ID` / `ORG_ID` and, for an `env`-derived backend URL, `ORG_LOGICAL_NAME` / `TENANT_NAME` -from its own environment. coder_eval keeps its user-facing names (`DELEGATE_`-namespaced first, then -`AUTH_TOKEN` / `TENANT_ID` / `ORG_ID` / `ORG_SLUG` / `TENANT_SLUG`) and `_host_auth_env` re-exports -them under the host's names. Confirmed live with no saved login: with the slugs the turn runs; without -them init fails with `env="alpha" requires org/tenant slugs`. `DELEGATE_ENV` becomes the `env` init -option; `DELEGATE_BACKEND_URL` becomes `backendUrl`, which the host ranks above `env`. +from its own environment, and `BACKEND_URL` / `INTEROP_URL` too. coder_eval uses the host's names +as its user-facing names and passes its environment through unchanged, so there is one set of names +and no re-export table to drift, and `coder_eval_uipath` pipelines use the same names. Confirmed live with no saved login: with the slugs the +turn runs; without them init fails with `env="alpha" requires org/tenant slugs`. The env slug is the +one value the host takes only as an init option, so `DELEGATE_SDK_ENV` becomes `env`. + +**Effort rides `sdk_options.effort`.** It is the same key as Claude Code, so one +`-D agent.sdk_options.effort=...` drives both agents, and the reports' Effort row reads it from +`get_sdk_options()`. The `-D` gate admits any registered kind whose config class declares +`sdk_options`; `DelegateAgentConfig` validates the keys against the host options it forwards. **`max_turns` stays client-side.** The host accepts `maxSteps` on `send`, but it does not stop the turn: live, `maxSteps: 2` ran 7 steps and only reported `maxStepsReached: true` in the `result`. diff --git a/.env.example b/.env.example index 238c5a35..e2e3280e 100644 --- a/.env.example +++ b/.env.example @@ -97,28 +97,25 @@ LOG_TO_FILE=false # Set to true to enable file logging # DELEGATE_STDIO_NODE_MODULES="/path/to/install/root" # DELEGATE_STDIO_PATH="/path/to/node_modules/@uipath/delegate-stdio/dist/delegate_stdio.mjs" # -# DELEGATE_ENV — cloud environment slug (alpha/staging/production), +# DELEGATE_SDK_ENV — cloud environment slug (alpha/staging/production), # used to derive the backend URL from the org/tenant slugs. -# DELEGATE_ENV="alpha" +# DELEGATE_SDK_ENV="alpha" # -# DELEGATE_BACKEND_URL — pin the full agent-service URL directly instead -# (wins over DELEGATE_ENV when both are set). -# DELEGATE_BACKEND_URL="https://cloud.uipath.com///delegate_" +# BACKEND_URL — pin the full agent-service URL directly instead +# (wins over DELEGATE_SDK_ENV when both are set). Read by the Node host. +# BACKEND_URL="https://cloud.uipath.com///delegate_" # # Auth: either the token set below, or a prior `delegate-cli login` -# that wrote ~/.aria/sdk-auth.json (read by the Node host itself, not -# by coder_eval). Each var also accepts a DELEGATE_-namespaced spelling -# (DELEGATE_AUTH_TOKEN, DELEGATE_TENANT_ID, ...), checked first -- prefer that -# spelling in a shared environment, since the bare names collide with what -# other tooling (npm, Vault, Terraform) commonly exports. +# that wrote ~/.aria/sdk-auth.json. The Node host reads all of these +# itself, under these exact names. # AUTH_TOKEN="..." # TENANT_ID="..." # ORG_ID="..." -# With DELEGATE_ENV (not DELEGATE_BACKEND_URL) also set these two -- the +# With DELEGATE_SDK_ENV (not BACKEND_URL) also set these two -- the # human-readable org/tenant names, not the GUIDs above -- or init fails with # 'env="alpha" requires org/tenant slugs'. See docs/agents/DELEGATE.md. -# ORG_SLUG="..." -# TENANT_SLUG="..." +# ORG_LOGICAL_NAME="..." +# TENANT_NAME="..." # Usage Telemetry (anonymous; on by default). # Disable entirely: diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index 124d1f57..7afd0981 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -1373,16 +1373,15 @@ jobs: if: steps.gate.outputs.present == 'true' env: DELEGATE_STDIO_NODE_MODULES: ${{ github.workspace }}/src/coder_eval/agents/delegate - DELEGATE_ENV: ${{ github.event.inputs.delegate_uipath_env || 'alpha' }} + DELEGATE_SDK_ENV: ${{ github.event.inputs.delegate_uipath_env || 'alpha' }} AUTH_TOKEN: ${{ steps.ropc.outputs.access_token }} TENANT_ID: ${{ secrets.DELEGATE_TENANT_ID }} ORG_ID: ${{ secrets.DELEGATE_ORG_ID }} - # Forwarded into `auth.organizationName` / `auth.tenantName` by - # DelegateAgent._build_init_options (see .claude/notes/agents.md § - # Delegate agent) — needed alongside the GUIDs above whenever - # `environment` (rather than `backendUrl`) resolves the backend. - ORG_SLUG: ${{ secrets.DELEGATE_ORG_SLUG }} - TENANT_SLUG: ${{ secrets.DELEGATE_TENANT_SLUG }} + # Read by the delegate-stdio host itself — needed alongside the GUIDs + # above whenever DELEGATE_SDK_ENV (rather than BACKEND_URL) resolves + # the backend (see .claude/notes/agents.md § Delegate agent). + ORG_LOGICAL_NAME: ${{ secrets.DELEGATE_ORG_SLUG }} + TENANT_NAME: ${{ secrets.DELEGATE_TENANT_SLUG }} run: .venv/bin/coder-eval run tasks/delegate/fizzbuzz_delegate.yaml --run-dir runs/delegate-live - name: Verify Delegate live run PASSED @@ -1406,12 +1405,12 @@ jobs: if: steps.gate.outputs.present == 'true' env: DELEGATE_STDIO_NODE_MODULES: ${{ github.workspace }}/src/coder_eval/agents/delegate - DELEGATE_ENV: ${{ github.event.inputs.delegate_uipath_env || 'alpha' }} + DELEGATE_SDK_ENV: ${{ github.event.inputs.delegate_uipath_env || 'alpha' }} AUTH_TOKEN: ${{ steps.ropc.outputs.access_token }} TENANT_ID: ${{ secrets.DELEGATE_TENANT_ID }} ORG_ID: ${{ secrets.DELEGATE_ORG_ID }} - ORG_SLUG: ${{ secrets.DELEGATE_ORG_SLUG }} - TENANT_SLUG: ${{ secrets.DELEGATE_TENANT_SLUG }} + ORG_LOGICAL_NAME: ${{ secrets.DELEGATE_ORG_SLUG }} + TENANT_NAME: ${{ secrets.DELEGATE_TENANT_SLUG }} run: | mkdir -p tmp .venv/bin/pytest tests/test_delegate_agent_live.py \ diff --git a/docs/USER_GUIDE.md b/docs/USER_GUIDE.md index 69d9aa6b..f3547d1d 100644 --- a/docs/USER_GUIDE.md +++ b/docs/USER_GUIDE.md @@ -429,7 +429,7 @@ Set these in `.env` (copy from `.env.example`). | `BEDROCK_SMALL_MODEL` | No | Cross-region Bedrock small/fast model ID | | `CODEX_API_KEY` / `CODEX_BASE_URL` / `CODEX_MODEL` / `CODEX_API_VERSION` | For Codex | Codex agent auth & endpoint routing — see [Codex Agent Guide](agents/CODEX.md#endpoint-routing). | | `GEMINI_API_KEY` / `ANTIGRAVITY_MODEL` | For Antigravity | Antigravity (Gemini) agent auth & model — see [Antigravity Agent Guide](agents/ANTIGRAVITY.md#setup). | -| `DELEGATE_ENV` / `DELEGATE_BACKEND_URL` / `AUTH_TOKEN` / `TENANT_ID` / `ORG_ID` / `ORG_SLUG` / `TENANT_SLUG` / `DELEGATE_STDIO_NODE_MODULES` / `DELEGATE_STDIO_PATH` | For Delegate | Delegate agent backend routing, auth & host install location — see [Delegate Agent Guide](agents/DELEGATE.md#setup). | +| `DELEGATE_SDK_ENV` / `BACKEND_URL` / `AUTH_TOKEN` / `TENANT_ID` / `ORG_ID` / `ORG_LOGICAL_NAME` / `TENANT_NAME` / `DELEGATE_STDIO_NODE_MODULES` / `DELEGATE_STDIO_PATH` | For Delegate | Delegate agent backend routing, auth & host install location — see [Delegate Agent Guide](agents/DELEGATE.md#setup). | | `UIPATH_PLUGIN_MARKETPLACE_DIR` | No | Conventional base directory for local plugins. Not special-cased by the framework: **any** `$VAR` / `${VAR}` referenced in a plugin `path` is expanded from the environment, so a plugin path like `$UIPATH_PLUGIN_MARKETPLACE_DIR/my-plugin` resolves against this variable. (An undefined variable in a plugin path logs a warning.) | | `PLUGIN_TOOLS_DIR` | No | A *separate* mechanism from the above: the canonical `node_modules/@uipath` directory used to pin UiPath CLI plugin discovery (not path substitution). When unset, the sandbox auto-derives it from the resolved `uip` binary. | | `CODER_EVAL_REMEDIATE_HOME_PLUGINS` | No | **DESTRUCTIVE.** Truthy deletes `$HOME/node_modules/@uipath` at sandbox setup to clear sibling-task pollution on dedicated eval hosts. Off by default; do **not** enable on developer workstations. | diff --git a/docs/agents/DELEGATE.md b/docs/agents/DELEGATE.md index df045d0b..26a5f2d8 100644 --- a/docs/agents/DELEGATE.md +++ b/docs/agents/DELEGATE.md @@ -39,17 +39,18 @@ On Windows, install into a short path. The interop binary sits deep inside `node | Variable | Purpose | |---|---| -| `DELEGATE_ENV` | Cloud environment slug (`alpha` / `staging` / `production`), sent as the host's `env` init option. The host derives the backend URL from the org/tenant slugs. | -| `DELEGATE_BACKEND_URL` | Pin the full agent-service URL directly (e.g. `https://cloud.uipath.com///delegate_`, or `http://localhost:5002` for a local backend). Wins over `DELEGATE_ENV` when both are set. | +| `DELEGATE_SDK_ENV` | Cloud environment slug (`alpha` / `staging` / `production`), sent as the host's `env` init option. The host derives the backend URL from the org/tenant slugs. | +| `BACKEND_URL` | Pin the full agent-service URL directly (e.g. `https://cloud.uipath.com///delegate_`, or `http://localhost:5002` for a local backend). The host reads it itself. Wins over `DELEGATE_SDK_ENV` when both are set. | +| `INTEROP_URL` | Connect to an interop instance that is already running, instead of letting the SDK spawn its own. The host reads it itself. | ### 3. Authenticate Either: -- **Environment token** — `AUTH_TOKEN`, `TENANT_ID`, `ORG_ID` env vars. Each also accepts a `DELEGATE_`-namespaced spelling (`DELEGATE_AUTH_TOKEN`, `DELEGATE_TENANT_ID`, `DELEGATE_ORG_ID`), which is checked first, because the bare names collide with what other tooling (npm, Vault, Terraform) commonly exports. When `DELEGATE_ENV` (not `DELEGATE_BACKEND_URL`) resolves the backend, also set `ORG_SLUG` and `TENANT_SLUG` (or `DELEGATE_ORG_SLUG` / `DELEGATE_TENANT_SLUG`). These are the human-readable org/tenant names, not the GUIDs. coder_eval exports all five values into the host's environment under the names the host reads (`AUTH_TOKEN`, `TENANT_ID`, `ORG_ID`, `ORG_LOGICAL_NAME`, `TENANT_NAME`). Without the slugs, init fails with `env="alpha" requires org/tenant slugs`. To skip the slugs, set `DELEGATE_BACKEND_URL` instead. -- **Saved login** — a prior `npx @uipath/delegate-cli login --env ` that wrote `~/.aria/sdk-auth.json`. The host reads and refreshes this file itself when no `AUTH_TOKEN` is supplied. This agent does not parse that file in Python, so auth-freshness logic lives in exactly one place. The saved login already carries the slugs, so `ORG_SLUG` / `TENANT_SLUG` are not needed. +- **Environment token** — `AUTH_TOKEN`, `TENANT_ID`, `ORG_ID` env vars. When `DELEGATE_SDK_ENV` (not `BACKEND_URL`) resolves the backend, also set `ORG_LOGICAL_NAME` and `TENANT_NAME`. These are the human-readable org/tenant names, not the GUIDs. All of these are the host's own names: coder_eval passes its environment to the host unchanged, and the host reads them itself. Without the slugs, init fails with `env="alpha" requires org/tenant slugs`. To skip the slugs, set `BACKEND_URL` instead. +- **Saved login** — a prior `npx @uipath/delegate-cli login --env ` that wrote `~/.aria/sdk-auth.json`. The host reads and refreshes this file itself when no `AUTH_TOKEN` is supplied. This agent does not parse that file in Python, so auth-freshness logic lives in exactly one place. The saved login already carries the slugs, so `ORG_LOGICAL_NAME` / `TENANT_NAME` are not needed. -The host also reads its own advanced variables directly, such as `DELEGATE_AUTH_TOKEN_FILE` (a token file that an external process keeps fresh), `INTEROP_URL` and `DELEGATE_STDIO_VERBOSE=1` (trace every frame to stderr). See the [package README](https://www.npmjs.com/package/@uipath/delegate-stdio). +The host also reads its own advanced variables directly, such as `DELEGATE_AUTH_TOKEN_FILE` (a token file that an external process keeps fresh) and `DELEGATE_STDIO_VERBOSE=1` (trace every frame to stderr). See the [package README](https://www.npmjs.com/package/@uipath/delegate-stdio). For runs longer than the token's lifetime (about one hour), the host refreshes the token itself. It uses the token file when `DELEGATE_AUTH_TOKEN_FILE` (or the older `AUTH_TOKEN_FILE`) is set. If no token file is set, it uses the `LLMGW_CLIENT_ID` / `LLMGW_CLIENT_SECRET` / `LLMGW_URL` S2S pair. The agent's shell tools inherit the host's environment, so when a token file is set, coder_eval removes the `LLMGW_*` variables from that environment: the host does not need them, and the code under test cannot read the client secret. When no token file is set, the variables stay, because the host needs them to refresh the token. @@ -67,7 +68,8 @@ coder-eval run tasks/delegate/hello_date_delegate.yaml --type delegate --model v agent: type: delegate model: virtuoso-1-5 - effort: high # low | medium | high | xhigh | max + sdk_options: + effort: high # low | medium | high | xhigh | max project_id: invoice-approval # client-side wiki-routing key; None means session-scoped wiki state session_id: abc-123 # pins the SDK session id a turn omits; project_id takes precedence enable_computer_use: false # default; screen tools off, file/shell/Office/PDF unaffected @@ -161,7 +163,7 @@ Run-limit semantics per harness: [Run-Limit Parity](HARNESS_PARITY.md). 1. **No multi-generation transcript splitting.** This agent builds one `AssistantMessage` per `communicate()` call, not one per backend round-trip. The host does send the signals a split needs (`isStepStart` on `message` events and per-round-trip `turnUsages` on `result`), but this agent does not use them yet. 2. **`max_turns` is enforced by coder_eval, not by the host.** The host's `maxSteps` option on `send` does not stop the turn (confirmed live: `maxSteps: 2` ran 7 steps and only reported `maxStepsReached: true`), so this agent does not send it. See [Run-Limit Parity](HARNESS_PARITY.md). 3. **Only three backend failures get a specific diagnosis.** A Cloudflare WAF block page is reported as a content-filter failure and is not retried: the same prompt or tool result is blocked again. An SSE connect timeout is reported as a connection failure and is retried. A session conflict ("A reply is already being generated") is retried in a new conversation. There is no first-response stall detection: a stalled turn ends at its turn timeout. Every other crash ends the turn as a retryable `AgentCrashError`. -4. **No `sdk_options` passthrough.** Unlike Claude Code, there is no allowlisted escape hatch for arbitrary SDK fields — only `effort`, `project_id`, `session_id`, and `enable_computer_use` are exposed as typed config fields. +4. **`sdk_options` accepts only `effort`.** Reasoning effort uses the same `sdk_options.effort` key as Claude Code, so `-D agent.sdk_options.effort=high` works for both agents. Any other key is a validation error. `project_id`, `session_id` and `enable_computer_use` are typed config fields. 5. **`enable_computer_use: true` requires local permissions and is unavailable on Linux.** macOS needs Accessibility + Screen Recording grants; the SDK throws unconditionally on Linux when this is enabled. ## Testing diff --git a/src/coder_eval/agents/delegate_agent.py b/src/coder_eval/agents/delegate_agent.py index d6d96f14..9d3227f4 100644 --- a/src/coder_eval/agents/delegate_agent.py +++ b/src/coder_eval/agents/delegate_agent.py @@ -88,15 +88,6 @@ _HOST_PACKAGE = "@uipath/delegate-stdio" _HOST_BUNDLE_REL_PATH = Path("node_modules") / "@uipath" / "delegate-stdio" / "dist" / "delegate_stdio.mjs" -_HOST_AUTH_ENV: tuple[tuple[str, str], ...] = ( - ("AUTH_TOKEN", "AUTH_TOKEN"), - ("TENANT_ID", "TENANT_ID"), - ("ORG_ID", "ORG_ID"), - ("ORG_LOGICAL_NAME", "ORG_SLUG"), - ("TENANT_NAME", "TENANT_SLUG"), -) -"""``(name the host reads, name coder_eval reads via _env)`` for each auth var.""" - _UNSUPPORTED_CONFIG_FIELDS: tuple[str, ...] = ( "allowed_tools", "disallowed_tools", @@ -126,22 +117,6 @@ def _tool_id(event: dict[str, Any]) -> str: return str(event.get("toolId") or "") -def _env(bare_name: str) -> str | None: - """Read a ``DELEGATE_``-namespaced auth var, falling back to the bare name. - - The bare spellings (``AUTH_TOKEN``, ``TENANT_ID``, ...) collide with names - other tooling (npm, Vault, Terraform) commonly exports, so the namespaced - spelling is checked first. ``_host_auth_env`` re-exports the value under - the name the host itself reads. - """ - return os.environ.get(f"DELEGATE_{bare_name}") or os.environ.get(bare_name) - - -def _host_auth_env() -> dict[str, str]: - """The auth vars to set in the host's environment, keyed by the host's names.""" - return {host_name: value for host_name, ours in _HOST_AUTH_ENV if (value := _env(ours))} - - _GATEWAY_S2S_ENV_VARS = ("LLMGW_CLIENT_ID", "LLMGW_CLIENT_SECRET", "LLMGW_URL") @@ -448,7 +423,6 @@ async def _spawn_and_init(self) -> None: env["PLUGIN_TOOLS_DIR"] = self._plugin_tools_dir # Respect an operator's own telemetry choice; only default it off. env.setdefault("DELEGATE_TELEMETRY_DISABLED", "1") - env.update(_host_auth_env()) if stripped := _strip_redundant_gateway_creds(env): logger.debug("delegate: a token file is configured; removed %s from the host env", ", ".join(stripped)) @@ -497,18 +471,14 @@ def _build_init_options(self) -> dict[str, Any]: } if self.config.model: options["model"] = self.config.model - if self.config.effort: - options["effort"] = self.config.effort + options.update(self.config.sdk_options) if self.config.project_id: options["projectId"] = self.config.project_id if self.config.session_id: options["sessionId"] = self.config.session_id if self._env_path_prepend: options["shellPathPrepend"] = list(self._env_path_prepend) - backend_url = os.environ.get("DELEGATE_BACKEND_URL") - if backend_url: - options["backendUrl"] = backend_url - environment = os.environ.get("DELEGATE_ENV") + environment = os.environ.get("DELEGATE_SDK_ENV") if environment: options["env"] = environment # list[LocalPluginConfig] is not list[dict[str, Any]] under list invariance. @@ -598,21 +568,21 @@ async def _teardown_host(self) -> None: await self._cancel_drain_tasks() self._process = None + def get_sdk_options(self) -> dict[str, Any] | None: + return dict(self.config.sdk_options) or None + def get_environment_info(self) -> dict[str, Any]: info: dict[str, Any] = { **super().get_environment_info(), "delegate_model": self.config.model, } - if self.config.effort: - info["delegate_effort"] = self.config.effort if self.config.enable_computer_use: info["delegate_enable_computer_use"] = True - # DELEGATE_BACKEND_URL wins over DELEGATE_ENV when both are set (see - # _build_init_options/docs/agents/DELEGATE.md), so recording delegate_env - # here too would assert a routing decision the SDK never made. Host only - # (never the full URL, which can carry embedded credentials). - backend_url = os.environ.get("DELEGATE_BACKEND_URL") - environment = os.environ.get("DELEGATE_ENV") + # The host ranks BACKEND_URL over the env slug (see docs/agents/DELEGATE.md), + # so recording delegate_env here too would assert a routing decision the SDK + # never made. Host only (never the full URL, which can carry embedded credentials). + backend_url = os.environ.get("BACKEND_URL") + environment = os.environ.get("DELEGATE_SDK_ENV") if backend_url: info["delegate_backend_url_host"] = urlparse(backend_url).hostname elif environment: diff --git a/src/coder_eval/models/agent_config.py b/src/coder_eval/models/agent_config.py index acf19584..93a8972c 100644 --- a/src/coder_eval/models/agent_config.py +++ b/src/coder_eval/models/agent_config.py @@ -393,6 +393,9 @@ class PiAgentConfig(BaseAgentConfig): ) +_DELEGATE_SDK_OPTION_FIELDS: frozenset[str] = frozenset({"effort"}) + + class DelegateAgentConfig(BaseAgentConfig): """Delegate agent configuration (UiPath Autopilot's Delegate agent). @@ -413,13 +416,13 @@ class DelegateAgentConfig(BaseAgentConfig): type: Literal[AgentKind.DELEGATE] # type: ignore[assignment] - effort: str | None = Field( - default=None, + sdk_options: dict[str, Any] = Field( + default_factory=dict, description=( - "Reasoning-effort tier forwarded to the backend as-is (documented values: " - "low/medium/high/xhigh/max). Not validated against a closed set here — the SDK " - "itself ignores a value it does not recognize, and a strict Literal would reject a " - "tier a future SDK release adds." + "Pass-through dict of delegate-stdio init options that coder_eval does not own " + f"directly. Allowed keys: {sorted(_DELEGATE_SDK_OPTION_FIELDS)}. 'effort' is the " + "reasoning-effort tier (documented values: low/medium/high/xhigh/max), forwarded " + "as-is: the SDK ignores a value it does not recognize." ), ) project_id: str | None = Field( @@ -448,6 +451,17 @@ class DelegateAgentConfig(BaseAgentConfig): ), ) + @field_validator("sdk_options") + @classmethod + def _validate_sdk_options_keys(cls, v: dict[str, Any]) -> dict[str, Any]: + unknown = sorted(set(v) - _DELEGATE_SDK_OPTION_FIELDS) + if unknown: + raise ValueError( + f"sdk_options keys {unknown} are not delegate-stdio init options coder_eval forwards " + + f"(valid keys: {sorted(_DELEGATE_SDK_OPTION_FIELDS)})" + ) + return v + class NoneAgentConfig(BaseAgentConfig): """No-op ("agentless") agent configuration. diff --git a/src/coder_eval/orchestration/overrides.py b/src/coder_eval/orchestration/overrides.py index de1f55d9..bbc7c207 100644 --- a/src/coder_eval/orchestration/overrides.py +++ b/src/coder_eval/orchestration/overrides.py @@ -89,6 +89,18 @@ def _assign_nested(patch: dict[str, Any], segments: list[str], value: Any) -> No cursor[segments[-1]] = value +def _kinds_accepting_sdk_options() -> list[str]: + from coder_eval.agents.registry import AgentRegistry + from coder_eval.plugins import ensure_plugins_loaded + + ensure_plugins_loaded() + return [ + kind + for kind in sorted(AgentRegistry.list_kinds()) + if (reg := AgentRegistry.get(kind)) is not None and "sdk_options" in reg.config_class.model_fields + ] + + def apply_overrides( task: TaskDefinition, overrides: Mapping[str, Any], @@ -132,15 +144,17 @@ def apply_overrides( if agent_patch: assert task.agent is not None, f"Task '{task.task_id}' has no agent config" - # Preserve the friendly "sdk_options only for claude-code" message before - # reconstruction, keyed on the type the agent is *becoming*. + # Preserve a friendly sdk_options message before reconstruction, keyed on + # the type the agent is *becoming*. if "sdk_options" in agent_patch: becoming = agent_patch.get("type", task.agent.type) type_value = becoming.value if isinstance(becoming, AgentKind) else becoming - if type_value != AgentKind.CLAUDE_CODE.value: + accepting = _kinds_accepting_sdk_options() + if type_value not in accepting: where = "no agent type is set" if type_value is None else f"agent type {type_value}" raise OverrideError( - f"sdk_options cannot be used with {where}. This option is only supported for claude-code agents." + f"sdk_options cannot be used with {where}. This option is only supported for " + + f"{', '.join(accepting)} agents." ) # Seed with only the explicitly-set fields (exclude_unset) so switching the # agent subclass via --type doesn't drag subclass-only defaults into a model diff --git a/tests/test_delegate_agent.py b/tests/test_delegate_agent.py index de85cadf..afdc9953 100644 --- a/tests/test_delegate_agent.py +++ b/tests/test_delegate_agent.py @@ -219,10 +219,18 @@ async def test_eof_during_init_raises_crash(self, patch_exec, tmp_path): await agent.start(str(tmp_path)) async def test_effort_and_project_id_forwarded(self, patch_exec, tmp_path): - _agent, proc = await _started_agent(patch_exec, [], tmp_path, effort="high", project_id="proj-1") + agent, proc = await _started_agent( + patch_exec, [], tmp_path, sdk_options={"effort": "high"}, project_id="proj-1" + ) options = proc.stdin.written[0]["options"] assert options["effort"] == "high" assert options["projectId"] == "proj-1" + assert agent.get_sdk_options() == {"effort": "high"} + + async def test_no_sdk_options_reports_none(self, patch_exec, tmp_path): + agent, proc = await _started_agent(patch_exec, [], tmp_path) + assert "effort" not in proc.stdin.written[0]["options"] + assert agent.get_sdk_options() is None async def test_enable_computer_use_default_false(self, patch_exec, tmp_path): _agent, proc = await _started_agent(patch_exec, [], tmp_path) @@ -241,11 +249,21 @@ async def test_unsupported_fields_warn(self, patch_exec, tmp_path, caplog): await _started_agent(patch_exec, [], tmp_path, system_prompt="be nice") assert any("has no Delegate SDK equivalent" in r.message for r in caplog.records) - async def test_delegate_env_becomes_the_env_option(self, patch_exec, tmp_path, monkeypatch): - monkeypatch.setenv("DELEGATE_ENV", "alpha") + async def test_delegate_sdk_env_becomes_the_env_option(self, patch_exec, tmp_path, monkeypatch): + monkeypatch.setenv("DELEGATE_SDK_ENV", "alpha") _agent, proc = await _started_agent(patch_exec, [], tmp_path) assert proc.stdin.written[0]["options"]["env"] == "alpha" + async def test_backend_url_is_left_for_the_host_to_read(self, patch_exec, tmp_path, monkeypatch): + monkeypatch.setenv("BACKEND_URL", "https://user:secret@backend.example/delegate_") + monkeypatch.setenv("DELEGATE_SDK_ENV", "alpha") + agent, proc = await _started_agent(patch_exec, [], tmp_path) + assert "backendUrl" not in proc.stdin.written[0]["options"] + assert proc.spawn_kwargs["env"]["BACKEND_URL"] == "https://user:secret@backend.example/delegate_" + info = agent.get_environment_info() + assert info["delegate_backend_url_host"] == "backend.example" + assert "delegate_env" not in info + async def test_skills_enabled_only_with_a_plugin(self, patch_exec, tmp_path): _agent, proc = await _started_agent(patch_exec, [], tmp_path) assert proc.stdin.written[0]["options"]["enableSkills"] is False @@ -259,21 +277,21 @@ async def test_skills_enabled_only_with_a_plugin(self, patch_exec, tmp_path): assert options["enableSkills"] is True assert options["bundledSkillsPath"] == str(plugin_dir / "skills") - async def test_auth_is_exported_under_the_hosts_own_env_names(self, patch_exec, tmp_path, monkeypatch): + async def test_auth_reaches_the_host_under_its_own_env_names(self, patch_exec, tmp_path, monkeypatch): """The host reads auth from its environment, so nothing auth-shaped goes into init options.""" - monkeypatch.setenv("DELEGATE_AUTH_TOKEN", "tok-1") - monkeypatch.setenv("AUTH_TOKEN", "stray-npm-token") - monkeypatch.setenv("TENANT_ID", "tenant-guid") - monkeypatch.setenv("ORG_ID", "org-guid") - monkeypatch.setenv("ORG_SLUG", "my-org") - monkeypatch.setenv("DELEGATE_TENANT_SLUG", "my-tenant") + auth = { + "AUTH_TOKEN": "tok-1", + "TENANT_ID": "tenant-guid", + "ORG_ID": "org-guid", + "ORG_LOGICAL_NAME": "my-org", + "TENANT_NAME": "my-tenant", + } + for name, value in auth.items(): + monkeypatch.setenv(name, value) + monkeypatch.setenv("DELEGATE_AUTH_TOKEN", "old-namespaced-token") _agent, proc = await _started_agent(patch_exec, [], tmp_path) env = proc.spawn_kwargs["env"] - assert env["AUTH_TOKEN"] == "tok-1" - assert env["TENANT_ID"] == "tenant-guid" - assert env["ORG_ID"] == "org-guid" - assert env["ORG_LOGICAL_NAME"] == "my-org" - assert env["TENANT_NAME"] == "my-tenant" + assert {name: env[name] for name in auth} == auth assert "auth" not in proc.stdin.written[0]["options"] @pytest.mark.parametrize( diff --git a/tests/test_delegate_agent_config.py b/tests/test_delegate_agent_config.py index d1bdb4f1..b0bb5b7f 100644 --- a/tests/test_delegate_agent_config.py +++ b/tests/test_delegate_agent_config.py @@ -25,7 +25,7 @@ def test_importable_from_models(): def test_validates_and_defaults(): cfg = DelegateAgentConfig(type="delegate") assert cfg.type == AgentKind.DELEGATE - assert cfg.effort is None + assert cfg.sdk_options == {} assert cfg.project_id is None assert cfg.session_id is None assert cfg.enable_computer_use is False @@ -35,7 +35,7 @@ def test_round_trips_through_model_dump(): cfg = DelegateAgentConfig( type="delegate", model="virtuoso-1-5", - effort="high", + sdk_options={"effort": "high"}, project_id="invoice-approval", session_id="abc-123", enable_computer_use=True, @@ -43,7 +43,7 @@ def test_round_trips_through_model_dump(): restored = DelegateAgentConfig.model_validate(cfg.model_dump()) assert restored == cfg assert restored.model == "virtuoso-1-5" - assert restored.effort == "high" + assert restored.sdk_options == {"effort": "high"} assert restored.project_id == "invoice-approval" assert restored.session_id == "abc-123" assert restored.enable_computer_use is True @@ -52,7 +52,13 @@ def test_round_trips_through_model_dump(): def test_effort_accepts_any_string_including_future_tiers(): # Deliberately permissive: the SDK ignores an unrecognized value, so a # strict Literal here would reject a tier a future SDK release adds. - assert DelegateAgentConfig(type="delegate", effort="ultra-max").effort == "ultra-max" + cfg = DelegateAgentConfig(type="delegate", sdk_options={"effort": "ultra-max"}) + assert cfg.sdk_options == {"effort": "ultra-max"} + + +def test_sdk_options_rejects_a_key_the_host_is_not_sent(): + with pytest.raises(ValidationError, match=r"valid keys: \['effort'\]"): + DelegateAgentConfig(type="delegate", sdk_options={"backendUrl": "http://x"}) def test_unknown_extra_field_rejected(): diff --git a/tests/test_delegate_agent_live.py b/tests/test_delegate_agent_live.py index c3c10cca..68f6daba 100644 --- a/tests/test_delegate_agent_live.py +++ b/tests/test_delegate_agent_live.py @@ -3,7 +3,7 @@ Hit a real Delegate backend through a real Node subprocess; skipped by default, only run with ``pytest -m live``. Needs Node + a real ``@uipath/delegate-stdio`` install, UiPath auth (``AUTH_TOKEN``/``TENANT_ID``/``ORG_ID`` or a saved login), -a backend (``DELEGATE_ENV``/``DELEGATE_BACKEND_URL``), and optionally +a backend (``DELEGATE_SDK_ENV``/``BACKEND_URL``), and optionally ``DELEGATE_MODEL``. A failure here while the unit tests pass usually means the host's frame shapes @@ -17,7 +17,7 @@ import pytest -from coder_eval.agents.delegate_agent import DelegateAgent, _env, _resolve_host_bundle +from coder_eval.agents.delegate_agent import DelegateAgent, _resolve_host_bundle from coder_eval.errors import AgentConfigError from coder_eval.models import AgentKind, parse_agent_config @@ -32,8 +32,8 @@ def _have_prerequisites() -> bool: _resolve_host_bundle() except AgentConfigError: return False - has_auth = bool(_env("AUTH_TOKEN")) or (Path.home() / ".aria" / "sdk-auth.json").is_file() - has_backend = bool(os.getenv("DELEGATE_ENV") or os.getenv("DELEGATE_BACKEND_URL")) + has_auth = bool(os.getenv("AUTH_TOKEN")) or (Path.home() / ".aria" / "sdk-auth.json").is_file() + has_backend = bool(os.getenv("DELEGATE_SDK_ENV") or os.getenv("BACKEND_URL")) return has_auth and has_backend diff --git a/tests/test_merge_characterization.py b/tests/test_merge_characterization.py index 2326b729..9d1c153c 100644 --- a/tests/test_merge_characterization.py +++ b/tests/test_merge_characterization.py @@ -306,7 +306,7 @@ def test_system_prompt_file_override_clears_sibling(self): def test_sdk_options_on_codex_raises_friendly(self): task = _live_task(agent={"type": "codex"}) - with pytest.raises(OverrideError, match="only supported for claude-code"): + with pytest.raises(OverrideError, match="only supported for claude-code, delegate agents"): apply_overrides(task, {"agent.sdk_options.effort": "high"}) def test_lineage_cli_source_for_touched_paths_only(self): diff --git a/tests/test_overrides_engine.py b/tests/test_overrides_engine.py index 6a1753f5..93381843 100644 --- a/tests/test_overrides_engine.py +++ b/tests/test_overrides_engine.py @@ -144,12 +144,17 @@ def test_agent_type_injection_switches_subclass(self): def test_sdk_options_on_codex_raises(self): task = _make_task(agent=parse_agent_config(type="codex")) - with pytest.raises(OverrideError, match="only supported for claude-code"): + with pytest.raises(OverrideError, match="only supported for claude-code, delegate agents"): apply_overrides(task, {"agent.sdk_options.effort": "high"}) + def test_sdk_options_effort_on_delegate_applies(self): + task = _make_task(agent=parse_agent_config(type="delegate")) + apply_overrides(task, {"agent.sdk_options.effort": "high"}) + assert task.agent.sdk_options == {"effort": "high"} + def test_sdk_options_with_agent_type_codex_raises(self): task = _make_task(agent=parse_agent_config(type="claude-code")) - with pytest.raises(OverrideError, match="only supported for claude-code"): + with pytest.raises(OverrideError, match="only supported for claude-code, delegate agents"): apply_overrides(task, {"agent.sdk_options.effort": "high"}, agent_type="codex") def test_unknown_root_raises(self): From 8359f4fcd42786a3b6cef3e53885df16b59b36e0 Mon Sep 17 00:00:00 2001 From: Mihai Chirculescu Date: Wed, 30 Sep 2026 15:15:30 +0300 Subject: [PATCH 06/20] fix(delegate): keep the usage of the calls under a max_turns cut A Delegate turn cut at max_turns or by an early stop never gets the host's result frame, so it lost all its token usage and cost, and the max_usd and max_total_tokens gates had nothing to check. The other harnesses keep the usage of the calls under the cap. The delegate-stdio host now writes one `usage` frame per backend round-trip, before the tool results of that round-trip. The adapter adds up the frames. The result's `usage` is the turn total, so it replaces the sum when it arrives. The max_turns call boundary does not change. The adapter warns when finished model calls report no usage: a completed turn with no usage, or a cut turn from a host that does not send the frame. A zero payload in the known buckets no longer warns as a renamed bucket. Co-Authored-By: Claude Opus 5.5 (1M context) --- .claude/notes/agents.md | 15 ++++ docs/agents/DELEGATE.md | 3 +- docs/agents/HARNESS_PARITY.md | 2 + src/coder_eval/agents/delegate_agent.py | 61 ++++++++++++++-- tests/test_delegate_agent.py | 94 ++++++++++++++++++++++++- 5 files changed, 165 insertions(+), 10 deletions(-) diff --git a/.claude/notes/agents.md b/.claude/notes/agents.md index 83c8110a..aab54b67 100644 --- a/.claude/notes/agents.md +++ b/.claude/notes/agents.md @@ -777,6 +777,21 @@ event stream and abandons the host when call N+1 opens. Once the `result` arrive (one entry per backend round-trip, confirmed equal to `assistantStepCount`) replaces the running estimate as `num_turns`. +**A cut turn keeps the usage of its finished calls.** The host has no interrupt command, so a turn +ended at `max_turns` or by a cooperative stop never gets its `result` frame. The host therefore +also writes one `usage` frame per backend round-trip, and the adapter sums them; the `result`'s +`usage` (their total) replaces that sum when it arrives. The order is what makes this work, and it +comes from reading the SDK and backend source, not from a live transcript: the backend sends a +request's `usage` chunk before its stream closes, the client runs the returned tools only after the +stream closes, and the host flushes new `usage` entries before it writes each event. So call N's +frame always precedes the `tool_result` that opens call N+1 and fires the cap. The in-flight call at +the cut has no usage, as on the other harnesses. A host from before the frame existed sends none, +and the adapter warns when a cut turn had finished calls but no usage. + +The frame is not a call boundary. It arrives before its call's tools run, so a cap that fired on +frame N would stop call N's tools, and one that waited for frame N+1 would let call N+1 run to its end. +The cap stays on "the previous call's tools have all returned". + **`enableSkills` must be sent explicitly.** The host's default is `false`, so a `bundledSkillsPath` alone loads nothing; the adapter sets `enableSkills` to whether a plugin resolved. diff --git a/docs/agents/DELEGATE.md b/docs/agents/DELEGATE.md index 26a5f2d8..0866e567 100644 --- a/docs/agents/DELEGATE.md +++ b/docs/agents/DELEGATE.md @@ -116,7 +116,7 @@ Each turn returns a `TurnRecord` with: - `agent_output` — the SDK's final response, falling back to accumulated streamed text. - `commands` — one `CommandTelemetry` per `tool_call`/`tool_result` pair. - `messages` — exactly **one** `AssistantMessage` per turn (see Known Limitations). -- `token_usage` — the `result` frame's `usage` (Anthropic convention: `input_tokens` excludes cache reads and writes). +- `token_usage` — the `result` frame's `usage` (Anthropic convention: `input_tokens` excludes cache reads and writes). A turn cut before that frame keeps the sum of the `usage` frames that the host sends for each model call. - `num_turns` — the length of the `result` frame's `turnUsages` (one entry per backend round-trip). - `model_used` — the `result` frame's `model`, falling back to the pinned `agent.model`. @@ -162,6 +162,7 @@ Run-limit semantics per harness: [Run-Limit Parity](HARNESS_PARITY.md). 1. **No multi-generation transcript splitting.** This agent builds one `AssistantMessage` per `communicate()` call, not one per backend round-trip. The host does send the signals a split needs (`isStepStart` on `message` events and per-round-trip `turnUsages` on `result`), but this agent does not use them yet. 2. **`max_turns` is enforced by coder_eval, not by the host.** The host's `maxSteps` option on `send` does not stop the turn (confirmed live: `maxSteps: 2` ran 7 steps and only reported `maxStepsReached: true`), so this agent does not send it. See [Run-Limit Parity](HARNESS_PARITY.md). + A turn cut at `max_turns`, or by a cooperative early stop, keeps the token usage and cost of every model call that finished before the cut, as on the other agents. The host sends a `usage` frame for each call before that call's tool results. The call in progress at the cut has no usage. A host that does not send `usage` frames reports usage only on its final `result` frame, which a cut turn never gets: that turn has no usage, and the agent logs a warning. 3. **Only three backend failures get a specific diagnosis.** A Cloudflare WAF block page is reported as a content-filter failure and is not retried: the same prompt or tool result is blocked again. An SSE connect timeout is reported as a connection failure and is retried. A session conflict ("A reply is already being generated") is retried in a new conversation. There is no first-response stall detection: a stalled turn ends at its turn timeout. Every other crash ends the turn as a retryable `AgentCrashError`. 4. **`sdk_options` accepts only `effort`.** Reasoning effort uses the same `sdk_options.effort` key as Claude Code, so `-D agent.sdk_options.effort=high` works for both agents. Any other key is a validation error. `project_id`, `session_id` and `enable_computer_use` are typed config fields. 5. **`enable_computer_use: true` requires local permissions and is unavailable on Linux.** macOS needs Accessibility + Screen Recording grants; the SDK throws unconditionally on Linux when this is enabled. diff --git a/docs/agents/HARNESS_PARITY.md b/docs/agents/HARNESS_PARITY.md index 73ffc19f..ad0f4906 100644 --- a/docs/agents/HARNESS_PARITY.md +++ b/docs/agents/HARNESS_PARITY.md @@ -571,6 +571,8 @@ Each harness finds the call boundary in its own stream (the table above): and the cap fires before call N+1 can run anything. A reply that announces several tools before their results counts once. If a tool never returns, the next call opens at the model's next text or new tool call instead. + The host sends each call's `usage` frame before that call's tool results, so + the calls under the cap keep their tokens and cost. Every harness enforces the cap on the same loop boundary as the cooperative early stop, then kills or cancels the in-flight turn so the cap stops diff --git a/src/coder_eval/agents/delegate_agent.py b/src/coder_eval/agents/delegate_agent.py index 9d3227f4..2e378779 100644 --- a/src/coder_eval/agents/delegate_agent.py +++ b/src/coder_eval/agents/delegate_agent.py @@ -12,7 +12,8 @@ stdin stdout {"cmd":"init","options":{...}} -> {"type":"init_ok"} {"cmd":"send","prompt":.., {"type":"event","event":{...}} (zero or more) - "sessionId":..} -> {"type":"result","response":..,"sessionId":.., + "sessionId":..} {"type":"usage","usage":{..}} (one per model call) + -> {"type":"result","response":..,"sessionId":.., "usage":{..},"turnUsages":[..],"model":..} {"cmd":"destroy"} -> {"type":"destroyed"} {"type":"error","message":..} (any command) @@ -226,14 +227,17 @@ def _resolve_bundled_skills_path(plugins: list[dict[str, Any]] | None) -> str | return str(Path(first["path"]) / "skills") +_USAGE_BUCKETS = ("input_tokens", "output_tokens", "cache_creation_input_tokens", "cache_read_input_tokens") + + def _parse_usage(raw: Any) -> TokenUsage | None: - """Parse the host's ``result.usage`` payload into ``TokenUsage``. + """Parse a host ``usage`` payload (a ``usage`` frame's, or the ``result``'s) into ``TokenUsage``. The host follows the Anthropic convention: ``input_tokens`` excludes cache reads and writes, which arrive separately as ``cache_read_input_tokens`` / ``cache_creation_input_tokens``. An absent or invalid bucket reads as 0, and - a payload with no recognised bucket returns ``None`` with a warning — a - renamed field degrades to zero tokens, never a crash. + an all-zero payload returns ``None``. One with no recognised bucket also + warns — a renamed field degrades to zero tokens, never a crash. """ if not isinstance(raw, dict): return None @@ -251,7 +255,7 @@ def _int(key: str) -> int: cache_read_input_tokens=_int("cache_read_input_tokens"), ) if usage.is_empty(): - if raw: + if raw and not any(bucket in raw for bucket in _USAGE_BUCKETS): logger.warning("delegate: usage payload matched none of the known bucket spellings: %r", sorted(raw)) return None return usage @@ -329,6 +333,7 @@ def __init__(self, *, iteration: int, user_input: str, model: str | None) -> Non self.results_incomplete = False self.model_used: str | None = model + # The sum of the per-call `usage` frames, until the `result` replaces it with the turn total. self.usage: TokenUsage | None = None self.final_response: str | None = None self.error_message: str | None = None @@ -673,6 +678,9 @@ def emit(event: StreamEvent) -> None: break if mtype == "error": await self._crash_on_host_error(state, collector, emit, str(msg.get("message", "unknown error"))) + if mtype == "usage": + self._add_call_usage(msg, state) + continue event = msg.get("event") if mtype != "event" or not isinstance(event, dict): logger.debug("delegate: ignoring host message %r", mtype) @@ -695,6 +703,7 @@ def emit(event: StreamEvent) -> None: status = AgentEndStatus.MAX_TURNS_EXHAUSTED else: status = AgentEndStatus.COMPLETED + self._warn_if_usage_missing(state, status) self._finalize_turn(state, status, emit) record = collector.build_turn_record() self._end_turn_ok() @@ -837,11 +846,49 @@ def _handle_tool_result( telemetry.result_status = "success" emit(ToolEndEvent(task_id=self.task_id, turn_id=state.turn_id, tool=telemetry, status=status)) + @staticmethod + def _add_call_usage(msg: dict[str, Any], state: _TurnState) -> None: + """Add one model call's ``usage`` frame to the turn's running total. + + The host writes a call's frame before any tool result that call caused, + so a turn cut at ``max_turns`` or by an early stop keeps the usage of + every call that finished. + """ + usage = _parse_usage(msg.get("usage")) + if usage is not None: + state.usage = usage if state.usage is None else state.usage + usage + + @staticmethod + def _warn_if_usage_missing(state: _TurnState, status: AgentEndStatus) -> None: + """Warn when model calls finished but no usage arrived, so the turn books no tokens or cost. + + A cut turn's in-flight call is not finished, so it is not counted. + """ + if state.usage is not None: + return + if status is AgentEndStatus.COMPLETED: + if state.api_calls > 0: + logger.warning( + "delegate: the host reported no token usage for a turn with %d model call(s); " + + "its tokens and cost are unknown", + state.api_calls, + ) + elif state.api_calls > 1: + logger.warning( + "delegate: turn ended (%s) with no usage frame from the host; tokens and cost for its %d " + + "finished model call(s) are unknown (a @uipath/delegate-stdio without per-call usage " + + "frames reports usage only on its result frame, which a cut turn never gets)", + status.value, + state.api_calls - 1, + ) + def _handle_result(self, msg: dict[str, Any], state: _TurnState) -> None: """Fold the host's terminal ``result`` frame into the turn state. - ``turnUsages`` has one entry per backend round-trip, so its length is the - authoritative call count and replaces the running estimate. + Its ``usage`` is the turn total, so it replaces the running sum of the + ``usage`` frames. ``turnUsages`` has one entry per backend round-trip, so + its length is the authoritative call count and replaces the running + estimate. """ response = msg.get("response") if isinstance(response, str): diff --git a/tests/test_delegate_agent.py b/tests/test_delegate_agent.py index afdc9953..d4cdb8ff 100644 --- a/tests/test_delegate_agent.py +++ b/tests/test_delegate_agent.py @@ -10,6 +10,7 @@ import asyncio import json +import logging import os from pathlib import Path from typing import Any @@ -39,6 +40,11 @@ def _result(**fields: Any) -> bytes: return _line({"type": "result", **fields}) +def _usage(input_tokens: int, output_tokens: int) -> bytes: + """One model call's ``usage`` frame, written before any tool result that call caused.""" + return _line({"type": "usage", "usage": {"input_tokens": input_tokens, "output_tokens": output_tokens}}) + + def _tool_call(tool_id: str, name: str = "shell", **args: Any) -> bytes: return _ev(type="tool_call", toolId=tool_id, toolName=name, toolArgs=args, toolStatus="pending") @@ -243,8 +249,6 @@ async def test_shell_path_prepend_forwarded(self, patch_exec, tmp_path): assert proc.stdin.written[0]["options"]["shellPathPrepend"] == ["/mocks/bin"] async def test_unsupported_fields_warn(self, patch_exec, tmp_path, caplog): - import logging - caplog.set_level(logging.WARNING) await _started_agent(patch_exec, [], tmp_path, system_prompt="be nice") assert any("has no Delegate SDK equivalent" in r.message for r in caplog.records) @@ -548,6 +552,83 @@ async def test_max_turns_exhausted(self, patch_exec, tmp_path): assert [c.tool_id for c in record.commands if c.result_status == "success"] == ["t0"] assert record.num_turns == 2 + @staticmethod + def _billed_round_trip(n: int) -> list[bytes]: + """A round-trip as the host writes it: the call's usage frame precedes its tool call and result.""" + return [ + _ev(type="message", content="", isStepStart=True), + _usage(10 * (n + 1), n + 1), + _tool_call(f"t{n}"), + _tool_result(f"t{n}"), + ] + + async def test_max_turns_cut_keeps_the_usage_of_the_calls_under_the_cap(self, patch_exec, tmp_path, caplog): + events = [ + *self._billed_round_trip(0), + *self._billed_round_trip(1), + *self._billed_round_trip(2), + _result(response="done", usage={"input_tokens": 60, "output_tokens": 6}), + ] + agent, _proc = await _started_agent(patch_exec, events, tmp_path, model="virtuoso-1-5") + with caplog.at_level(logging.WARNING): + record = await agent.communicate("hi", max_turns=2) + assert record.max_turns_exhausted is True + assert record.num_turns == 3 + assert record.token_usage is not None + assert (record.token_usage.uncached_input_tokens, record.token_usage.output_tokens) == (30, 3) + assert record.token_usage.total_cost_usd is not None + assert not any("usage" in r.message for r in caplog.records) + + async def test_early_stop_keeps_the_usage_of_the_finished_calls(self, patch_exec, tmp_path): + events = [*self._billed_round_trip(0), _ev(type="message", content="next", isStepStart=True)] + agent, _proc = await _started_agent(patch_exec, events, tmp_path) + seen = {"n": 0} + + def should_stop() -> bool: + seen["n"] += 1 + return seen["n"] == 4 + + record = await agent.communicate("hi", should_stop=should_stop) + assert record.token_usage is not None + assert (record.token_usage.uncached_input_tokens, record.token_usage.output_tokens) == (10, 1) + + async def test_result_usage_replaces_the_frame_sum(self, patch_exec, tmp_path): + """The result's usage is the turn total, so adding it to the frames would count each call twice.""" + events = [ + *self._billed_round_trip(0), + _ev(type="message", content="done", isStepStart=True), + _usage(20, 2), + _result(response="done", usage={"input_tokens": 30, "output_tokens": 3}), + ] + agent, _proc = await _started_agent(patch_exec, events, tmp_path) + record = await agent.communicate("hi") + assert record.token_usage is not None + assert (record.token_usage.uncached_input_tokens, record.token_usage.output_tokens) == (30, 3) + + async def test_cut_turn_from_a_host_without_usage_frames_warns(self, patch_exec, tmp_path, caplog): + events = [*self._round_trip(0), *self._round_trip(1), _result(response="done", usage={"output_tokens": 5})] + agent, _proc = await _started_agent(patch_exec, events, tmp_path) + with caplog.at_level(logging.WARNING): + record = await agent.communicate("hi", max_turns=1) + assert record.max_turns_exhausted is True + assert record.token_usage is None + assert any("1 finished model call(s)" in r.message for r in caplog.records) + + async def test_early_stop_before_any_finished_call_does_not_warn(self, patch_exec, tmp_path, caplog): + events = [_ev(type="message", content="partial", isStepStart=True)] + agent, _proc = await _started_agent(patch_exec, events, tmp_path) + with caplog.at_level(logging.WARNING): + await agent.communicate("hi", should_stop=lambda: True) + assert not any("usage" in r.message for r in caplog.records) + + async def test_completed_turn_without_usage_warns(self, patch_exec, tmp_path, caplog): + events = [_ev(type="message", content="hi", isStepStart=True), _result(response="hi", usage=None)] + agent, _proc = await _started_agent(patch_exec, events, tmp_path) + with caplog.at_level(logging.WARNING): + record = await agent.communicate("hi") + assert record.token_usage is None + assert any("no token usage for a turn with 1 model call(s)" in r.message for r in caplog.records) + async def test_max_turns_counts_round_trips_not_text_chunks(self, patch_exec, tmp_path): """The SDK streams a reply as several message events; they are one round-trip.""" events = [ @@ -681,6 +762,15 @@ async def test_usage_all_zero_is_none_and_warns(self, patch_exec, tmp_path, capl assert record.token_usage is None assert any("usage payload matched none" in r.message for r in caplog.records) + async def test_zero_usage_frame_in_known_buckets_does_not_warn(self, patch_exec, tmp_path, caplog): + """A call can report zero tokens; that is not a renamed bucket.""" + events = [_usage(0, 0), _result(response="done")] + agent, _ = await _started_agent(patch_exec, events, tmp_path) + with caplog.at_level("WARNING"): + record = await agent.communicate("hi") + assert record.token_usage is None + assert not any("usage payload matched none" in r.message for r in caplog.records) + async def test_cost_wired_into_record_for_configured_model(self, patch_exec, tmp_path): events = [_result(response="done", usage={"input_tokens": 1_000_000, "output_tokens": 1_000_000})] agent, _ = await _started_agent(patch_exec, events, tmp_path, model="virtuoso-1-5") From c4f35541fef36d1c778f660f117b4c01e1778fef Mon Sep 17 00:00:00 2001 From: Mihai Chirculescu Date: Thu, 1 Oct 2026 12:31:17 +0300 Subject: [PATCH 07/20] fix(delegate): keep tool rows whose result has no open call The Delegate SDK sends no tool_call for a tool name it cannot resolve, only the failed tool_result. That result carries toolName and echoes the args in toolResult.args, so the synthesized row now takes both from it instead of recording "unknown" with no parameters (29 rows in nightly 13599116). The SDK also starts a new tool while earlier results are still pending, and those results arrive later. A new call no longer force-closes the tools that are still open, so the late results match their own rows instead of becoming "unknown" rows. The call counting is unchanged. Co-Authored-By: Claude Opus 5.5 (1M context) --- src/coder_eval/agents/delegate_agent.py | 27 ++++++++++----- tests/test_delegate_agent.py | 45 +++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 8 deletions(-) diff --git a/src/coder_eval/agents/delegate_agent.py b/src/coder_eval/agents/delegate_agent.py index 2e378779..e0199401 100644 --- a/src/coder_eval/agents/delegate_agent.py +++ b/src/coder_eval/agents/delegate_agent.py @@ -118,6 +118,18 @@ def _tool_id(event: dict[str, Any]) -> str: return str(event.get("toolId") or "") +def _orphan_tool_name(event: dict[str, Any], tool_id: str) -> str: + """The name on a ``tool_result`` with no open call; the SDK falls back to the tool id when it has none.""" + name = event.get("toolName") + return name if isinstance(name, str) and name and name != tool_id else "unknown" + + +def _orphan_tool_args(output: Any) -> dict[str, Any]: + """The call's args, which a failed lookup echoes back in its ``toolResult``.""" + args = output.get("args") if isinstance(output, dict) else None + return args if isinstance(args, dict) else {} + + _GATEWAY_S2S_ENV_VARS = ("LLMGW_CLIENT_ID", "LLMGW_CLIENT_SECRET", "LLMGW_URL") @@ -328,8 +340,8 @@ def __init__(self, *, iteration: int, user_input: str, model: str | None) -> Non # after the first opens once the previous call's tools have all returned. self.api_calls = 0 # A result arrived while other tools were still open, so the next call is not - # counted yet. If the model speaks or calls a new tool first, those tools never - # returned and the next call has begun. + # counted yet. If the model speaks or calls a new tool first, the next call has + # begun; the open tools stay open, because the SDK can still deliver their results. self.results_incomplete = False self.model_used: str | None = model @@ -742,7 +754,6 @@ def _handle_event(self, event: dict[str, Any], state: _TurnState, emit: Callable elif state.results_incomplete and ( event_type in _TEXT_EVENT_TYPES or (event_type == "tool_call" and _tool_id(event) not in state.open_tools) ): - self._close_open_tools(state, emit) state.api_calls += 1 state.results_incomplete = False @@ -810,19 +821,19 @@ def _handle_tool_result( ) -> None: tool_id = _tool_id(event) telemetry = state.open_tools.pop(tool_id, None) + output = event.get("toolResult") if telemetry is None: - # A result with no matching open call (id mismatch or unknown shape). - # Never drop it: synthesize a telemetry row rather than lose the - # event. + # The SDK sends no tool_call for a tool name it cannot resolve, only + # this result, so the row takes its name and args from the result. state.sequence += 1 telemetry = CommandTelemetry( - tool_name="unknown", + tool_name=_orphan_tool_name(event, tool_id), tool_id=tool_id or str(uuid.uuid4()), assistant_turn_index=state.message_events, timestamp=datetime.now(), + parameters=_orphan_tool_args(output), sequence_number=state.sequence, ) - output = event.get("toolResult") if isinstance(output, dict) and isinstance(output.get("content"), str): output = output["content"] completed = datetime.now() diff --git a/tests/test_delegate_agent.py b/tests/test_delegate_agent.py index d4cdb8ff..e167a42b 100644 --- a/tests/test_delegate_agent.py +++ b/tests/test_delegate_agent.py @@ -473,6 +473,51 @@ async def test_orphaned_tool_call_is_force_closed(self, patch_exec, tmp_path): record = await agent.communicate("hi") assert record.commands[0].result_status == "unknown" + async def test_result_for_an_unresolved_tool_takes_its_name_and_args(self, patch_exec, tmp_path): + """The SDK sends no tool_call for a tool name it cannot resolve, only the failed result.""" + not_found = _ev( + type="tool_result", + toolId="s/x", + toolName="Bash", + toolResult={ + "responseType": "error", + "content": "Tool Bash not found.", + "name": "Bash", + "args": {"command": "ls"}, + }, + toolStatus="failed", + ) + agent, _ = await _started_agent(patch_exec, [not_found, _result(response="done")], tmp_path) + record = await agent.communicate("hi") + [command] = record.commands + assert (command.tool_name, command.parameters, command.result_status) == ("Bash", {"command": "ls"}, "error") + + async def test_result_with_only_the_sdk_id_fallback_name_stays_unknown(self, patch_exec, tmp_path): + orphan = _ev(type="tool_result", toolId="s/x", toolName="s/x", toolResult="ok", toolStatus="completed") + agent, _ = await _started_agent(patch_exec, [orphan, _result(response="done")], tmp_path) + record = await agent.communicate("hi") + assert record.commands[0].tool_name == "unknown" + + async def test_a_result_after_a_new_tool_call_still_matches_its_call(self, patch_exec, tmp_path): + """The SDK can start a new tool while earlier results are pending; those results still arrive.""" + events = [ + _tool_call("a", path="x.md"), + _tool_call("b", path="y.md"), + _tool_result("a"), + _tool_call("c", command="ls"), + _tool_result("b"), + _tool_result("c"), + _result(response="done"), + ] + agent, _ = await _started_agent(patch_exec, events, tmp_path) + record = await agent.communicate("hi") + assert [(c.tool_id, c.tool_name, c.result_status) for c in record.commands] == [ + ("a", "shell", "success"), + ("b", "shell", "success"), + ("c", "shell", "success"), + ] + assert record.commands[1].parameters == {"path": "y.md"} + async def test_cooperative_stop_ends_cleanly(self, patch_exec, tmp_path): events = [ _ev(type="message", content="partial", isStepStart=True), From 6a007271338b1dec381076006d707f9257142b10 Mon Sep 17 00:00:00 2001 From: Mihai Chirculescu Date: Thu, 1 Oct 2026 15:14:38 +0300 Subject: [PATCH 08/20] fix(delegate): make crash retry independent of the stderr tail Fixes from the review of nightly 13599116: - No crash reason carries the host's stderr tail now; it is logged at WARNING. The tail holds the sandbox path, so the task id decided the category: "Delegate backend error: terminated" was retried on four tasks and ended skill-review-agents-lowcode-guardrail-unknown-validator as a non-retryable AGENT_INVALID_OUTPUT, because "guardrail" matched. - Only an init error that a retry cannot fix (missing or rejected auth, missing org/tenant slugs, an unknown env) raises AgentConfigError. Other init errors and the 60 s init deadline are retryable. - A stdout drain cancelled at teardown no longer logs "stdout drain failed" at ERROR (396 times on the Linux slice). - get_sdk_options() returns the init options sent to the host. The reports use it in place of agent_config, so the pass-through dict hid the Model row on every run that set an effort. - The install search also looks in agents/delegate/, as the docs say. - _force_kill_host drops the process handle itself, so kill() clears it too, also when the reap times out. HARNESS_PARITY.md no longer describes the out-of-tree delegate-sdk agent as a live agent; its untimed tool records are now a historical note. Co-Authored-By: Claude Opus 5.5 (1M context) --- .claude/notes/agents.md | 27 +++++- docs/agents/DELEGATE.md | 6 +- docs/agents/HARNESS_PARITY.md | 36 +++----- src/coder_eval/agents/delegate_agent.py | 114 ++++++++++++++---------- tests/test_delegate_agent.py | 81 ++++++++++++++--- 5 files changed, 180 insertions(+), 84 deletions(-) diff --git a/.claude/notes/agents.md b/.claude/notes/agents.md index aab54b67..1c9e0572 100644 --- a/.claude/notes/agents.md +++ b/.claude/notes/agents.md @@ -770,6 +770,13 @@ one value the host takes only as an init option, so `DELEGATE_SDK_ENV` becomes ` `get_sdk_options()`. The `-D` gate admits any registered kind whose config class declares `sdk_options`; `DelegateAgentConfig` validates the keys against the host options it forwards. +`get_sdk_options()` returns the whole `init` options dict sent to the host, not the user's +pass-through dict. Both reports use `EvaluationResult.sdk_options` in place of `agent_config` +whenever it is truthy, so a pass-through-only dict hid the Model row from any run that set an +effort. The init dict carries `model` and `effort`. The reports then show Permission Mode as N/A +and Allowed Tools as "(all)", which is correct for this SDK. There is no Plugins row, because the +init dict names the skills directory (`bundledSkillsPath`), not a `plugins` list. + **`max_turns` stays client-side.** The host accepts `maxSteps` on `send`, but it does not stop the turn: live, `maxSteps: 2` ran 7 steps and only reported `maxStepsReached: true` in the `result`. Forwarding it would suggest a cap that does not exist, so the adapter keeps counting calls from the @@ -824,9 +831,20 @@ window from the turn's own start, so the head and tail both measure ~0. is replaced either way. A `session_id` pinned in config still reaches the new host as an init option, so a pinned run does not recover from this. -A rewritten reason omits the stderr tail — the tail logs the SDK's own watchdog lines, and one -"timeout" in it would re-route the crash to `AGENT_TIMEOUT` (the `timeout` rule runs before both -"content filter" and "connection"). The tail is logged at WARNING instead. +**No crash reason carries the stderr tail.** `errors/categorization.py` matches substrings of the +reason, and the tail is incidental text: the SDK's own watchdog lines, PIDs, token lifetimes, and +the host's `Working directory:` line, which holds the sandbox path and so the task id. In build +13599116 the same transient `Delegate backend error: terminated` was retried on four tasks and +ended `skill-review-agents-lowcode-guardrail-unknown-validator` as a non-retryable +`AGENT_INVALID_OUTPUT`, because "guardrail" in its path matched the content-filter rule. +`_log_stderr_tail` logs the tail at WARNING instead, on every crash path. + +**Only an init error that a retry cannot fix is non-retryable.** An init `error` frame raises +`AgentConfigError` only when it matches `_INIT_CONFIG_ERROR_MARKERS`: missing or rejected auth +(the out-of-tree adapter's auth markers, kept as they were), missing org/tenant slugs, or an +unknown env slug. Any other init error, and the 60 s init deadline, raise a retryable +`AgentCrashError`, so a backend 5xx or a slow token refresh during `agent.initialize()` gets +`execute_with_retry('Agent start')`'s retries instead of ending the task at once. **`LLMGW_*` leaves the host env only when a token file is configured.** The host's shells inherit its env, so the gateway client secret there is readable by the code under test. But the host's own @@ -850,6 +868,9 @@ a `"send"` still nominally in flight instead of respawning — risking a stale r the new turn's. `tests/test_delegate_agent.py`'s `test_timeout_elapsing_mid_read_still_raises_turn_timeout_error` pins the fix. The host survives a failed `send` (it answers the next `destroy` with `destroyed`), but the adapter still does not reuse it, so no late event from the failed turn can leak into the retry. +`_force_kill_host` drops the handle itself, before its reap, so every path that kills the host +clears it, even when the 5 s reap times out. Each call site used to pair the kill with its own +clear, and `kill()` did not. **Skills mapping is deliberately NOT the shared `agents/_skills.py` resolver.** That resolver enumerates individual skill directories for a repeated `--skill `-style CLI argument (OpenCode/Pi's shape). diff --git a/docs/agents/DELEGATE.md b/docs/agents/DELEGATE.md index 0866e567..88a86a70 100644 --- a/docs/agents/DELEGATE.md +++ b/docs/agents/DELEGATE.md @@ -24,7 +24,7 @@ Under the hood, this agent spawns the host that the public [`@uipath/delegate-st npm install @uipath/delegate-stdio ``` -**You usually don't need to set anything about where.** When neither `DELEGATE_STDIO_NODE_MODULES` nor `DELEGATE_STDIO_PATH` is set, coder_eval auto-locates the install by walking up from the current directory through its ancestors (and home) — the same way Node resolves modules. Running the install inside `src/coder_eval/agents/delegate/` (this agent's own directory, which ships a `package.json` naming the dependency) is a convenient default location. +**You usually don't need to set anything about where.** When neither `DELEGATE_STDIO_NODE_MODULES` nor `DELEGATE_STDIO_PATH` is set, coder_eval searches for the install in this order: the current directory and its ancestors (the same way Node resolves modules), then `src/coder_eval/agents/delegate/`, then your home directory. `src/coder_eval/agents/delegate/` is this agent's own directory, and it ships a `package.json` that names the dependency. An install there is found from any current directory. To override the auto-search, set **one** of: @@ -130,9 +130,11 @@ A wall-clock deadline (`timeout`) is enforced both between reads (a top-of-loop On any crash (host death, an `error` frame during a turn, or an unexpected exception), the agent: 1. Sets `pending_turn` to a `crashed=True` TurnRecord with captured telemetry. -2. Raises `AgentCrashError` (retryable) or `AgentConfigError` (non-retryable — missing Node/SDK install, or an SDK init rejection such as a missing `backendUrl`). +2. Raises `AgentCrashError` (retryable) or `AgentConfigError` (non-retryable — missing Node/SDK install, or an init error that a retry cannot fix: missing or rejected auth, missing org/tenant slugs, or an unknown `DELEGATE_SDK_ENV`). Any other init error, and an init that does not respond within 60 s, is retryable. 3. The orchestrator reads `pending_turn` and calls `discard_pending_turn()` to roll back state. +The crash reason never includes the host's stderr. The agent logs the last 20 stderr lines at WARNING instead, because the error categorizer matches words in the reason, and stderr contains the sandbox path, which contains the task id. + A dead host is always detected and its handle cleared, so a retried `communicate()` call always respawns a fresh host rather than hang or cross-wire a stale response. ### Skills Discovery diff --git a/docs/agents/HARNESS_PARITY.md b/docs/agents/HARNESS_PARITY.md index ad0f4906..b46f45d5 100644 --- a/docs/agents/HARNESS_PARITY.md +++ b/docs/agents/HARNESS_PARITY.md @@ -498,27 +498,6 @@ needed to drive it. ### Known divergences -- **This page's `delegate` column is a DIFFERENT agent** from the one described - in the bullet immediately below. `delegate` (this repo, `AgentKind.DELEGATE`) - drives the public `@uipath/delegate-stdio` Node host; - `delegate-sdk` (the bullet below) is an older, UiPath-internal-only agent in - the separate `coder_eval_uipath` plugin, driving the non-public - `@uipath/delegate-stdio` package. They are not the same adapter code and this page - does not claim their timing behavior matches. -- **Delegate (`delegate-sdk`, out of tree)** records `duration_ms` but no - execution bounds, so its tool calls cannot be placed on a timeline. Its - coverage is ~88%. Mirror the Codex change in `coder_eval_uipath` - (audit P3-1). **The consequence is now the same on both surfaces:** such a - call contributes to NO bucket. Python has always dropped it - (`timing.main_thread_tool_spans` filters on `is not None`), and - `evalboard/lib/timing.ts::toolExecutionMs` no longer folds the bare duration - into its union — a duration with no bounds cannot be placed on the timeline, - so unioning it double-books whatever it overlapped and can drive the - four-bucket residual negative. Its time reads as **Unaccounted**, which is - what that cell means: measured, but not placeable. Codex was in the same - state until `_item_timing` landed on 2026-09-10 (0% bounded before, 100% - after), so on historical codex runs ~8 h in aggregate moves out of Tool exec - and into Unaccounted; that population is closed and no new record joins it. - **Antigravity books orphan-poll waiting as agent duration.** A task can spend `0.8 × turn_timeout` waiting on a tool call that never reaches DONE — 14 tasks and 9.6h of one 83h run. Only CLOSED tool intervals are subtracted, so that @@ -536,9 +515,22 @@ needed to drive it. cost of normalizing it is that the event drives the live renderers, so moving it changes the turn boundaries users watch during a run. -All three are deliberately deferred; see `c/time-bugs-audit.md` for the +Both are deliberately deferred; see `c/time-bugs-audit.md` for the measurements. +**Older records can have a tool `duration_ms` with no execution bounds.** Codex +records from before `_item_timing` landed on 2026-09-10 have this shape, and so do +records from `delegate-sdk`, the out-of-tree agent in `coder_eval_uipath` that the +built-in `delegate` replaced. Such a tool call cannot be placed on a timeline, so it +contributes to NO bucket on either surface. Python drops it +(`timing.main_thread_tool_spans` filters on `is not None`), and +`evalboard/lib/timing.ts::toolExecutionMs` does not fold the bare duration into its +union: a duration with no bounds would double-book whatever it overlapped and could +make the four-bucket residual negative. Its time reads as **Unaccounted**, which is +what that cell means: measured, but not placeable. On historical codex runs, this +moves ~8 h in aggregate out of Tool exec and into Unaccounted. No built-in agent +writes this shape now. + ## `max_turns` counts model API calls on every harness One turn is one main-thread model API call, the unit Claude Code's `--max-turns` diff --git a/src/coder_eval/agents/delegate_agent.py b/src/coder_eval/agents/delegate_agent.py index e0199401..e565a529 100644 --- a/src/coder_eval/agents/delegate_agent.py +++ b/src/coder_eval/agents/delegate_agent.py @@ -88,6 +88,8 @@ _HOST_PACKAGE = "@uipath/delegate-stdio" _HOST_BUNDLE_REL_PATH = Path("node_modules") / "@uipath" / "delegate-stdio" / "dist" / "delegate_stdio.mjs" +_AGENT_INSTALL_ROOT = Path(__file__).resolve().parent / "delegate" +"""This agent's own directory; it ships a ``package.json`` that names the host package.""" _UNSUPPORTED_CONFIG_FIELDS: tuple[str, ...] = ( "allowed_tools", @@ -107,6 +109,21 @@ indefinitely, unlike every turn-scoped read, which is deadline-bounded.""" _SIGKILL: signal.Signals = getattr(signal, "SIGKILL", signal.SIGTERM) +_INIT_CONFIG_ERROR_MARKERS = ( + "auth required", + "authentication", + "unauthorized", + "invalid credentials", + "invalid token", + "expired", + "401", + "403", + "requires org/tenant slugs", + "unknown env", +) +"""Substrings of an init ``error`` message that a retry cannot fix: missing or rejected +auth, or a bad ``DELEGATE_SDK_ENV``. Any other init error is retryable.""" + # Event types (inside the host's `event` frames) that carry model-turn content. # `session_start` / `done` are informational; anything else is logged and ignored. _TEXT_EVENT_TYPES = frozenset({"thinking", "message"}) @@ -152,17 +169,16 @@ def _strip_redundant_gateway_creds(env: dict[str, str]) -> tuple[str, ...]: def _candidate_install_roots() -> list[Path]: - """Ancestor-walk search roots: cwd, its ancestors, and home. + """Search roots, in order: cwd and its ancestors, ``_AGENT_INSTALL_ROOT``, then home. - Mirrors Node's own module resolution so an ``npm install`` run in the - launch directory, any ancestor, or (where npm lands a package when the cwd - has no ``package.json``) home, is found with zero configuration. + The cwd walk mirrors Node's own module resolution. Home is where npm lands a + package when the cwd has no ``package.json``. """ cwd = Path.cwd().resolve() roots: list[Path] = [cwd, *cwd.parents] - home = Path.home().resolve() - if home not in roots: - roots.append(home) + for root in (_AGENT_INSTALL_ROOT, Path.home().resolve()): + if root not in roots: + roots.append(root) return roots @@ -171,9 +187,8 @@ def _resolve_host_bundle() -> Path: Resolution order: ``DELEGATE_STDIO_PATH`` (explicit file path) -> ``DELEGATE_STDIO_NODE_MODULES`` (explicit install root, probed exactly) -> - ancestor walk from cwd (plus home), so an ``npm install`` anywhere in that - chain — including this module's own ``agents/delegate/`` directory, which - ships a ``package.json`` naming the dependency — is found automatically. + ``_candidate_install_roots()``, so an ``npm install`` in this agent's own + ``agents/delegate/`` directory is found from any cwd. Raises: AgentConfigError: no install found anywhere searched. @@ -207,10 +222,10 @@ def _resolve_host_bundle() -> Path: searched_block = "\n ".join(str(p) for p in searched) raise AgentConfigError( - f"{_HOST_PACKAGE} not found. Searched the cwd, its ancestors, and home:\n" + f"{_HOST_PACKAGE} not found. Searched the cwd, its ancestors, this agent's directory, and home:\n" + f" {searched_block}\n" + f"Run `npm install {_HOST_PACKAGE}` (plain, public install — no token needed), " - + "e.g. in this framework's own agents/delegate/ directory, or set DELEGATE_STDIO_NODE_MODULES " + + f"e.g. in {_AGENT_INSTALL_ROOT}, or set DELEGATE_STDIO_NODE_MODULES " + "to the install root, or DELEGATE_STDIO_PATH to the dist/delegate_stdio.mjs file directly. " + "See docs/agents/DELEGATE.md." ) @@ -392,6 +407,7 @@ def __init__( self._state = AgentState.WORKING self._host_bundle: Path | None = None + self._init_options: dict[str, Any] | None = None self._process: asyncio.subprocess.Process | None = None self._stdout_task: asyncio.Task[None] | None = None self._stderr_task: asyncio.Task[None] | None = None @@ -465,21 +481,22 @@ async def _spawn_and_init(self) -> None: self._stdout_task = asyncio.create_task(self._drain_stdout(self._process.stdout)) self._stderr_task = asyncio.create_task(self._drain_stderr(self._process.stderr)) - init_options = self._build_init_options() - await self._send_command({"cmd": "init", "options": init_options}) + self._init_options = self._build_init_options() + await self._send_command({"cmd": "init", "options": self._init_options}) try: ack = await asyncio.wait_for(self._read_until(("init_ok", "error")), timeout=_INIT_TIMEOUT_SEC) except TimeoutError as exc: await self._force_kill_host() - self._process = None - raise AgentConfigError( + raise AgentCrashError( f"Delegate SDK init did not respond within {_INIT_TIMEOUT_SEC:.0f}s " + "(hung auth refresh or backend connect?)." ) from exc if ack.get("type") == "error": await self._force_kill_host() - self._process = None - raise AgentConfigError(f"Delegate SDK init failed: {ack.get('message', 'unknown error')}") + message = str(ack.get("message", "unknown error")) + if any(marker in message.lower() for marker in _INIT_CONFIG_ERROR_MARKERS): + raise AgentConfigError(f"Delegate SDK init failed: {message}") + raise AgentCrashError(f"Delegate SDK init failed: {message}") def _build_init_options(self) -> dict[str, Any]: options: dict[str, Any] = { @@ -528,7 +545,13 @@ def kill_sync(self) -> None: self._sweep_pgid() async def _force_kill_host(self) -> None: + """SIGKILL the host and drop its handle, so the next ``communicate()`` respawns. + + The handle is dropped even when the reap times out: a dying host must never + receive the next ``send``. + """ proc = self._process + self._process = None if proc is not None and proc.returncode is None: with contextlib.suppress(ProcessLookupError): proc.kill() @@ -583,10 +606,10 @@ async def _teardown_host(self) -> None: await asyncio.wait_for(self._process.wait(), timeout=_STOP_TIMEOUT_SEC) await self._force_kill_host() await self._cancel_drain_tasks() - self._process = None def get_sdk_options(self) -> dict[str, Any] | None: - return dict(self.config.sdk_options) or None + """The ``init`` options sent to the last host spawned, or ``None`` before ``start()``.""" + return self._init_options def get_environment_info(self) -> dict[str, Any]: info: dict[str, Any] = { @@ -702,11 +725,11 @@ def emit(event: StreamEvent) -> None: if max_turns is not None and state.api_calls > max_turns: state.max_turns_exhausted = True - await self._abandon_host_after_loop_exit() + await self._force_kill_host() break if should_stop is not None and should_stop(): stopped_early = True - await self._abandon_host_after_loop_exit() + await self._force_kill_host() break if stopped_early: @@ -738,7 +761,6 @@ def finalize_on_cancel( # from a dead stdin) -- force-kill so a retry always respawns # rather than write into, or read stale events from, this host. await self._force_kill_host() - self._process = None self._crash_turn(state, collector, emit, f"Delegate turn failed: {e!s}", cause=e) raise # unreachable — _crash_turn is NoReturn @@ -1009,19 +1031,25 @@ def _finalize_turn( ) ) - async def _abandon_host_after_loop_exit(self) -> None: - """Force-kill after ``max_turns``/cooperative-stop; next turn respawns.""" - await self._force_kill_host() - self._process = None + def _log_stderr_tail(self, reason: str) -> None: + """Log the host's stderr tail beside ``reason``. + + Never put the tail in a raised reason: ``errors/categorization.py`` matches + substrings of the reason, and the tail holds incidental text (the sandbox + path, which names the task) that can match a non-retryable rule. + + Rationale: .claude/notes/agents.md § Delegate agent + """ + logger.warning("delegate: %s. stderr tail:\n%s", reason, self._stderr_tail()) async def _abandon_host_and_crash( self, state: _TurnState, collector: EventCollector, emit: Callable[[StreamEvent], None], - message: str, + reason: str, ) -> NoReturn: - """Force-kill the current host, drop the handle, and crash the turn. + """Force-kill the current host and crash the turn with ``reason``. Shared by every mid-``communicate()`` failure kernel that must not let the NEXT ``communicate()`` reuse this host: reuse risks writing a @@ -1029,8 +1057,8 @@ async def _abandon_host_and_crash( fires after this failure as the retry's own result. """ await self._force_kill_host() - self._process = None - self._crash_turn(state, collector, emit, f"{message}. stderr tail:\n{self._stderr_tail()}") + self._log_stderr_tail(reason) + self._crash_turn(state, collector, emit, reason) async def _crash_on_host_error( self, @@ -1041,10 +1069,8 @@ async def _crash_on_host_error( ) -> NoReturn: """Crash the turn on a host ``error`` frame; the host is never reused. - A rewritten reason goes out without the stderr tail, which can itself - say "timeout" and so undo the rewrite's categorization; the tail is - logged instead. A session conflict also drops the remembered session id, - because a retry into the same conversation can only conflict again. + A session conflict also drops the remembered session id, because a retry + into the same conversation can only conflict again. """ reason = _describe_host_error(message) if reason is None: @@ -1054,11 +1080,8 @@ async def _crash_on_host_error( self._session_id, ) self._session_id = None - await self._abandon_host_and_crash(state, collector, emit, f"Delegate send failed: {message}") - logger.warning("delegate: %s stderr tail:\n%s", reason, self._stderr_tail()) - await self._force_kill_host() - self._process = None - self._crash_turn(state, collector, emit, reason) + reason = f"Delegate send failed: {message}" + await self._abandon_host_and_crash(state, collector, emit, reason) def _crash_turn( self, @@ -1081,9 +1104,6 @@ async def _timeout_turn( self, state: _TurnState, collector: EventCollector, emit: Callable[[StreamEvent], None], timeout: float ) -> NoReturn: await self._force_kill_host() - # Drop the handle so the NEXT communicate() respawns rather than reuse - # a killed process (or, worse, a "send" still nominally in flight). - self._process = None def finalize(status: AgentEndStatus, *, crashed: bool = False, crash_reason: str | None = None) -> None: self._finalize_turn(state, status, emit, crashed=crashed, crash_reason=crash_reason) @@ -1113,9 +1133,9 @@ async def _read_until(self, accepted_types: tuple[str, ...]) -> dict[str, Any]: # As in communicate()'s loop: the host may still be alive # (a buffer overrun or drain exception, not necessarily exit). await self._force_kill_host() - self._process = None - tail = self._stderr_tail() - raise AgentCrashError(f"Delegate host exited before responding. stderr tail:\n{tail}") + reason = "Delegate host exited before responding" + self._log_stderr_tail(reason) + raise AgentCrashError(reason) if msg.get("type") in accepted_types: return msg logger.debug("delegate: ignoring host message %r during init", msg.get("type")) @@ -1145,6 +1165,8 @@ async def _drain_stdout(self, stream: asyncio.StreamReader) -> None: continue if isinstance(obj, dict): await self._stdout_queue.put(obj) + except asyncio.CancelledError: + raise except BaseException: logger.exception("delegate: stdout drain failed") await self._stdout_queue.put(None) diff --git a/tests/test_delegate_agent.py b/tests/test_delegate_agent.py index e167a42b..11a8f2a4 100644 --- a/tests/test_delegate_agent.py +++ b/tests/test_delegate_agent.py @@ -24,6 +24,7 @@ from coder_eval.errors.categories import ErrorCategory from coder_eval.errors.categorization import categorize_error from coder_eval.models import AgentKind, DelegateAgentConfig +from coder_eval.reports.markdown import collect_agent_settings_rows from coder_eval.streaming.events import AgentEndEvent, AgentEndStatus, AgentStartEvent @@ -211,13 +212,34 @@ async def test_init_ok_completes_start(self, patch_exec, tmp_path): assert sent["options"]["workingDirectory"] == str(tmp_path) assert proc.spawn_args == ("node", str(Path("/opt/delegate_stdio.mjs"))) - async def test_init_error_raises_agent_config_error(self, patch_exec, tmp_path): - patch_exec([_line({"type": "error", "message": "backendUrl is required", "stack": "Error: ..."})]) + @pytest.mark.parametrize( + ("host_message", "error_type"), + [ + ("Auth required: set AUTH_TOKEN/TENANT_ID/ORG_ID env vars", AgentConfigError), + ('env="alpha" requires org/tenant slugs. Set ORG_LOGICAL_NAME and TENANT_NAME', AgentConfigError), + ("Token endpoint https://cloud.example/token returned HTTP 503: unavailable", AgentCrashError), + ("fetch failed", AgentCrashError), + ], + ids=["no-auth", "no-slugs", "token-endpoint-5xx", "network"], + ) + async def test_only_an_init_error_a_retry_cannot_fix_is_non_retryable( + self, patch_exec, tmp_path, host_message, error_type + ): + patch_exec([_line({"type": "error", "message": host_message, "stack": "Error: ..."})]) agent = DelegateAgent(_config()) - with pytest.raises(AgentConfigError, match="backendUrl is required"): + with pytest.raises(error_type, match="Delegate SDK init failed"): await agent.start(str(tmp_path)) assert agent._process is None + async def test_init_timeout_is_retryable(self, patch_exec, tmp_path, monkeypatch): + monkeypatch.setattr(agent_module, "_INIT_TIMEOUT_SEC", 0.05) + patch_exec([], hang_after=True) + agent = DelegateAgent(_config()) + with pytest.raises(AgentCrashError, match="did not respond") as excinfo: + await agent.start(str(tmp_path)) + assert categorize_error(excinfo.value, {"component": "agent"}) is ErrorCategory.AGENT_CRASH + assert agent._process is None + async def test_eof_during_init_raises_crash(self, patch_exec, tmp_path): patch_exec([]) # EOF immediately agent = DelegateAgent(_config()) @@ -225,18 +247,22 @@ async def test_eof_during_init_raises_crash(self, patch_exec, tmp_path): await agent.start(str(tmp_path)) async def test_effort_and_project_id_forwarded(self, patch_exec, tmp_path): - agent, proc = await _started_agent( + _agent, proc = await _started_agent( patch_exec, [], tmp_path, sdk_options={"effort": "high"}, project_id="proj-1" ) options = proc.stdin.written[0]["options"] assert options["effort"] == "high" assert options["projectId"] == "proj-1" - assert agent.get_sdk_options() == {"effort": "high"} - async def test_no_sdk_options_reports_none(self, patch_exec, tmp_path): - agent, proc = await _started_agent(patch_exec, [], tmp_path) - assert "effort" not in proc.stdin.written[0]["options"] - assert agent.get_sdk_options() is None + async def test_sdk_options_are_the_init_options_sent(self, patch_exec, tmp_path): + """The reports prefer ``sdk_options`` over ``agent_config``, so it must carry the model too.""" + assert DelegateAgent(_config()).get_sdk_options() is None + agent, proc = await _started_agent( + patch_exec, [], tmp_path, model="virtuoso-1-5", sdk_options={"effort": "high"} + ) + assert agent.get_sdk_options() == proc.stdin.written[0]["options"] + rows = dict(collect_agent_settings_rows(agent.get_sdk_options() or {}, is_sdk=True)) + assert (rows["Model"], rows["Effort"]) == ("virtuoso-1-5", "high") async def test_enable_computer_use_default_false(self, patch_exec, tmp_path): _agent, proc = await _started_agent(patch_exec, [], tmp_path) @@ -421,6 +447,21 @@ async def test_known_host_errors_are_recategorized(self, patch_exec, tmp_path, h assert proc._killed assert agent._process is None + async def test_crash_category_ignores_the_stderr_tail(self, patch_exec, tmp_path, caplog): + """The tail names the sandbox, so a task id containing "guardrail" must not make a crash non-retryable.""" + patch_exec( + [_line({"type": "init_ok"}), _line({"type": "error", "message": "Delegate backend error: terminated"})], + [b"[handleInit] Working directory: /work/skill-lowcode-guardrail-validator\n"], + ) + agent = DelegateAgent(_config(), task_id="t1") + await agent.start(str(tmp_path)) + await asyncio.sleep(0) + assert agent._stderr_lines + with caplog.at_level(logging.WARNING), pytest.raises(AgentCrashError) as excinfo: + await agent.communicate("hi") + assert categorize_error(excinfo.value, {"component": "agent"}) is ErrorCategory.AGENT_CRASH + assert any("guardrail-validator" in r.getMessage() for r in caplog.records) + async def test_session_conflict_drops_the_session_id(self, patch_exec, tmp_path): events = [ _line( @@ -863,6 +904,17 @@ async def test_stop_sends_destroy_and_marks_finished(self, patch_exec, tmp_path) assert proc.stdin.written[-1] == {"cmd": "destroy"} assert agent.pending_turn is None + async def test_stop_cancels_a_blocked_stdout_drain_without_an_error_log( + self, patch_exec, tmp_path, monkeypatch, caplog + ): + monkeypatch.setattr(agent_module, "_STOP_TIMEOUT_SEC", 0.01) + patch_exec([_line({"type": "init_ok"})], hang_after=True) + agent = DelegateAgent(_config(), task_id="t1") + await agent.start(str(tmp_path)) + with caplog.at_level(logging.ERROR): + await agent.stop() + assert not [r for r in caplog.records if r.levelno >= logging.ERROR] + class TestKill: def test_kill_sync_signals_running_process(self, patch_exec, tmp_path, monkeypatch): @@ -903,6 +955,12 @@ async def test_force_kill_host_sweeps_process_group(self, patch_exec, tmp_path, if os.name == "posix": assert killpg_calls == [(proc.pid, agent_module._SIGKILL)] + async def test_kill_drops_the_handle_so_the_next_turn_respawns(self, patch_exec, tmp_path): + agent, proc = await _started_agent(patch_exec, [], tmp_path) + await agent.kill() + assert proc._killed + assert agent._process is None + class TestRegistration: def test_registered_as_delegate(self): @@ -917,7 +975,8 @@ def test_create_agent_dispatches(self): assert isinstance(agent, DelegateAgent) -def test_install_target_names_the_host_package(): +def test_install_target_names_the_host_package_and_is_searched(): """``agents/delegate/package.json`` is the documented ``npm install`` target the resolver finds.""" - package_json = Path(agent_module.__file__).parent / "delegate" / "package.json" + package_json = agent_module._AGENT_INSTALL_ROOT / "package.json" assert agent_module._HOST_PACKAGE in json.loads(package_json.read_text())["dependencies"] + assert agent_module._AGENT_INSTALL_ROOT in agent_module._candidate_install_roots() From 9890cd9c68cbc96d10734c0b4058d8a722747447 Mon Sep 17 00:00:00 2001 From: Mihai Chirculescu Date: Thu, 1 Oct 2026 16:31:34 +0300 Subject: [PATCH 09/20] feat(delegate)!: send auth to the host as an init option; read the DELEGATE_* names again The adapter read the delegate-stdio host's own variable names and passed its environment through unchanged. The bare names collide with other tooling (a BACKEND_URL exported for another service routed the host there), and every agent shell command inherited the token. The adapter now reads coder_eval's names again, as main does: DELEGATE_SDK_PATH, DELEGATE_SDK_NODE_MODULES, DELEGATE_ENV, DELEGATE_BACKEND_URL, and DELEGATE_AUTH_TOKEN / DELEGATE_TENANT_ID / DELEGATE_ORG_ID / DELEGATE_ORG_SLUG / DELEGATE_TENANT_SLUG, each with the bare spelling as a fallback. It sends the auth and the slugs as the host's new `auth` init option, DELEGATE_BACKEND_URL as `backendUrl`, and DELEGATE_ENV as `env`. It removes AUTH_TOKEN, TENANT_ID, ORG_ID, ORG_LOGICAL_NAME, TENANT_NAME, BACKEND_URL and DELEGATE_AUTH_TOKEN from the host's environment. - DELEGATE_SDK_PATH must name delegate-stdio's dist/delegate_stdio.mjs. An old value that names delegate-sdk's dist/index.mjs is an error. - A config-class init error names coder_eval's variables beside the host's message, which names the host's. - get_sdk_options() redacts the credentials: `auth` becomes its field names and `backendUrl` its host, because the run records it. BREAKING CHANGE: needs a @uipath/delegate-stdio release that accepts the `auth` init option; 1.202.1 and older ignore it and fail init with "Auth required" unless a saved login exists. Verified live against alpha with a host built from Autopilot's chore/move-delegate-sdk-2 branch. Co-Authored-By: Claude Opus 5.5 (1M context) --- .claude/notes/agents.md | 33 ++++-- .env.example | 44 ++++---- .github/workflows/pr-checks.yml | 42 ++++---- docs/USER_GUIDE.md | 2 +- docs/agents/DELEGATE.md | 18 ++-- src/coder_eval/agents/delegate_agent.py | 113 ++++++++++++++++---- tests/test_delegate_agent.py | 133 ++++++++++++++++++------ tests/test_delegate_agent_live.py | 10 +- tests/test_error_handling.py | 2 +- 9 files changed, 286 insertions(+), 111 deletions(-) diff --git a/.claude/notes/agents.md b/.claude/notes/agents.md index 1c9e0572..17175cb6 100644 --- a/.claude/notes/agents.md +++ b/.claude/notes/agents.md @@ -757,13 +757,29 @@ An earlier version of this agent wrapped `@uipath/delegate-sdk` in a first-party (`delegate_host.mjs`) because `delegate-stdio` was not yet public. That host and its protocol are gone; git history has them. -**Auth goes through the host's environment, not the init options.** The host reads `AUTH_TOKEN` / -`TENANT_ID` / `ORG_ID` and, for an `env`-derived backend URL, `ORG_LOGICAL_NAME` / `TENANT_NAME` -from its own environment, and `BACKEND_URL` / `INTEROP_URL` too. coder_eval uses the host's names -as its user-facing names and passes its environment through unchanged, so there is one set of names -and no re-export table to drift, and `coder_eval_uipath` pipelines use the same names. Confirmed live with no saved login: with the slugs the -turn runs; without them init fails with `env="alpha" requires org/tenant slugs`. The env slug is the -one value the host takes only as an init option, so `DELEGATE_SDK_ENV` becomes `env`. +**Auth and the backend reach the host as init options, not through its environment.** coder_eval +keeps its own names: `_env` reads the `DELEGATE_`-namespaced spelling first, then the bare +`AUTH_TOKEN` / `TENANT_ID` / `ORG_ID` / `ORG_SLUG` / `TENANT_SLUG`, and `_auth_option` sends the +values as the host's `auth` init option. `DELEGATE_BACKEND_URL` becomes `backendUrl` and +`DELEGATE_ENV` becomes `env`. It does not adopt the host's own names (`AUTH_TOKEN`, `TENANT_ID`, +`ORG_ID`, `ORG_LOGICAL_NAME`, `TENANT_NAME`, `BACKEND_URL`), because the bare names collide with +other tooling and the CI secrets are named after the `DELEGATE_` spellings. + +The host gives each `auth` field priority over its environment variable, and the SDK reads none of +these names. It takes the credentials from the `TokenAuthProvider` the host builds, and an explicit +`backendUrl` replaces the `BACKEND_URL` default its bundle reads at load. So `_HOST_ENV_REMOVED` +takes the host's names, and the token under either spelling, out of the host's environment. The +agent's shell tools inherit that environment, so a static token there is readable by the code under +test. And a `BACKEND_URL` or `ORG_LOGICAL_NAME` that another tool exports can no longer route the +host. A refresh source (token file, `LLMGW_*` pair, saved login) still writes its fresh token into +the host's own `process.env.AUTH_TOKEN`. + +The option needs a host release that accepts it. An older host ignores it and fails init with +`Auth required` unless a saved login exists; `_INIT_CONFIG_ERROR_HINT` names coder_eval's variables +beside the host's message, which names the host's. Confirmed live against alpha with a host built +from the Autopilot branch and `DELEGATE_STDIO_VERBOSE=1`: the host resolved auth from the option, +not the saved login, and neither its stderr nor the recorded `sdk_options` contained the token. +Without the slugs, init fails with `env="alpha" requires org/tenant slugs`. **Effort rides `sdk_options.effort`.** It is the same key as Claude Code, so one `-D agent.sdk_options.effort=...` drives both agents, and the reports' Effort row reads it from @@ -771,7 +787,8 @@ one value the host takes only as an init option, so `DELEGATE_SDK_ENV` becomes ` `sdk_options`; `DelegateAgentConfig` validates the keys against the host options it forwards. `get_sdk_options()` returns the whole `init` options dict sent to the host, not the user's -pass-through dict. Both reports use `EvaluationResult.sdk_options` in place of `agent_config` +pass-through dict, with the credentials redacted: `auth` becomes its field names and `backendUrl` +its host, because the result is persisted in `task.json`. Both reports use `EvaluationResult.sdk_options` in place of `agent_config` whenever it is truthy, so a pass-through-only dict hid the Model row from any run that set an effort. The init dict carries `model` and `effort`. The reports then show Permission Mode as N/A and Allowed Tools as "(all)", which is correct for this SDK. There is no Plugins row, because the diff --git a/.env.example b/.env.example index e2e3280e..deed0120 100644 --- a/.env.example +++ b/.env.example @@ -89,33 +89,39 @@ LOG_TO_FILE=false # Set to true to enable file logging # CODER_EVAL_REMEDIATE_HOME_PLUGINS=0 # Delegate agent settings (UiPath Autopilot's Delegate agent, requires Node.js -# on PATH plus `npm install @uipath/delegate-stdio` — a public package, -# no token needed). See docs/agents/DELEGATE.md. +# on PATH plus `npm install @uipath/delegate-stdio`, a release that accepts the +# `auth` init option). See docs/agents/DELEGATE.md. # -# DELEGATE_STDIO_NODE_MODULES / DELEGATE_STDIO_PATH — override the auto-located -# install (ancestor-walk from cwd, plus home); set at most one. -# DELEGATE_STDIO_NODE_MODULES="/path/to/install/root" -# DELEGATE_STDIO_PATH="/path/to/node_modules/@uipath/delegate-stdio/dist/delegate_stdio.mjs" +# DELEGATE_SDK_NODE_MODULES / DELEGATE_SDK_PATH — override the auto-located +# install (cwd and its ancestors, src/coder_eval/agents/delegate/, then home); +# set at most one. +# DELEGATE_SDK_NODE_MODULES="/path/to/install/root" +# DELEGATE_SDK_PATH="/path/to/node_modules/@uipath/delegate-stdio/dist/delegate_stdio.mjs" # -# DELEGATE_SDK_ENV — cloud environment slug (alpha/staging/production), +# DELEGATE_ENV — cloud environment slug (alpha/staging/production), # used to derive the backend URL from the org/tenant slugs. -# DELEGATE_SDK_ENV="alpha" +# DELEGATE_ENV="alpha" # -# BACKEND_URL — pin the full agent-service URL directly instead -# (wins over DELEGATE_SDK_ENV when both are set). Read by the Node host. -# BACKEND_URL="https://cloud.uipath.com///delegate_" +# DELEGATE_BACKEND_URL — pin the full agent-service URL directly instead +# (wins over DELEGATE_ENV when both are set). +# DELEGATE_BACKEND_URL="https://cloud.uipath.com///delegate_" # # Auth: either the token set below, or a prior `delegate-cli login` -# that wrote ~/.aria/sdk-auth.json. The Node host reads all of these -# itself, under these exact names. -# AUTH_TOKEN="..." -# TENANT_ID="..." -# ORG_ID="..." -# With DELEGATE_SDK_ENV (not BACKEND_URL) also set these two -- the +# that wrote ~/.aria/sdk-auth.json (read by the Node host itself, not +# by coder_eval). coder_eval sends these values to the host as its `auth` +# init option, so the token stays out of the environment that the agent's +# shell tools inherit. Each var also accepts the bare spelling (AUTH_TOKEN, +# TENANT_ID, ORG_ID, ORG_SLUG, TENANT_SLUG) as a fallback -- prefer the +# DELEGATE_ spelling in a shared environment, since the bare names collide +# with what other tooling (npm, Vault, Terraform) commonly exports. +# DELEGATE_AUTH_TOKEN="..." +# DELEGATE_TENANT_ID="..." +# DELEGATE_ORG_ID="..." +# With DELEGATE_ENV (not DELEGATE_BACKEND_URL) also set these two -- the # human-readable org/tenant names, not the GUIDs above -- or init fails with # 'env="alpha" requires org/tenant slugs'. See docs/agents/DELEGATE.md. -# ORG_LOGICAL_NAME="..." -# TENANT_NAME="..." +# DELEGATE_ORG_SLUG="..." +# DELEGATE_TENANT_SLUG="..." # Usage Telemetry (anonymous; on by default). # Disable entirely: diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index 7afd0981..331e39f2 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -1187,10 +1187,10 @@ jobs: # and implemented in coder_eval_uipath/eval_runner/scripts/ci/refresh-auth.sh — # rather than a long-lived AUTH_TOKEN secret, since a minted token's ~1h TTL # would make a static secret stale between runs. A dedicated bot user's - # username/password is the durable credential; the Delegate SDK reads the - # minted token straight from AUTH_TOKEN/TENANT_ID/ORG_ID (see - # docs/agents/DELEGATE.md), so no sdk-auth.json or client-secret refresh chain - # is needed here. + # username/password is the durable credential; the agent sends the minted + # token (DELEGATE_AUTH_TOKEN, with DELEGATE_TENANT_ID / DELEGATE_ORG_ID) to the + # host as its `auth` init option (see docs/agents/DELEGATE.md), so no + # sdk-auth.json or client-secret refresh chain is needed here. name: Delegate Live E2E (ROPC) runs-on: uipath-ubuntu-24.04 timeout-minutes: 15 @@ -1372,16 +1372,16 @@ jobs: - name: Run fizzbuzz delegate task (assert PASS) if: steps.gate.outputs.present == 'true' env: - DELEGATE_STDIO_NODE_MODULES: ${{ github.workspace }}/src/coder_eval/agents/delegate - DELEGATE_SDK_ENV: ${{ github.event.inputs.delegate_uipath_env || 'alpha' }} - AUTH_TOKEN: ${{ steps.ropc.outputs.access_token }} - TENANT_ID: ${{ secrets.DELEGATE_TENANT_ID }} - ORG_ID: ${{ secrets.DELEGATE_ORG_ID }} - # Read by the delegate-stdio host itself — needed alongside the GUIDs - # above whenever DELEGATE_SDK_ENV (rather than BACKEND_URL) resolves - # the backend (see .claude/notes/agents.md § Delegate agent). - ORG_LOGICAL_NAME: ${{ secrets.DELEGATE_ORG_SLUG }} - TENANT_NAME: ${{ secrets.DELEGATE_TENANT_SLUG }} + DELEGATE_SDK_NODE_MODULES: ${{ github.workspace }}/src/coder_eval/agents/delegate + DELEGATE_ENV: ${{ github.event.inputs.delegate_uipath_env || 'alpha' }} + DELEGATE_AUTH_TOKEN: ${{ steps.ropc.outputs.access_token }} + DELEGATE_TENANT_ID: ${{ secrets.DELEGATE_TENANT_ID }} + DELEGATE_ORG_ID: ${{ secrets.DELEGATE_ORG_ID }} + # Needed alongside the GUIDs above whenever DELEGATE_ENV (rather than + # DELEGATE_BACKEND_URL) resolves the backend; the agent sends all five to + # the host as its `auth` init option (see .claude/notes/agents.md § Delegate agent). + DELEGATE_ORG_SLUG: ${{ secrets.DELEGATE_ORG_SLUG }} + DELEGATE_TENANT_SLUG: ${{ secrets.DELEGATE_TENANT_SLUG }} run: .venv/bin/coder-eval run tasks/delegate/fizzbuzz_delegate.yaml --run-dir runs/delegate-live - name: Verify Delegate live run PASSED @@ -1404,13 +1404,13 @@ jobs: - name: Run Delegate live unit tests (assert PASS) if: steps.gate.outputs.present == 'true' env: - DELEGATE_STDIO_NODE_MODULES: ${{ github.workspace }}/src/coder_eval/agents/delegate - DELEGATE_SDK_ENV: ${{ github.event.inputs.delegate_uipath_env || 'alpha' }} - AUTH_TOKEN: ${{ steps.ropc.outputs.access_token }} - TENANT_ID: ${{ secrets.DELEGATE_TENANT_ID }} - ORG_ID: ${{ secrets.DELEGATE_ORG_ID }} - ORG_LOGICAL_NAME: ${{ secrets.DELEGATE_ORG_SLUG }} - TENANT_NAME: ${{ secrets.DELEGATE_TENANT_SLUG }} + DELEGATE_SDK_NODE_MODULES: ${{ github.workspace }}/src/coder_eval/agents/delegate + DELEGATE_ENV: ${{ github.event.inputs.delegate_uipath_env || 'alpha' }} + DELEGATE_AUTH_TOKEN: ${{ steps.ropc.outputs.access_token }} + DELEGATE_TENANT_ID: ${{ secrets.DELEGATE_TENANT_ID }} + DELEGATE_ORG_ID: ${{ secrets.DELEGATE_ORG_ID }} + DELEGATE_ORG_SLUG: ${{ secrets.DELEGATE_ORG_SLUG }} + DELEGATE_TENANT_SLUG: ${{ secrets.DELEGATE_TENANT_SLUG }} run: | mkdir -p tmp .venv/bin/pytest tests/test_delegate_agent_live.py \ diff --git a/docs/USER_GUIDE.md b/docs/USER_GUIDE.md index f3547d1d..08495350 100644 --- a/docs/USER_GUIDE.md +++ b/docs/USER_GUIDE.md @@ -429,7 +429,7 @@ Set these in `.env` (copy from `.env.example`). | `BEDROCK_SMALL_MODEL` | No | Cross-region Bedrock small/fast model ID | | `CODEX_API_KEY` / `CODEX_BASE_URL` / `CODEX_MODEL` / `CODEX_API_VERSION` | For Codex | Codex agent auth & endpoint routing — see [Codex Agent Guide](agents/CODEX.md#endpoint-routing). | | `GEMINI_API_KEY` / `ANTIGRAVITY_MODEL` | For Antigravity | Antigravity (Gemini) agent auth & model — see [Antigravity Agent Guide](agents/ANTIGRAVITY.md#setup). | -| `DELEGATE_SDK_ENV` / `BACKEND_URL` / `AUTH_TOKEN` / `TENANT_ID` / `ORG_ID` / `ORG_LOGICAL_NAME` / `TENANT_NAME` / `DELEGATE_STDIO_NODE_MODULES` / `DELEGATE_STDIO_PATH` | For Delegate | Delegate agent backend routing, auth & host install location — see [Delegate Agent Guide](agents/DELEGATE.md#setup). | +| `DELEGATE_ENV` / `DELEGATE_BACKEND_URL` / `DELEGATE_AUTH_TOKEN` / `DELEGATE_TENANT_ID` / `DELEGATE_ORG_ID` / `DELEGATE_ORG_SLUG` / `DELEGATE_TENANT_SLUG` / `DELEGATE_SDK_NODE_MODULES` / `DELEGATE_SDK_PATH` | For Delegate | Delegate agent backend routing, auth & host install location — see [Delegate Agent Guide](agents/DELEGATE.md#setup). | | `UIPATH_PLUGIN_MARKETPLACE_DIR` | No | Conventional base directory for local plugins. Not special-cased by the framework: **any** `$VAR` / `${VAR}` referenced in a plugin `path` is expanded from the environment, so a plugin path like `$UIPATH_PLUGIN_MARKETPLACE_DIR/my-plugin` resolves against this variable. (An undefined variable in a plugin path logs a warning.) | | `PLUGIN_TOOLS_DIR` | No | A *separate* mechanism from the above: the canonical `node_modules/@uipath` directory used to pin UiPath CLI plugin discovery (not path substitution). When unset, the sandbox auto-derives it from the resolved `uip` binary. | | `CODER_EVAL_REMEDIATE_HOME_PLUGINS` | No | **DESTRUCTIVE.** Truthy deletes `$HOME/node_modules/@uipath` at sandbox setup to clear sibling-task pollution on dedicated eval hosts. Off by default; do **not** enable on developer workstations. | diff --git a/docs/agents/DELEGATE.md b/docs/agents/DELEGATE.md index 88a86a70..98f7b82b 100644 --- a/docs/agents/DELEGATE.md +++ b/docs/agents/DELEGATE.md @@ -24,14 +24,14 @@ Under the hood, this agent spawns the host that the public [`@uipath/delegate-st npm install @uipath/delegate-stdio ``` -**You usually don't need to set anything about where.** When neither `DELEGATE_STDIO_NODE_MODULES` nor `DELEGATE_STDIO_PATH` is set, coder_eval searches for the install in this order: the current directory and its ancestors (the same way Node resolves modules), then `src/coder_eval/agents/delegate/`, then your home directory. `src/coder_eval/agents/delegate/` is this agent's own directory, and it ships a `package.json` that names the dependency. An install there is found from any current directory. +**You usually don't need to set anything about where.** When neither `DELEGATE_SDK_NODE_MODULES` nor `DELEGATE_SDK_PATH` is set, coder_eval searches for the install in this order: the current directory and its ancestors (the same way Node resolves modules), then `src/coder_eval/agents/delegate/`, then your home directory. `src/coder_eval/agents/delegate/` is this agent's own directory, and it ships a `package.json` that names the dependency. An install there is found from any current directory. To override the auto-search, set **one** of: | Variable | Purpose | |---|---| -| `DELEGATE_STDIO_NODE_MODULES` | Install root that holds `node_modules/@uipath/...`. | -| `DELEGATE_STDIO_PATH` | Absolute path straight to `@uipath/delegate-stdio/dist/delegate_stdio.mjs`. | +| `DELEGATE_SDK_NODE_MODULES` | Install root that holds `node_modules/@uipath/...`. | +| `DELEGATE_SDK_PATH` | Absolute path straight to `@uipath/delegate-stdio/dist/delegate_stdio.mjs`. Any other file, such as `@uipath/delegate-sdk`'s `dist/index.mjs`, is an error. | On Windows, install into a short path. The interop binary sits deep inside `node_modules`, and a path longer than 260 characters makes its spawn fail with `ENOENT`. @@ -39,16 +39,18 @@ On Windows, install into a short path. The interop binary sits deep inside `node | Variable | Purpose | |---|---| -| `DELEGATE_SDK_ENV` | Cloud environment slug (`alpha` / `staging` / `production`), sent as the host's `env` init option. The host derives the backend URL from the org/tenant slugs. | -| `BACKEND_URL` | Pin the full agent-service URL directly (e.g. `https://cloud.uipath.com///delegate_`, or `http://localhost:5002` for a local backend). The host reads it itself. Wins over `DELEGATE_SDK_ENV` when both are set. | +| `DELEGATE_ENV` | Cloud environment slug (`alpha` / `staging` / `production`), sent as the host's `env` init option. The host derives the backend URL from the org/tenant slugs. | +| `DELEGATE_BACKEND_URL` | Pin the full agent-service URL directly (e.g. `https://cloud.uipath.com///delegate_`, or `http://localhost:5002` for a local backend), sent as the host's `backendUrl` init option. Wins over `DELEGATE_ENV` when both are set. | | `INTEROP_URL` | Connect to an interop instance that is already running, instead of letting the SDK spawn its own. The host reads it itself. | ### 3. Authenticate Either: -- **Environment token** — `AUTH_TOKEN`, `TENANT_ID`, `ORG_ID` env vars. When `DELEGATE_SDK_ENV` (not `BACKEND_URL`) resolves the backend, also set `ORG_LOGICAL_NAME` and `TENANT_NAME`. These are the human-readable org/tenant names, not the GUIDs. All of these are the host's own names: coder_eval passes its environment to the host unchanged, and the host reads them itself. Without the slugs, init fails with `env="alpha" requires org/tenant slugs`. To skip the slugs, set `BACKEND_URL` instead. -- **Saved login** — a prior `npx @uipath/delegate-cli login --env ` that wrote `~/.aria/sdk-auth.json`. The host reads and refreshes this file itself when no `AUTH_TOKEN` is supplied. This agent does not parse that file in Python, so auth-freshness logic lives in exactly one place. The saved login already carries the slugs, so `ORG_LOGICAL_NAME` / `TENANT_NAME` are not needed. +- **Environment token** — `DELEGATE_AUTH_TOKEN`, `DELEGATE_TENANT_ID`, `DELEGATE_ORG_ID` env vars. When `DELEGATE_ENV` (not `DELEGATE_BACKEND_URL`) resolves the backend, also set `DELEGATE_ORG_SLUG` and `DELEGATE_TENANT_SLUG`. These are the human-readable org/tenant names, not the GUIDs. Without the slugs, init fails with `env="alpha" requires org/tenant slugs`. To skip the slugs, set `DELEGATE_BACKEND_URL` instead. Each variable also accepts the bare spelling (`AUTH_TOKEN`, `TENANT_ID`, `ORG_ID`, `ORG_SLUG`, `TENANT_SLUG`) as a fallback. Prefer the `DELEGATE_` spelling in a shared environment, because the bare names collide with what other tooling (npm, Vault, Terraform) commonly exports. +- **Saved login** — a prior `npx @uipath/delegate-cli login --env ` that wrote `~/.aria/sdk-auth.json`. The host reads and refreshes this file itself when no token is supplied. This agent does not parse that file in Python, so auth-freshness logic lives in exactly one place. The saved login already carries the slugs, so `DELEGATE_ORG_SLUG` / `DELEGATE_TENANT_SLUG` are not needed. + +coder_eval sends these values to the host as its `auth` init option, on stdin. It also removes the host's own variable names (`AUTH_TOKEN`, `TENANT_ID`, `ORG_ID`, `ORG_LOGICAL_NAME`, `TENANT_NAME`, `BACKEND_URL`) and `DELEGATE_AUTH_TOKEN` from the host's environment. So the agent's shell commands cannot read the token, and a variable that another tool exports does not change where the host connects. The `auth` init option needs a `@uipath/delegate-stdio` release that accepts it: version 1.202.1 and older ignore it, and then init fails with `Auth required` unless a saved login exists. The host also reads its own advanced variables directly, such as `DELEGATE_AUTH_TOKEN_FILE` (a token file that an external process keeps fresh) and `DELEGATE_STDIO_VERBOSE=1` (trace every frame to stderr). See the [package README](https://www.npmjs.com/package/@uipath/delegate-stdio). @@ -130,7 +132,7 @@ A wall-clock deadline (`timeout`) is enforced both between reads (a top-of-loop On any crash (host death, an `error` frame during a turn, or an unexpected exception), the agent: 1. Sets `pending_turn` to a `crashed=True` TurnRecord with captured telemetry. -2. Raises `AgentCrashError` (retryable) or `AgentConfigError` (non-retryable — missing Node/SDK install, or an init error that a retry cannot fix: missing or rejected auth, missing org/tenant slugs, or an unknown `DELEGATE_SDK_ENV`). Any other init error, and an init that does not respond within 60 s, is retryable. +2. Raises `AgentCrashError` (retryable) or `AgentConfigError` (non-retryable — missing Node/SDK install, or an init error that a retry cannot fix: missing or rejected auth, missing org/tenant slugs, or an unknown `DELEGATE_ENV`). Any other init error, and an init that does not respond within 60 s, is retryable. 3. The orchestrator reads `pending_turn` and calls `discard_pending_turn()` to roll back state. The crash reason never includes the host's stderr. The agent logs the last 20 stderr lines at WARNING instead, because the error categorizer matches words in the reason, and stderr contains the sandbox path, which contains the task id. diff --git a/src/coder_eval/agents/delegate_agent.py b/src/coder_eval/agents/delegate_agent.py index e565a529..20cb9838 100644 --- a/src/coder_eval/agents/delegate_agent.py +++ b/src/coder_eval/agents/delegate_agent.py @@ -87,7 +87,8 @@ # --- Host resolution --------------------------------------------------------- _HOST_PACKAGE = "@uipath/delegate-stdio" -_HOST_BUNDLE_REL_PATH = Path("node_modules") / "@uipath" / "delegate-stdio" / "dist" / "delegate_stdio.mjs" +_HOST_BUNDLE_NAME = "delegate_stdio.mjs" +_HOST_BUNDLE_REL_PATH = Path("node_modules") / "@uipath" / "delegate-stdio" / "dist" / _HOST_BUNDLE_NAME _AGENT_INSTALL_ROOT = Path(__file__).resolve().parent / "delegate" """This agent's own directory; it ships a ``package.json`` that names the host package.""" @@ -122,7 +123,16 @@ "unknown env", ) """Substrings of an init ``error`` message that a retry cannot fix: missing or rejected -auth, or a bad ``DELEGATE_SDK_ENV``. Any other init error is retryable.""" +auth, or a bad ``DELEGATE_ENV``. Any other init error is retryable.""" + +_INIT_CONFIG_ERROR_HINT = ( + "coder_eval sends the auth and the org/tenant slugs as the host's `auth` init option, read from " + + "DELEGATE_AUTH_TOKEN / DELEGATE_TENANT_ID / DELEGATE_ORG_ID / DELEGATE_ORG_SLUG / DELEGATE_TENANT_SLUG " + + "(or the same names without DELEGATE_), and keeps the host's own AUTH_TOKEN / TENANT_ID / ORG_ID / " + + "ORG_LOGICAL_NAME / TENANT_NAME out of its environment. A @uipath/delegate-stdio without the `auth` " + + "init option ignores it. See docs/agents/DELEGATE.md." +) +"""Appended to a config-class init error, whose host message names the host's own variables.""" # Event types (inside the host's `event` frames) that carry model-turn content. # `session_start` / `done` are informational; anything else is logged and ignored. @@ -147,6 +157,44 @@ def _orphan_tool_args(output: Any) -> dict[str, Any]: return args if isinstance(args, dict) else {} +def _env(bare_name: str) -> str | None: + """Read a ``DELEGATE_``-namespaced variable, falling back to the bare name. + + The bare spellings (``AUTH_TOKEN``, ``TENANT_ID``, ...) collide with names + other tooling (npm, Vault, Terraform) commonly exports, so the namespaced + spelling is checked first. + """ + return os.environ.get(f"DELEGATE_{bare_name}") or os.environ.get(bare_name) + + +_AUTH_OPTION_FIELDS: tuple[tuple[str, str], ...] = ( + ("accessToken", "AUTH_TOKEN"), + ("tenantId", "TENANT_ID"), + ("organizationId", "ORG_ID"), + ("orgLogicalName", "ORG_SLUG"), + ("tenantName", "TENANT_SLUG"), +) +"""``(field of the host's auth init option, name read through _env)``.""" + + +def _auth_option() -> dict[str, str]: + """The host's ``auth`` init option, built from the variables that are set.""" + return {field: value for field, name in _AUTH_OPTION_FIELDS if (value := _env(name))} + + +_HOST_ENV_REMOVED = ( + "AUTH_TOKEN", + "TENANT_ID", + "ORG_ID", + "ORG_LOGICAL_NAME", + "TENANT_NAME", + "BACKEND_URL", + "DELEGATE_AUTH_TOKEN", +) +"""Removed from the host's environment. The host reads the first six itself, but +coder_eval sends their values as init options instead; and the agent's shell tools +inherit the host env, so no spelling of the token may stay in it.""" + _GATEWAY_S2S_ENV_VARS = ("LLMGW_CLIENT_ID", "LLMGW_CLIENT_SECRET", "LLMGW_URL") @@ -185,31 +233,38 @@ def _candidate_install_roots() -> list[Path]: def _resolve_host_bundle() -> Path: """Locate the installed ``@uipath/delegate-stdio``'s ``dist/delegate_stdio.mjs``. - Resolution order: ``DELEGATE_STDIO_PATH`` (explicit file path) -> - ``DELEGATE_STDIO_NODE_MODULES`` (explicit install root, probed exactly) -> + Resolution order: ``DELEGATE_SDK_PATH`` (explicit file path) -> + ``DELEGATE_SDK_NODE_MODULES`` (explicit install root, probed exactly) -> ``_candidate_install_roots()``, so an ``npm install`` in this agent's own ``agents/delegate/`` directory is found from any cwd. Raises: - AgentConfigError: no install found anywhere searched. + AgentConfigError: no install found anywhere searched, or ``DELEGATE_SDK_PATH`` + names another file, such as ``@uipath/delegate-sdk``'s ``dist/index.mjs``. """ - explicit = os.environ.get("DELEGATE_STDIO_PATH") + explicit = os.environ.get("DELEGATE_SDK_PATH") if explicit: path = Path(explicit).expanduser().resolve() if not path.is_file(): raise AgentConfigError( - f"DELEGATE_STDIO_PATH={path} does not point to a file. Point it at " - + f"{_HOST_PACKAGE}'s dist/delegate_stdio.mjs." + f"DELEGATE_SDK_PATH={path} does not point to a file. Point it at " + + f"{_HOST_PACKAGE}'s dist/{_HOST_BUNDLE_NAME}." + ) + if path.name != _HOST_BUNDLE_NAME: + raise AgentConfigError( + f"DELEGATE_SDK_PATH={path} must point at {_HOST_PACKAGE}'s dist/{_HOST_BUNDLE_NAME}, " + + "not at @uipath/delegate-sdk's dist/index.mjs or another file. " + + f"Run `npm install {_HOST_PACKAGE}`; see docs/agents/DELEGATE.md." ) return path - root_override = os.environ.get("DELEGATE_STDIO_NODE_MODULES") + root_override = os.environ.get("DELEGATE_SDK_NODE_MODULES") if root_override: path = (Path(root_override).expanduser().resolve() / _HOST_BUNDLE_REL_PATH).resolve() if not path.is_file(): raise AgentConfigError( - f"DELEGATE_STDIO_NODE_MODULES={root_override}: {_HOST_PACKAGE} not found at {path}. " - + f"Run `npm install {_HOST_PACKAGE}` there, or set DELEGATE_STDIO_PATH directly." + f"DELEGATE_SDK_NODE_MODULES={root_override}: {_HOST_PACKAGE} not found at {path}. " + + f"Run `npm install {_HOST_PACKAGE}` there, or set DELEGATE_SDK_PATH directly." ) return path @@ -225,8 +280,8 @@ def _resolve_host_bundle() -> Path: f"{_HOST_PACKAGE} not found. Searched the cwd, its ancestors, this agent's directory, and home:\n" + f" {searched_block}\n" + f"Run `npm install {_HOST_PACKAGE}` (plain, public install — no token needed), " - + f"e.g. in {_AGENT_INSTALL_ROOT}, or set DELEGATE_STDIO_NODE_MODULES " - + "to the install root, or DELEGATE_STDIO_PATH to the dist/delegate_stdio.mjs file directly. " + + f"e.g. in {_AGENT_INSTALL_ROOT}, or set DELEGATE_SDK_NODE_MODULES " + + f"to the install root, or DELEGATE_SDK_PATH to the dist/{_HOST_BUNDLE_NAME} file directly. " + "See docs/agents/DELEGATE.md." ) @@ -458,6 +513,8 @@ async def _spawn_and_init(self) -> None: env.setdefault("DELEGATE_TELEMETRY_DISABLED", "1") if stripped := _strip_redundant_gateway_creds(env): logger.debug("delegate: a token file is configured; removed %s from the host env", ", ".join(stripped)) + for name in _HOST_ENV_REMOVED: + env.pop(name, None) await self._cancel_drain_tasks() self._stdout_queue = asyncio.Queue() @@ -495,7 +552,7 @@ async def _spawn_and_init(self) -> None: await self._force_kill_host() message = str(ack.get("message", "unknown error")) if any(marker in message.lower() for marker in _INIT_CONFIG_ERROR_MARKERS): - raise AgentConfigError(f"Delegate SDK init failed: {message}") + raise AgentConfigError(f"Delegate SDK init failed: {message} {_INIT_CONFIG_ERROR_HINT}") raise AgentCrashError(f"Delegate SDK init failed: {message}") def _build_init_options(self) -> dict[str, Any]: @@ -512,9 +569,14 @@ def _build_init_options(self) -> dict[str, Any]: options["sessionId"] = self.config.session_id if self._env_path_prepend: options["shellPathPrepend"] = list(self._env_path_prepend) - environment = os.environ.get("DELEGATE_SDK_ENV") + backend_url = os.environ.get("DELEGATE_BACKEND_URL") + if backend_url: + options["backendUrl"] = backend_url + environment = os.environ.get("DELEGATE_ENV") if environment: options["env"] = environment + if auth := _auth_option(): + options["auth"] = auth # list[LocalPluginConfig] is not list[dict[str, Any]] under list invariance. skills_path = _resolve_bundled_skills_path(self.config.plugins) # type: ignore[arg-type] options["enableSkills"] = skills_path is not None @@ -608,8 +670,19 @@ async def _teardown_host(self) -> None: await self._cancel_drain_tasks() def get_sdk_options(self) -> dict[str, Any] | None: - """The ``init`` options sent to the last host spawned, or ``None`` before ``start()``.""" - return self._init_options + """The ``init`` options sent to the last host spawned, or ``None`` before ``start()``. + + Credentials are redacted, because the result is persisted with the run: + ``auth`` becomes the names of its fields, and ``backendUrl`` its host. + """ + if self._init_options is None: + return None + options = dict(self._init_options) + if "auth" in options: + options["auth"] = sorted(options["auth"]) + if "backendUrl" in options: + options["backendUrl"] = urlparse(options["backendUrl"]).hostname + return options def get_environment_info(self) -> dict[str, Any]: info: dict[str, Any] = { @@ -618,11 +691,11 @@ def get_environment_info(self) -> dict[str, Any]: } if self.config.enable_computer_use: info["delegate_enable_computer_use"] = True - # The host ranks BACKEND_URL over the env slug (see docs/agents/DELEGATE.md), + # The host ranks backendUrl over the env slug (see docs/agents/DELEGATE.md), # so recording delegate_env here too would assert a routing decision the SDK # never made. Host only (never the full URL, which can carry embedded credentials). - backend_url = os.environ.get("BACKEND_URL") - environment = os.environ.get("DELEGATE_SDK_ENV") + backend_url = os.environ.get("DELEGATE_BACKEND_URL") + environment = os.environ.get("DELEGATE_ENV") if backend_url: info["delegate_backend_url_host"] = urlparse(backend_url).hostname elif environment: diff --git a/tests/test_delegate_agent.py b/tests/test_delegate_agent.py index 11a8f2a4..ebc513f5 100644 --- a/tests/test_delegate_agent.py +++ b/tests/test_delegate_agent.py @@ -146,6 +146,29 @@ async def fake_exec(*args: Any, **kwargs: Any) -> _FakeProcess: return _install +_DELEGATE_ENV_NAMES = ( + "DELEGATE_SDK_PATH", + "DELEGATE_SDK_NODE_MODULES", + "DELEGATE_ENV", + "DELEGATE_BACKEND_URL", + *( + f"{prefix}{name}" + for prefix in ("", "DELEGATE_") + for name in ("AUTH_TOKEN", "TENANT_ID", "ORG_ID", "ORG_SLUG", "TENANT_SLUG") + ), + "ORG_LOGICAL_NAME", + "TENANT_NAME", + "BACKEND_URL", +) + + +@pytest.fixture(autouse=True) +def _clean_delegate_env(monkeypatch: pytest.MonkeyPatch) -> None: + """The adapter builds init options from these names, so the developer's own values must not leak in.""" + for name in _DELEGATE_ENV_NAMES: + monkeypatch.delenv(name, raising=False) + + def _config(**overrides: Any) -> DelegateAgentConfig: return DelegateAgentConfig(type=AgentKind.DELEGATE, **overrides) @@ -163,31 +186,39 @@ class TestResolveHostBundle: def test_explicit_path_env_var(self, tmp_path, monkeypatch): entry = tmp_path / "delegate_stdio.mjs" entry.write_text("#!/usr/bin/env node") - monkeypatch.setenv("DELEGATE_STDIO_PATH", str(entry)) + monkeypatch.setenv("DELEGATE_SDK_PATH", str(entry)) assert _resolve_host_bundle() == entry.resolve() + def test_explicit_path_to_another_file_raises(self, tmp_path, monkeypatch): + sdk_entry = tmp_path / "node_modules" / "@uipath" / "delegate-sdk" / "dist" / "index.mjs" + sdk_entry.parent.mkdir(parents=True) + sdk_entry.write_text("export {}") + monkeypatch.setenv("DELEGATE_SDK_PATH", str(sdk_entry)) + with pytest.raises(AgentConfigError, match="must point at @uipath/delegate-stdio"): + _resolve_host_bundle() + def test_explicit_path_missing_file_raises(self, tmp_path, monkeypatch): - monkeypatch.setenv("DELEGATE_STDIO_PATH", str(tmp_path / "nope.mjs")) + monkeypatch.setenv("DELEGATE_SDK_PATH", str(tmp_path / "nope.mjs")) with pytest.raises(AgentConfigError, match="does not point to a file"): _resolve_host_bundle() def test_node_modules_override(self, tmp_path, monkeypatch): - monkeypatch.delenv("DELEGATE_STDIO_PATH", raising=False) - monkeypatch.setenv("DELEGATE_STDIO_NODE_MODULES", str(tmp_path)) + monkeypatch.delenv("DELEGATE_SDK_PATH", raising=False) + monkeypatch.setenv("DELEGATE_SDK_NODE_MODULES", str(tmp_path)) entry = tmp_path / "node_modules" / "@uipath" / "delegate-stdio" / "dist" / "delegate_stdio.mjs" entry.parent.mkdir(parents=True) entry.write_text("#!/usr/bin/env node") assert _resolve_host_bundle() == entry.resolve() def test_node_modules_override_missing_raises(self, tmp_path, monkeypatch): - monkeypatch.delenv("DELEGATE_STDIO_PATH", raising=False) - monkeypatch.setenv("DELEGATE_STDIO_NODE_MODULES", str(tmp_path)) + monkeypatch.delenv("DELEGATE_SDK_PATH", raising=False) + monkeypatch.setenv("DELEGATE_SDK_NODE_MODULES", str(tmp_path)) with pytest.raises(AgentConfigError, match="not found"): _resolve_host_bundle() def test_no_config_and_not_found_raises_with_search_list(self, tmp_path, monkeypatch): - monkeypatch.delenv("DELEGATE_STDIO_PATH", raising=False) - monkeypatch.delenv("DELEGATE_STDIO_NODE_MODULES", raising=False) + monkeypatch.delenv("DELEGATE_SDK_PATH", raising=False) + monkeypatch.delenv("DELEGATE_SDK_NODE_MODULES", raising=False) monkeypatch.chdir(tmp_path) monkeypatch.setattr(os, "getcwd", lambda: str(tmp_path)) # A real npm install under the developer's actual home directory must @@ -231,6 +262,14 @@ async def test_only_an_init_error_a_retry_cannot_fix_is_non_retryable( await agent.start(str(tmp_path)) assert agent._process is None + async def test_a_config_init_error_names_the_variables_coder_eval_reads(self, patch_exec, tmp_path): + """The host's message names the host's own variables, which coder_eval keeps out of its env.""" + message = 'env="alpha" requires org/tenant slugs. Set ORG_LOGICAL_NAME and TENANT_NAME env vars' + patch_exec([_line({"type": "error", "message": message})]) + agent = DelegateAgent(_config()) + with pytest.raises(AgentConfigError, match="DELEGATE_ORG_SLUG / DELEGATE_TENANT_SLUG"): + await agent.start(str(tmp_path)) + async def test_init_timeout_is_retryable(self, patch_exec, tmp_path, monkeypatch): monkeypatch.setattr(agent_module, "_INIT_TIMEOUT_SEC", 0.05) patch_exec([], hang_after=True) @@ -279,21 +318,30 @@ async def test_unsupported_fields_warn(self, patch_exec, tmp_path, caplog): await _started_agent(patch_exec, [], tmp_path, system_prompt="be nice") assert any("has no Delegate SDK equivalent" in r.message for r in caplog.records) - async def test_delegate_sdk_env_becomes_the_env_option(self, patch_exec, tmp_path, monkeypatch): - monkeypatch.setenv("DELEGATE_SDK_ENV", "alpha") - _agent, proc = await _started_agent(patch_exec, [], tmp_path) + async def test_delegate_env_becomes_the_env_option(self, patch_exec, tmp_path, monkeypatch): + monkeypatch.setenv("DELEGATE_ENV", "alpha") + agent, proc = await _started_agent(patch_exec, [], tmp_path) assert proc.stdin.written[0]["options"]["env"] == "alpha" + assert agent.get_environment_info()["delegate_env"] == "alpha" - async def test_backend_url_is_left_for_the_host_to_read(self, patch_exec, tmp_path, monkeypatch): - monkeypatch.setenv("BACKEND_URL", "https://user:secret@backend.example/delegate_") - monkeypatch.setenv("DELEGATE_SDK_ENV", "alpha") + async def test_delegate_backend_url_becomes_the_backend_url_option(self, patch_exec, tmp_path, monkeypatch): + monkeypatch.setenv("DELEGATE_BACKEND_URL", "https://user:secret@backend.example/delegate_") + monkeypatch.setenv("DELEGATE_ENV", "alpha") agent, proc = await _started_agent(patch_exec, [], tmp_path) - assert "backendUrl" not in proc.stdin.written[0]["options"] - assert proc.spawn_kwargs["env"]["BACKEND_URL"] == "https://user:secret@backend.example/delegate_" + assert proc.stdin.written[0]["options"]["backendUrl"] == "https://user:secret@backend.example/delegate_" info = agent.get_environment_info() assert info["delegate_backend_url_host"] == "backend.example" assert "delegate_env" not in info + async def test_the_host_never_reads_its_own_auth_or_backend_names(self, patch_exec, tmp_path, monkeypatch): + """A BACKEND_URL or ORG_LOGICAL_NAME exported for another tool must not route the host.""" + host_names = ("AUTH_TOKEN", "TENANT_ID", "ORG_ID", "ORG_LOGICAL_NAME", "TENANT_NAME", "BACKEND_URL") + for name in host_names: + monkeypatch.setenv(name, "value-for-another-tool") + _agent, proc = await _started_agent(patch_exec, [], tmp_path) + env = proc.spawn_kwargs["env"] + assert not [name for name in host_names if name in env] + async def test_skills_enabled_only_with_a_plugin(self, patch_exec, tmp_path): _agent, proc = await _started_agent(patch_exec, [], tmp_path) assert proc.stdin.written[0]["options"]["enableSkills"] is False @@ -307,23 +355,50 @@ async def test_skills_enabled_only_with_a_plugin(self, patch_exec, tmp_path): assert options["enableSkills"] is True assert options["bundledSkillsPath"] == str(plugin_dir / "skills") - async def test_auth_reaches_the_host_under_its_own_env_names(self, patch_exec, tmp_path, monkeypatch): - """The host reads auth from its environment, so nothing auth-shaped goes into init options.""" - auth = { - "AUTH_TOKEN": "tok-1", - "TENANT_ID": "tenant-guid", - "ORG_ID": "org-guid", - "ORG_LOGICAL_NAME": "my-org", - "TENANT_NAME": "my-tenant", - } - for name, value in auth.items(): + async def test_auth_reaches_the_host_as_the_auth_init_option(self, patch_exec, tmp_path, monkeypatch): + """The token goes to the host on stdin only, so the agent's shells cannot read it from their env.""" + for name, value in { + "DELEGATE_AUTH_TOKEN": "tok-1", + "DELEGATE_TENANT_ID": "tenant-guid", + "DELEGATE_ORG_ID": "org-guid", + "DELEGATE_ORG_SLUG": "my-org", + "DELEGATE_TENANT_SLUG": "my-tenant", + }.items(): monkeypatch.setenv(name, value) - monkeypatch.setenv("DELEGATE_AUTH_TOKEN", "old-namespaced-token") _agent, proc = await _started_agent(patch_exec, [], tmp_path) - env = proc.spawn_kwargs["env"] - assert {name: env[name] for name in auth} == auth + assert proc.stdin.written[0]["options"]["auth"] == { + "accessToken": "tok-1", + "tenantId": "tenant-guid", + "organizationId": "org-guid", + "orgLogicalName": "my-org", + "tenantName": "my-tenant", + } + assert "tok-1" not in proc.spawn_kwargs["env"].values() + + async def test_the_namespaced_name_wins_and_the_bare_name_is_the_fallback(self, patch_exec, tmp_path, monkeypatch): + monkeypatch.setenv("DELEGATE_AUTH_TOKEN", "namespaced-token") + monkeypatch.setenv("AUTH_TOKEN", "bare-token") + monkeypatch.setenv("ORG_SLUG", "bare-org") + _agent, proc = await _started_agent(patch_exec, [], tmp_path) + auth = proc.stdin.written[0]["options"]["auth"] + assert auth == {"accessToken": "namespaced-token", "orgLogicalName": "bare-org"} + + async def test_no_auth_variables_send_no_auth_option(self, patch_exec, tmp_path): + """The host then falls back to the saved login.""" + _agent, proc = await _started_agent(patch_exec, [], tmp_path) assert "auth" not in proc.stdin.written[0]["options"] + async def test_sdk_options_redact_the_credentials(self, patch_exec, tmp_path, monkeypatch): + """The run records sdk_options, so the token and a credential-bearing URL must not reach it.""" + monkeypatch.setenv("DELEGATE_AUTH_TOKEN", "tok-1") + monkeypatch.setenv("DELEGATE_TENANT_ID", "tenant-guid") + monkeypatch.setenv("DELEGATE_BACKEND_URL", "https://user:secret@backend.example/delegate_") + agent, _proc = await _started_agent(patch_exec, [], tmp_path) + options = agent.get_sdk_options() or {} + assert options["auth"] == ["accessToken", "tenantId"] + assert options["backendUrl"] == "backend.example" + assert "tok-1" not in json.dumps(options) + @pytest.mark.parametrize( ("token_file_env", "stripped"), [ diff --git a/tests/test_delegate_agent_live.py b/tests/test_delegate_agent_live.py index 68f6daba..ff65246c 100644 --- a/tests/test_delegate_agent_live.py +++ b/tests/test_delegate_agent_live.py @@ -2,8 +2,9 @@ Hit a real Delegate backend through a real Node subprocess; skipped by default, only run with ``pytest -m live``. Needs Node + a real ``@uipath/delegate-stdio`` -install, UiPath auth (``AUTH_TOKEN``/``TENANT_ID``/``ORG_ID`` or a saved login), -a backend (``DELEGATE_SDK_ENV``/``BACKEND_URL``), and optionally +install, UiPath auth (``DELEGATE_AUTH_TOKEN``/``DELEGATE_TENANT_ID``/``DELEGATE_ORG_ID``, or the +same names without ``DELEGATE_``, or a saved login), a backend +(``DELEGATE_ENV``/``DELEGATE_BACKEND_URL``), and optionally ``DELEGATE_MODEL``. A failure here while the unit tests pass usually means the host's frame shapes @@ -32,8 +33,9 @@ def _have_prerequisites() -> bool: _resolve_host_bundle() except AgentConfigError: return False - has_auth = bool(os.getenv("AUTH_TOKEN")) or (Path.home() / ".aria" / "sdk-auth.json").is_file() - has_backend = bool(os.getenv("DELEGATE_SDK_ENV") or os.getenv("BACKEND_URL")) + has_token = bool(os.getenv("DELEGATE_AUTH_TOKEN") or os.getenv("AUTH_TOKEN")) + has_auth = has_token or (Path.home() / ".aria" / "sdk-auth.json").is_file() + has_backend = bool(os.getenv("DELEGATE_ENV") or os.getenv("DELEGATE_BACKEND_URL")) return has_auth and has_backend diff --git a/tests/test_error_handling.py b/tests/test_error_handling.py index b3b6e711..8ea01dd3 100644 --- a/tests/test_error_handling.py +++ b/tests/test_error_handling.py @@ -252,7 +252,7 @@ def test_categorize_agent_config_error_message_does_not_misroute(self): """The typed check wins even when the message would match a retryable string pattern.""" # "connection" would otherwise route to the retryable AGENT_API_ERROR. result = categorize_error( - AgentConfigError("DELEGATE_STDIO_PATH unset; cannot establish connection"), + AgentConfigError("DELEGATE_SDK_PATH unset; cannot establish connection"), {"component": "agent"}, ) assert result == ErrorCategory.AGENT_CONFIG_ERROR From 4563218a7c0489f9fe75a356dde9ec8c557f9a1a Mon Sep 17 00:00:00 2001 From: Mihai Chirculescu Date: Thu, 1 Oct 2026 16:45:51 +0300 Subject: [PATCH 10/20] docs(notes): shorten the Delegate agent section Keep each decision and its reason; drop the narration and the repeated detail. Co-Authored-By: Claude Opus 5.5 (1M context) --- .claude/notes/agents.md | 249 ++++++++++++++-------------------------- 1 file changed, 85 insertions(+), 164 deletions(-) diff --git a/.claude/notes/agents.md b/.claude/notes/agents.md index 17175cb6..a5cc250e 100644 --- a/.claude/notes/agents.md +++ b/.claude/notes/agents.md @@ -743,167 +743,89 @@ release adding an unclassified field fails loudly instead of silently passing th ## Delegate agent -`DelegateAgent` drives UiPath Autopilot's Delegate agent — reasoning in the UiPath backend, -tools executing locally through the SDK's bundled interop process. It spawns the host that the -public `@uipath/delegate-stdio` npm package ships (`dist/delegate_stdio.mjs`); that package depends -on `@uipath/delegate-sdk` and the platform's `@uipath/delegate-runtime-*` interop binaries, so one -`npm install` is the whole prerequisite. The package README is the wire-protocol SSOT; every frame -shape the adapter reads (`event` wrapper, `toolArgs`/`toolResult`/`toolStatus`, `isStepStart`, -`result.usage`/`turnUsages`/`model`, top-level `error`, `destroyed`) was confirmed against a live -`1.202.1` transcript before the port, and `tests/test_delegate_agent.py`'s frame builders replay -those shapes. - -An earlier version of this agent wrapped `@uipath/delegate-sdk` in a first-party host -(`delegate_host.mjs`) because `delegate-stdio` was not yet public. That host and its protocol are -gone; git history has them. - -**Auth and the backend reach the host as init options, not through its environment.** coder_eval -keeps its own names: `_env` reads the `DELEGATE_`-namespaced spelling first, then the bare -`AUTH_TOKEN` / `TENANT_ID` / `ORG_ID` / `ORG_SLUG` / `TENANT_SLUG`, and `_auth_option` sends the -values as the host's `auth` init option. `DELEGATE_BACKEND_URL` becomes `backendUrl` and -`DELEGATE_ENV` becomes `env`. It does not adopt the host's own names (`AUTH_TOKEN`, `TENANT_ID`, -`ORG_ID`, `ORG_LOGICAL_NAME`, `TENANT_NAME`, `BACKEND_URL`), because the bare names collide with -other tooling and the CI secrets are named after the `DELEGATE_` spellings. - -The host gives each `auth` field priority over its environment variable, and the SDK reads none of -these names. It takes the credentials from the `TokenAuthProvider` the host builds, and an explicit -`backendUrl` replaces the `BACKEND_URL` default its bundle reads at load. So `_HOST_ENV_REMOVED` -takes the host's names, and the token under either spelling, out of the host's environment. The -agent's shell tools inherit that environment, so a static token there is readable by the code under -test. And a `BACKEND_URL` or `ORG_LOGICAL_NAME` that another tool exports can no longer route the -host. A refresh source (token file, `LLMGW_*` pair, saved login) still writes its fresh token into -the host's own `process.env.AUTH_TOKEN`. - -The option needs a host release that accepts it. An older host ignores it and fails init with -`Auth required` unless a saved login exists; `_INIT_CONFIG_ERROR_HINT` names coder_eval's variables -beside the host's message, which names the host's. Confirmed live against alpha with a host built -from the Autopilot branch and `DELEGATE_STDIO_VERBOSE=1`: the host resolved auth from the option, -not the saved login, and neither its stderr nor the recorded `sdk_options` contained the token. -Without the slugs, init fails with `env="alpha" requires org/tenant slugs`. - -**Effort rides `sdk_options.effort`.** It is the same key as Claude Code, so one -`-D agent.sdk_options.effort=...` drives both agents, and the reports' Effort row reads it from -`get_sdk_options()`. The `-D` gate admits any registered kind whose config class declares -`sdk_options`; `DelegateAgentConfig` validates the keys against the host options it forwards. - -`get_sdk_options()` returns the whole `init` options dict sent to the host, not the user's -pass-through dict, with the credentials redacted: `auth` becomes its field names and `backendUrl` -its host, because the result is persisted in `task.json`. Both reports use `EvaluationResult.sdk_options` in place of `agent_config` -whenever it is truthy, so a pass-through-only dict hid the Model row from any run that set an -effort. The init dict carries `model` and `effort`. The reports then show Permission Mode as N/A -and Allowed Tools as "(all)", which is correct for this SDK. There is no Plugins row, because the -init dict names the skills directory (`bundledSkillsPath`), not a `plugins` list. - -**`max_turns` stays client-side.** The host accepts `maxSteps` on `send`, but it does not stop the -turn: live, `maxSteps: 2` ran 7 steps and only reported `maxStepsReached: true` in the `result`. -Forwarding it would suggest a cap that does not exist, so the adapter keeps counting calls from the -event stream and abandons the host when call N+1 opens. Once the `result` arrives, `len(turnUsages)` -(one entry per backend round-trip, confirmed equal to `assistantStepCount`) replaces the running -estimate as `num_turns`. - -**A cut turn keeps the usage of its finished calls.** The host has no interrupt command, so a turn -ended at `max_turns` or by a cooperative stop never gets its `result` frame. The host therefore -also writes one `usage` frame per backend round-trip, and the adapter sums them; the `result`'s -`usage` (their total) replaces that sum when it arrives. The order is what makes this work, and it -comes from reading the SDK and backend source, not from a live transcript: the backend sends a -request's `usage` chunk before its stream closes, the client runs the returned tools only after the -stream closes, and the host flushes new `usage` entries before it writes each event. So call N's -frame always precedes the `tool_result` that opens call N+1 and fires the cap. The in-flight call at -the cut has no usage, as on the other harnesses. A host from before the frame existed sends none, -and the adapter warns when a cut turn had finished calls but no usage. - -The frame is not a call boundary. It arrives before its call's tools run, so a cap that fired on -frame N would stop call N's tools, and one that waited for frame N+1 would let call N+1 run to its end. -The cap stays on "the previous call's tools have all returned". - -**`enableSkills` must be sent explicitly.** The host's default is `false`, so a `bundledSkillsPath` -alone loads nothing; the adapter sets `enableSkills` to whether a plugin resolved. - -**No multi-generation transcript splitting (yet).** `DelegateAgent` builds exactly ONE -`AssistantMessage` per `communicate()` call. The host now carries what a per-round-trip split needs — -`isStepStart` on `message` events and per-round-trip `turnUsages` on `result` — but the adapter only -uses `isStepStart` to merge streamed deltas into one text block. `timing.close_window` opens that one -window from the turn's own start, so the head and tail both measure ~0. - -**Known host failures are recategorized.** The out-of-tree `delegate-sdk` adapter drove this same -`delegate-stdio` host against this same backend, so three of its failure signatures are ported, in -`_describe_host_error` / `_crash_on_host_error`. Each routes a host `error` frame to the category -`errors/categorization.py` should give it: - -- **Cloudflare WAF block page** (`Continue with UiPath Platform`, "not available in - your country"). A 403 from a managed WAF rule that matched shell-like text in the request body — - for example a skill doc's `python -c "...open(...,'w')..."` echoed back by a file read. It is not a - geo or auth block. The same payload is blocked again on retry, so the reason is rewritten to carry - "content filter" (`AGENT_INVALID_OUTPUT`, not retried). Two markers, because the host truncates its - message near 50 KB and the country sentence sits after ~48 KB of inline font CSS; the `` is - in the first ~300 bytes. -- **SSE connect timeout.** The request got no response headers inside the SDK's connect watchdog - (30 s) on each of its three internal attempts. The raw "timeout" would route to `AGENT_TIMEOUT`, - which is not retried, but no headers means the backend never started the turn — a transient - availability window. The rewrite carries "connection" (`AGENT_API_ERROR`, retried with backoff) and - defangs "timeout". -- **Session conflict** ("A reply is already being generated"). A backend 409 for a send into a - conversation whose previous generation still runs. A retry into that conversation can only - conflict again, so the remembered `_session_id` is dropped and the retry starts a new one. The host - is replaced either way. A `session_id` pinned in config still reaches the new host as an init - option, so a pinned run does not recover from this. - -**No crash reason carries the stderr tail.** `errors/categorization.py` matches substrings of the -reason, and the tail is incidental text: the SDK's own watchdog lines, PIDs, token lifetimes, and -the host's `Working directory:` line, which holds the sandbox path and so the task id. In build -13599116 the same transient `Delegate backend error: terminated` was retried on four tasks and -ended `skill-review-agents-lowcode-guardrail-unknown-validator` as a non-retryable -`AGENT_INVALID_OUTPUT`, because "guardrail" in its path matched the content-filter rule. -`_log_stderr_tail` logs the tail at WARNING instead, on every crash path. - -**Only an init error that a retry cannot fix is non-retryable.** An init `error` frame raises -`AgentConfigError` only when it matches `_INIT_CONFIG_ERROR_MARKERS`: missing or rejected auth -(the out-of-tree adapter's auth markers, kept as they were), missing org/tenant slugs, or an -unknown env slug. Any other init error, and the 60 s init deadline, raise a retryable -`AgentCrashError`, so a backend 5xx or a slow token refresh during `agent.initialize()` gets -`execute_with_retry('Agent start')`'s retries instead of ending the task at once. - -**`LLMGW_*` leaves the host env only when a token file is configured.** The host's shells inherit -its env, so the gateway client secret there is readable by the code under test. But the host's own -`selectTokenSource` uses that S2S pair as its refresh source whenever no token file is set, so -stripping it unconditionally would end token refresh an hour into a long run. With a token file -the file wins and the pair is dead weight, so `_strip_redundant_gateway_creds` removes it — mirroring -the host's lookup exactly (`DELEGATE_AUTH_TOKEN_FILE ?? AUTH_TOKEN_FILE`, split on the path -delimiter, blank entries ignored; an empty `DELEGATE_AUTH_TOKEN_FILE` shadows the legacy name). -Removing it in every case needs the out-of-tree adapter's S2S token-file refresher, which mints -adapter-side and hands the host a file. - -Not ported: the first-response stall-timeout+resend (opt-in, and a stall already ends as a turn -timeout) and that S2S token-file refresher. Port either once its need is observed here. - -**The process handle must be cleared on every path that leaves the host dead or dying** — EOF, an -`error` frame, a timeout (both the top-of-loop pre-check AND a timeout elapsing while blocked inside -`asyncio.wait_for`), cooperative stop, and `max_turns` exhaustion. A real bug shipped once during this -agent's own development: a timeout elapsing mid-read fell through to the generic crash path instead of -`TurnTimeoutError`, and left the process handle set, so the NEXT `communicate()` call reused a host with -a `"send"` still nominally in flight instead of respawning — risking a stale response being consumed as -the new turn's. `tests/test_delegate_agent.py`'s `test_timeout_elapsing_mid_read_still_raises_turn_timeout_error` -pins the fix. The host survives a failed `send` (it answers the next `destroy` with `destroyed`), but -the adapter still does not reuse it, so no late event from the failed turn can leak into the retry. -`_force_kill_host` drops the handle itself, before its reap, so every path that kills the host -clears it, even when the 5 s reap times out. Each call site used to pair the kill with its own -clear, and `kill()` did not. - -**Skills mapping is deliberately NOT the shared `agents/_skills.py` resolver.** That resolver enumerates -individual skill directories for a repeated `--skill <dir>`-style CLI argument (OpenCode/Pi's shape). -The Delegate SDK's `bundledSkillsPath` wants exactly ONE parent directory whose children are skill -folders — a genuinely different shape — so `_resolve_bundled_skills_path` maps `<plugin.path>/skills`, -first plugin wins, instead of force-fitting the shared helper. - -**Registered unconditionally**, exactly like `codex`/`antigravity` — there is no conditional-registration -gate keyed on an optional-dependency extra anywhere in this codebase; `opencode`/`pi` declare an EMPTY -extra purely as packaging-metadata documentation, and `delegate` follows that same shape (no new pip -package — Node/`@uipath/delegate-stdio` is the real prerequisite, resolved lazily in `start()`). - -**Windows path length.** The interop binary sits deep under `node_modules/@uipath/delegate-runtime-win32-x64/`; -an install root long enough to push that path past 260 characters makes the SDK's spawn fail with -`ENOENT` even though the file exists. Seen live from a scratch directory; the docs tell users to -install into a short path. +`DelegateAgent` drives UiPath Autopilot's Delegate agent: reasoning in the UiPath backend, tools run +locally. It spawns the host from the public `@uipath/delegate-stdio` npm package +(`dist/delegate_stdio.mjs`), which pulls in `@uipath/delegate-sdk` and the interop binaries. The +package README is the wire-protocol SSOT. Every frame shape the adapter reads was confirmed against a +live `1.202.1` transcript; `tests/test_delegate_agent.py`'s frame builders replay those shapes. (An +earlier first-party `delegate_host.mjs` wrapper is gone; see git history.) + +**Auth goes to the host as the `auth` init option, not through its env.** `_env` reads the +`DELEGATE_*` name first, then the bare `AUTH_TOKEN` / `TENANT_ID` / `ORG_ID` / `ORG_SLUG` / +`TENANT_SLUG`. `DELEGATE_BACKEND_URL` → `backendUrl`, `DELEGATE_ENV` → `env`. We do not adopt the +host's own names (`ORG_LOGICAL_NAME`, `BACKEND_URL`, …): bare names collide with other tooling, and CI +secrets use the `DELEGATE_*` names. `_HOST_ENV_REMOVED` strips the host's names and the token from the +host env, because the agent's shells inherit it (token leak to code under test) and a stray +`BACKEND_URL` would reroute the host. A refresh source still writes the fresh token into the host's +own `process.env.AUTH_TOKEN`. An older host ignores the option and fails init with `Auth required`; +`_INIT_CONFIG_ERROR_HINT` names our variables beside the host's message. `env=<slug>` without org/tenant +slugs fails init. + +**`LLMGW_*` leaves the host env only when a token file is set.** Without a token file, the host's +`selectTokenSource` uses that S2S pair to refresh, so stripping it always would end refresh after an +hour. `_strip_redundant_gateway_creds` mirrors the host's lookup exactly +(`DELEGATE_AUTH_TOKEN_FILE ?? AUTH_TOKEN_FILE`, path-delimiter split, blanks ignored). + +**Effort rides `sdk_options.effort`** — the same key as Claude Code, so one `-D` drives both. +`get_sdk_options()` returns the whole `init` dict (so the reports show Model and Effort), with +credentials redacted: `auth` → its field names, `backendUrl` → its host. It is persisted in +`task.json`. + +**`enableSkills` must be sent explicitly**: the host default is `false`, so `bundledSkillsPath` alone +loads nothing. `_resolve_bundled_skills_path` maps `<plugin.path>/skills` (first plugin wins), not the +shared `agents/_skills.py` resolver: the SDK wants ONE parent directory, not a list of skill dirs. + +**`max_turns` stays client-side.** The host's `maxSteps` does not stop the turn (live: `maxSteps: 2` +ran 7 steps). The adapter counts calls from the event stream and abandons the host when call N+1 +opens; `len(turnUsages)` from the `result` then replaces the estimate as `num_turns`. + +**A cut turn keeps the usage of its finished calls.** A turn cut by `max_turns` or a stop gets no +`result`, so the adapter sums the per-round-trip `usage` frames; the `result`'s total replaces the sum +when it arrives. Ordering (from SDK/backend source, not a live transcript): call N's `usage` frame +always precedes the `tool_result` that opens call N+1. The frame is NOT a call boundary — it arrives +before its call's tools run — so the cap stays on "the previous call's tools have all returned". The +adapter warns when a cut turn had finished calls but no usage (an old host). + +**One `AssistantMessage` per `communicate()`.** `isStepStart` and `turnUsages` would allow a +per-round-trip split; today `isStepStart` only merges streamed deltas. `close_window` opens the one +window at turn start, so head and tail measure ~0. + +**Known host failures are recategorized** in `_describe_host_error` / `_crash_on_host_error`, ported +from the out-of-tree `delegate-sdk` adapter that drove the same host and backend: + +- **Cloudflare WAF block page** (`<title>Continue with UiPath Platform` or "not available in + your country"): a WAF rule matched shell-like text in the request, not a geo/auth block. A retry + is blocked again, so the reason carries "content filter" (`AGENT_INVALID_OUTPUT`). Two markers, + because the host truncates near 50 KB and the country sentence sits after ~48 KB of CSS. +- **SSE connect timeout**: no response headers, so the backend never started the turn. Rewritten to + "connection" (`AGENT_API_ERROR`, retried) with "timeout" removed, which would be `AGENT_TIMEOUT` + (not retried). +- **Session conflict** ("A reply is already being generated", 409): `_session_id` is dropped so the + retry starts a new conversation. A `session_id` pinned in config does not recover. + +Not ported: stall-timeout+resend and the S2S token-file refresher. Port them when the need shows. + +**No crash reason carries the stderr tail.** Categorization matches substrings of the reason, and the +tail holds incidental text (the sandbox path, so the task id). In build 13599116 "guardrail" in a task +path turned a transient error into non-retryable `AGENT_INVALID_OUTPUT`. `_log_stderr_tail` logs the +tail at WARNING instead. + +**Only an init error that a retry cannot fix is non-retryable.** `AgentConfigError` only on +`_INIT_CONFIG_ERROR_MARKERS` (auth missing/rejected, missing slugs, unknown env). Any other init error +and the 60 s init deadline raise a retryable `AgentCrashError`. + +**Clear the process handle on every path that leaves the host dead or dying** — EOF, `error` frame, +timeout (pre-check AND mid-`wait_for`), stop, `max_turns`. A shipped bug left it set after a mid-read +timeout, so the next turn reused a host with a `send` in flight +(`test_timeout_elapsing_mid_read_still_raises_turn_timeout_error` pins it). `_force_kill_host` clears +the handle before its reap. The adapter never reuses a host after a failed `send`, so no late event +leaks into the retry. + +**Registered unconditionally**, like `codex`/`antigravity`; its extra is empty. Node and +`@uipath/delegate-stdio` are the real prerequisite, resolved lazily in `start()`. + +**Windows path length**: an install root that pushes the interop binary path past 260 characters makes +the spawn fail with `ENOENT`. The docs tell users to install into a short path. ### Delegate agent pricing @@ -937,9 +859,8 @@ introduced or should silently "fix" by picking one number over the other. `AgentKind.DELEGATE` is excluded from `tests/test_agent_golden_master.py`'s `_NO_GOLDEN_COVERAGE` allowlist rather than given fixture scenarios: a golden snapshot pins a byte-identical `TurnRecord` -for a scripted event stream. The frame shapes are now confirmed against a live `delegate-stdio` -transcript (see "Delegate agent" above), so this exemption can be lifted by recording real scenarios; -until then `tests/test_delegate_agent.py` replays those shapes directly. +for a scripted event stream. The frame shapes are now confirmed live, so record real scenarios to +lift this; until then `tests/test_delegate_agent.py` replays the shapes. This does NOT extend to `tests/test_timing_identity_contract.py::test_delegate_buckets_tile_the_turn`, which IS a real, unexempted case: the ms-exact four-bucket identity depends only on this agent's own From 0ea0d998bacd092e6c9972281dec18e61ac80511 Mon Sep 17 00:00:00 2001 From: Mihai Chirculescu Date: Thu, 1 Oct 2026 17:27:15 +0300 Subject: [PATCH 11/20] fix(delegate): record the SDK's LoadSkill call as the canonical Skill call Since UiPath/Autopilot#6477 the Delegate SDK loads a catalog skill with LoadSkill {"name", "plugin"} instead of reading its SKILL.md, and the delegate-stdio host passes the name through unaliased. skill_triggered reads only `Skill` + `skill` or a `skills//` path, so it never saw a Delegate skill load. DelegateAgent now maps LoadSkill {name} to Skill {skill}. The rename is keyed by the host's tool name: the host already reports ExecuteSkillApi as `Skill`, and that call's `name` is an API call, not a skill load. _tool_call's parameters are positional-only so a test can pass a `name=` tool arg. Co-Authored-By: Claude Opus 5.5 (1M context) --- .claude/notes/agents.md | 7 +++++- docs/agents/DELEGATE.md | 2 ++ src/coder_eval/agents/delegate_agent.py | 21 +++++++++++++++--- tests/test_delegate_agent.py | 29 ++++++++++++++++++++++++- 4 files changed, 54 insertions(+), 5 deletions(-) diff --git a/.claude/notes/agents.md b/.claude/notes/agents.md index a5cc250e..fb116b8b 100644 --- a/.claude/notes/agents.md +++ b/.claude/notes/agents.md @@ -500,7 +500,7 @@ shell-aware `parameters["command"]` extraction in `criteria/command_executed.py` to raw-JSON matching — so the same task scores differently per harness. Unknown names pass through unchanged. -Three cases are worth knowing: +Four cases are worth knowing: - **OpenCode's tool set varies by MODEL within the one harness.** A live 174-task run showed DeepSeek using `write`/`edit` 199 times and `apply_patch` 0, while GPT-5.6 used @@ -513,6 +513,11 @@ Three cases are worth knowing: - **Pi's search tool is `find`** (glob-by-pattern), not `glob`; there is no `glob` tool in its built-in set, so mapping `find` to the canonical `Glob` is what keeps `command_executed` and `commands_efficiency` comparable. +- **Delegate's host maps most names itself, but not `LoadSkill`.** `delegate-stdio` reports + `ReadFile` as `Read` and `ExecuteSkillApi` as `Skill`, so `DelegateAgent` maps only + `LoadSkill {name}` to `Skill {skill}`; without it `skill_triggered` never sees a Delegate skill + load. The rename is keyed by the host's name: keyed by `Skill`, it would also turn + ExecuteSkillApi's `name`, an API call, into a skill engagement. Antigravity additionally strips the result payload out of a tool call's arguments: the harness folds result fields into the same `args` dict at DONE. Beyond a static key list, diff --git a/docs/agents/DELEGATE.md b/docs/agents/DELEGATE.md index 98f7b82b..7a767a2c 100644 --- a/docs/agents/DELEGATE.md +++ b/docs/agents/DELEGATE.md @@ -93,6 +93,8 @@ success_criteria: A `plugins:` entry with `type: local` and a `path` is mounted as `/skills` (the Delegate SDK's `bundledSkillsPath`, which expects one directory whose direct children are skill folders), and turns on `enableSkills`. With no plugin, skills stay off. If more than one plugin is configured, the first wins and a warning is logged. +The SDK loads a catalog skill with `LoadSkill {"name", "plugin"}`. The adapter records it as `Skill {"skill", "plugin"}`, the call Claude Code makes, so `skill_triggered` and other skill-loaded criteria see the load. + ## Architecture ### Class Hierarchy diff --git a/src/coder_eval/agents/delegate_agent.py b/src/coder_eval/agents/delegate_agent.py index 20cb9838..e0a58f93 100644 --- a/src/coder_eval/agents/delegate_agent.py +++ b/src/coder_eval/agents/delegate_agent.py @@ -140,11 +140,25 @@ _FAILED_TOOL_STATUSES = frozenset({"failed", "interrupted"}) """``toolStatus`` values the SDK's ``tool_result`` carries for a tool that did not complete.""" +# The host reports most tools under their canonical (Claude) names already +# (ReadFile -> Read, ...), but not LoadSkill, which loads a catalog skill by name. +# Keyed by the host's name, not the canonical one: the host also reports +# ExecuteSkillApi as `Skill`, and that call's `name` is an API call, not a skill load. +# Rationale: .claude/notes/agents.md § Tool-name and argument normalization +_TOOL_NAME_MAP: dict[str, str] = {"LoadSkill": "Skill"} +_DELEGATE_ARG_RENAME: dict[str, dict[str, str]] = {"LoadSkill": {"name": "skill"}} + def _tool_id(event: dict[str, Any]) -> str: return str(event.get("toolId") or "") +def _canonical_tool(tool_name: str, params: dict[str, Any]) -> tuple[str, dict[str, Any]]: + """Map a host tool name and its argument keys onto the canonical cross-agent vocabulary.""" + rename = _DELEGATE_ARG_RENAME.get(tool_name, {}) + return _TOOL_NAME_MAP.get(tool_name, tool_name), {rename.get(key, key): value for key, value in params.items()} + + def _orphan_tool_name(event: dict[str, Any], tool_id: str) -> str: """The name on a ``tool_result`` with no open call; the SDK falls back to the tool id when it has none.""" name = event.get("toolName") @@ -891,9 +905,10 @@ def _append_message_text(state: _TurnState, text: str, *, starts_step: bool) -> def _handle_tool_call(self, event: dict[str, Any], state: _TurnState, emit: Callable[[StreamEvent], None]) -> None: tool_id = _tool_id(event) or str(uuid.uuid4()) - tool_name = str(event.get("toolName") or "unknown") - parameters = event.get("toolArgs") - parameters = parameters if isinstance(parameters, dict) else {} + raw_args = event.get("toolArgs") + tool_name, parameters = _canonical_tool( + str(event.get("toolName") or "unknown"), raw_args if isinstance(raw_args, dict) else {} + ) state.sequence += 1 telemetry = CommandTelemetry( tool_name=tool_name, diff --git a/tests/test_delegate_agent.py b/tests/test_delegate_agent.py index ebc513f5..a548ecf9 100644 --- a/tests/test_delegate_agent.py +++ b/tests/test_delegate_agent.py @@ -20,6 +20,7 @@ from coder_eval.agents import delegate_agent as agent_module from coder_eval.agents.delegate_agent import DelegateAgent, _resolve_host_bundle from coder_eval.agents.registry import AgentRegistry, create_agent +from coder_eval.criteria.skill_triggered import _engaged_skill_names from coder_eval.errors import AgentConfigError, AgentCrashError, TurnTimeoutError from coder_eval.errors.categories import ErrorCategory from coder_eval.errors.categorization import categorize_error @@ -46,7 +47,7 @@ def _usage(input_tokens: int, output_tokens: int) -> bytes: return _line({"type": "usage", "usage": {"input_tokens": input_tokens, "output_tokens": output_tokens}}) -def _tool_call(tool_id: str, name: str = "shell", **args: Any) -> bytes: +def _tool_call(tool_id: str, name: str = "shell", /, **args: Any) -> bytes: return _ev(type="tool_call", toolId=tool_id, toolName=name, toolArgs=args, toolStatus="pending") @@ -608,6 +609,32 @@ async def test_result_for_an_unresolved_tool_takes_its_name_and_args(self, patch [command] = record.commands assert (command.tool_name, command.parameters, command.result_status) == ("Bash", {"command": "ls"}, "error") + async def test_load_skill_is_recorded_as_the_canonical_skill_call(self, patch_exec, tmp_path): + """Skill criteria read `Skill` + `skill`; the host sends `LoadSkill` + `name`.""" + events = [ + _tool_call("s1", "LoadSkill", name="uipath-rpa", plugin=""), + _tool_result("s1"), + _result(response="done"), + ] + agent, _ = await _started_agent(patch_exec, events, tmp_path) + record = await agent.communicate("hi") + [command] = record.commands + assert (command.tool_name, command.parameters) == ("Skill", {"skill": "uipath-rpa", "plugin": ""}) + assert _engaged_skill_names(command) == {"uipath-rpa"} + + async def test_skill_api_call_keeps_its_name_arg(self, patch_exec, tmp_path): + """The host already reports ExecuteSkillApi as `Skill`; its `name` is not a skill load.""" + events = [ + _tool_call("s1", "Skill", name="process-knowledge", method="search"), + _tool_result("s1"), + _result(response="done"), + ] + agent, _ = await _started_agent(patch_exec, events, tmp_path) + record = await agent.communicate("hi") + [command] = record.commands + assert (command.tool_name, command.parameters) == ("Skill", {"name": "process-knowledge", "method": "search"}) + assert _engaged_skill_names(command) == set() + async def test_result_with_only_the_sdk_id_fallback_name_stays_unknown(self, patch_exec, tmp_path): orphan = _ev(type="tool_result", toolId="s/x", toolName="s/x", toolResult="ok", toolStatus="completed") agent, _ = await _started_agent(patch_exec, [orphan, _result(response="done")], tmp_path) From 4292979b3b5a249963f84710f9efa1198ab3eafc Mon Sep 17 00:00:00 2001 From: Mihai Chirculescu Date: Sat, 3 Oct 2026 12:14:41 +0300 Subject: [PATCH 12/20] feat(delegate)!: find a global delegate-stdio install; rename DELEGATE_SDK_PATH to DELEGATE_STDIO_PATH `npm install -g @uipath/delegate-stdio` is now the whole user setup. The resolver finds a global install through the package's `delegate-stdio` bin on PATH: on POSIX the shim resolves to the bundle, on Windows the bundle sits in node_modules beside the .cmd shim. A local install still wins. DELEGATE_SDK_PATH becomes DELEGATE_STDIO_PATH, the name the Autopilot RPA pipeline already exports; it stays for CI pinning only. Remove DELEGATE_SDK_NODE_MODULES and the home probe. Co-Authored-By: Claude Opus 5.5 (1M context) --- .claude/notes/agents.md | 10 +++ .env.example | 10 +-- .github/workflows/pr-checks.yml | 2 - docs/USER_GUIDE.md | 2 +- docs/agents/DELEGATE.md | 18 ++--- pyproject.toml | 2 +- src/coder_eval/agents/delegate/package.json | 2 +- src/coder_eval/agents/delegate_agent.py | 73 ++++++++++----------- tests/test_delegate_agent.py | 67 ++++++++++++------- tests/test_error_handling.py | 2 +- 10 files changed, 100 insertions(+), 88 deletions(-) diff --git a/.claude/notes/agents.md b/.claude/notes/agents.md index fb116b8b..1a64ef1d 100644 --- a/.claude/notes/agents.md +++ b/.claude/notes/agents.md @@ -829,6 +829,16 @@ leaks into the retry. **Registered unconditionally**, like `codex`/`antigravity`; its extra is empty. Node and `@uipath/delegate-stdio` are the real prerequisite, resolved lazily in `start()`. +**Host resolution: `DELEGATE_STDIO_PATH`, then local installs, then a global one.** Local means +the cwd, its ancestors and `agents/delegate/`; a local install wins, as in Node. A global install +is found through the package's `delegate-stdio` bin shim on `PATH`, not `npm root -g`, so the +resolver runs no npm subprocess. On POSIX the shim is a symlink to the bundle. On Windows it is a +`.cmd` file beside the prefix's `node_modules`. We never execute the `.cmd`: that would make +`cmd.exe` parse the arguments again. The home probe and `DELEGATE_SDK_NODE_MODULES` are gone. The +global search covers "install once, run from anywhere", and the root override only repeated the +path override. The explicit path stays for CI: a source build has no install root, and an install +in a temporary directory for each run keeps the runs on a shared host on their own versions. + **Windows path length**: an install root that pushes the interop binary path past 260 characters makes the spawn fail with `ENOENT`. The docs tell users to install into a short path. diff --git a/.env.example b/.env.example index deed0120..67dcd642 100644 --- a/.env.example +++ b/.env.example @@ -89,14 +89,8 @@ LOG_TO_FILE=false # Set to true to enable file logging # CODER_EVAL_REMEDIATE_HOME_PLUGINS=0 # Delegate agent settings (UiPath Autopilot's Delegate agent, requires Node.js -# on PATH plus `npm install @uipath/delegate-stdio`, a release that accepts the -# `auth` init option). See docs/agents/DELEGATE.md. -# -# DELEGATE_SDK_NODE_MODULES / DELEGATE_SDK_PATH — override the auto-located -# install (cwd and its ancestors, src/coder_eval/agents/delegate/, then home); -# set at most one. -# DELEGATE_SDK_NODE_MODULES="/path/to/install/root" -# DELEGATE_SDK_PATH="/path/to/node_modules/@uipath/delegate-stdio/dist/delegate_stdio.mjs" +# on PATH plus `npm install -g @uipath/delegate-stdio`, a release that accepts the +# `auth` init option). coder_eval finds the install itself. See docs/agents/DELEGATE.md. # # DELEGATE_ENV — cloud environment slug (alpha/staging/production), # used to derive the backend URL from the org/tenant slugs. diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index 331e39f2..14fb02da 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -1372,7 +1372,6 @@ jobs: - name: Run fizzbuzz delegate task (assert PASS) if: steps.gate.outputs.present == 'true' env: - DELEGATE_SDK_NODE_MODULES: ${{ github.workspace }}/src/coder_eval/agents/delegate DELEGATE_ENV: ${{ github.event.inputs.delegate_uipath_env || 'alpha' }} DELEGATE_AUTH_TOKEN: ${{ steps.ropc.outputs.access_token }} DELEGATE_TENANT_ID: ${{ secrets.DELEGATE_TENANT_ID }} @@ -1404,7 +1403,6 @@ jobs: - name: Run Delegate live unit tests (assert PASS) if: steps.gate.outputs.present == 'true' env: - DELEGATE_SDK_NODE_MODULES: ${{ github.workspace }}/src/coder_eval/agents/delegate DELEGATE_ENV: ${{ github.event.inputs.delegate_uipath_env || 'alpha' }} DELEGATE_AUTH_TOKEN: ${{ steps.ropc.outputs.access_token }} DELEGATE_TENANT_ID: ${{ secrets.DELEGATE_TENANT_ID }} diff --git a/docs/USER_GUIDE.md b/docs/USER_GUIDE.md index 08495350..b27377f6 100644 --- a/docs/USER_GUIDE.md +++ b/docs/USER_GUIDE.md @@ -429,7 +429,7 @@ Set these in `.env` (copy from `.env.example`). | `BEDROCK_SMALL_MODEL` | No | Cross-region Bedrock small/fast model ID | | `CODEX_API_KEY` / `CODEX_BASE_URL` / `CODEX_MODEL` / `CODEX_API_VERSION` | For Codex | Codex agent auth & endpoint routing — see [Codex Agent Guide](agents/CODEX.md#endpoint-routing). | | `GEMINI_API_KEY` / `ANTIGRAVITY_MODEL` | For Antigravity | Antigravity (Gemini) agent auth & model — see [Antigravity Agent Guide](agents/ANTIGRAVITY.md#setup). | -| `DELEGATE_ENV` / `DELEGATE_BACKEND_URL` / `DELEGATE_AUTH_TOKEN` / `DELEGATE_TENANT_ID` / `DELEGATE_ORG_ID` / `DELEGATE_ORG_SLUG` / `DELEGATE_TENANT_SLUG` / `DELEGATE_SDK_NODE_MODULES` / `DELEGATE_SDK_PATH` | For Delegate | Delegate agent backend routing, auth & host install location — see [Delegate Agent Guide](agents/DELEGATE.md#setup). | +| `DELEGATE_ENV` / `DELEGATE_BACKEND_URL` / `DELEGATE_AUTH_TOKEN` / `DELEGATE_TENANT_ID` / `DELEGATE_ORG_ID` / `DELEGATE_ORG_SLUG` / `DELEGATE_TENANT_SLUG` | For Delegate | Delegate agent backend routing & auth — see [Delegate Agent Guide](agents/DELEGATE.md#setup). | | `UIPATH_PLUGIN_MARKETPLACE_DIR` | No | Conventional base directory for local plugins. Not special-cased by the framework: **any** `$VAR` / `${VAR}` referenced in a plugin `path` is expanded from the environment, so a plugin path like `$UIPATH_PLUGIN_MARKETPLACE_DIR/my-plugin` resolves against this variable. (An undefined variable in a plugin path logs a warning.) | | `PLUGIN_TOOLS_DIR` | No | A *separate* mechanism from the above: the canonical `node_modules/@uipath` directory used to pin UiPath CLI plugin discovery (not path substitution). When unset, the sandbox auto-derives it from the resolved `uip` binary. | | `CODER_EVAL_REMEDIATE_HOME_PLUGINS` | No | **DESTRUCTIVE.** Truthy deletes `$HOME/node_modules/@uipath` at sandbox setup to clear sibling-task pollution on dedicated eval hosts. Off by default; do **not** enable on developer workstations. | diff --git a/docs/agents/DELEGATE.md b/docs/agents/DELEGATE.md index 7a767a2c..cb62a35d 100644 --- a/docs/agents/DELEGATE.md +++ b/docs/agents/DELEGATE.md @@ -15,25 +15,19 @@ Under the hood, this agent spawns the host that the public [`@uipath/delegate-st ## Setup -### 1. Install Node and the Delegate host +### 1. Install the Delegate host -1. **Node.js** on your `PATH`. -2. **`@uipath/delegate-stdio`**, a public npm package (no token, no custom registry). Install it plain: +With **Node.js** on your `PATH`, install the host once: ```bash -npm install @uipath/delegate-stdio +npm install -g @uipath/delegate-stdio ``` -**You usually don't need to set anything about where.** When neither `DELEGATE_SDK_NODE_MODULES` nor `DELEGATE_SDK_PATH` is set, coder_eval searches for the install in this order: the current directory and its ancestors (the same way Node resolves modules), then `src/coder_eval/agents/delegate/`, then your home directory. `src/coder_eval/agents/delegate/` is this agent's own directory, and it ships a `package.json` that names the dependency. An install there is found from any current directory. +That is all. The package is public (no token, no custom registry). coder_eval finds the global install by itself, so you set no path. -To override the auto-search, set **one** of: +A local `npm install @uipath/delegate-stdio` also works: in your project, in a parent directory, or in `src/coder_eval/agents/delegate/` (it ships a `package.json`). A local install wins over a global one. On Windows, keep a local install in a short path: a path longer than 260 characters to the interop binary makes its spawn fail with `ENOENT`. -| Variable | Purpose | -|---|---| -| `DELEGATE_SDK_NODE_MODULES` | Install root that holds `node_modules/@uipath/...`. | -| `DELEGATE_SDK_PATH` | Absolute path straight to `@uipath/delegate-stdio/dist/delegate_stdio.mjs`. Any other file, such as `@uipath/delegate-sdk`'s `dist/index.mjs`, is an error. | - -On Windows, install into a short path. The interop binary sits deep inside `node_modules`, and a path longer than 260 characters makes its spawn fail with `ENOENT`. +> **CI only.** To pin one exact build, for example a build from source or an install in a temporary directory, set `DELEGATE_STDIO_PATH` to its `dist/delegate_stdio.mjs`. ### 2. Choose a backend diff --git a/pyproject.toml b/pyproject.toml index 014b74d3..eada6273 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -85,7 +85,7 @@ dev = [ # tasks that invoke `uv run uipath eval ...`) # - the Delegate agent's home in the packaging metadata (`agent.type: delegate`). # Like `opencode`/`pi` below, its real prerequisite is a Node package -# (`npm install @uipath/delegate-stdio`, public — no token needed), +# (`npm install -g @uipath/delegate-stdio`, public — no token needed), # so nothing new is added to THIS list; it just shares this extra's name. # Without this extra, the framework still installs and runs; UiPath-dependent # code paths fail at dispatch with a clear hint pointing back here. diff --git a/src/coder_eval/agents/delegate/package.json b/src/coder_eval/agents/delegate/package.json index e4d9276c..478a6d09 100644 --- a/src/coder_eval/agents/delegate/package.json +++ b/src/coder_eval/agents/delegate/package.json @@ -1,7 +1,7 @@ { "name": "coder-eval-delegate-host", "private": true, - "description": "npm install target for the delegate agent's Node prerequisite (@uipath/delegate-stdio, which pulls in @uipath/delegate-sdk). Installing here, or in any ancestor directory, lets coder_eval's ancestor-walk resolver find it with zero configuration.", + "description": "npm install target for the delegate agent's Node prerequisite (@uipath/delegate-stdio, which pulls in @uipath/delegate-sdk). coder_eval finds an install here from any cwd; a global `npm install -g` is found too.", "type": "module", "dependencies": { "@uipath/delegate-stdio": "^1.202.1" diff --git a/src/coder_eval/agents/delegate_agent.py b/src/coder_eval/agents/delegate_agent.py index e0a58f93..4b3a5d2c 100644 --- a/src/coder_eval/agents/delegate_agent.py +++ b/src/coder_eval/agents/delegate_agent.py @@ -19,7 +19,7 @@ {"type":"error","message":..} (any command) Prerequisites are documented in ``docs/agents/DELEGATE.md`` and enforced with a -clear ``AgentConfigError`` at ``start()`` (Node.js, ``npm install +clear ``AgentConfigError`` at ``start()`` (Node.js, ``npm install -g @uipath/delegate-stdio``, UiPath auth). Rationale: .claude/notes/agents.md § Delegate agent @@ -88,6 +88,7 @@ _HOST_PACKAGE = "@uipath/delegate-stdio" _HOST_BUNDLE_NAME = "delegate_stdio.mjs" +_HOST_BIN_NAME = "delegate-stdio" _HOST_BUNDLE_REL_PATH = Path("node_modules") / "@uipath" / "delegate-stdio" / "dist" / _HOST_BUNDLE_NAME _AGENT_INSTALL_ROOT = Path(__file__).resolve().parent / "delegate" """This agent's own directory; it ships a ``package.json`` that names the host package.""" @@ -231,71 +232,69 @@ def _strip_redundant_gateway_creds(env: dict[str, str]) -> tuple[str, ...]: def _candidate_install_roots() -> list[Path]: - """Search roots, in order: cwd and its ancestors, ``_AGENT_INSTALL_ROOT``, then home. + """Local search roots, in order: cwd and its ancestors, then ``_AGENT_INSTALL_ROOT``. - The cwd walk mirrors Node's own module resolution. Home is where npm lands a - package when the cwd has no ``package.json``. + The cwd walk mirrors Node's own module resolution. """ cwd = Path.cwd().resolve() roots: list[Path] = [cwd, *cwd.parents] - for root in (_AGENT_INSTALL_ROOT, Path.home().resolve()): - if root not in roots: - roots.append(root) + if _AGENT_INSTALL_ROOT not in roots: + roots.append(_AGENT_INSTALL_ROOT) return roots +def _global_install_candidates() -> list[Path]: + """Bundle paths implied by an ``npm install -g``, found through the package's shim on PATH. + + On POSIX the shim is a symlink to the bundle; on Windows it is a ``.cmd`` file that + sits beside the global prefix's ``node_modules``. + """ + shim = shutil.which(_HOST_BIN_NAME) + if shim is None: + return [] + shim_path = Path(shim) + return [shim_path.resolve(), (shim_path.parent / _HOST_BUNDLE_REL_PATH).resolve()] + + def _resolve_host_bundle() -> Path: """Locate the installed ``@uipath/delegate-stdio``'s ``dist/delegate_stdio.mjs``. - Resolution order: ``DELEGATE_SDK_PATH`` (explicit file path) -> - ``DELEGATE_SDK_NODE_MODULES`` (explicit install root, probed exactly) -> - ``_candidate_install_roots()``, so an ``npm install`` in this agent's own - ``agents/delegate/`` directory is found from any cwd. + Resolution order: ``DELEGATE_STDIO_PATH`` (explicit file path) -> + ``_candidate_install_roots()`` -> a global install (``_global_install_candidates()``). Raises: - AgentConfigError: no install found anywhere searched, or ``DELEGATE_SDK_PATH`` + AgentConfigError: no install found anywhere searched, or ``DELEGATE_STDIO_PATH`` names another file, such as ``@uipath/delegate-sdk``'s ``dist/index.mjs``. """ - explicit = os.environ.get("DELEGATE_SDK_PATH") + explicit = os.environ.get("DELEGATE_STDIO_PATH") if explicit: path = Path(explicit).expanduser().resolve() if not path.is_file(): raise AgentConfigError( - f"DELEGATE_SDK_PATH={path} does not point to a file. Point it at " + f"DELEGATE_STDIO_PATH={path} does not point to a file. Point it at " + f"{_HOST_PACKAGE}'s dist/{_HOST_BUNDLE_NAME}." ) if path.name != _HOST_BUNDLE_NAME: raise AgentConfigError( - f"DELEGATE_SDK_PATH={path} must point at {_HOST_PACKAGE}'s dist/{_HOST_BUNDLE_NAME}, " + f"DELEGATE_STDIO_PATH={path} must point at {_HOST_PACKAGE}'s dist/{_HOST_BUNDLE_NAME}, " + "not at @uipath/delegate-sdk's dist/index.mjs or another file. " - + f"Run `npm install {_HOST_PACKAGE}`; see docs/agents/DELEGATE.md." - ) - return path - - root_override = os.environ.get("DELEGATE_SDK_NODE_MODULES") - if root_override: - path = (Path(root_override).expanduser().resolve() / _HOST_BUNDLE_REL_PATH).resolve() - if not path.is_file(): - raise AgentConfigError( - f"DELEGATE_SDK_NODE_MODULES={root_override}: {_HOST_PACKAGE} not found at {path}. " - + f"Run `npm install {_HOST_PACKAGE}` there, or set DELEGATE_SDK_PATH directly." + + "See docs/agents/DELEGATE.md." ) return path - searched: list[Path] = [] - for root in _candidate_install_roots(): - candidate = (root / _HOST_BUNDLE_REL_PATH).resolve() - searched.append(candidate) - if candidate.is_file(): + candidates = [(root / _HOST_BUNDLE_REL_PATH).resolve() for root in _candidate_install_roots()] + candidates += _global_install_candidates() + for candidate in candidates: + if candidate.name == _HOST_BUNDLE_NAME and candidate.is_file(): return candidate - searched_block = "\n ".join(str(p) for p in searched) + searched_block = "\n ".join(str(p) for p in candidates) raise AgentConfigError( - f"{_HOST_PACKAGE} not found. Searched the cwd, its ancestors, this agent's directory, and home:\n" + f"{_HOST_PACKAGE} not found. Install it with `npm install -g {_HOST_PACKAGE}` " + + "(public, no token needed). Searched the cwd, its ancestors, this agent's directory, " + + f"and the `{_HOST_BIN_NAME}` command on PATH:\n" + f" {searched_block}\n" - + f"Run `npm install {_HOST_PACKAGE}` (plain, public install — no token needed), " - + f"e.g. in {_AGENT_INSTALL_ROOT}, or set DELEGATE_SDK_NODE_MODULES " - + f"to the install root, or DELEGATE_SDK_PATH to the dist/{_HOST_BUNDLE_NAME} file directly. " + + f"To use a build elsewhere, set DELEGATE_STDIO_PATH to its dist/{_HOST_BUNDLE_NAME}. " + "See docs/agents/DELEGATE.md." ) @@ -496,7 +495,7 @@ async def start( if shutil.which("node") is None: raise AgentConfigError( "Node.js was not found on PATH. Install it (https://nodejs.org/), " - + f"then `npm install {_HOST_PACKAGE}`. See docs/agents/DELEGATE.md." + + f"then `npm install -g {_HOST_PACKAGE}`. See docs/agents/DELEGATE.md." ) self._host_bundle = _resolve_host_bundle() diff --git a/tests/test_delegate_agent.py b/tests/test_delegate_agent.py index a548ecf9..77a2e5e2 100644 --- a/tests/test_delegate_agent.py +++ b/tests/test_delegate_agent.py @@ -148,8 +148,7 @@ async def fake_exec(*args: Any, **kwargs: Any) -> _FakeProcess: _DELEGATE_ENV_NAMES = ( - "DELEGATE_SDK_PATH", - "DELEGATE_SDK_NODE_MODULES", + "DELEGATE_STDIO_PATH", "DELEGATE_ENV", "DELEGATE_BACKEND_URL", *( @@ -183,49 +182,67 @@ async def _started_agent( return agent, proc +def _write_bundle(root: Path) -> Path: + entry = root / "node_modules" / "@uipath" / "delegate-stdio" / "dist" / "delegate_stdio.mjs" + entry.parent.mkdir(parents=True) + entry.write_text("#!/usr/bin/env node") + return entry + + class TestResolveHostBundle: + @pytest.fixture + def no_installs(self, tmp_path, monkeypatch) -> Path: + """Pin every search root to an empty ``tmp_path`` and hide any real global install.""" + monkeypatch.setattr(agent_module, "_candidate_install_roots", lambda: [tmp_path]) + monkeypatch.setattr("shutil.which", lambda _name: None) + return tmp_path + def test_explicit_path_env_var(self, tmp_path, monkeypatch): entry = tmp_path / "delegate_stdio.mjs" entry.write_text("#!/usr/bin/env node") - monkeypatch.setenv("DELEGATE_SDK_PATH", str(entry)) + monkeypatch.setenv("DELEGATE_STDIO_PATH", str(entry)) assert _resolve_host_bundle() == entry.resolve() def test_explicit_path_to_another_file_raises(self, tmp_path, monkeypatch): sdk_entry = tmp_path / "node_modules" / "@uipath" / "delegate-sdk" / "dist" / "index.mjs" sdk_entry.parent.mkdir(parents=True) sdk_entry.write_text("export {}") - monkeypatch.setenv("DELEGATE_SDK_PATH", str(sdk_entry)) + monkeypatch.setenv("DELEGATE_STDIO_PATH", str(sdk_entry)) with pytest.raises(AgentConfigError, match="must point at @uipath/delegate-stdio"): _resolve_host_bundle() def test_explicit_path_missing_file_raises(self, tmp_path, monkeypatch): - monkeypatch.setenv("DELEGATE_SDK_PATH", str(tmp_path / "nope.mjs")) + monkeypatch.setenv("DELEGATE_STDIO_PATH", str(tmp_path / "nope.mjs")) with pytest.raises(AgentConfigError, match="does not point to a file"): _resolve_host_bundle() - def test_node_modules_override(self, tmp_path, monkeypatch): - monkeypatch.delenv("DELEGATE_SDK_PATH", raising=False) - monkeypatch.setenv("DELEGATE_SDK_NODE_MODULES", str(tmp_path)) - entry = tmp_path / "node_modules" / "@uipath" / "delegate-stdio" / "dist" / "delegate_stdio.mjs" - entry.parent.mkdir(parents=True) - entry.write_text("#!/usr/bin/env node") + def test_install_in_a_cwd_ancestor_is_found(self, tmp_path, monkeypatch): + entry = _write_bundle(tmp_path) + nested = tmp_path / "project" / "sub" + nested.mkdir(parents=True) + monkeypatch.chdir(nested) assert _resolve_host_bundle() == entry.resolve() - def test_node_modules_override_missing_raises(self, tmp_path, monkeypatch): - monkeypatch.delenv("DELEGATE_SDK_PATH", raising=False) - monkeypatch.setenv("DELEGATE_SDK_NODE_MODULES", str(tmp_path)) - with pytest.raises(AgentConfigError, match="not found"): - _resolve_host_bundle() + def test_global_install_found_through_a_posix_symlink_shim(self, no_installs, tmp_path, monkeypatch): + entry = _write_bundle(tmp_path / "prefix" / "lib") + monkeypatch.setattr("shutil.which", lambda _name: str(entry)) + assert _resolve_host_bundle() == entry.resolve() - def test_no_config_and_not_found_raises_with_search_list(self, tmp_path, monkeypatch): - monkeypatch.delenv("DELEGATE_SDK_PATH", raising=False) - monkeypatch.delenv("DELEGATE_SDK_NODE_MODULES", raising=False) - monkeypatch.chdir(tmp_path) - monkeypatch.setattr(os, "getcwd", lambda: str(tmp_path)) - # A real npm install under the developer's actual home directory must - # not make this test flaky -- pin every search root to tmp_path. - monkeypatch.setattr(agent_module, "_candidate_install_roots", lambda: [tmp_path]) - with pytest.raises(AgentConfigError, match="Searched the cwd"): + def test_global_install_found_beside_a_windows_cmd_shim(self, no_installs, tmp_path, monkeypatch): + prefix = tmp_path / "npm" + entry = _write_bundle(prefix) + monkeypatch.setattr("shutil.which", lambda _name: str(prefix / "delegate-stdio.cmd")) + assert _resolve_host_bundle() == entry.resolve() + + def test_local_install_wins_over_global(self, no_installs, tmp_path, monkeypatch): + local = _write_bundle(tmp_path) + global_prefix = tmp_path / "npm" + _write_bundle(global_prefix) + monkeypatch.setattr("shutil.which", lambda _name: str(global_prefix / "delegate-stdio.cmd")) + assert _resolve_host_bundle() == local.resolve() + + def test_not_found_raises_with_install_hint(self, no_installs): + with pytest.raises(AgentConfigError, match=r"npm install -g @uipath/delegate-stdio"): _resolve_host_bundle() diff --git a/tests/test_error_handling.py b/tests/test_error_handling.py index 8ea01dd3..b3b6e711 100644 --- a/tests/test_error_handling.py +++ b/tests/test_error_handling.py @@ -252,7 +252,7 @@ def test_categorize_agent_config_error_message_does_not_misroute(self): """The typed check wins even when the message would match a retryable string pattern.""" # "connection" would otherwise route to the retryable AGENT_API_ERROR. result = categorize_error( - AgentConfigError("DELEGATE_SDK_PATH unset; cannot establish connection"), + AgentConfigError("DELEGATE_STDIO_PATH unset; cannot establish connection"), {"component": "agent"}, ) assert result == ErrorCategory.AGENT_CONFIG_ERROR From 403386367891c9e5df9814a86065dfebd0aae171 Mon Sep 17 00:00:00 2001 From: Mihai Chirculescu Date: Sat, 3 Oct 2026 12:34:18 +0300 Subject: [PATCH 13/20] chore(delegate): require @uipath/delegate-stdio 1.203.0 1.203.0 is the first release that accepts the `auth` init option and writes a `usage` frame for each model call. An older host ignores `auth`, and coder_eval keeps the host's own token variables out of its env, so the token path cannot work on 1.202.1. Remove the text that explains an older host: the init-error hint, the cut-turn usage warning, the Delegate guide and the notes. Co-Authored-By: Claude Opus 5.5 (1M context) --- .claude/notes/agents.md | 14 ++++++++------ docs/agents/DELEGATE.md | 4 ++-- src/coder_eval/agents/delegate/package.json | 2 +- src/coder_eval/agents/delegate_agent.py | 6 ++---- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.claude/notes/agents.md b/.claude/notes/agents.md index 1a64ef1d..93a88db0 100644 --- a/.claude/notes/agents.md +++ b/.claude/notes/agents.md @@ -751,9 +751,11 @@ release adding an unclassified field fails loudly instead of silently passing th `DelegateAgent` drives UiPath Autopilot's Delegate agent: reasoning in the UiPath backend, tools run locally. It spawns the host from the public `@uipath/delegate-stdio` npm package (`dist/delegate_stdio.mjs`), which pulls in `@uipath/delegate-sdk` and the interop binaries. The -package README is the wire-protocol SSOT. Every frame shape the adapter reads was confirmed against a -live `1.202.1` transcript; `tests/test_delegate_agent.py`'s frame builders replay those shapes. (An -earlier first-party `delegate_host.mjs` wrapper is gone; see git history.) +package README is the wire-protocol SSOT. `agents/delegate/package.json` sets the floor at `1.203.0`, the +first release with the `auth` init option and the `usage` frame (UiPath/Autopilot#6656). The other frame +shapes were confirmed against a live `1.202.1` transcript; `tests/test_delegate_agent.py`'s frame +builders replay those shapes. (An earlier first-party `delegate_host.mjs` wrapper is gone; see git +history.) **Auth goes to the host as the `auth` init option, not through its env.** `_env` reads the `DELEGATE_*` name first, then the bare `AUTH_TOKEN` / `TENANT_ID` / `ORG_ID` / `ORG_SLUG` / @@ -762,8 +764,8 @@ host's own names (`ORG_LOGICAL_NAME`, `BACKEND_URL`, …): bare names collide wi secrets use the `DELEGATE_*` names. `_HOST_ENV_REMOVED` strips the host's names and the token from the host env, because the agent's shells inherit it (token leak to code under test) and a stray `BACKEND_URL` would reroute the host. A refresh source still writes the fresh token into the host's -own `process.env.AUTH_TOKEN`. An older host ignores the option and fails init with `Auth required`; -`_INIT_CONFIG_ERROR_HINT` names our variables beside the host's message. `env=` without org/tenant +own `process.env.AUTH_TOKEN`. `_INIT_CONFIG_ERROR_HINT` names our variables beside the host's message, +which names the host's own. `env=` without org/tenant slugs fails init. **`LLMGW_*` leaves the host env only when a token file is set.** Without a token file, the host's @@ -789,7 +791,7 @@ opens; `len(turnUsages)` from the `result` then replaces the estimate as `num_tu when it arrives. Ordering (from SDK/backend source, not a live transcript): call N's `usage` frame always precedes the `tool_result` that opens call N+1. The frame is NOT a call boundary — it arrives before its call's tools run — so the cap stays on "the previous call's tools have all returned". The -adapter warns when a cut turn had finished calls but no usage (an old host). +adapter warns when a cut turn had finished calls but no usage. **One `AssistantMessage` per `communicate()`.** `isStepStart` and `turnUsages` would allow a per-round-trip split; today `isStepStart` only merges streamed deltas. `close_window` opens the one diff --git a/docs/agents/DELEGATE.md b/docs/agents/DELEGATE.md index cb62a35d..fb0a74b3 100644 --- a/docs/agents/DELEGATE.md +++ b/docs/agents/DELEGATE.md @@ -23,7 +23,7 @@ With **Node.js** on your `PATH`, install the host once: npm install -g @uipath/delegate-stdio ``` -That is all. The package is public (no token, no custom registry). coder_eval finds the global install by itself, so you set no path. +That is all. The package is public (no token, no custom registry). coder_eval finds the global install by itself, so you set no path. coder_eval needs version 1.203.0 or later, the first release that accepts the `auth` init option and writes a `usage` frame for each model call. A local `npm install @uipath/delegate-stdio` also works: in your project, in a parent directory, or in `src/coder_eval/agents/delegate/` (it ships a `package.json`). A local install wins over a global one. On Windows, keep a local install in a short path: a path longer than 260 characters to the interop binary makes its spawn fail with `ENOENT`. @@ -44,7 +44,7 @@ Either: - **Environment token** — `DELEGATE_AUTH_TOKEN`, `DELEGATE_TENANT_ID`, `DELEGATE_ORG_ID` env vars. When `DELEGATE_ENV` (not `DELEGATE_BACKEND_URL`) resolves the backend, also set `DELEGATE_ORG_SLUG` and `DELEGATE_TENANT_SLUG`. These are the human-readable org/tenant names, not the GUIDs. Without the slugs, init fails with `env="alpha" requires org/tenant slugs`. To skip the slugs, set `DELEGATE_BACKEND_URL` instead. Each variable also accepts the bare spelling (`AUTH_TOKEN`, `TENANT_ID`, `ORG_ID`, `ORG_SLUG`, `TENANT_SLUG`) as a fallback. Prefer the `DELEGATE_` spelling in a shared environment, because the bare names collide with what other tooling (npm, Vault, Terraform) commonly exports. - **Saved login** — a prior `npx @uipath/delegate-cli login --env ` that wrote `~/.aria/sdk-auth.json`. The host reads and refreshes this file itself when no token is supplied. This agent does not parse that file in Python, so auth-freshness logic lives in exactly one place. The saved login already carries the slugs, so `DELEGATE_ORG_SLUG` / `DELEGATE_TENANT_SLUG` are not needed. -coder_eval sends these values to the host as its `auth` init option, on stdin. It also removes the host's own variable names (`AUTH_TOKEN`, `TENANT_ID`, `ORG_ID`, `ORG_LOGICAL_NAME`, `TENANT_NAME`, `BACKEND_URL`) and `DELEGATE_AUTH_TOKEN` from the host's environment. So the agent's shell commands cannot read the token, and a variable that another tool exports does not change where the host connects. The `auth` init option needs a `@uipath/delegate-stdio` release that accepts it: version 1.202.1 and older ignore it, and then init fails with `Auth required` unless a saved login exists. +coder_eval sends these values to the host as its `auth` init option, on stdin. It also removes the host's own variable names (`AUTH_TOKEN`, `TENANT_ID`, `ORG_ID`, `ORG_LOGICAL_NAME`, `TENANT_NAME`, `BACKEND_URL`) and `DELEGATE_AUTH_TOKEN` from the host's environment. So the agent's shell commands cannot read the token, and a variable that another tool exports does not change where the host connects. The host also reads its own advanced variables directly, such as `DELEGATE_AUTH_TOKEN_FILE` (a token file that an external process keeps fresh) and `DELEGATE_STDIO_VERBOSE=1` (trace every frame to stderr). See the [package README](https://www.npmjs.com/package/@uipath/delegate-stdio). diff --git a/src/coder_eval/agents/delegate/package.json b/src/coder_eval/agents/delegate/package.json index 478a6d09..661349ad 100644 --- a/src/coder_eval/agents/delegate/package.json +++ b/src/coder_eval/agents/delegate/package.json @@ -4,6 +4,6 @@ "description": "npm install target for the delegate agent's Node prerequisite (@uipath/delegate-stdio, which pulls in @uipath/delegate-sdk). coder_eval finds an install here from any cwd; a global `npm install -g` is found too.", "type": "module", "dependencies": { - "@uipath/delegate-stdio": "^1.202.1" + "@uipath/delegate-stdio": "^1.203.0" } } diff --git a/src/coder_eval/agents/delegate_agent.py b/src/coder_eval/agents/delegate_agent.py index 4b3a5d2c..dabbafd7 100644 --- a/src/coder_eval/agents/delegate_agent.py +++ b/src/coder_eval/agents/delegate_agent.py @@ -130,8 +130,7 @@ "coder_eval sends the auth and the org/tenant slugs as the host's `auth` init option, read from " + "DELEGATE_AUTH_TOKEN / DELEGATE_TENANT_ID / DELEGATE_ORG_ID / DELEGATE_ORG_SLUG / DELEGATE_TENANT_SLUG " + "(or the same names without DELEGATE_), and keeps the host's own AUTH_TOKEN / TENANT_ID / ORG_ID / " - + "ORG_LOGICAL_NAME / TENANT_NAME out of its environment. A @uipath/delegate-stdio without the `auth` " - + "init option ignores it. See docs/agents/DELEGATE.md." + + "ORG_LOGICAL_NAME / TENANT_NAME out of its environment. See docs/agents/DELEGATE.md." ) """Appended to a config-class init error, whose host message names the host's own variables.""" @@ -996,8 +995,7 @@ def _warn_if_usage_missing(state: _TurnState, status: AgentEndStatus) -> None: elif state.api_calls > 1: logger.warning( "delegate: turn ended (%s) with no usage frame from the host; tokens and cost for its %d " - + "finished model call(s) are unknown (a @uipath/delegate-stdio without per-call usage " - + "frames reports usage only on its result frame, which a cut turn never gets)", + + "finished model call(s) are unknown", status.value, state.api_calls - 1, ) From e4d64e5e37a810d07b6eeb8d3f1b1ad000eba3cc Mon Sep 17 00:00:00 2001 From: Mihai Chirculescu Date: Sat, 3 Oct 2026 12:35:03 +0300 Subject: [PATCH 14/20] fix(delegate): keep a non-retryable init error terminal on a respawn A respawn in communicate() turned every AgentConfigError into a retryable AgentCrashError. After start(), the only source of that error is the init classifier, which marks the errors that a retry cannot fix. So an expired token ended the task at start() but was retried after a mid-run respawn. Let the error go up as it is. Co-Authored-By: Claude Opus 5.5 (1M context) --- .claude/notes/agents.md | 4 +++- src/coder_eval/agents/delegate_agent.py | 9 +-------- tests/test_delegate_agent.py | 22 ++++++++++++++++++++++ 3 files changed, 26 insertions(+), 9 deletions(-) diff --git a/.claude/notes/agents.md b/.claude/notes/agents.md index 93a88db0..abc243f7 100644 --- a/.claude/notes/agents.md +++ b/.claude/notes/agents.md @@ -819,7 +819,9 @@ tail at WARNING instead. **Only an init error that a retry cannot fix is non-retryable.** `AgentConfigError` only on `_INIT_CONFIG_ERROR_MARKERS` (auth missing/rejected, missing slugs, unknown env). Any other init error -and the 60 s init deadline raise a retryable `AgentCrashError`. +and the 60 s init deadline raise a retryable `AgentCrashError`. A mid-run respawn keeps the same +classification: it once made every `AgentConfigError` retryable, so an expired token ended the task at +`start()` but was retried later. **Clear the process handle on every path that leaves the host dead or dying** — EOF, `error` frame, timeout (pre-check AND mid-`wait_for`), stop, `max_turns`. A shipped bug left it set after a mid-read diff --git a/src/coder_eval/agents/delegate_agent.py b/src/coder_eval/agents/delegate_agent.py index dabbafd7..c94f1ab5 100644 --- a/src/coder_eval/agents/delegate_agent.py +++ b/src/coder_eval/agents/delegate_agent.py @@ -733,14 +733,7 @@ async def communicate( if self._process is None or self._process.returncode is not None: # A prior cooperative stop or crash left no live host: respawn # fresh rather than fail fast. - try: - await self._spawn_and_init() - except AgentConfigError as exc: - # Node/the SDK install were already validated once in start(); - # a failure respawning mid-run is a transient backend/auth - # hiccup, not a missing prerequisite -- make it retryable - # instead of ending the task outright. - raise AgentCrashError(str(exc)) from exc + await self._spawn_and_init() self._begin_turn() collector = EventCollector() diff --git a/tests/test_delegate_agent.py b/tests/test_delegate_agent.py index 77a2e5e2..58c7816c 100644 --- a/tests/test_delegate_agent.py +++ b/tests/test_delegate_agent.py @@ -905,6 +905,28 @@ async def test_respawns_after_crash_on_next_turn(self, patch_exec, tmp_path): record = await agent.communicate("hi again") assert record.agent_output == "recovered" + @pytest.mark.parametrize( + ("host_message", "error_type"), + [ + ("Auth required: token expired", AgentConfigError), + ("fetch failed", AgentCrashError), + ], + ids=["auth", "network"], + ) + async def test_respawn_init_error_keeps_the_start_classification( + self, patch_exec, tmp_path, host_message, error_type + ): + """The same init error must be as retryable on a mid-run respawn as at ``start()``.""" + agent, _ = await _started_agent(patch_exec, [], tmp_path) + with pytest.raises(AgentCrashError): + await agent.communicate("hi") + await agent.discard_pending_turn() + + patch_exec([_line({"type": "error", "message": host_message})]) + with pytest.raises(error_type, match="Delegate SDK init failed"): + await agent.communicate("hi again") + assert agent._process is None + async def test_emits_exactly_one_start_and_end_event(self, patch_exec, tmp_path): events = [_result(response="done")] agent, _ = await _started_agent(patch_exec, events, tmp_path) From 03b00a8242a9a63d085672acc9abe8252d753ef8 Mon Sep 17 00:00:00 2001 From: Mihai Chirculescu Date: Sat, 3 Oct 2026 12:36:48 +0300 Subject: [PATCH 15/20] fix(delegate): match an init 401/403 as a whole word The init classifier matched the bare substrings "401", "403" and "expired". A port (127.0.0.1:54013), a GUID or an unrelated "expired" in a transient init error made it a non-retryable AgentConfigError. Match the status codes with a word-boundary regex, and narrow "expired" to token expiry ("token expired", "signature has expired", "jwt expired"). Co-Authored-By: Claude Opus 5.5 (1M context) --- .claude/notes/agents.md | 9 +++++---- src/coder_eval/agents/delegate_agent.py | 18 ++++++++++++++---- tests/test_delegate_agent.py | 17 ++++++++++++++++- 3 files changed, 35 insertions(+), 9 deletions(-) diff --git a/.claude/notes/agents.md b/.claude/notes/agents.md index abc243f7..af6690e3 100644 --- a/.claude/notes/agents.md +++ b/.claude/notes/agents.md @@ -818,10 +818,11 @@ path turned a transient error into non-retryable `AGENT_INVALID_OUTPUT`. `_log_s tail at WARNING instead. **Only an init error that a retry cannot fix is non-retryable.** `AgentConfigError` only on -`_INIT_CONFIG_ERROR_MARKERS` (auth missing/rejected, missing slugs, unknown env). Any other init error -and the 60 s init deadline raise a retryable `AgentCrashError`. A mid-run respawn keeps the same -classification: it once made every `AgentConfigError` retryable, so an expired token ended the task at -`start()` but was retried later. +`_INIT_CONFIG_ERROR_MARKERS` (auth missing/rejected, missing slugs, unknown env) or on a whole-word +401/403. A bare `"401"` substring also matched ports and GUIDs, such as `127.0.0.1:54013`, and ended a +transient failure with no retry. Any other init error and the 60 s init deadline raise a retryable +`AgentCrashError`. A mid-run respawn keeps the same classification: it once made every +`AgentConfigError` retryable, so an expired token ended the task at `start()` but was retried later. **Clear the process handle on every path that leaves the host dead or dying** — EOF, `error` frame, timeout (pre-check AND mid-`wait_for`), stop, `max_turns`. A shipped bug left it set after a mid-read diff --git a/src/coder_eval/agents/delegate_agent.py b/src/coder_eval/agents/delegate_agent.py index c94f1ab5..03f7833f 100644 --- a/src/coder_eval/agents/delegate_agent.py +++ b/src/coder_eval/agents/delegate_agent.py @@ -117,14 +117,24 @@ "unauthorized", "invalid credentials", "invalid token", - "expired", - "401", - "403", + "token expired", + "signature has expired", + "jwt expired", "requires org/tenant slugs", "unknown env", ) """Substrings of an init ``error`` message that a retry cannot fix: missing or rejected auth, or a bad ``DELEGATE_ENV``. Any other init error is retryable.""" +_INIT_CONFIG_ERROR_STATUS = re.compile(r"\b40[13]\b") +"""An HTTP 401/403 as a whole word, so a port or a GUID that contains the digits does not match.""" + + +def _is_config_init_error(message: str) -> bool: + lowered = message.lower() + return any(marker in lowered for marker in _INIT_CONFIG_ERROR_MARKERS) or bool( + _INIT_CONFIG_ERROR_STATUS.search(message) + ) + _INIT_CONFIG_ERROR_HINT = ( "coder_eval sends the auth and the org/tenant slugs as the host's `auth` init option, read from " @@ -563,7 +573,7 @@ async def _spawn_and_init(self) -> None: if ack.get("type") == "error": await self._force_kill_host() message = str(ack.get("message", "unknown error")) - if any(marker in message.lower() for marker in _INIT_CONFIG_ERROR_MARKERS): + if _is_config_init_error(message): raise AgentConfigError(f"Delegate SDK init failed: {message} {_INIT_CONFIG_ERROR_HINT}") raise AgentCrashError(f"Delegate SDK init failed: {message}") diff --git a/tests/test_delegate_agent.py b/tests/test_delegate_agent.py index 58c7816c..be880bc8 100644 --- a/tests/test_delegate_agent.py +++ b/tests/test_delegate_agent.py @@ -266,10 +266,25 @@ async def test_init_ok_completes_start(self, patch_exec, tmp_path): [ ("Auth required: set AUTH_TOKEN/TENANT_ID/ORG_ID env vars", AgentConfigError), ('env="alpha" requires org/tenant slugs. Set ORG_LOGICAL_NAME and TENANT_NAME', AgentConfigError), + ("401 Invalid token: Signature has expired", AgentConfigError), + ("Request failed with status code 403", AgentConfigError), ("Token endpoint https://cloud.example/token returned HTTP 503: unavailable", AgentCrashError), ("fetch failed", AgentCrashError), + ("connect ECONNREFUSED 127.0.0.1:54013", AgentCrashError), + ("tenant c7a3f401-0000-4000-8000-000000000403 is not reachable", AgentCrashError), + ("cache entry expired before the backend replied", AgentCrashError), + ], + ids=[ + "no-auth", + "no-slugs", + "expired-token", + "status-403", + "token-endpoint-5xx", + "network", + "port-with-401", + "guid-with-401-403", + "unrelated-expired", ], - ids=["no-auth", "no-slugs", "token-endpoint-5xx", "network"], ) async def test_only_an_init_error_a_retry_cannot_fix_is_non_retryable( self, patch_exec, tmp_path, host_message, error_type From 496f3e97003841eb73337462fdd3703c1743bdbd Mon Sep 17 00:00:00 2001 From: Mihai Chirculescu Date: Sat, 3 Oct 2026 12:39:34 +0300 Subject: [PATCH 16/20] docs: say that the sdk_options keys and record depend on the agent type The task guide still said that sdk_options needs `type: claude-code`. The override guard now accepts each agent type whose config declares the field, and Delegate accepts `effort`. EvaluationResult.sdk_options and REPORT_SCHEMA.md still called the record a ClaudeAgentOptions dump, but Delegate now writes its redacted host init options there. Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/REPORT_SCHEMA.md | 6 ++++-- docs/TASK_DEFINITION_GUIDE.md | 25 +++++++++++++++---------- experiments/default.yaml | 3 ++- src/coder_eval/models/results.py | 6 ++++-- 4 files changed, 25 insertions(+), 15 deletions(-) diff --git a/docs/REPORT_SCHEMA.md b/docs/REPORT_SCHEMA.md index f0fb59e1..8605e964 100644 --- a/docs/REPORT_SCHEMA.md +++ b/docs/REPORT_SCHEMA.md @@ -139,8 +139,10 @@ The authoritative per-replicate record. **Errors** (populated on failure): `error_message`, `error_details`, `error_log_tail` (carries the Docker build-log tail for `BUILD_FAILED`). -**Config/environment:** `environment_info`, `agent_config`, `sdk_options` (raw -`ClaudeAgentOptions` dump), `sandbox_path`, `task_config` +**Config/environment:** `environment_info`, `agent_config`, `sdk_options` (the +options the agent sent to its SDK; the shape depends on the agent: a raw +`ClaudeAgentOptions` dump on Claude Code, the host's `init` options with +credentials redacted on Delegate, `null` on the other agents), `sandbox_path`, `task_config` (`{resolved, source_yaml, source_file, lineage}` — `lineage` maps each field to `{value, source, source_detail}` so you can trace which config layer set it). `environment_info.system_prompt_semantics` (`"append"` / `"replace"` / diff --git a/docs/TASK_DEFINITION_GUIDE.md b/docs/TASK_DEFINITION_GUIDE.md index af9e5482..373210d5 100644 --- a/docs/TASK_DEFINITION_GUIDE.md +++ b/docs/TASK_DEFINITION_GUIDE.md @@ -170,18 +170,23 @@ agent: - "Write" - "Bash" model: "claude-sonnet-5" # Optional: specific model - sdk_options: # Optional: Claude Code SDK pass-through - effort: high # any non-framework-managed ClaudeAgentOptions field + sdk_options: # Optional: agent SDK pass-through (keys depend on `type`) + effort: high # claude-code: any non-framework-managed ClaudeAgentOptions field ``` -**`sdk_options`** is a typed pass-through dict for Claude Code SDK -`ClaudeAgentOptions` fields that Coder Eval doesn't own directly. Keys are -validated against the SDK's dataclass at YAML load; framework-managed keys -(`model`, `allowed_tools`, `permission_mode`, `hooks`, `mcp_servers`, …) -are rejected. Deep-merged across the 5-layer config chain. Override via -CLI with the repeatable `-D agent.sdk_options.KEY=VALUE`. -**Requires `type: "claude-code"`** to function; on other agent types it raises -an error. +**`sdk_options`** is a pass-through dict for the agent SDK options that Coder +Eval doesn't own directly. Only an agent type whose config declares the field +accepts it; on another agent type it raises an error. Each agent type +validates its own keys at YAML load: + +- `claude-code` — `ClaudeAgentOptions` fields, validated against the SDK's + dataclass; framework-managed keys (`model`, `allowed_tools`, + `permission_mode`, `hooks`, `mcp_servers`, …) are rejected. See the + [Claude Code guide](agents/CLAUDE_CODE.md). +- `delegate` — only `effort`. See the [Delegate guide](agents/DELEGATE.md). + +Deep-merged across the 5-layer config chain. Override via CLI with the +repeatable `-D agent.sdk_options.KEY=VALUE`. **Permission Modes:** - `default` — Default permission handling diff --git a/experiments/default.yaml b/experiments/default.yaml index 28bde5ae..e0571efd 100644 --- a/experiments/default.yaml +++ b/experiments/default.yaml @@ -70,7 +70,8 @@ defaults: ignore_patterns: [] # Claude Code SDK pass-through options — anything coder_eval doesn't own - # directly (e.g. `effort`, `max_thinking_tokens`). Keys must be + # directly (e.g. `effort`, `max_thinking_tokens`). Other agent types take + # their own keys (see docs/TASK_DEFINITION_GUIDE.md). Here keys must be # ClaudeAgentOptions fields and must not be framework-managed (model, # allowed_tools, permission_mode, hooks, mcp_servers, ...). Deep-merged # across experiment / task / variant / --sdk-option layers, so a higher diff --git a/src/coder_eval/models/results.py b/src/coder_eval/models/results.py index f778130f..1277c87b 100644 --- a/src/coder_eval/models/results.py +++ b/src/coder_eval/models/results.py @@ -672,10 +672,12 @@ class EvaluationResult(BaseModel): description="Agent configuration used for the evaluation (from task YAML)", ) - # SDK options (raw dump of all ClaudeAgentOptions fields including defaults) sdk_options: dict[str, Any] | None = Field( default=None, - description="Raw SDK options dump from ClaudeAgentOptions (all fields including defaults)", + description=( + "Options the agent sent to its SDK; the shape depends on the agent " + "(Claude Code: raw ClaudeAgentOptions dump; Delegate: host init options, credentials redacted)" + ), ) # Artifacts From a4cb6fdd7fd3b5228e558d0a031bab3413f098f6 Mon Sep 17 00:00:00 2001 From: Mihai Chirculescu Date: Sat, 3 Oct 2026 12:40:17 +0300 Subject: [PATCH 17/20] docs(delegate): describe the host env scrub as defense in depth The guide said that the agent's shell commands "cannot read the token". The scrub removes the token from the host env only. Code under test can still read the token file, the saved login, the LLMGW_* secret when no token file is set, the AUTH_TOKEN that the host writes on refresh, and the coder_eval process env. List these gaps. Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/agents/DELEGATE.md | 9 ++++++++- src/coder_eval/agents/delegate_agent.py | 3 ++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/docs/agents/DELEGATE.md b/docs/agents/DELEGATE.md index fb0a74b3..3e272efb 100644 --- a/docs/agents/DELEGATE.md +++ b/docs/agents/DELEGATE.md @@ -44,7 +44,14 @@ Either: - **Environment token** — `DELEGATE_AUTH_TOKEN`, `DELEGATE_TENANT_ID`, `DELEGATE_ORG_ID` env vars. When `DELEGATE_ENV` (not `DELEGATE_BACKEND_URL`) resolves the backend, also set `DELEGATE_ORG_SLUG` and `DELEGATE_TENANT_SLUG`. These are the human-readable org/tenant names, not the GUIDs. Without the slugs, init fails with `env="alpha" requires org/tenant slugs`. To skip the slugs, set `DELEGATE_BACKEND_URL` instead. Each variable also accepts the bare spelling (`AUTH_TOKEN`, `TENANT_ID`, `ORG_ID`, `ORG_SLUG`, `TENANT_SLUG`) as a fallback. Prefer the `DELEGATE_` spelling in a shared environment, because the bare names collide with what other tooling (npm, Vault, Terraform) commonly exports. - **Saved login** — a prior `npx @uipath/delegate-cli login --env ` that wrote `~/.aria/sdk-auth.json`. The host reads and refreshes this file itself when no token is supplied. This agent does not parse that file in Python, so auth-freshness logic lives in exactly one place. The saved login already carries the slugs, so `DELEGATE_ORG_SLUG` / `DELEGATE_TENANT_SLUG` are not needed. -coder_eval sends these values to the host as its `auth` init option, on stdin. It also removes the host's own variable names (`AUTH_TOKEN`, `TENANT_ID`, `ORG_ID`, `ORG_LOGICAL_NAME`, `TENANT_NAME`, `BACKEND_URL`) and `DELEGATE_AUTH_TOKEN` from the host's environment. So the agent's shell commands cannot read the token, and a variable that another tool exports does not change where the host connects. +coder_eval sends these values to the host as its `auth` init option, on stdin. It also removes the host's own variable names (`AUTH_TOKEN`, `TENANT_ID`, `ORG_ID`, `ORG_LOGICAL_NAME`, `TENANT_NAME`, `BACKEND_URL`) and `DELEGATE_AUTH_TOKEN` from the host's environment. So the agent's shell commands do not get the token in their environment, and a variable that another tool exports does not change where the host connects. + +This is defense in depth, not a security boundary. Code under test runs as your user, so it can still get to a credential through: + +- the token file that `DELEGATE_AUTH_TOKEN_FILE` names, or the saved login in `~/.aria/sdk-auth.json`; +- the `LLMGW_*` client secret, when no token file is set (see below); +- the host's own `AUTH_TOKEN`, which the host writes into its environment when it refreshes the token; +- the environment of the coder_eval process, which still holds `DELEGATE_AUTH_TOKEN` (on Linux, through `/proc//environ`). The host also reads its own advanced variables directly, such as `DELEGATE_AUTH_TOKEN_FILE` (a token file that an external process keeps fresh) and `DELEGATE_STDIO_VERBOSE=1` (trace every frame to stderr). See the [package README](https://www.npmjs.com/package/@uipath/delegate-stdio). diff --git a/src/coder_eval/agents/delegate_agent.py b/src/coder_eval/agents/delegate_agent.py index 03f7833f..a358bfbd 100644 --- a/src/coder_eval/agents/delegate_agent.py +++ b/src/coder_eval/agents/delegate_agent.py @@ -217,7 +217,8 @@ def _auth_option() -> dict[str, str]: ) """Removed from the host's environment. The host reads the first six itself, but coder_eval sends their values as init options instead; and the agent's shell tools -inherit the host env, so no spelling of the token may stay in it.""" +inherit the host env, so no spelling of the token may stay in it. This is defense in +depth; docs/agents/DELEGATE.md lists the known gaps.""" _GATEWAY_S2S_ENV_VARS = ("LLMGW_CLIENT_ID", "LLMGW_CLIENT_SECRET", "LLMGW_URL") From 669455dcc104e1478bb19a23b991ac83bfaca789 Mon Sep 17 00:00:00 2001 From: Mihai Chirculescu Date: Sat, 3 Oct 2026 12:40:47 +0300 Subject: [PATCH 18/20] test(golden): give the Delegate golden-coverage exemption a true reason The reason pointed at "UNVERIFIED" field shapes in delegate_agent.py, which this branch removed: the frame shapes are now confirmed against a live transcript. Delegate is exempt only because no delegate-stdio scenarios are recorded yet, as the notes already say. Co-Authored-By: Claude Opus 5.5 (1M context) --- tests/test_agent_golden_master.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/test_agent_golden_master.py b/tests/test_agent_golden_master.py index 12273127..c367395f 100644 --- a/tests/test_agent_golden_master.py +++ b/tests/test_agent_golden_master.py @@ -236,10 +236,8 @@ async def test_pi_reconciliation_invariant(scenario, tmp_path): _NO_GOLDEN_COVERAGE: dict[AgentKind, str] = { AgentKind.NONE: "agentless backend — runs no model, streams nothing", AgentKind.UNKNOWN: "sentinel for an undeterminable type — never registered", - # SDK event field shapes UNVERIFIED against a live backend; a snapshot built - # on guessed names would pin the guess as "correct". See delegate_agent.py. - # Rationale: .claude/notes/agents.md § Delegate agent - AgentKind.DELEGATE: "field shapes UNVERIFIED against a live backend — see delegate_agent.py", + # Rationale: .claude/notes/agents.md § Delegate agent golden-master and timing-identity coverage + AgentKind.DELEGATE: "no recorded delegate-stdio scenarios yet — tests/test_delegate_agent.py replays the frames", } From 404b09949fa1f5e42fbe532395d4c793b11d5078 Mon Sep 17 00:00:00 2001 From: Mihai Chirculescu Date: Sat, 3 Oct 2026 12:41:52 +0300 Subject: [PATCH 19/20] test(overrides): pin the sdk_options guard on the registry, not its contents Three tests matched the literal "claude-code, delegate agents". With a plugin installed that registers another kind with sdk_options, such as coder_eval_uipath's studio-web, the message lists more kinds and the tests fail. Match the fixed part of the message instead. Add a test that a throwaway registered kind whose config declares sdk_options accepts -D agent.sdk_options.*. This is the generic contract that lets coder_eval_uipath remove its override patch. Co-Authored-By: Claude Opus 5.5 (1M context) --- tests/test_merge_characterization.py | 2 +- tests/test_overrides_engine.py | 29 ++++++++++++++++++++++++++-- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/tests/test_merge_characterization.py b/tests/test_merge_characterization.py index 9d1c153c..74ac8924 100644 --- a/tests/test_merge_characterization.py +++ b/tests/test_merge_characterization.py @@ -306,7 +306,7 @@ def test_system_prompt_file_override_clears_sibling(self): def test_sdk_options_on_codex_raises_friendly(self): task = _live_task(agent={"type": "codex"}) - with pytest.raises(OverrideError, match="only supported for claude-code, delegate agents"): + with pytest.raises(OverrideError, match=r"agent type codex\. This option is only supported for "): apply_overrides(task, {"agent.sdk_options.effort": "high"}) def test_lineage_cli_source_for_touched_paths_only(self): diff --git a/tests/test_overrides_engine.py b/tests/test_overrides_engine.py index 93381843..0eb24ef1 100644 --- a/tests/test_overrides_engine.py +++ b/tests/test_overrides_engine.py @@ -144,7 +144,7 @@ def test_agent_type_injection_switches_subclass(self): def test_sdk_options_on_codex_raises(self): task = _make_task(agent=parse_agent_config(type="codex")) - with pytest.raises(OverrideError, match="only supported for claude-code, delegate agents"): + with pytest.raises(OverrideError, match=r"agent type codex\. This option is only supported for "): apply_overrides(task, {"agent.sdk_options.effort": "high"}) def test_sdk_options_effort_on_delegate_applies(self): @@ -152,9 +152,34 @@ def test_sdk_options_effort_on_delegate_applies(self): apply_overrides(task, {"agent.sdk_options.effort": "high"}) assert task.agent.sdk_options == {"effort": "high"} + def test_sdk_options_applies_to_any_registered_kind_that_declares_it(self): + """A plugin kind whose config declares ``sdk_options`` passes the guard, not only the built-ins.""" + from typing import Any, Literal + + from pydantic import Field + + from coder_eval.agents.registry import AgentRegistry + from coder_eval.models import BaseAgentConfig + + class _PluginConfig(BaseAgentConfig): + type: Literal["sdk-options-plugin"] # type: ignore[assignment] + sdk_options: dict[str, Any] = Field(default_factory=dict) + + class _PluginAgent: + def __init__(self, config, route=None, **kwargs): + self.config = config + + AgentRegistry.register("sdk-options-plugin", _PluginConfig)(_PluginAgent) + try: + task = _make_task(agent=parse_agent_config(type="sdk-options-plugin")) + apply_overrides(task, {"agent.sdk_options.mode": "fast"}) + assert task.agent.sdk_options == {"mode": "fast"} + finally: + AgentRegistry._registry.pop("sdk-options-plugin", None) + def test_sdk_options_with_agent_type_codex_raises(self): task = _make_task(agent=parse_agent_config(type="claude-code")) - with pytest.raises(OverrideError, match="only supported for claude-code, delegate agents"): + with pytest.raises(OverrideError, match=r"agent type codex\. This option is only supported for "): apply_overrides(task, {"agent.sdk_options.effort": "high"}, agent_type="codex") def test_unknown_root_raises(self): From 94e4a5aa9ee13a3d9e81b82d34d80258aee57232 Mon Sep 17 00:00:00 2001 From: Mihai Chirculescu Date: Sat, 3 Oct 2026 13:13:11 +0300 Subject: [PATCH 20/20] fix(delegate): reject an sdk_options.effort that is not a string `effort: 5` or `effort: [high]` passed validation. The host then logged the value as not a tier and used the model's default effort, while task.json still recorded the bad value. Reject a value that is not a string at load. The host stays the only list of tiers, so a new tier needs no coder_eval change. Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/agents/DELEGATE.md | 2 +- src/coder_eval/models/agent_config.py | 6 ++++-- tests/test_delegate_agent_config.py | 6 ++++++ 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/docs/agents/DELEGATE.md b/docs/agents/DELEGATE.md index 3e272efb..cf6c452d 100644 --- a/docs/agents/DELEGATE.md +++ b/docs/agents/DELEGATE.md @@ -171,7 +171,7 @@ Run-limit semantics per harness: [Run-Limit Parity](HARNESS_PARITY.md). 2. **`max_turns` is enforced by coder_eval, not by the host.** The host's `maxSteps` option on `send` does not stop the turn (confirmed live: `maxSteps: 2` ran 7 steps and only reported `maxStepsReached: true`), so this agent does not send it. See [Run-Limit Parity](HARNESS_PARITY.md). A turn cut at `max_turns`, or by a cooperative early stop, keeps the token usage and cost of every model call that finished before the cut, as on the other agents. The host sends a `usage` frame for each call before that call's tool results. The call in progress at the cut has no usage. A host that does not send `usage` frames reports usage only on its final `result` frame, which a cut turn never gets: that turn has no usage, and the agent logs a warning. 3. **Only three backend failures get a specific diagnosis.** A Cloudflare WAF block page is reported as a content-filter failure and is not retried: the same prompt or tool result is blocked again. An SSE connect timeout is reported as a connection failure and is retried. A session conflict ("A reply is already being generated") is retried in a new conversation. There is no first-response stall detection: a stalled turn ends at its turn timeout. Every other crash ends the turn as a retryable `AgentCrashError`. -4. **`sdk_options` accepts only `effort`.** Reasoning effort uses the same `sdk_options.effort` key as Claude Code, so `-D agent.sdk_options.effort=high` works for both agents. Any other key is a validation error. `project_id`, `session_id` and `enable_computer_use` are typed config fields. +4. **`sdk_options` accepts only `effort`.** Reasoning effort uses the same `sdk_options.effort` key as Claude Code, so `-D agent.sdk_options.effort=high` works for both agents. Any other key, or an `effort` that is not a string, is a validation error. The host checks the tier itself: it logs a tier it does not recognize and uses the model's default. `project_id`, `session_id` and `enable_computer_use` are typed config fields. 5. **`enable_computer_use: true` requires local permissions and is unavailable on Linux.** macOS needs Accessibility + Screen Recording grants; the SDK throws unconditionally on Linux when this is enabled. ## Testing diff --git a/src/coder_eval/models/agent_config.py b/src/coder_eval/models/agent_config.py index 93a8972c..27ce5974 100644 --- a/src/coder_eval/models/agent_config.py +++ b/src/coder_eval/models/agent_config.py @@ -421,8 +421,8 @@ class DelegateAgentConfig(BaseAgentConfig): description=( "Pass-through dict of delegate-stdio init options that coder_eval does not own " f"directly. Allowed keys: {sorted(_DELEGATE_SDK_OPTION_FIELDS)}. 'effort' is the " - "reasoning-effort tier (documented values: low/medium/high/xhigh/max), forwarded " - "as-is: the SDK ignores a value it does not recognize." + "reasoning-effort tier (documented values: low/medium/high/xhigh/max): it must be a " + "string, forwarded as-is, and the host ignores a tier it does not recognize." ), ) project_id: str | None = Field( @@ -460,6 +460,8 @@ def _validate_sdk_options_keys(cls, v: dict[str, Any]) -> dict[str, Any]: f"sdk_options keys {unknown} are not delegate-stdio init options coder_eval forwards " + f"(valid keys: {sorted(_DELEGATE_SDK_OPTION_FIELDS)})" ) + if "effort" in v and not isinstance(v["effort"], str): + raise ValueError(f"sdk_options.effort must be a string tier, got {v['effort']!r}") return v diff --git a/tests/test_delegate_agent_config.py b/tests/test_delegate_agent_config.py index b0bb5b7f..9426cd04 100644 --- a/tests/test_delegate_agent_config.py +++ b/tests/test_delegate_agent_config.py @@ -56,6 +56,12 @@ def test_effort_accepts_any_string_including_future_tiers(): assert cfg.sdk_options == {"effort": "ultra-max"} +@pytest.mark.parametrize("value", [5, ["high"], {"tier": "high"}, None], ids=["int", "list", "dict", "null"]) +def test_effort_rejects_a_value_that_is_not_a_string(value): + with pytest.raises(ValidationError, match=r"sdk_options\.effort must be a string"): + DelegateAgentConfig(type="delegate", sdk_options={"effort": value}) + + def test_sdk_options_rejects_a_key_the_host_is_not_sent(): with pytest.raises(ValidationError, match=r"valid keys: \['effort'\]"): DelegateAgentConfig(type="delegate", sdk_options={"backendUrl": "http://x"})