From d04930a35b9d408738e2f00fd4175a79818f039d Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Sat, 22 Aug 2026 03:20:32 +0000 Subject: [PATCH 1/5] Add Kernel browser runtime to Harbor adapter control arm --- CHANGELOG.md | 1 + docs/harbor.md | 19 + src/clawbench/eval/harbor_adapter.py | 230 ++++++++-- .../run_support/browser_runtime/providers.py | 10 + src/clawbench/runtime/harbor/Dockerfile | 2 +- .../runtime/harbor/kernel-browser.py | 295 ++++++++++++ src/clawbench/runtime/harbor/start-runtime.sh | 15 + tests/test_harbor_adapter.py | 2 +- tests/test_harbor_kernel_control.py | 424 ++++++++++++++++++ 9 files changed, 963 insertions(+), 35 deletions(-) create mode 100644 src/clawbench/runtime/harbor/kernel-browser.py create mode 100644 tests/test_harbor_kernel_control.py diff --git a/CHANGELOG.md b/CHANGELOG.md index c02b71cf..cd343c8d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ### Added - Added Kernel as a managed remote browser runtime with live view and downloaded replay recordings. Thanks to @[rgarcia](https://github.com/rgarcia). - Added `clawbench-analyze` entrypoint for aggregate batch error analysis. +- Added a `--browser-runtime kernel` mode to the Harbor adapter that runs each task against one Kernel cloud browser, exposing only a credential-free CDP bridge to the agent, and finalizes the replay and deletes the browser during verification. ### Changed - Updated the harbor adaptor to support the full V2 lenient & strict and reports numeric results. diff --git a/docs/harbor.md b/docs/harbor.md index fba41381..32c8491a 100644 --- a/docs/harbor.md +++ b/docs/harbor.md @@ -108,6 +108,25 @@ uvx --from harbor==0.15.0 harbor run \ --jobs-dir ./harbor-jobs/hermes-deepseek-flash ``` +## Kernel browser runtime (control arm) + +By default each Harbor trial runs Chromium inside its own container. Pass `--browser-runtime kernel` to the adapter to run the same tasks against one Kernel cloud browser per task instead: + +```bash +uv run clawbench-harbor-adapt \ + --output-dir ./harbor-datasets/clawbench-v2-kernel \ + --browser-runtime kernel \ + --browser-runtime-options '{"stealth": false}' \ + --task-ids v2-1134-chapter-finder-redcross \ + --overwrite +``` + +During task setup, the environment creates exactly one Kernel browser and replay, starts the ClawBench runtime server against it, and exposes only the local credential-free CDP bridge (`http://127.0.0.1:7878`) to the agent — the Kernel API key is never visible to the benchmark agent. Session identity and cleanup metadata land in `/my-info/kernel_browser.json`. During verification the provider replay is finalized, `recording.mp4` is downloaded into `/data`, and the browser is deleted (idempotently, including failure paths via a setup trap). + +Generated tasks register a pinned Playwright MCP package (`@playwright/mcp@0.0.79`) pointed at the CDP bridge, so Harbor's stock Claude Code and Codex agents drive the Kernel browser with native Playwright MCP tool calls — structurally identical to ClawBench's native Claude/Codex harnesses. + +Export `KERNEL_API_KEY` (and optionally `KERNEL_BASE_URL` for non-production gateways) before `harbor run`; no extra flags are needed. + ## Making it fast A full V2 sweep is 129 containerized browser sessions, each capped by the task's `time_limit`. Serial, that is a very long night. What actually moves the needle, in order: diff --git a/src/clawbench/eval/harbor_adapter.py b/src/clawbench/eval/harbor_adapter.py index 041c6d2c..2f95db18 100644 --- a/src/clawbench/eval/harbor_adapter.py +++ b/src/clawbench/eval/harbor_adapter.py @@ -11,11 +11,24 @@ from pathlib import Path from typing import Any +from clawbench.runner.run_support.browser_runtime.providers import ( + BrowserRuntimeError, + _parse_options, +) from clawbench.runner.run_support.task import build_instruction, validate_task_data from clawbench.utils.paths import RUNTIME_ROOT, asset_path DEFAULT_CASES_DIR = asset_path("test-cases", "v2") STEP_NAME = "run" +HARBOR_BROWSER_RUNTIMES = ("local", "kernel") +# Remote browser runtimes connect through the runtime server's local, +# credential-free CDP bridge instead of a container-local Chromium. +REMOTE_BRIDGE_CDP_URL = "http://127.0.0.1:7878" +LOCAL_CDP_URL = "http://127.0.0.1:9223" +# Pinned Playwright MCP package Harbor's stock Claude Code and Codex agents +# use to drive the ClawBench browser through the CDP bridge. +PLAYWRIGHT_MCP_PACKAGE = "@playwright/mcp" +PLAYWRIGHT_MCP_VERSION = "0.0.79" def sanitize_task_name(raw: str) -> str: @@ -96,6 +109,16 @@ def copy_environment(env_dir: Path) -> None: copytree_filtered(RUNTIME_ROOT / "shared", env_dir / "shared") copytree_filtered(RUNTIME_ROOT / "harbor", env_dir / "harbor") (env_dir / "harbor" / "Dockerfile").unlink(missing_ok=True) + # The Kernel lifecycle scripts reuse the same provider implementation as + # the native runner. + shutil.copy2( + Path(__file__).resolve().parents[1] + / "runner" + / "run_support" + / "browser_runtime" + / "providers.py", + env_dir / "harbor" / "browser_runtime_providers.py", + ) shutil.copy2( Path(__file__).resolve().parents[1] / "runner" @@ -108,18 +131,71 @@ def copy_environment(env_dir: Path) -> None: chmod_executable(script) -def harbor_instruction(task: dict[str, Any]) -> str: +def playwright_mcp_server(cdp_url: str) -> dict[str, Any]: + return { + "name": "playwright", + "transport": "stdio", + "command": "npx", + "args": [ + "-y", + f"{PLAYWRIGHT_MCP_PACKAGE}@{PLAYWRIGHT_MCP_VERSION}", + "--cdp-endpoint", + cdp_url, + ], + } + + +def mcp_servers_toml(servers: list[dict[str, Any]]) -> str: + if not servers: + return "" + blocks = [] + for server in servers: + args = ", ".join(json.dumps(arg) for arg in server["args"]) + blocks.append( + "[[environment.mcp_servers]]\n" + f"name = {json.dumps(server['name'])}\n" + f"transport = {json.dumps(server['transport'])}\n" + f"command = {json.dumps(server['command'])}\n" + f"args = [{args}]\n" + ) + return "\n" + "\n".join(blocks) + + +def harbor_instruction(task: dict[str, Any], *, browser_runtime: str = "local") -> str: instruction = build_instruction(task) - return ( - instruction + "\n\n---\n" + cdp_url = REMOTE_BRIDGE_CDP_URL if browser_runtime == "kernel" else LOCAL_CDP_URL + runtime_section = ( "Harbor browser runtime:\n" - "- Use the existing Chromium session exposed by Chrome DevTools Protocol.\n" - "- CDP endpoint: http://127.0.0.1:9223\n" + "- Use the existing browser session exposed by Chrome DevTools Protocol.\n" + f"- CDP endpoint: {cdp_url}\n" "- CDP environment variables are also set for the agent process: " "CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL.\n" - "- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html\n" - "- Do not launch a separate browser. Complete the task through the existing browser session.\n" - "---\n" + ) + if browser_runtime == "local": + runtime_section += "- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html\n" + restrictions = ( + "Task constraints (matching the native ClawBench harness rules):\n" + f"- Time limit: {task.get('time_limit')} minutes. The harness stops the " + "run when the limit elapses, so finish and submit before then.\n" + "- Complete the task entirely in the browser; do not launch or use any " + "other browser.\n" + "- Use only Playwright MCP browser tools plus reading files under " + "./my-info/ to accomplish the task.\n" + "- Do NOT make direct HTTP/network requests for task completion via shell " + "tools, scripts, API calls, or SMTP — every task action must go through " + "the browser.\n" + "- Submit through the browser: perform the task's final submission action " + "in the browser so the request happens on the page.\n" + "- Stop after submission: once the final action is submitted, stop and do " + "not start other work.\n" + ) + return ( + instruction + + "\n\n---\n" + + runtime_section + + "- Do not launch a separate browser. Complete the task through the existing browser session.\n" + + "---\n\n" + + restrictions ) @@ -130,12 +206,37 @@ def task_toml( dataset_name: str, timeout_sec: int, task_dir_name: str, + browser_runtime: str = "local", + browser_runtime_options: str | None = None, ) -> str: escaped_description = json.dumps(description) escaped_dataset = json.dumps(dataset_name) escaped_source = json.dumps(task_dir_name) escaped_package = json.dumps(package_name) - return f"""schema_version = "1.3" + cdp_url = REMOTE_BRIDGE_CDP_URL if browser_runtime == "kernel" else LOCAL_CDP_URL + kernel_env = "" + mcp_servers = "" + if browser_runtime == "kernel": + runtime_options_line = ( + f"\nCLAWBENCH_BROWSER_RUNTIME_OPTIONS = {json.dumps(browser_runtime_options)}" + if browser_runtime_options + else '\nCLAWBENCH_BROWSER_RUNTIME_OPTIONS = "${CLAWBENCH_BROWSER_RUNTIME_OPTIONS:-}"' + ) + kernel_env = ( + '\nCLAWBENCH_HARBOR_BROWSER_RUNTIME = "kernel"' + '\nKERNEL_API_KEY = "${KERNEL_API_KEY}"' + '\nKERNEL_BASE_URL = "${KERNEL_BASE_URL:-}"' + + runtime_options_line + + '\nCLAWBENCH_RECORDING_MODE = "provider-download"' + ) + mcp_servers = mcp_servers_toml([playwright_mcp_server(REMOTE_BRIDGE_CDP_URL)]) + healthcheck_command = ( + "curl -sf http://127.0.0.1:7878/api/status | grep -q '" + + '\\"eval_interceptor_ready\\":true' + + f"' && curl -sf {cdp_url}/json/version >/dev/null" + ) + return ( + f"""schema_version = "1.3" source = "clawbench-v2" artifacts = ["/data"] @@ -147,20 +248,24 @@ def task_toml( [metadata] dataset = {escaped_dataset} source_task = {escaped_source} +browser_runtime = "{browser_runtime}" [environment] build_timeout_sec = 1200.0 network_mode = "public" -workdir = "/app" +# The container root doubles as the step workdir so every exec runs with a +# cwd that exists even on overlay drivers that cannot resolve image-created +# directories during `docker exec`. +workdir = "/" [environment.env] PURELY_MAIL_API_KEY = "${{PURELY_MAIL_API_KEY}}" PURELY_MAIL_DOMAIN = "${{PURELY_MAIL_DOMAIN}}" -CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" -BROWSER_CDP_URL = "http://127.0.0.1:9223" -CDP_URL = "http://127.0.0.1:9223" -CHROME_CDP_URL = "http://127.0.0.1:9223" -PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_CDP_URL = "{cdp_url}" +BROWSER_CDP_URL = "{cdp_url}" +CDP_URL = "{cdp_url}" +CHROME_CDP_URL = "{cdp_url}" +PLAYWRIGHT_CDP_URL = "{cdp_url}"{kernel_env} CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" CLAWBENCH_JUDGE_BASE_URL = "${{CLAWBENCH_JUDGE_BASE_URL:-}}" @@ -178,37 +283,59 @@ def task_toml( timeout_sec = 300.0 [steps.healthcheck] -command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\\\"eval_interceptor_ready\\\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +command = "{healthcheck_command}" interval_sec = 2.0 timeout_sec = 5.0 start_period_sec = 2.0 start_interval_sec = 1.0 retries = 30 """ + + mcp_servers + ) -def setup_script() -> str: - return """#!/bin/bash +def setup_script(browser_runtime: str = "local") -> str: + kernel_setup = "" + readiness = ( + " if curl -sf http://127.0.0.1:7878/api/status >/dev/null \\\n" + " && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then\n" + ) + if browser_runtime == "kernel": + readiness = ( + " if curl -sf http://127.0.0.1:7878/api/status >/dev/null \\\n" + " && curl -sf http://127.0.0.1:7878/json/version >/dev/null; then\n" + ) + kernel_setup = ( + "# Create the Kernel browser and replay before the runtime server" + " starts so it can bridge the provider CDP endpoint.\n" + "/app/src/runtime-server/.venv/bin/python /app/src/harbor/kernel-browser.py start\n" + "export CLAWBENCH_BROWSER_CDP_URL_FILE=/tmp/clawbench-run/kernel-cdp-url\n" + "\n" + "cleanup_browser() {\n" + " /app/src/runtime-server/.venv/bin/python /app/src/harbor/kernel-browser.py cleanup || true\n" + "}\n" + "trap cleanup_browser EXIT\n" + "\n" + ) + return f"""#!/bin/bash set -euo pipefail -mkdir -p /data /logs/verifier /app/extra_info -cp /app/eval-schema.json /eval-schema.json +mkdir -p /data /logs/verifier /extra_info /app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py \ - --task-json /app/task.json \ - --extra-info-dir /app/extra_info \ - --output-dir /app/my-info + --task-json /task.json \ + --extra-info-dir /extra_info \ + --output-dir /my-info # Harbor installs its stock agent before step setup. Wrap that executable so # ClawBench's existing /data/.stop-requested signal ends the agent cleanly. /app/src/harbor/wrap-harbor-agent.sh -/app/src/harbor/start-runtime.sh +{kernel_setup}/app/src/harbor/start-runtime.sh for _ in $(seq 1 60); do - if curl -sf http://127.0.0.1:7878/api/status >/dev/null \ - && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then - rm -f /app/setup.sh +{readiness} rm -f /app/setup.sh + trap - EXIT exit 0 fi sleep 1 @@ -219,15 +346,21 @@ def setup_script() -> str: """ -def test_script() -> str: - return """#!/bin/bash +def test_script(browser_runtime: str = "local") -> str: + kernel_finalize = "" + if browser_runtime == "kernel": + kernel_finalize = ( + "# Stop the provider replay, download the recording, delete the browser.\n" + "/app/src/runtime-server/.venv/bin/python /app/src/harbor/kernel-browser.py finalize\n" + ) + return f"""#!/bin/bash set -euo pipefail curl -sf -X POST http://127.0.0.1:7878/api/stop || true curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true sleep 2 rm -f /data/.stop-requested -rm -rf /logs/verifier/data +{kernel_finalize}rm -rf /logs/verifier/data cp -a /data /logs/verifier/data /app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py @@ -269,6 +402,8 @@ def write_harbor_task( output_name: str, org: str, dataset_name: str, + browser_runtime: str = "local", + browser_runtime_options: str | None = None, ) -> Path: dest = output_root / output_name if dest.exists(): @@ -296,15 +431,19 @@ def write_harbor_task( dataset_name=dataset_name, timeout_sec=timeout_sec, task_dir_name=task_dir.name, + browser_runtime=browser_runtime, + browser_runtime_options=browser_runtime_options, ) ) - (step_dir / "instruction.md").write_text(harbor_instruction(task)) + (step_dir / "instruction.md").write_text( + harbor_instruction(task, browser_runtime=browser_runtime) + ) (workdir / "eval-schema.json").write_text(json.dumps(task["eval_schema"], indent=2)) (workdir / "task.json").write_text(json.dumps(task, indent=2, ensure_ascii=False)) copy_extra_info(task, task_dir, workdir / "extra_info") - write_text_executable(workdir / "setup.sh", setup_script()) + write_text_executable(workdir / "setup.sh", setup_script(browser_runtime)) (tests_dir / "task.json").write_text(json.dumps(task, indent=2, ensure_ascii=False)) - write_text_executable(tests_dir / "test.sh", test_script()) + write_text_executable(tests_dir / "test.sh", test_script(browser_runtime)) write_text_executable(solution_dir / "solve.sh", solve_script()) copy_environment(env_dir) return dest @@ -343,6 +482,18 @@ def build_parser() -> argparse.ArgumentParser: action="store_true", help="Overwrite an existing output directory", ) + parser.add_argument( + "--browser-runtime", + choices=HARBOR_BROWSER_RUNTIMES, + default="local", + help="Browser runtime for the generated tasks; kernel creates one " + "Kernel browser per task during Harbor setup", + ) + parser.add_argument( + "--browser-runtime-options", + default=None, + help="JSON object with runtime options, e.g. '{\"stealth\":true}'", + ) return parser @@ -350,6 +501,13 @@ def main(argv: list[str] | None = None) -> int: parser = build_parser() args = parser.parse_args(argv) + options: dict[str, Any] = {} + if args.browser_runtime_options: + try: + options = _parse_options(args.browser_runtime_options) + except BrowserRuntimeError as exc: + parser.error(str(exc)) + cases_dir = (args.cases_dir or DEFAULT_CASES_DIR).resolve() default_cases = args.cases_dir is None if not cases_dir.exists(): @@ -388,10 +546,16 @@ def main(argv: list[str] | None = None) -> int: output_name=out_name, org=args.org, dataset_name=args.dataset_name, + browser_runtime=args.browser_runtime, + browser_runtime_options=(json.dumps(options) if options else None), ) ) print(f"Wrote {len(written)} Harbor task(s) to {output_dir}") + if args.browser_runtime != "local": + print(f"Browser runtime: {args.browser_runtime}") + if options: + print(f"Runtime options: {json.dumps(options)}") return 0 diff --git a/src/clawbench/runner/run_support/browser_runtime/providers.py b/src/clawbench/runner/run_support/browser_runtime/providers.py index 999da3a7..c01ebb2d 100644 --- a/src/clawbench/runner/run_support/browser_runtime/providers.py +++ b/src/clawbench/runner/run_support/browser_runtime/providers.py @@ -558,6 +558,16 @@ def _delete(self, session_id: str) -> str: raise return "deleted" + def session_exists(self, session_id: str) -> bool: + """Return True if the provider still reports this browser session.""" + try: + self._request("GET", f"/browsers/{session_id}") + except _KernelApiError as e: + if e.status == 404: + return False + raise + return True + def _stop_replay(self, session_id: str, replay_id: str) -> None: try: self._request( diff --git a/src/clawbench/runtime/harbor/Dockerfile b/src/clawbench/runtime/harbor/Dockerfile index cd7c1921..d804a557 100644 --- a/src/clawbench/runtime/harbor/Dockerfile +++ b/src/clawbench/runtime/harbor/Dockerfile @@ -8,7 +8,7 @@ RUN echo 'APT::Sandbox::User "root";' > /etc/apt/apt.conf.d/01disable-sandbox \ && printf '#!/bin/sh\n/usr/bin/dpkg-statoverride.real "$@" 2>/dev/null || true\n' \ > /usr/bin/dpkg-statoverride && chmod +x /usr/bin/dpkg-statoverride \ && apt-get update && apt-get install -y --no-install-recommends \ - chromium xvfb ffmpeg socat curl git x11vnc xclip \ + chromium xvfb ffmpeg socat curl git x11vnc xclip nodejs npm \ libegl1 libgbm1 \ fonts-noto-color-emoji fonts-noto-cjk \ && mv /usr/bin/dpkg-statoverride.real /usr/bin/dpkg-statoverride \ diff --git a/src/clawbench/runtime/harbor/kernel-browser.py b/src/clawbench/runtime/harbor/kernel-browser.py new file mode 100644 index 00000000..41655368 --- /dev/null +++ b/src/clawbench/runtime/harbor/kernel-browser.py @@ -0,0 +1,295 @@ +#!/usr/bin/env python3 +"""Kernel browser lifecycle for the ClawBench Harbor control arm. + +Mirrors the native runner's browser-runtime phases (start / finalize / +cleanup) inside a Harbor task container, reusing the same provider +implementation the native runner uses. The Kernel API key and the real +provider CDP URL stay in root-only files under /tmp/clawbench-run; the +benchmark agent only ever sees the credential-free CDP bridge exposed by +the ClawBench runtime server. +""" + +from __future__ import annotations + +import argparse +import json +import os +import shutil +import sys +import time +from pathlib import Path +from typing import Any + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +try: # Vendored next to this script inside Harbor task environments. + from browser_runtime_providers import ( # type: ignore[import-not-found] + BrowserRuntimeError, + BrowserSession, + KernelRuntimeProvider, + redact_cdp_url, + ) +except ImportError: # Source checkout / tests. + from clawbench.runner.run_support.browser_runtime.providers import ( + BrowserRuntimeError, + BrowserSession, + KernelRuntimeProvider, + redact_cdp_url, + ) + +DATA_DIR = Path(os.environ.get("CLAWBENCH_DATA_DIR", "/data")) +STATE_DIR = Path("/tmp/clawbench-run") +STATE_FILE = STATE_DIR / "kernel-browser-state.json" +CDP_URL_FILE = STATE_DIR / "kernel-cdp-url" +METADATA_FILE = ( + Path(os.environ.get("CLAWBENCH_MY_INFO_DIR", "/my-info")) / "kernel_browser.json" +) +LIFECYCLE_FILE = DATA_DIR / "kernel-browser-lifecycle.json" +TASK_FILE = Path("/task.json") +RUNTIME_SERVER_BRIDGE_URL = "http://127.0.0.1:7878" + + +def _load_provider() -> KernelRuntimeProvider: + api_key = os.environ.get("KERNEL_API_KEY", "") + if not api_key: + raise SystemExit("KERNEL_API_KEY is required for the kernel browser runtime") + options: dict[str, Any] = {} + raw_options = os.environ.get("CLAWBENCH_BROWSER_RUNTIME_OPTIONS", "") + if raw_options: + options = json.loads(raw_options) + if not isinstance(options, dict): + raise SystemExit("CLAWBENCH_BROWSER_RUNTIME_OPTIONS must be a JSON object") + return KernelRuntimeProvider( + api_key=api_key, + options=options, + api_url=os.environ.get("KERNEL_BASE_URL") or "https://api.onkernel.com", + ) + + +def _load_state() -> dict[str, Any] | None: + if STATE_FILE.is_file(): + return json.loads(STATE_FILE.read_text()) + # Fall back to the agent-visible metadata so cleanup still works if the + # state file was lost but the browser was created. + if METADATA_FILE.is_file(): + metadata = json.loads(METADATA_FILE.read_text()) + if metadata.get("session_id"): + return {"metadata": metadata, "events": []} + return None + + +def _save_state(state: dict[str, Any]) -> None: + STATE_DIR.mkdir(parents=True, exist_ok=True) + STATE_FILE.write_text(json.dumps(state, indent=2)) + STATE_FILE.chmod(0o600) + if state.get("cdp_url"): + CDP_URL_FILE.write_text(str(state["cdp_url"])) + CDP_URL_FILE.chmod(0o600) + + +def _public_metadata(state: dict[str, Any]) -> dict[str, Any]: + """Credential-free view of the session for the agent-visible file.""" + metadata = state.get("metadata", {}) + inner = ( + metadata.get("metadata") if isinstance(metadata.get("metadata"), dict) else {} + ) + return { + "provider": "kernel", + "mode": "remote", + "runtime": "kernel", + "session_id": metadata.get("session_id"), + "replay_id": inner.get("replay_id") or metadata.get("replay_id"), + "region": inner.get("region") or metadata.get("region"), + "stealth": inner.get("stealth") or metadata.get("stealth"), + "timeout_seconds": ( + inner.get("timeout_seconds") or metadata.get("timeout_seconds") + ), + "cdp_url": redact_cdp_url(metadata["cdp_url"]) + if metadata.get("cdp_url") + else None, + "viewer_url": "[REDACTED]" if metadata.get("viewer_url") else None, + "cdp_bridge_url": RUNTIME_SERVER_BRIDGE_URL, + "recording_mode": "provider-download", + "status": state.get("status"), + "cleanup_status": state.get("cleanup_status"), + "cleanup_error": state.get("cleanup_error"), + "deletion_verified": state.get("deletion_verified"), + "events": state.get("events", []), + } + + +def _write_metadata(state: dict[str, Any]) -> None: + METADATA_FILE.parent.mkdir(parents=True, exist_ok=True) + METADATA_FILE.write_text(json.dumps(_public_metadata(state), indent=2)) + + +def _record(state: dict[str, Any], event: str, **fields: Any) -> None: + entry = {"event": event, "ts": time.time(), **fields} + state.setdefault("events", []).append(entry) + print(json.dumps(entry), flush=True) + + +def _task_time_limit_s() -> int: + task = json.loads(TASK_FILE.read_text()) + return int(float(task["time_limit"]) * 60) + + +def cmd_start() -> int: + if STATE_FILE.is_file(): + state = json.loads(STATE_FILE.read_text()) + if state.get("status") == "created": + print("Kernel browser already created for this trial", flush=True) + _write_metadata(state) + return 0 + + provider = _load_provider() + task = json.loads(TASK_FILE.read_text()) + session = provider.start(task, _task_time_limit_s()) + state: dict[str, Any] = { + "status": "created", + "cdp_url": session.cdp_url, + "metadata": session.to_metadata(), + "events": [], + } + # to_metadata redacts the CDP URL; keep the real one for the runtime + # server only, in the 0600 state file. + state["metadata"]["cdp_url"] = session.cdp_url + _record(state, "browser_created", session_id=session.session_id) + _save_state(state) + _write_metadata(state) + print( + f"Kernel browser ready; CDP bridge at {RUNTIME_SERVER_BRIDGE_URL}", flush=True + ) + return 0 + + +def _download_recording(provider: KernelRuntimeProvider, state: dict[str, Any]) -> None: + staging = STATE_DIR / "finalize-output" + shutil.rmtree(staging, ignore_errors=True) + session = _session_from_state(state) + provider.finalize(session, staging) + recording = staging / "data" / "recording.mp4" + if recording.is_file(): + dest = DATA_DIR / "recording.mp4" + dest.parent.mkdir(parents=True, exist_ok=True) + shutil.move(str(recording), dest) + state["recording_bytes"] = dest.stat().st_size + + +def _session_from_state(state: dict[str, Any]) -> BrowserSession: + metadata = state.get("metadata", {}) + inner = ( + metadata.get("metadata") if isinstance(metadata.get("metadata"), dict) else {} + ) + return BrowserSession( + provider="kernel", + mode="remote", + session_id=metadata.get("session_id"), + cdp_url=metadata.get("cdp_url", ""), + metadata={ + "replay_id": inner.get("replay_id") or metadata.get("replay_id"), + }, + recording_mode="provider-download", + ) + + +def cmd_finalize() -> int: + state = _load_state() + if state is None: + print("No Kernel browser was created; nothing to finalize", flush=True) + return 0 + if state.get("status") == "deleted": + print("Kernel browser already finalized and deleted", flush=True) + return 0 + + provider = _load_provider() + session_id = state.get("metadata", {}).get("session_id") + try: + _download_recording(provider, state) + _record(state, "replay_finalized", recording_bytes=state.get("recording_bytes")) + state["status"] = "finalized" + _write_metadata(state) + except BrowserRuntimeError as e: + # The replay is lost, but deletion must still proceed. + state["cleanup_error"] = f"replay finalization failed: {e}" + _record(state, "replay_finalization_failed", error=str(e)) + + try: + provider.cleanup(_session_from_state(state)) + state["cleanup_status"] = "deleted" + except BrowserRuntimeError as e: + state["cleanup_error"] = str(e) + _record(state, "finalize_failed", error=str(e)) + _save_state(state) + _write_metadata(state) + return 1 + + _verify_deletion(provider, state) + state["status"] = "deleted" + _record( + state, + "browser_deleted", + session_id=session_id, + cleanup_status=state.get("cleanup_status"), + deletion_verified=state.get("deletion_verified"), + ) + _save_state(state) + _write_metadata(state) + _write_lifecycle(state) + return 0 + + +def cmd_cleanup() -> int: + """Cleanup-only path (e.g. setup failed after the browser was created).""" + state = _load_state() + if state is None or state.get("status") == "deleted": + print("No Kernel browser to clean up", flush=True) + return 0 + + provider = _load_provider() + try: + provider.cleanup(_session_from_state(state)) + state["cleanup_status"] = "deleted" + except BrowserRuntimeError as e: + state["cleanup_error"] = str(e) + _record(state, "cleanup_failed", error=str(e)) + _write_metadata(state) + return 1 + + _verify_deletion(provider, state) + state["status"] = "deleted" + _record(state, "browser_deleted", cleanup_status=state.get("cleanup_status")) + _save_state(state) + _write_metadata(state) + _write_lifecycle(state) + return 0 + + +def _verify_deletion(provider: KernelRuntimeProvider, state: dict[str, Any]) -> None: + session_id = state.get("metadata", {}).get("session_id") + if not session_id: + state["deletion_verified"] = False + return + try: + state["deletion_verified"] = not provider.session_exists(session_id) + except BrowserRuntimeError as e: + state["deletion_verified"] = False + state["cleanup_error"] = f"deletion check failed: {e}" + + +def _write_lifecycle(state: dict[str, Any]) -> None: + LIFECYCLE_FILE.parent.mkdir(parents=True, exist_ok=True) + LIFECYCLE_FILE.write_text(json.dumps(_public_metadata(state), indent=2)) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("command", choices=["start", "finalize", "cleanup"]) + args = parser.parse_args() + return {"start": cmd_start, "finalize": cmd_finalize, "cleanup": cmd_cleanup}[ + args.command + ]() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/clawbench/runtime/harbor/start-runtime.sh b/src/clawbench/runtime/harbor/start-runtime.sh index 2e0749d3..62bd59b2 100644 --- a/src/clawbench/runtime/harbor/start-runtime.sh +++ b/src/clawbench/runtime/harbor/start-runtime.sh @@ -8,6 +8,15 @@ if [ -f /tmp/clawbench-run/runtime.pid ] && kill -0 "$(cat /tmp/clawbench-run/ru exit 0 fi +# Remote browser runtimes (e.g. Kernel) hand us a provider CDP URL through a +# file instead of launching local Chromium. The runtime server proxies that +# WebSocket behind a credential-free local bridge on port 7878. +REMOTE_MODE=false +if [ -n "${CLAWBENCH_BROWSER_CDP_URL_FILE:-}" ] && [ -f "${CLAWBENCH_BROWSER_CDP_URL_FILE}" ]; then + REMOTE_MODE=true +fi + +if [ "$REMOTE_MODE" = false ]; then export DISPLAY="${DISPLAY:-:99}" Xvfb "$DISPLAY" -screen 0 1920x1080x24 >/tmp/clawbench-run/xvfb.log 2>&1 & echo "$!" > /tmp/clawbench-run/xvfb.pid @@ -118,6 +127,12 @@ sleep 1 /opt/novnc/utils/novnc_proxy --vnc localhost:5900 --listen 6080 >/tmp/clawbench-run/novnc.log 2>&1 & echo "$!" > /tmp/clawbench-run/novnc.pid +fi + +if [ "$REMOTE_MODE" = true ]; then + # Provider replays replace local X11 recording. + export CLAWBENCH_RECORDING_MODE="${CLAWBENCH_RECORDING_MODE:-provider-download}" +fi echo "$$" > /tmp/clawbench-run/runtime.pid echo "CDP ready at http://127.0.0.1:9223" diff --git a/tests/test_harbor_adapter.py b/tests/test_harbor_adapter.py index 09852732..c7afe7b8 100644 --- a/tests/test_harbor_adapter.py +++ b/tests/test_harbor_adapter.py @@ -109,7 +109,7 @@ def test_write_harbor_task_emits_expected_tree_and_extra_info(tmp_path: Path) -> assert ( config["task"]["name"] == "clawbench/v2-047-daily-life-personal-care-taskrabbit" ) - assert config["environment"]["workdir"] == "/app" + assert config["environment"]["workdir"] == "/" assert config["environment"]["network_mode"] == "public" assert config["environment"]["env"]["BROWSER_CDP_URL"] == "http://127.0.0.1:9223" assert config["environment"]["env"]["PLAYWRIGHT_CDP_URL"] == "http://127.0.0.1:9223" diff --git a/tests/test_harbor_kernel_control.py b/tests/test_harbor_kernel_control.py new file mode 100644 index 00000000..9c88a5a4 --- /dev/null +++ b/tests/test_harbor_kernel_control.py @@ -0,0 +1,424 @@ +"""Regression tests for the Harbor ClawBench Kernel control arm.""" + +from __future__ import annotations + +import importlib.util +import json +import sys +import tomllib +from pathlib import Path + +import pytest + +from clawbench.eval.harbor_adapter import ( + PLAYWRIGHT_MCP_PACKAGE, + PLAYWRIGHT_MCP_VERSION, + main as adapt_main, + write_harbor_task, +) + +REPO_ROOT = Path(__file__).resolve().parents[1] +KERNEL_BROWSER_SCRIPT = ( + REPO_ROOT / "src" / "clawbench" / "runtime" / "harbor" / "kernel-browser.py" +) + + +def _task() -> dict: + return { + "metadata": { + "task_id": 1134, + "description": "Find nearest Red Cross chapter", + }, + "instruction": "Find the nearest Red Cross chapter to zip code 90210.", + "eval_schema": {"url_pattern": "redcross\\.org/chapter", "method": "GET"}, + "time_limit": 30, + "extra_info": [], + } + + +@pytest.fixture() +def adapted_kernel_task(tmp_path: Path) -> Path: + case = tmp_path / "case" + case.mkdir() + (case / "task.json").write_text(json.dumps(_task())) + return write_harbor_task( + task_dir=case, + task=_task(), + output_root=tmp_path / "out", + output_name="v2-1134-chapter-finder-redcross", + org="clawbench", + dataset_name="v2", + browser_runtime="kernel", + ) + + +def test_kernel_runtime_selection_writes_bridge_env_and_pinned_mcp( + adapted_kernel_task: Path, +) -> None: + config = tomllib.loads((adapted_kernel_task / "task.toml").read_text()) + + assert config["metadata"]["browser_runtime"] == "kernel" + env = config["environment"]["env"] + assert env["CLAWBENCH_HARBOR_BROWSER_RUNTIME"] == "kernel" + assert env["PLAYWRIGHT_CDP_URL"] == "http://127.0.0.1:7878" + assert env["CLAWBENCH_CDP_URL"] == "http://127.0.0.1:7878" + assert env["CLAWBENCH_RECORDING_MODE"] == "provider-download" + assert env["KERNEL_API_KEY"] == "${KERNEL_API_KEY}" + + servers = config["environment"]["mcp_servers"] + assert len(servers) == 1 + server = servers[0] + assert server["name"] == "playwright" + assert server["transport"] == "stdio" + assert server["command"] == "npx" + assert server["args"][0] == "-y" + # Pinned package so the control arm is reproducible. + assert server["args"][1] == f"{PLAYWRIGHT_MCP_PACKAGE}@{PLAYWRIGHT_MCP_VERSION}" + assert "--cdp-endpoint" in server["args"] + assert "http://127.0.0.1:7878" in server["args"] + + healthcheck = config["steps"][0]["healthcheck"]["command"] + assert "7878/json/version" in healthcheck + + +def test_kernel_setup_and_test_scripts_wire_lifecycle( + adapted_kernel_task: Path, +) -> None: + workdir = adapted_kernel_task / "steps" / "run" / "workdir" + tests = adapted_kernel_task / "steps" / "run" / "tests" + + setup = (workdir / "setup.sh").read_text() + assert "kernel-browser.py start" in setup + assert "export CLAWBENCH_BROWSER_CDP_URL_FILE=" in setup + assert "trap cleanup_browser EXIT" in setup + assert "kernel-browser.py cleanup" in setup + assert "127.0.0.1:7878/json/version" in setup + + test = (tests / "test.sh").read_text() + assert "kernel-browser.py finalize" in test + + env_dir = adapted_kernel_task / "environment" + assert (env_dir / "harbor" / "kernel-browser.py").is_file() + assert (env_dir / "harbor" / "browser_runtime_providers.py").is_file() + + +def test_local_runtime_default_has_no_kernel_hooks(tmp_path: Path) -> None: + case = tmp_path / "case" + case.mkdir() + (case / "task.json").write_text(json.dumps(_task())) + out = write_harbor_task( + task_dir=case, + task=_task(), + output_root=tmp_path / "out", + output_name="local", + org="clawbench", + dataset_name="v2", + ) + + config = tomllib.loads((out / "task.toml").read_text()) + assert config["metadata"]["browser_runtime"] == "local" + assert "mcp_servers" not in config["environment"] + assert config["environment"]["env"]["PLAYWRIGHT_CDP_URL"] == "http://127.0.0.1:9223" + assert config["environment"]["env"].get("CLAWBENCH_HARBOR_BROWSER_RUNTIME") is None + + setup = (out / "steps" / "run" / "workdir" / "setup.sh").read_text() + assert "kernel-browser" not in setup + assert (out / "environment" / "harbor" / "browser_runtime_providers.py").is_file() + + +def test_instruction_ports_native_restrictions(adapted_kernel_task: Path) -> None: + instruction = (adapted_kernel_task / "steps" / "run" / "instruction.md").read_text() + # Native ClawBench prompt text (source of truth) survives verbatim. + assert "entirely through the browser" in instruction + assert "Do NOT use command-line tools, scripts, or direct API/SMTP calls" in ( + instruction + ) + # Ported harness restrictions. + assert "Time limit: 30 minutes" in instruction + assert "Playwright MCP browser tools" in instruction + assert "Do NOT make direct HTTP/network requests" in instruction + assert "Submit through the browser" in instruction + assert "Stop after submission" in instruction + # Credential-free bridge endpoint, never a provider URL. + assert "http://127.0.0.1:7878" in instruction + assert "ws://" not in instruction + + +def test_adapter_cli_rejects_invalid_runtime_options_json(tmp_path: Path) -> None: + with pytest.raises(SystemExit) as excinfo: + adapt_main( + [ + "--output-dir", + str(tmp_path / "out"), + "--limit", + "1", + "--overwrite", + "--browser-runtime", + "kernel", + "--browser-runtime-options", + "not-json", + ] + ) + assert excinfo.value.code == 2 + + +def test_adapter_cli_bakes_runtime_options_into_task_toml( + tmp_path: Path, +) -> None: + adapt_main( + [ + "--output-dir", + str(tmp_path / "out"), + "--limit", + "1", + "--overwrite", + "--browser-runtime", + "kernel", + "--browser-runtime-options", + '{"stealth": true}', + ] + ) + task_toml = next((tmp_path / "out").glob("*/task.toml")) + config = tomllib.loads(task_toml.read_text()) + assert config["environment"]["env"]["CLAWBENCH_BROWSER_RUNTIME_OPTIONS"] == ( + '{"stealth": true}' + ) + + +# --------------------------------------------------------------------------- +# kernel-browser.py lifecycle script +# --------------------------------------------------------------------------- + + +def _load_kernel_browser_module(monkeypatch: pytest.MonkeyPatch, home: Path): + spec = importlib.util.spec_from_file_location( + "kernel_browser_under_test", KERNEL_BROWSER_SCRIPT + ) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + monkeypatch.setattr(module, "STATE_DIR", home / "clawbench-run") + monkeypatch.setattr( + module, "STATE_FILE", home / "clawbench-run" / "kernel-browser-state.json" + ) + monkeypatch.setattr( + module, "CDP_URL_FILE", home / "clawbench-run" / "kernel-cdp-url" + ) + monkeypatch.setattr( + module, "METADATA_FILE", home / "my-info" / "kernel_browser.json" + ) + monkeypatch.setattr( + module, "LIFECYCLE_FILE", home / "data" / "kernel-browser-lifecycle.json" + ) + monkeypatch.setattr(module, "TASK_FILE", home / "task.json") + monkeypatch.setattr(module, "DATA_DIR", home / "data") + sys.modules[spec.name] = module + return module + + +class FakeSession: + def __init__(self) -> None: + from clawbench.runner.run_support.browser_runtime.providers import ( + BrowserSession, + ) + + self._session = BrowserSession( + provider="kernel", + mode="remote", + session_id="sess-123", + cdp_url="wss://kernel.example/browser/sess-123/cdp?token=secret-token", + viewer_url="https://kernel.example/live/sess-123", + viewer_url_sensitive=True, + metadata={"replay_id": "replay-9", "region": "us-east"}, + recording_mode="provider-download", + ) + + def __getattr__(self, name: str): + return getattr(self._session, name) + + +class FakeProvider: + name = "kernel" + + def __init__(self, fail_cleanup: bool = False) -> None: + self.fail_cleanup = fail_cleanup + self.finalize_calls = 0 + self.cleanup_calls = 0 + self.exists_calls: list[str] = [] + self.deleted = False + + def start(self, task: dict, time_limit_s: int) -> FakeSession: + assert time_limit_s == 1800 + return FakeSession() + + def finalize(self, session: FakeSession, output_dir: Path) -> None: + self.finalize_calls += 1 + recording = Path(output_dir) / "data" / "recording.mp4" + recording.parent.mkdir(parents=True, exist_ok=True) + recording.write_bytes(b"mp4-bytes") + session.metadata["recording_bytes"] = len(b"mp4-bytes") + + def cleanup(self, session: FakeSession) -> None: + self.cleanup_calls += 1 + if self.fail_cleanup: + from clawbench.runner.run_support.browser_runtime.providers import ( + BrowserRuntimeError, + ) + + raise BrowserRuntimeError("delete failed") + self.deleted = True + + def session_exists(self, session_id: str) -> bool: + self.exists_calls.append(session_id) + return not self.deleted + + +@pytest.fixture() +def kernel_env(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + (tmp_path / "task.json").write_text(json.dumps(_task())) + for var in ( + "KERNEL_API_KEY", + "KERNEL_BASE_URL", + "CLAWBENCH_BROWSER_RUNTIME_OPTIONS", + ): + monkeypatch.delenv(var, raising=False) + return tmp_path + + +def test_start_writes_state_and_credential_free_metadata( + kernel_env: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("KERNEL_API_KEY", "k-test-key") + module = _load_kernel_browser_module(monkeypatch, kernel_env) + provider = FakeProvider() + monkeypatch.setattr(module, "_load_provider", lambda: provider) + + assert module.cmd_start() == 0 + + state = json.loads(module.STATE_FILE.read_text()) + assert state["status"] == "created" + assert state["cdp_url"].startswith("wss://kernel.example") + assert oct(module.STATE_FILE.stat().st_mode & 0o777) == "0o600" + assert oct(module.CDP_URL_FILE.stat().st_mode & 0o777) == "0o600" + + metadata = json.loads(module.METADATA_FILE.read_text()) + assert metadata["session_id"] == "sess-123" + assert metadata["replay_id"] == "replay-9" + assert metadata["cdp_bridge_url"] == "http://127.0.0.1:7878" + blob = module.METADATA_FILE.read_text() + assert "k-test-key" not in blob + assert "secret-token" not in blob + + +def test_finalize_downloads_recording_deletes_browser_and_is_idempotent( + kernel_env: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + module = _load_kernel_browser_module(monkeypatch, kernel_env) + provider = FakeProvider() + monkeypatch.setattr(module, "_load_provider", lambda: provider) + assert module.cmd_start() == 0 + + assert module.cmd_finalize() == 0 + + recording = kernel_env / "data" / "recording.mp4" + assert recording.read_bytes() == b"mp4-bytes" + lifecycle = json.loads(module.LIFECYCLE_FILE.read_text()) + assert lifecycle["status"] == "deleted" + assert lifecycle["deletion_verified"] is True + assert provider.cleanup_calls == 1 + assert provider.exists_calls == ["sess-123"] + + # Idempotent rerun makes no further API calls. + assert module.cmd_finalize() == 0 + assert provider.cleanup_calls == 1 + + +def test_finalize_reports_failure_when_cleanup_fails( + kernel_env: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + module = _load_kernel_browser_module(monkeypatch, kernel_env) + provider = FakeProvider(fail_cleanup=True) + monkeypatch.setattr(module, "_load_provider", lambda: provider) + assert module.cmd_start() == 0 + + assert module.cmd_finalize() == 1 + state = json.loads(module.STATE_FILE.read_text()) + assert state["cleanup_error"] == "delete failed" + + +def test_cleanup_recovers_from_agent_visible_metadata_alone( + kernel_env: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + module = _load_kernel_browser_module(monkeypatch, kernel_env) + provider = FakeProvider() + monkeypatch.setattr(module, "_load_provider", lambda: provider) + assert module.cmd_start() == 0 + module.STATE_FILE.unlink() + + assert module.cmd_cleanup() == 0 + assert provider.cleanup_calls == 1 + lifecycle = json.loads(module.LIFECYCLE_FILE.read_text()) + assert lifecycle["status"] == "deleted" + assert lifecycle["deletion_verified"] is True + + +def test_commands_are_noops_without_a_created_browser( + kernel_env: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + module = _load_kernel_browser_module(monkeypatch, kernel_env) + provider = FakeProvider() + monkeypatch.setattr(module, "_load_provider", lambda: provider) + + assert module.cmd_finalize() == 0 + assert module.cmd_cleanup() == 0 + assert provider.cleanup_calls == 0 + + +def test_kernel_session_exists_reports_404_as_missing( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from clawbench.runner.run_support.browser_runtime.providers import ( + KernelRuntimeProvider, + _KernelApiError, + ) + + provider = KernelRuntimeProvider(api_key="k", options={}) + + def fake_request(method: str, path: str, payload=None) -> bytes: + raise _KernelApiError("not found", status=404) + + monkeypatch.setattr(provider, "_request", fake_request) + assert provider.session_exists("gone") is False + + def ok_request(method: str, path: str, payload=None) -> bytes: + if method == "GET": + return b"{}" + raise AssertionError(method) + + monkeypatch.setattr(provider, "_request", ok_request) + assert provider.session_exists("alive") is True + + +def test_finalize_still_deletes_browser_when_replay_download_fails( + kernel_env: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + from clawbench.runner.run_support.browser_runtime.providers import ( + BrowserRuntimeError, + ) + + module = _load_kernel_browser_module(monkeypatch, kernel_env) + provider = FakeProvider() + monkeypatch.setattr(module, "_load_provider", lambda: provider) + + def failing_finalize(session, output_dir): # noqa: ANN001 + raise BrowserRuntimeError("replay stuck processing") + + monkeypatch.setattr(provider, "finalize", failing_finalize) + assert module.cmd_start() == 0 + + assert module.cmd_finalize() == 0 + lifecycle = json.loads(module.LIFECYCLE_FILE.read_text()) + assert lifecycle["status"] == "deleted" + assert lifecycle["deletion_verified"] is True + assert "replay finalization failed" in lifecycle["cleanup_error"] + assert provider.cleanup_calls == 1 From a7e4b594ddfaf42205eb1f4c8f3e91c71d6ebc72 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Sat, 22 Aug 2026 03:32:49 +0000 Subject: [PATCH 2/5] Start runtime server in remote browser mode --- src/clawbench/runtime/harbor/start-runtime.sh | 25 ++++++++++--------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/src/clawbench/runtime/harbor/start-runtime.sh b/src/clawbench/runtime/harbor/start-runtime.sh index 62bd59b2..9232b87f 100644 --- a/src/clawbench/runtime/harbor/start-runtime.sh +++ b/src/clawbench/runtime/harbor/start-runtime.sh @@ -16,17 +16,24 @@ if [ -n "${CLAWBENCH_BROWSER_CDP_URL_FILE:-}" ] && [ -f "${CLAWBENCH_BROWSER_CDP REMOTE_MODE=true fi -if [ "$REMOTE_MODE" = false ]; then -export DISPLAY="${DISPLAY:-:99}" -Xvfb "$DISPLAY" -screen 0 1920x1080x24 >/tmp/clawbench-run/xvfb.log 2>&1 & -echo "$!" > /tmp/clawbench-run/xvfb.pid -sleep 1 - cd /app/src/runtime-server uv run --no-sync uvicorn server:app --host 0.0.0.0 --port 7878 >/tmp/clawbench-run/runtime-server.log 2>&1 & echo "$!" > /tmp/clawbench-run/runtime-server.pid sleep 1 +if [ "$REMOTE_MODE" = true ]; then + # Provider replays replace local X11 recording. + export CLAWBENCH_RECORDING_MODE="${CLAWBENCH_RECORDING_MODE:-provider-download}" + echo "CDP bridge ready at http://127.0.0.1:7878" + echo "$$" > /tmp/clawbench-run/runtime.pid + exit 0 +fi + +export DISPLAY="${DISPLAY:-:99}" +Xvfb "$DISPLAY" -screen 0 1920x1080x24 >/tmp/clawbench-run/xvfb.log 2>&1 & +echo "$!" > /tmp/clawbench-run/xvfb.pid +sleep 1 + mkdir -p /tmp/chrome-profile/Default cat > /tmp/chrome-profile/Default/Preferences <<'PREFS' { @@ -127,12 +134,6 @@ sleep 1 /opt/novnc/utils/novnc_proxy --vnc localhost:5900 --listen 6080 >/tmp/clawbench-run/novnc.log 2>&1 & echo "$!" > /tmp/clawbench-run/novnc.pid -fi - -if [ "$REMOTE_MODE" = true ]; then - # Provider replays replace local X11 recording. - export CLAWBENCH_RECORDING_MODE="${CLAWBENCH_RECORDING_MODE:-provider-download}" -fi echo "$$" > /tmp/clawbench-run/runtime.pid echo "CDP ready at http://127.0.0.1:9223" From baec19230b60aa5fb305fea4fc4d7a3287a53340 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Sat, 22 Aug 2026 18:21:45 +0000 Subject: [PATCH 3/5] Fix Kernel-backed Red Cross benchmark --- docs/harbor.md | 2 +- .../run_support/browser_runtime/providers.py | 6 +--- .../runtime/harbor/kernel-browser.py | 4 ++- tests/test_browser_runtime.py | 3 +- tests/test_harbor_kernel_control.py | 29 ++++++++++++++++++- 5 files changed, 34 insertions(+), 10 deletions(-) diff --git a/docs/harbor.md b/docs/harbor.md index 32c8491a..0dbba906 100644 --- a/docs/harbor.md +++ b/docs/harbor.md @@ -116,7 +116,7 @@ By default each Harbor trial runs Chromium inside its own container. Pass `--bro uv run clawbench-harbor-adapt \ --output-dir ./harbor-datasets/clawbench-v2-kernel \ --browser-runtime kernel \ - --browser-runtime-options '{"stealth": false}' \ + --browser-runtime-options '{"stealth": true}' \ --task-ids v2-1134-chapter-finder-redcross \ --overwrite ``` diff --git a/src/clawbench/runner/run_support/browser_runtime/providers.py b/src/clawbench/runner/run_support/browser_runtime/providers.py index c01ebb2d..9a852082 100644 --- a/src/clawbench/runner/run_support/browser_runtime/providers.py +++ b/src/clawbench/runner/run_support/browser_runtime/providers.py @@ -585,13 +585,9 @@ def start(self, task: dict[str, Any], time_limit_s: int) -> BrowserSession: timeout_seconds = min(259200, max(10, time_limit_s + 120)) payload = { **self.options, + "stealth": self.options.get("stealth", True), "headless": False, "timeout_seconds": timeout_seconds, - "viewport": { - "width": 1920, - "height": 1080, - "refresh_rate": 25, - }, } try: result = self._request_json("POST", "/browsers", payload) diff --git a/src/clawbench/runtime/harbor/kernel-browser.py b/src/clawbench/runtime/harbor/kernel-browser.py index 41655368..7a346e51 100644 --- a/src/clawbench/runtime/harbor/kernel-browser.py +++ b/src/clawbench/runtime/harbor/kernel-browser.py @@ -100,7 +100,9 @@ def _public_metadata(state: dict[str, Any]) -> dict[str, Any]: "session_id": metadata.get("session_id"), "replay_id": inner.get("replay_id") or metadata.get("replay_id"), "region": inner.get("region") or metadata.get("region"), - "stealth": inner.get("stealth") or metadata.get("stealth"), + "stealth": ( + inner["stealth"] if "stealth" in inner else metadata.get("stealth") + ), "timeout_seconds": ( inner.get("timeout_seconds") or metadata.get("timeout_seconds") ), diff --git a/tests/test_browser_runtime.py b/tests/test_browser_runtime.py index 11dccdd0..85e6568c 100644 --- a/tests/test_browser_runtime.py +++ b/tests/test_browser_runtime.py @@ -398,7 +398,7 @@ def fake_urlopen( monkeypatch.setattr(urllib.request, "urlopen", fake_urlopen) provider = KernelRuntimeProvider( api_key="kernel-secret", - options={"stealth": True, "region": "us-east"}, + options={"region": "us-east"}, replay_poll_interval_s=0, replay_poll_timeout_s=1, ) @@ -413,7 +413,6 @@ def fake_urlopen( "region": "us-east", "headless": False, "timeout_seconds": 1920, - "viewport": {"width": 1920, "height": 1080, "refresh_rate": 25}, } assert session.provider == "kernel" assert session.recording_mode == "provider-download" diff --git a/tests/test_harbor_kernel_control.py b/tests/test_harbor_kernel_control.py index 9c88a5a4..1b9e7343 100644 --- a/tests/test_harbor_kernel_control.py +++ b/tests/test_harbor_kernel_control.py @@ -4,6 +4,7 @@ import importlib.util import json +import re import sys import tomllib from pathlib import Path @@ -52,6 +53,27 @@ def adapted_kernel_task(tmp_path: Path) -> Path: ) +def test_redcross_task_intercepts_zip_lookup_not_chapter_finder_page() -> None: + task_path = ( + REPO_ROOT + / "test-cases" + / "v2" + / "v2-1134-chapter-finder-redcross" + / "task.json" + ) + schema = json.loads(task_path.read_text())["eval_schema"] + + assert not re.search( + schema["url_pattern"], + "https://www.redcross.org/find-your-local-chapter.html", + ) + assert re.search( + schema["url_pattern"], + "https://www.redcross.org/api/lookup/v1/region-mappings/90210?type=RCO", + ) + assert schema["params"] == {"type": "RCO"} + + def test_kernel_runtime_selection_writes_bridge_env_and_pinned_mcp( adapted_kernel_task: Path, ) -> None: @@ -229,7 +251,11 @@ def __init__(self) -> None: cdp_url="wss://kernel.example/browser/sess-123/cdp?token=secret-token", viewer_url="https://kernel.example/live/sess-123", viewer_url_sensitive=True, - metadata={"replay_id": "replay-9", "region": "us-east"}, + metadata={ + "replay_id": "replay-9", + "region": "us-east", + "stealth": False, + }, recording_mode="provider-download", ) @@ -304,6 +330,7 @@ def test_start_writes_state_and_credential_free_metadata( metadata = json.loads(module.METADATA_FILE.read_text()) assert metadata["session_id"] == "sess-123" assert metadata["replay_id"] == "replay-9" + assert metadata["stealth"] is False assert metadata["cdp_bridge_url"] == "http://127.0.0.1:7878" blob = module.METADATA_FILE.read_text() assert "k-test-key" not in blob From c7feaa2435ca8115c0762c44e13885fe5adf3e98 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Mon, 31 Aug 2026 16:21:51 +0000 Subject: [PATCH 4/5] Skip POSIX mode assertions on Windows --- tests/test_harbor_kernel_control.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_harbor_kernel_control.py b/tests/test_harbor_kernel_control.py index 1b9e7343..df63f971 100644 --- a/tests/test_harbor_kernel_control.py +++ b/tests/test_harbor_kernel_control.py @@ -324,8 +324,9 @@ def test_start_writes_state_and_credential_free_metadata( state = json.loads(module.STATE_FILE.read_text()) assert state["status"] == "created" assert state["cdp_url"].startswith("wss://kernel.example") - assert oct(module.STATE_FILE.stat().st_mode & 0o777) == "0o600" - assert oct(module.CDP_URL_FILE.stat().st_mode & 0o777) == "0o600" + if sys.platform != "win32": + assert oct(module.STATE_FILE.stat().st_mode & 0o777) == "0o600" + assert oct(module.CDP_URL_FILE.stat().st_mode & 0o777) == "0o600" metadata = json.loads(module.METADATA_FILE.read_text()) assert metadata["session_id"] == "sess-123" From 4c06c6b7b3fdb508abd4946cd6197598c4a5a6f1 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Wed, 2 Sep 2026 01:46:06 +0000 Subject: [PATCH 5/5] Fail closed on PurelyMail API errors --- CHANGELOG.md | 1 + src/clawbench/runner/run_support/email.py | 9 ++- src/clawbench/runtime/harbor/cleanup-email.py | 9 ++- src/clawbench/runtime/harbor/prepare-task.py | 9 ++- tests/test_purelymail_errors.py | 73 +++++++++++++++++++ 5 files changed, 98 insertions(+), 3 deletions(-) create mode 100644 tests/test_purelymail_errors.py diff --git a/CHANGELOG.md b/CHANGELOG.md index cd343c8d..5dc18dcb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/). - Updated the harbor adaptor to support the full V2 lenient & strict and reports numeric results. ### Fixed +- Fail task setup when PurelyMail returns an API error instead of emitting credentials for an account that was not created. - Fixed an issue where malformed per-run metadata could prevent `batch-summary.json` from being written and, when configured, uploaded. - Fixed the issue that an invalid judge model would lose the `run-meta.json` file. diff --git a/src/clawbench/runner/run_support/email.py b/src/clawbench/runner/run_support/email.py index 79a98a02..77ffa28e 100644 --- a/src/clawbench/runner/run_support/email.py +++ b/src/clawbench/runner/run_support/email.py @@ -18,7 +18,14 @@ def purelymail_request(endpoint: str, body: dict, api_key: str) -> dict: method="POST", ) with urlopen(req, timeout=15) as resp: - return json.loads(resp.read()) + result = json.loads(resp.read()) + if not isinstance(result, dict): + raise RuntimeError(f"PurelyMail {endpoint} returned an invalid response") + if result.get("type") == "error": + code = f" ({result['code']})" if result.get("code") else "" + message = f": {result['message']}" if result.get("message") else "" + raise RuntimeError(f"PurelyMail {endpoint} failed{code}{message}") + return result def create_email(api_key: str, domain: str) -> tuple[str, str]: diff --git a/src/clawbench/runtime/harbor/cleanup-email.py b/src/clawbench/runtime/harbor/cleanup-email.py index 0d676722..b49ae04c 100644 --- a/src/clawbench/runtime/harbor/cleanup-email.py +++ b/src/clawbench/runtime/harbor/cleanup-email.py @@ -28,7 +28,14 @@ def purelymail_request(endpoint: str, body: dict, api_key: str) -> dict: method="POST", ) with urlopen(req, timeout=15) as resp: - return json.loads(resp.read()) + result = json.loads(resp.read()) + if not isinstance(result, dict): + raise RuntimeError(f"PurelyMail {endpoint} returned an invalid response") + if result.get("type") == "error": + code = f" ({result['code']})" if result.get("code") else "" + message = f": {result['message']}" if result.get("message") else "" + raise RuntimeError(f"PurelyMail {endpoint} failed{code}{message}") + return result def main() -> int: diff --git a/src/clawbench/runtime/harbor/prepare-task.py b/src/clawbench/runtime/harbor/prepare-task.py index c60c6b91..9bc457eb 100644 --- a/src/clawbench/runtime/harbor/prepare-task.py +++ b/src/clawbench/runtime/harbor/prepare-task.py @@ -30,7 +30,14 @@ def purelymail_request(endpoint: str, body: dict, api_key: str) -> dict: method="POST", ) with urlopen(req, timeout=15) as resp: - return json.loads(resp.read()) + result = json.loads(resp.read()) + if not isinstance(result, dict): + raise RuntimeError(f"PurelyMail {endpoint} returned an invalid response") + if result.get("type") == "error": + code = f" ({result['code']})" if result.get("code") else "" + message = f": {result['message']}" if result.get("message") else "" + raise RuntimeError(f"PurelyMail {endpoint} failed{code}{message}") + return result def create_email(api_key: str, domain: str) -> tuple[str, str]: diff --git a/tests/test_purelymail_errors.py b/tests/test_purelymail_errors.py new file mode 100644 index 00000000..45bce147 --- /dev/null +++ b/tests/test_purelymail_errors.py @@ -0,0 +1,73 @@ +from __future__ import annotations + +import importlib.util +import json +from pathlib import Path + +import pytest + +from clawbench.runner.run_support import email as native_email + +REPO_ROOT = Path(__file__).resolve().parents[1] + + +class FakeResponse: + def __init__(self, payload: object) -> None: + self.payload = payload + + def __enter__(self): + return self + + def __exit__(self, *_args) -> None: + return None + + def read(self) -> bytes: + return json.dumps(self.payload).encode() + + +def load_harbor_script(name: str): + path = REPO_ROOT / "src" / "clawbench" / "runtime" / "harbor" / name + spec = importlib.util.spec_from_file_location(name.replace("-", "_"), path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.mark.parametrize( + "module", + [ + native_email, + pytest.param(load_harbor_script("prepare-task.py"), id="harbor-prepare"), + pytest.param(load_harbor_script("cleanup-email.py"), id="harbor-cleanup"), + ], +) +def test_purelymail_api_errors_fail_closed(monkeypatch, module) -> None: + monkeypatch.setattr( + module, + "urlopen", + lambda *_args, **_kwargs: FakeResponse( + { + "type": "error", + "code": "invalidToken", + "message": "Token not valid.", + } + ), + ) + + with pytest.raises( + RuntimeError, + match=r"PurelyMail createUser failed \(invalidToken\): Token not valid\.", + ): + module.purelymail_request("createUser", {}, "secret-token") + + +def test_purelymail_rejects_non_object_responses(monkeypatch) -> None: + monkeypatch.setattr( + native_email, + "urlopen", + lambda *_args, **_kwargs: FakeResponse(["unexpected"]), + ) + + with pytest.raises(RuntimeError, match="returned an invalid response"): + native_email.purelymail_request("createUser", {}, "secret-token")