diff --git a/.github/workflows/publish-harbor-image.yml b/.github/workflows/publish-harbor-image.yml new file mode 100644 index 00000000..84fc009d --- /dev/null +++ b/.github/workflows/publish-harbor-image.yml @@ -0,0 +1,49 @@ +name: publish-harbor-image + +# Builds the prebuilt runtime image every committed Harbor task points at +# ([environment].docker_image in harbor/datasets/clawbench-*/*/task.toml). +# Runs on release tags and on demand; pushes to Docker Hub (clawbench/ namespace). +# Requires repo secrets DOCKERHUB_USERNAME and DOCKERHUB_TOKEN. +on: + push: + tags: ["v*"] + workflow_dispatch: + inputs: + tag: + description: "Image tag (defaults to pyproject version)" + required: false + +permissions: + contents: read + +jobs: + build-and-push: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + + - name: Resolve tag + id: meta + run: | + VERSION="$(grep -m1 '^version' pyproject.toml | sed -E 's/.*"([^"]+)".*/\1/')" + TAG="${{ github.event.inputs.tag }}" + echo "tag=${TAG:-$VERSION}" >> "$GITHUB_OUTPUT" + echo "image=docker.io/clawbench/clawbench-harbor-runtime" >> "$GITHUB_OUTPUT" + + - uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - uses: docker/setup-buildx-action@v3 + + - uses: docker/build-push-action@v6 + with: + context: src/clawbench/runtime + file: src/clawbench/runtime/harbor/Dockerfile + push: true + tags: | + ${{ steps.meta.outputs.image }}:${{ steps.meta.outputs.tag }} + ${{ steps.meta.outputs.image }}:latest + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/.github/workflows/validate-harbor.yml b/.github/workflows/validate-harbor.yml new file mode 100644 index 00000000..76d59b6f --- /dev/null +++ b/.github/workflows/validate-harbor.yml @@ -0,0 +1,91 @@ +name: validate-harbor + +on: + pull_request: + branches: [main] + paths: + - "harbor/**" + - "registry.json" + - "scripts/harbor/**" + - "src/clawbench/eval/harbor_adapter.py" + - "src/clawbench/runtime/harbor/**" + - "test-cases/v2/**" + - ".github/workflows/validate-harbor.yml" + push: + branches: [main] + paths: + - "harbor/**" + - "registry.json" + - "scripts/harbor/**" + - "src/clawbench/eval/harbor_adapter.py" + - "test-cases/v2/**" + workflow_dispatch: + +jobs: + validate-harbor: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + + - uses: astral-sh/setup-uv@v6 + with: + python-version: "3.11" + + - name: Install ClawBench + Harbor + run: | + uv sync --frozen + uv pip install "harbor==0.22.0" + + - name: Committed Harbor datasets are up to date + run: | + set -euo pipefail + VERSION="$(grep -m1 '^version' pyproject.toml | sed -E 's/.*"([^"]+)".*/\1/')" + for SUITE in v2 v1; do + uv run clawbench-harbor-adapt \ + --suite "$SUITE" \ + --output-dir "/tmp/clawbench-$SUITE" \ + --docker-image "clawbench/clawbench-harbor-runtime:$VERSION" \ + --overwrite + diff -r "/tmp/clawbench-$SUITE" "harbor/datasets/clawbench-$SUITE" \ + || { echo "::error::harbor/datasets/clawbench-$SUITE is stale — run scripts/harbor/regenerate.sh"; exit 1; } + uv run python scripts/harbor/build_registry.py \ + --dataset-dir "harbor/datasets/clawbench-$SUITE" --name "clawbench-$SUITE" \ + --version "$VERSION" --output /tmp/registry.json + done + diff /tmp/registry.json registry.json \ + || { echo "::error::registry.json is stale — run scripts/harbor/regenerate.sh"; exit 1; } + + - name: Every task loads with Harbor's TaskConfig + run: | + uv run python - <<'PY' + import sys + from pathlib import Path + from harbor.models.task.task import Task + bad = [] + total = 0 + for root in (Path("harbor/datasets/clawbench-v2"), Path("harbor/datasets/clawbench-v1")): + for d in sorted(p.parent for p in root.glob("*/task.toml")): + total += 1 + try: + Task(task_dir=d).config + except Exception as exc: # noqa: BLE001 + bad.append(f"{d.name}: {exc}") + print(f"loaded {total - len(bad)} / {total} tasks") + if bad: + print("\n".join(bad)); sys.exit(1) + PY + + - name: Manifests and registry agree on task names + run: | + uv run python - <<'PY' + import json, tomllib, sys + registry = {e["name"]: e for e in json.load(open("registry.json"))} + pairs = [("harbor/dataset.toml", "clawbench-v2"), ("harbor/dataset-v1.toml", "clawbench-v1")] + for path, name in pairs: + manifest = tomllib.loads(open(path).read()) + m = {t["name"].split("/", 1)[1] for t in manifest["tasks"]} + r = {t["name"] for t in registry[name]["tasks"]} + if m != r: + print(path, "manifest-only:", sorted(m - r)); print("registry-only:", sorted(r - m)); sys.exit(1) + print(f"{len(m)} tasks agree for {name}") + PY diff --git a/.gitignore b/.gitignore index cd12fa11..6095e5d9 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,13 @@ build/ run-logs/ harbor-datasets/ harbor-jobs/ + +# macOS AppleDouble sidecars (created when files are copied from a Mac) +._* + +# Harbor: full generated datasets are build output (see harbor/README.md); +# only the committed prebuilt-mode dataset under harbor/datasets/ is tracked. +harbor/datasets/*/ +!harbor/datasets/clawbench-v2/ +harbor/jobs/ +harbor/trials/ diff --git a/AGENTS.md b/AGENTS.md index 12783aff..75ccdd6f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -32,6 +32,12 @@ ClawBench/ models.yaml # Model API configs (gitignored -- copy from example) models.example.yaml # Template with placeholder keys model.schema.json # JSON schema for model entries + harbor/ # Harbor-format packaging: registry, dataset manifest, adapter package + README.md # How to run / publish ClawBench through Harbor + dataset.toml # Hub manifest (tiger-ai-lab/clawbench-v2) + adapter/ # harbor-framework/harbor adapters/clawbench package + datasets/clawbench-{v1,v2}/ # Committed prebuilt-mode Harbor tasks (generated, CI-checked) + registry.json # Harbor git registry (harbor run --repo TIGER-AI-Lab/ClawBench) test-cases/ task.schema.json # JSON schema for task.json v1/ # V1: 152 task directories diff --git a/CHANGELOG.md b/CHANGELOG.md index 8803ac90..8d464a67 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ## [Unreleased] ### Added - Added Kernel as a managed remote browser runtime with live view and downloaded replay recordings. Thanks to @[rgarcia](https://github.com/rgarcia). +- Added a `harbor/` package: an in-repo Harbor registry (`registry.json` + committed `harbor/datasets/clawbench-{v1,v2}/`) so `harbor run --repo TIGER-AI-Lab/ClawBench -d clawbench-v2` works without conversion, a `dataset.toml` manifest for publishing to the Harbor Hub, and a Harbor-official adapter package (`harbor/adapter/`) for upstreaming to `harbor-framework/harbor`. +- Added `clawbench-harbor-adapt --docker-image` (prebuilt-image mode): generated tasks reference the published `clawbench/clawbench-harbor-runtime` Docker Hub image instead of shipping a 280 KB `environment/` build context each (a one-line `FROM` Dockerfile remains, as Harbor requires `environment/` to exist). +- Added `clawbench-harbor-adapt --suite v1|v2`: the V1 corpus (152 tasks) is now exported to Harbor alongside V2. + +### Fixed +- README: removed the duplicated `## Star History` heading (EN + zh-CN). ## [0.9.2] - 2026-08-18 ### Added diff --git a/README.md b/README.md index cbea4eaf..617dfaee 100644 --- a/README.md +++ b/README.md @@ -157,9 +157,9 @@ order food, book travel, apply for jobs, write reviews, manage projects.
-❓ **Have a question**
-[FAQ](#faq) · [Open an issue](https://github.com/TIGER-AI-Lab/ClawBench/issues/new/choose)
-Or ask on the HF dataset page +🧭 **Run it via Harbor**
+[`harbor run --repo TIGER-AI-Lab/ClawBench`](harbor/README.md)
+[FAQ](#faq) · [Open an issue](https://github.com/TIGER-AI-Lab/ClawBench/issues/new/choose) @@ -333,6 +333,7 @@ Full registry: [`src/clawbench/runtime/harnesses/harnesses.yaml`](src/clawbench/ | --- | --- | | Use a managed remote browser instead of a local container | [`docs/browser-runtimes.md`](docs/browser-runtimes.md) — Kernel and Browserbase setup, options, and recordings | | Run V2 through the Harbor framework (and run it fast) | [`docs/harbor.md`](docs/harbor.md) — conversion, judge wiring, concurrency, troubleshooting | +| Pull ClawBench as a Harbor dataset (`harbor run --repo TIGER-AI-Lab/ClawBench -d clawbench-v2`) | [`harbor/README.md`](harbor/README.md) — in-repo registry, Hub publishing, adapter package | | See every CLI command and flag | [`docs/cli.md`](docs/cli.md) |
@@ -1021,15 +1022,17 @@ Open to contributions — new test cases, bug fixes, or evaluation submissions f ## Star History -## Star History - +

- Star History Chart + Star History Chart +

+ +

Live chart — click to open on star-history.com. Sister projects: HarnessBench · RewardHarness

## License & Acknowledgments diff --git a/docs/README.zh-CN.md b/docs/README.zh-CN.md index 138a60cb..e9caee85 100644 --- a/docs/README.zh-CN.md +++ b/docs/README.zh-CN.md @@ -105,9 +105,9 @@ -❓ **有问题**
-[FAQ](#faq) · [提 issue](https://github.com/TIGER-AI-Lab/ClawBench/issues/new/choose)
-也可在 HF 数据集页提问 +🧭 **用 Harbor 跑**
+[`harbor run --repo TIGER-AI-Lab/ClawBench`](../harbor/README.md)
+[FAQ](#faq) · [提 issue](https://github.com/TIGER-AI-Lab/ClawBench/issues/new/choose) @@ -279,6 +279,7 @@ harness 是驱动浏览器的智能体框架,和模型是两个独立维度。 | --- | --- | | 用托管的远程浏览器代替本地容器 | [`docs/browser-runtimes.md`](browser-runtimes.md) —— Kernel 和 Browserbase 配置、参数与录制 | | 用 Harbor 框架跑 V2(并且跑得快) | [`docs/harbor.md`](harbor.md) —— 转换、judge 配置、并发、排错 | +| 把 ClawBench 当作 Harbor 数据集直接拉取(`harbor run --repo TIGER-AI-Lab/ClawBench -d clawbench-v2`) | [`harbor/README.md`](../harbor/README.md) —— 仓库内 registry、发布到 Hub、adapter 包 | | 查所有 CLI 命令和参数 | [`docs/cli.md`](cli.md) |
@@ -960,15 +961,17 @@ Pi 的 `agent.log`、`proxy.log` 等 harness 诊断日志不会复制到最终 ## Star 历史 -## Star History - +

- Star History Chart + Star History Chart +

+ +

实时图表 —— 点击在 star-history.com 打开。姊妹项目:HarnessBench · RewardHarness

## 许可证与致谢 diff --git a/docs/harbor.md b/docs/harbor.md index 9a0453e2..7cd9b3a5 100644 --- a/docs/harbor.md +++ b/docs/harbor.md @@ -4,6 +4,7 @@ **Use Harbor when** you already run other benchmarks through it, you want Harbor's agent registry (`-a openclaw`, `-a hermes`, …) instead of ClawBench's own harnesses, or you need Harbor's retry/attempt semantics. **Use `clawbench-batch` instead** when you just want to score a model on ClawBench — it is the shorter path and it is what the leaderboard uses. +- [Zero-conversion path: the git registry](#zero-conversion-path-the-git-registry) - [Prerequisites](#prerequisites) - [1. Convert V2 into a Harbor dataset](#1-convert-v2-into-a-harbor-dataset) - [2. Wire up the judge](#2-wire-up-the-judge) @@ -12,6 +13,18 @@ - [What the generated environment contains](#what-the-generated-environment-contains) - [Troubleshooting](#troubleshooting) +## Zero-conversion path: the git registry + +The repo ships pre-generated Harbor datasets ([`harbor/datasets/clawbench-v2/`](../harbor/datasets/clawbench-v2/) — leaderboard corpus — and [`harbor/datasets/clawbench-v1/`](../harbor/datasets/clawbench-v1/), select with `-d clawbench-v1`) and a [`registry.json`](../registry.json), so Harbor can pull ClawBench straight from GitHub: + +```bash +harbor run --repo TIGER-AI-Lab/ClawBench -d clawbench-v2 -a hermes -m deepseek/deepseek-v4-flash \ + --env-file .env --ve CLAWBENCH_JUDGE_BASE_URL=... --ve CLAWBENCH_JUDGE_API_KEY=... \ + --ve CLAWBENCH_JUDGE_MODEL=deepseek-v4-pro --ve CLAWBENCH_JUDGE_API_TYPE=openai-completions +``` + +Those tasks reference the published runtime image `clawbench/clawbench-harbor-runtime:` (Docker Hub) instead of building one per task. Publishing to the Harbor Hub and upstreaming the adapter are covered in [`harbor/README.md`](../harbor/README.md). The rest of this page is the local-conversion path, which builds the image from source and is what you want when modifying the runtime. + ## Prerequisites - **Docker.** Harbor runs use Harbor's Docker provider, so Docker must be available even if you normally use Podman for native ClawBench runs. @@ -42,6 +55,7 @@ uv run clawbench-harbor-adapt \ | `--task-ids …` | Convert specific tasks by directory name or numeric `task_id` | | `--cases-dir ` | Convert a corpus other than V2 (defaults to `test-cases/v2/`) | | `--dataset-name`, `--org` | Metadata written into the generated `task.toml` | +| `--docker-image ` | Prebuilt mode: reference a published runtime image instead of copying `environment/` into every task (used for the committed dataset) | ## 2. Wire up the judge @@ -59,7 +73,7 @@ Use `deepseek-v4-pro` if you want numbers comparable to the published leaderboar ## 3. Run it ```bash -uvx --from harbor==0.15.0 harbor run \ +uvx --from harbor==0.22.0 harbor run \ -p ./harbor-datasets/clawbench-v2 \ -a "" \ -m "" \ @@ -70,7 +84,7 @@ uvx --from harbor==0.15.0 harbor run \ --ve CLAWBENCH_JUDGE_API_TYPE="${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" ``` -Drop `uvx --from harbor==0.15.0` if Harbor is already installed. +Drop `uvx --from harbor==0.22.0` if Harbor is already installed. Generated tasks use `schema_version = "1.3"` and load unchanged on Harbor 0.15 – 0.22. ### OpenClaw through an OpenAI-compatible endpoint @@ -78,7 +92,7 @@ Drop `uvx --from harbor==0.15.0` if Harbor is already installed. export OPENAI_BASE_URL="https://openrouter.ai/api/v1" export OPENAI_API_KEY="$OPENROUTER_API_KEY" -uvx --from harbor==0.15.0 harbor run \ +uvx --from harbor==0.22.0 harbor run \ -p ./harbor-datasets/clawbench-v2 \ -a openclaw \ -m openai/deepseek/deepseek-v4-flash \ @@ -96,7 +110,7 @@ uvx --from harbor==0.15.0 harbor run \ ```bash export OPENROUTER_API_KEY="your-openrouter-key" -uvx --from harbor==0.15.0 harbor run \ +uvx --from harbor==0.22.0 harbor run \ -p ./harbor-datasets/clawbench-v2 \ -a hermes \ -m deepseek/deepseek-v4-flash \ @@ -115,7 +129,7 @@ A full V2 sweep is 129 containerized browser sessions, each capped by the task's **1. Raise concurrency.** `-n / --n-concurrent` is the single biggest lever: ```bash -uvx --from harbor==0.15.0 harbor run -p ./harbor-datasets/clawbench-v2 \ +uvx --from harbor==0.22.0 harbor run -p ./harbor-datasets/clawbench-v2 \ -a hermes -m deepseek/deepseek-v4-flash -n 8 --env-file .env --ve ... ``` @@ -124,7 +138,7 @@ Each trial is a full Chromium container, so budget roughly **1 CPU core and ~2 G **2. Build the image once.** The first trial builds the ClawBench environment image; parallel cold starts all build at once. Warm the cache with the smoke dataset before the real sweep: ```bash -uvx --from harbor==0.15.0 harbor run -p ./harbor-datasets/clawbench-v2-smoke \ +uvx --from harbor==0.22.0 harbor run -p ./harbor-datasets/clawbench-v2-smoke \ -a hermes -m deepseek/deepseek-v4-flash --env-file .env --ve ... ``` diff --git a/harbor/README.md b/harbor/README.md new file mode 100644 index 00000000..33198689 --- /dev/null +++ b/harbor/README.md @@ -0,0 +1,81 @@ +# ClawBench × Harbor + +This directory packages ClawBench in the [Harbor](https://github.com/harbor-framework/harbor) task format so anyone who already evaluates agents with `harbor run` can score them on ClawBench without learning our CLI. It has three parts, each aimed at a different distribution channel: + +| Part | Channel | Who uses it | +| --- | --- | --- | +| [`../registry.json`](../registry.json) + [`datasets/clawbench-v2/`](datasets/clawbench-v2/) | **Git registry** — `harbor run --repo TIGER-AI-Lab/ClawBench -d clawbench-v2` | Anyone, today, no publishing step | +| [`dataset.toml`](dataset.toml) | **Harbor Hub** — `harbor publish` → `harbor run -d tiger-ai-lab/clawbench-v2` | Maintainers publish once; users pull by name | +| [`adapter/`](adapter/) | **Upstream adapter** — PR to `harbor-framework/harbor/adapters/clawbench` + `laude-institute/harbor-datasets` | Gets ClawBench into the official adapter list / registry and eligible for Harbor-Index | + +The task content is *generated* from [`test-cases/v2/`](../test-cases/v2/) by [`clawbench-harbor-adapt`](../docs/harbor.md); nothing here is hand-edited. `scripts/harbor/regenerate.sh` rebuilds all of it and CI fails if the committed copy is stale. + +## Run ClawBench through Harbor (git registry) + +```bash +uv tool install harbor # Harbor ≥ 0.22 + +harbor run --repo TIGER-AI-Lab/ClawBench -d clawbench-v2 \ + -a hermes -m deepseek/deepseek-v4-flash \ + --env-file .env \ + --ve CLAWBENCH_JUDGE_BASE_URL=... --ve CLAWBENCH_JUDGE_API_KEY=... \ + --ve CLAWBENCH_JUDGE_MODEL=deepseek-v4-pro --ve CLAWBENCH_JUDGE_API_TYPE=openai-completions \ + -n 8 +``` + +- Pin a release with `--repo TIGER-AI-Lab/ClawBench@v0.9.2`. +- `-i 'v2-9*'` / `-x '...'` include or exclude tasks by glob. +- Judge wiring, concurrency, and troubleshooting: [`docs/harbor.md`](../docs/harbor.md). + +**Runtime image.** Committed tasks are in *prebuilt mode*: `task.toml` points `[environment].docker_image` at `clawbench/clawbench-harbor-runtime (Docker Hub):` instead of shipping a 280 KB `environment/` build context per task (each task keeps a one-line `environment/Dockerfile` that `FROM`s the image, as Harbor requires the directory to exist) (129 × 280 KB ≈ 35 MB otherwise). The image is built from [`src/clawbench/runtime/harbor/Dockerfile`](../src/clawbench/runtime/harbor/Dockerfile) by `scripts/harbor/build-runtime-image.sh` and published by the `publish-harbor-image` workflow on release tags / manual dispatch (repo secrets `DOCKERHUB_USERNAME` / `DOCKERHUB_TOKEN`). If you want tasks that build locally instead, generate them yourself: + +```bash +uv run clawbench-harbor-adapt --output-dir ./harbor-datasets/clawbench-v2 --overwrite +harbor run -p ./harbor-datasets/clawbench-v2 -a hermes -m deepseek/deepseek-v4-flash ... +``` + +## Publish to the Harbor Hub (maintainers) + +The Hub is the registry behind `harbor run -d /` and the leaderboards on hub.harborframework.com. One-time setup, then one command per release: + +```bash +harbor auth login # GitHub OAuth; creates org = your GitHub login +harbor auth org create tiger-ai-lab # or ask an existing member to add you + +# Task names in dataset.toml are /; the org must be one you own on the Hub. +scripts/harbor/regenerate.sh # refresh datasets/, registry.json, dataset.toml + +harbor publish harbor/dataset.toml harbor/dataset-v1.toml --public -t v0.9.2 +harbor run -d tiger-ai-lab/clawbench-v2@v2 -a hermes -m ... # anyone, anywhere +``` + +`harbor publish` uploads the task directories referenced by the manifest, so the Hub copy does not depend on this git repo. Visibility defaults to private; `--public` applies to new packages only (`harbor dataset visibility` for existing ones). + +## Upstream to harbor-framework (adapter + registry + Harbor-Index) + +Harbor's official adapters live in `harbor-framework/harbor/adapters//` and their generated datasets in `laude-institute/harbor-datasets/datasets//`; `registry.json` in the harbor repo ties the two together. The process (from [harborframework.com/docs/datasets/adapters](https://www.harborframework.com/docs/datasets/adapters)): + +1. `adapter/` here is already in the `harbor adapter init` layout (README template sections, `adapter_metadata.json`, `parity_experiment.json`, `run_clawbench.yaml`, `src/clawbench_adapter/{adapter,main}.py`, `task-template/`). Copy it to a harbor fork as `adapters/clawbench/`, branch `clawbench-adapter`. +2. `harbor adapter review -p adapters/clawbench --skip-ai` — structural validation (required files, JSON schemas, README sections). +3. Oracle check: ClawBench has no scripted browser oracle (live sites), so document that `solution/solve.sh` is a no-op and that verifier correctness is demonstrated by the parity run instead. +4. Open `[WIP] Adapter: clawbench` on harbor-framework/harbor; coordinate parity on Discord `#adapters-announcements` (Lin Shi, adapters lead). 2077AI-sponsored parity API keys: `adapters/parity_api_instructions.md`. +5. **Parity experiment** — same agent on both sides: `clawbench-batch --harness hermes` (original) vs `harbor run -a hermes` (adapter), ≥3 runs each, report mean ± sample SEM for Intercepted and Reward-lenient; match when the run ranges overlap. Fill `parity_experiment.json`, upload artifacts to `harborframework/parity-experiments` on Hugging Face. +6. PR generated tasks to `laude-institute/harbor-datasets` under `datasets/clawbench/` with a `dataset.toml` + README; add the `clawbench` entry to harbor's `registry.json`. +7. Retitle `[Ready for Review] Adapter: clawbench`, request review from @Slimshilin. + +Harbor-Index (harbor-index.org) is curated *from* registered adapters (tasks with ≤33 % frontier solve rate, broken-task audit, human review) — there is no direct submission; step 6 is the prerequisite. + +## Layout + +``` +harbor/ +├── README.md this file +├── dataset.toml Hub manifest: tiger-ai-lab/clawbench-v2 +├── dataset-v1.toml Hub manifest: tiger-ai-lab/clawbench-v1 +├── job-config.yaml harbor run -c harbor/job-config.yaml (local docker, hermes) +├── adapter/ harbor-framework/harbor adapters/clawbench package +└── datasets/ + ├── clawbench-v2/ 129 generated prebuilt-mode tasks (leaderboard corpus) + └── clawbench-v1/ 152 generated prebuilt-mode tasks (V1 corpus) +../registry.json git registry consumed by --repo +``` diff --git a/harbor/adapter/.python-version b/harbor/adapter/.python-version new file mode 100644 index 00000000..2c073331 --- /dev/null +++ b/harbor/adapter/.python-version @@ -0,0 +1 @@ +3.11 diff --git a/harbor/adapter/adapter_metadata.json b/harbor/adapter/adapter_metadata.json new file mode 100644 index 00000000..166779d6 --- /dev/null +++ b/harbor/adapter/adapter_metadata.json @@ -0,0 +1,42 @@ +[ + { + "adapter_name": "clawbench", + "adapter_builders": [ + "Yuxuan Zhang (mtrxcop@gmail.com)" + ], + "original_benchmark": [ + { + "split": "v2", + "size": 129, + "harness": "agent", + "supported_agents": [ + "openclaw", + "hermes", + "opencode", + "claude-code", + "codex", + "browser-use", + "claw-code", + "pi", + "webbrain" + ], + "adaptable": true, + "notes": "V2 is the leaderboard corpus (129 tasks, live consumer websites). V1 (152 tasks) is not adapted: its eval schemas predate the two-stage judge." + } + ], + "harbor_adapter": [ + { + "split": "v2", + "adapted_benchmark_size": 129, + "parity_benchmark_size": null, + "parity_sampling_rate": null, + "registry_benchmark_size": 129, + "added_agents": [], + "parity_matching_agents": [], + "parity_unmatching_agents": null, + "parity_costs": null, + "notes": "Parity pending. Plan: hermes + deepseek/deepseek-v4-flash, 3 runs per side, metrics Intercepted and Reward (lenient judge, deepseek-v4-pro)." + } + ] + } +] diff --git a/harbor/adapter/parity_experiment.json b/harbor/adapter/parity_experiment.json new file mode 100644 index 00000000..345f1384 --- /dev/null +++ b/harbor/adapter/parity_experiment.json @@ -0,0 +1,34 @@ +[ + { + "adapter_name": "clawbench", + "agent": "hermes@", + "model": "deepseek/deepseek-v4-flash", + "date": "YYYY-MM-DD", + "adapted_benchmark_size": 129, + "parity_benchmark_size": 129, + "number_of_runs": 3, + "notes": "Original side: clawbench-batch --harness hermes --cases-suite v2 --all-cases, rescored with clawbench-rescore --judge-model deepseek-v4-pro --rubric both. Harbor side: harbor run -c adapters/clawbench/run_clawbench.yaml -a hermes with the same judge env. Timeouts / infra failures count as reward 0.", + "original_parity_repo": "https://github.com/TIGER-AI-Lab/ClawBench", + "adapter_pr": [], + "dataset_pr": [], + "parity_pr": [], + "metrics": [ + { + "benchmark_name": "clawbench-v2", + "metric": "Reward (lenient judge)", + "original": "", + "harbor": "", + "original_runs": [], + "harbor_runs": [] + }, + { + "benchmark_name": "clawbench-v2", + "metric": "Intercepted", + "original": "", + "harbor": "", + "original_runs": [], + "harbor_runs": [] + } + ] + } +] diff --git a/harbor/adapter/pyproject.toml b/harbor/adapter/pyproject.toml new file mode 100644 index 00000000..f952719a --- /dev/null +++ b/harbor/adapter/pyproject.toml @@ -0,0 +1,22 @@ +[project] +name = "harbor-clawbench-adapter" +version = "0.1.0" +description = "Harbor adapter for ClawBench: everyday tasks on live websites, scored by request interception + LLM judge" +readme = "README.md" +authors = [ + { name = "Yuxuan Zhang", email = "mtrxcop@gmail.com" } +] +requires-python = ">=3.11" +dependencies = [ + "clawbench-eval>=0.9.2", +] + +[project.scripts] +clawbench-adapter = "clawbench_adapter.main:main" + +[build-system] +requires = ["hatchling>=1.27"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["src/clawbench_adapter"] diff --git a/harbor/adapter/run_clawbench.yaml b/harbor/adapter/run_clawbench.yaml new file mode 100644 index 00000000..fc4b258b --- /dev/null +++ b/harbor/adapter/run_clawbench.yaml @@ -0,0 +1,25 @@ +jobs_dir: jobs +n_attempts: 1 +timeout_multiplier: 1.0 +orchestrator: + type: local + n_concurrent_trials: 4 + quiet: false +environment: + type: docker + force_build: false + delete: true + # Judge credentials (required for a non-zero reward): + # env: + # CLAWBENCH_JUDGE_BASE_URL: ${CLAWBENCH_JUDGE_BASE_URL} + # CLAWBENCH_JUDGE_API_KEY: ${CLAWBENCH_JUDGE_API_KEY} + # CLAWBENCH_JUDGE_MODEL: deepseek-v4-pro + # CLAWBENCH_JUDGE_API_TYPE: openai-completions + # PURELY_MAIL_API_KEY: ${PURELY_MAIL_API_KEY} + # PURELY_MAIL_DOMAIN: ${PURELY_MAIL_DOMAIN} +agents: + - name: hermes + # - name: openclaw + # - name: claude-code +datasets: + - path: datasets/clawbench diff --git a/harbor/adapter/src/clawbench_adapter/__init__.py b/harbor/adapter/src/clawbench_adapter/__init__.py new file mode 100644 index 00000000..77c95cd6 --- /dev/null +++ b/harbor/adapter/src/clawbench_adapter/__init__.py @@ -0,0 +1 @@ +"""Harbor adapter for ClawBench (https://claw-bench.com).""" diff --git a/harbor/adapter/src/clawbench_adapter/adapter.py b/harbor/adapter/src/clawbench_adapter/adapter.py new file mode 100644 index 00000000..3f93e65c --- /dev/null +++ b/harbor/adapter/src/clawbench_adapter/adapter.py @@ -0,0 +1,51 @@ +"""ClawBench → Harbor adapter. + +The conversion itself lives in the ``clawbench-eval`` package +(``clawbench.eval.harbor_adapter``) so that the adapter and the native runner +can never disagree about instruction text, eval schemas, or the verifier. This +class is the thin ``Adapter`` wrapper the Harbor adapter guide +expects, with the ``task-template/`` directory kept alongside for reference. +""" + +from __future__ import annotations + +from pathlib import Path + +from clawbench.eval.harbor_adapter import main as convert + +TEMPLATE_DIR = Path(__file__).parent / "task-template" + + +class ClawBenchAdapter: + def __init__( + self, + output_dir: Path, + *, + overwrite: bool = False, + limit: int | None = None, + task_ids: list[str] | None = None, + cases_dir: Path | None = None, + org: str = "clawbench", + docker_image: str | None = None, + ) -> None: + self.output_dir = output_dir + self.overwrite = overwrite + self.limit = limit + self.task_ids = task_ids or [] + self.cases_dir = cases_dir + self.org = org + self.docker_image = docker_image + + def run(self) -> int: + argv = ["--output-dir", str(self.output_dir), "--org", self.org] + if self.overwrite: + argv.append("--overwrite") + if self.limit is not None: + argv += ["--limit", str(self.limit)] + if self.task_ids: + argv += ["--task-ids", ",".join(self.task_ids)] + if self.cases_dir is not None: + argv += ["--cases-dir", str(self.cases_dir)] + if self.docker_image: + argv += ["--docker-image", self.docker_image] + return convert(argv) diff --git a/harbor/adapter/src/clawbench_adapter/main.py b/harbor/adapter/src/clawbench_adapter/main.py new file mode 100644 index 00000000..26e8de69 --- /dev/null +++ b/harbor/adapter/src/clawbench_adapter/main.py @@ -0,0 +1,52 @@ +"""Main entry point for the ClawBench adapter. Do not modify the standard flags. + +Constructs ClawBenchAdapter and calls run() to generate tasks in the Harbor +format at the configured output directory. +""" + +from __future__ import annotations + +import argparse +from pathlib import Path + +from .adapter import ClawBenchAdapter + +# Default output dir: /datasets/clawbench +DEFAULT_OUTPUT_DIR = Path(__file__).resolve().parents[4] / "datasets" / "clawbench" + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--output-dir", type=Path, default=DEFAULT_OUTPUT_DIR) + parser.add_argument("--limit", type=int, default=None) + parser.add_argument("--overwrite", action="store_true") + parser.add_argument("--task-ids", nargs="+", default=None) + # Adapter-specific flags + parser.add_argument( + "--cases-dir", + type=Path, + default=None, + help="ClawBench corpus (defaults to the bundled test-cases/v2)", + ) + parser.add_argument("--org", default="clawbench", help="Task name org prefix") + parser.add_argument( + "--docker-image", + default=None, + help="Prebuilt runtime image; omit to ship environment/ per task", + ) + args = parser.parse_args() + + adapter = ClawBenchAdapter( + args.output_dir, + overwrite=args.overwrite, + limit=args.limit, + task_ids=args.task_ids, + cases_dir=args.cases_dir, + org=args.org, + docker_image=args.docker_image, + ) + raise SystemExit(adapter.run()) + + +if __name__ == "__main__": + main() diff --git a/harbor/adapter/src/clawbench_adapter/task-template/environment/Dockerfile b/harbor/adapter/src/clawbench_adapter/task-template/environment/Dockerfile new file mode 100644 index 00000000..77241eea --- /dev/null +++ b/harbor/adapter/src/clawbench_adapter/task-template/environment/Dockerfile @@ -0,0 +1,4 @@ +# Default mode copies src/clawbench/runtime/harbor/Dockerfile here (Chromium + +# recorder/interceptor runtime-server + noVNC). Prebuilt mode omits this +# directory and sets [environment].docker_image instead. +FROM ghcr.io/tiger-ai-lab/clawbench-harbor-runtime:latest diff --git a/harbor/adapter/src/clawbench_adapter/task-template/instruction.md b/harbor/adapter/src/clawbench_adapter/task-template/instruction.md new file mode 100644 index 00000000..2d22d024 --- /dev/null +++ b/harbor/adapter/src/clawbench_adapter/task-template/instruction.md @@ -0,0 +1,12 @@ +{instruction} + +You have access to the following files under `/my-info/`: `alex_green_personal_info.json` (the persona: name, address, phone, disposable email + password), `resume.pdf`. Use them wherever the task needs personal details or a login. + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/adapter/src/clawbench_adapter/task-template/solution/solve.sh b/harbor/adapter/src/clawbench_adapter/task-template/solution/solve.sh new file mode 100755 index 00000000..efb726f3 --- /dev/null +++ b/harbor/adapter/src/clawbench_adapter/task-template/solution/solve.sh @@ -0,0 +1,5 @@ +#!/bin/bash +set -euo pipefail +# ClawBench tasks run against live consumer websites; there is no scripted +# browser oracle. Verifier correctness is established by parity runs instead. +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/adapter/src/clawbench_adapter/task-template/task.toml b/harbor/adapter/src/clawbench_adapter/task-template/task.toml new file mode 100644 index 00000000..6dc01a73 --- /dev/null +++ b/harbor/adapter/src/clawbench_adapter/task-template/task.toml @@ -0,0 +1,51 @@ +# Rendered per task by clawbench.eval.harbor_adapter; {placeholders} are filled from task.json. +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/{task_id}" +description = "{description}" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "{source_task}" + +[environment] +docker_image = "ghcr.io/tiger-ai-lab/clawbench-harbor-runtime:{clawbench_version}" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "clawbench/{task_id}" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/adapter/src/clawbench_adapter/task-template/tests/test.sh b/harbor/adapter/src/clawbench_adapter/task-template/tests/test.sh new file mode 100755 index 00000000..4dbe2f57 --- /dev/null +++ b/harbor/adapter/src/clawbench_adapter/task-template/tests/test.sh @@ -0,0 +1,14 @@ +#!/bin/bash +set -euo pipefail +# Stop the recorder, snapshot /data, then score: +# Stage 1 interception.json must match the task's eval-schema (url_pattern + method) +# Stage 2 LLM judge (CLAWBENCH_JUDGE_*) confirms the intercepted payload fulfils the instruction +# reward = 1.0 iff both stages pass; written to /logs/verifier/reward.txt + reward.json +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/dataset-v1.toml b/harbor/dataset-v1.toml new file mode 100644 index 00000000..bd1c8073 --- /dev/null +++ b/harbor/dataset-v1.toml @@ -0,0 +1,620 @@ +# Dataset manifest for tiger-ai-lab/clawbench-v1 +# Add tasks using: harbor add / +# Publish using: harbor publish + +[dataset] +name = "tiger-ai-lab/clawbench-v1" +version = "1.0.0" +description = "ClawBench V1: 152 everyday tasks on live consumer websites (original corpus), scored by request interception + LLM judge. https://claw-bench.com" +keywords = [] +[[dataset.authors]] +name = "Yuxuan Zhang" +email = "mtrxcop@gmail.com" + +[[tasks]] +name = "clawbench/001-daily-life-food-uber-eats" +digest = "sha256:a0718052e76d95b24c69dfcfbd609378000667f49c2d77bf9e23d2d26f628839" + +[[tasks]] +name = "clawbench/002-daily-life-food-doordash" +digest = "sha256:4db7e7a04cb2bc9cdccc095ffbef0c0134d930513d7a83768cebb5925c235ef7" + +[[tasks]] +name = "clawbench/004-daily-life-food-instacart" +digest = "sha256:cfb7fd798558c1ff220a0673abb5b9a6acf3a86a124a28e2dc34ea8750a6e869" + +[[tasks]] +name = "clawbench/006-daily-life-food-uber-eats" +digest = "sha256:a0b8803a32188618d626fa378cb7b3b9e86010ec6467b1d2ae65b17255bf2f33" + +[[tasks]] +name = "clawbench/007-daily-life-food-instacart" +digest = "sha256:d9516f24031bf70b824c74733550343bd5dc0f2f1be51cb59410fe6c958cf8a1" + +[[tasks]] +name = "clawbench/011-daily-life-housing-zillow" +digest = "sha256:d9e075a8c2709db45c3770d3f6925cedc886455c6a5338330c36aee17dfac348" + +[[tasks]] +name = "clawbench/015-daily-life-housing-craigslist" +digest = "sha256:ae928c55afc843ce24e3e174b649d3ab6b7738c2a359aecbc56aa068c4db99a7" + +[[tasks]] +name = "clawbench/035-daily-life-health-medical-betterhelp" +digest = "sha256:2c609e7c250933278a38bd46d02f42d229fb0054cbc6c29a713cc757fa853f12" + +[[tasks]] +name = "clawbench/041-daily-life-pets-rover" +digest = "sha256:251f5a62f52fe9d032d9996ec417dcf3f24ac8265d32d1c25c64bdf5e98aaafc" + +[[tasks]] +name = "clawbench/043-daily-life-pets-rover" +digest = "sha256:29473dbab0691f7b27fe5d980ce31eb30039609d198380390923350182930ed7" + +[[tasks]] +name = "clawbench/045-daily-life-personal-care-booksy" +digest = "sha256:23d308ec2fa25d7b03db764ecf5a3dbe9f2f91b387d70e409d48626614693f7f" + +[[tasks]] +name = "clawbench/047-daily-life-personal-care-taskrabbit" +digest = "sha256:5abda618bd23798dc7f5740970323f40814f01acc7e38c3d2b4b87b3b1d31c01" + +[[tasks]] +name = "clawbench/086-job-search-hr-cv-autofill-greenhouse-meta" +digest = "sha256:fd7c0ab94d89442adc2cb84b4e359644b593d190aeeb28093612b66e39e3de4e" + +[[tasks]] +name = "clawbench/089-job-search-hr-cv-autofill-simplify-jobs" +digest = "sha256:26f5b2bd7be9f14835a6e252f7022b274ed2c858145d340eb498ef29b02e64b8" + +[[tasks]] +name = "clawbench/091-job-search-hr-job-apply-indeed" +digest = "sha256:6a0ae3287457fa20a789a3fa350e666bc29010f3e19deb228dee56cbadd536d5" + +[[tasks]] +name = "clawbench/120-office-secretary-tasks-email-mgmt-purelymail" +digest = "sha256:bf38e927a4650a7d64df376d17f23e09beeada88c13df13da31fc3b0dd9f07d8" + +[[tasks]] +name = "clawbench/121-office-secretary-tasks-email-mgmt-purelymail" +digest = "sha256:e5ebb7b9f589d8316b4765161cdaeb03d527beb4ed30fad6dc909495e8e4c379" + +[[tasks]] +name = "clawbench/128-office-secretary-tasks-email-mgmt-purelymail" +digest = "sha256:ae9ca8c0d654aa2c6d2d9a99035cf905cecebd6ad491e0c2b71490704287bfbc" + +[[tasks]] +name = "clawbench/134-office-secretary-tasks-calendar-calendly" +digest = "sha256:69d23784aa5732c87452e1b267a871567f37cec8babb4be00375d7eccee3b6c6" + +[[tasks]] +name = "clawbench/137-office-secretary-tasks-calendar-doodle" +digest = "sha256:550e7dd5d320216f076c6fca4723cb61ba620319514550fc4343a319347e7930" + +[[tasks]] +name = "clawbench/139-office-secretary-tasks-calendar-calendly" +digest = "sha256:b1b60853c610ae12c93ce2f93bb7f920b673e93428f51e29b7297d31438f830f" + +[[tasks]] +name = "clawbench/142-office-secretary-tasks-collab-trello" +digest = "sha256:115ec1299daad8f28c79e23902bb97d44aba7d7854a5c3216625cb271a39f5e6" + +[[tasks]] +name = "clawbench/179-dev-tech-github-ops-github" +digest = "sha256:3edca9cd49744dd4d43bd8b891d0b621af7f95b97ce2162ec58ad3d65bae9dd6" + +[[tasks]] +name = "clawbench/180-dev-tech-github-ops-github" +digest = "sha256:2d9d46ea190f9fe6c35add5dc92abd5cde5a3bc1e4bd52cac80d4bd69d88f01f" + +[[tasks]] +name = "clawbench/215-academia-research-paper-tables-overleaf" +digest = "sha256:9d279c6a051b8f86e2a49387c4f8d4f06d140f37d248cdf99fc0d331dd350acc" + +[[tasks]] +name = "clawbench/242-academia-research-research-tools-overleaf" +digest = "sha256:4a665e39e11c19e992e097bcbf9bd4692217f6967eee406e73d3ee90ba5ca92e" + +[[tasks]] +name = "clawbench/246-academia-research-research-tools-zotero" +digest = "sha256:2004337a81ffbacd9abf6de7833131c4fa7613641c74f246f8db107619010a01" + +[[tasks]] +name = "clawbench/247-academia-research-research-tools-semantic-scholar" +digest = "sha256:e0c6067e455b2e80cdc275c8abc622d7d4209ec962043b1694dde196dbcf8891" + +[[tasks]] +name = "clawbench/265-education-learning-general-coursera" +digest = "sha256:6e122854d66492d22f02dd59182c594b4fc9af1305bb9afad5a4eee054d45ef3" + +[[tasks]] +name = "clawbench/266-education-learning-general-leetcode" +digest = "sha256:1dfe6dfbe32751db214168c1baf65a52254da7bf95af10aeedd046868fa46ada" + +[[tasks]] +name = "clawbench/273-education-learning-general-edx" +digest = "sha256:93f53ee7aef8c42c01db71b1ccc49436082dbe371ab45e86fdcbb501dff52779" + +[[tasks]] +name = "clawbench/274-education-learning-general-udemy" +digest = "sha256:7b76d7c1d2e8dba62a71a9fc1c8a4ee8dbfcb5944d8687a7fac20132b49f3b7f" + +[[tasks]] +name = "clawbench/279-travel-general-airbnb" +digest = "sha256:ea28e5f458054c27b1bca7cbc66f839f813d3287d43241465992ba06936f1195" + +[[tasks]] +name = "clawbench/280-travel-general-booking-com" +digest = "sha256:d5f24dd42e7796631f741d018c013e9b4e509879e326218d9a77daa74524384b" + +[[tasks]] +name = "clawbench/363-entertainment-hobbies-general-ticketmaster" +digest = "sha256:7be92dab1f74a12ccfb7da62ab915701f741b2e266c5bd9f4dcb295ee5d14216" + +[[tasks]] +name = "clawbench/369-entertainment-hobbies-general-goodreads" +digest = "sha256:acd1dd31ef49e15aa9f5957592d00c223549d3de0615ae374b37fd104113207e" + +[[tasks]] +name = "clawbench/372-entertainment-hobbies-general-eventbrite" +digest = "sha256:eee9c4d5ba9622492dff030ee2c68dad4406a0e82c94bdbb8512a48ba5e2943c" + +[[tasks]] +name = "clawbench/403-personal-management-account-security-1password-web" +digest = "sha256:a762c460f79d38c00bd57837b069354b966c64f5d5ae732cb99f2ba3e83fecf0" + +[[tasks]] +name = "clawbench/413-personal-management-personal-tools-todoist" +digest = "sha256:22d0518f3794a76b32a4f594265c7559eac799e04827ddb7df4192510d296e27" + +[[tasks]] +name = "clawbench/468-rating-voting-general-glassdoor" +digest = "sha256:6ee8db3aff699af2e760c67a606dcf5e809ff4ab08b4fd8e97c8ed9cd00afdc5" + +[[tasks]] +name = "clawbench/469-rating-voting-general-tripadvisor" +digest = "sha256:cc223e04f686a0b62a7840314fa16cfd41c6e2808754481e611cbe27880fe13b" + +[[tasks]] +name = "clawbench/470-rating-voting-general-trustpilot" +digest = "sha256:7e59091dd1c5ff1b435f573f482a95f2a267eb6e094dcf41b6f4f2fcfb417b47" + +[[tasks]] +name = "clawbench/474-rating-voting-general-capterra" +digest = "sha256:d1e5f8a4418599182cae1e3497e139d7fd9f6093f761fc111d6bf490b4c7ac5d" + +[[tasks]] +name = "clawbench/475-rating-voting-general-g2" +digest = "sha256:721b1ee8b1eb93eea1637694a754ad496e582bfa809f0a57db3d9cf9168efa15" + +[[tasks]] +name = "clawbench/482-creation-init-general-confluence" +digest = "sha256:c94f9fb3ece627ca8821c8523a7a67031a77f8c98143ac0008ff93dd7ef5dbb6" + +[[tasks]] +name = "clawbench/483-creation-init-general-airtable" +digest = "sha256:15b2d94ca863da27ea839399e1d48c8d70159f00bd5024a78e92fa7e50b96c5c" + +[[tasks]] +name = "clawbench/484-creation-init-general-clickup" +digest = "sha256:306ea1650f0bf282ff0e566fb8c971db06d491552b0dbd20e62822d6ce071c35" + +[[tasks]] +name = "clawbench/485-creation-init-general-webflow" +digest = "sha256:aacffaf3e5cb669a62b8fad4b5089afb965eeca4b220ebe23f6d0d44c02f4a8d" + +[[tasks]] +name = "clawbench/486-creation-init-general-mailchimp" +digest = "sha256:904b8ea8a77e03a934e6ec520a0f8f5b326714553522bba095d63ecbe4354305" + +[[tasks]] +name = "clawbench/487-creation-init-general-typeform" +digest = "sha256:a4d6c72424f63ff02cb490c0be34ad15112d9c99cfded09c4bada3ee079f9f0f" + +[[tasks]] +name = "clawbench/488-creation-init-general-substack" +digest = "sha256:33fc78e60af7cfd8885a800a7a5c0677f74aa613889b750bff9be2eb531e502a" + +[[tasks]] +name = "clawbench/489-creation-init-general-ghost" +digest = "sha256:f0f9e00aa3879d1430d52e76ccff13b0c46f94f5f77108c0d75c0441ca65d45a" + +[[tasks]] +name = "clawbench/501-creation-init-general-asana" +digest = "sha256:97c96051713794614fb9850512737da93e91283f676a0b35e369a8cc1710295c" + +[[tasks]] +name = "clawbench/529-daily-life-shopping-delivery-king-arthur-baking" +digest = "sha256:55a0c00443715dfcd105f79e677aa919b5e34547e6a8221ac41a99d75d4f342f" + +[[tasks]] +name = "clawbench/533-daily-life-utilities-inmyarea" +digest = "sha256:12cf6186049f0a0d96709e50f10e67ef68acf0f27ac9e325fc70a0abddf6a9f2" + +[[tasks]] +name = "clawbench/535-daily-life-home-home-depot" +digest = "sha256:2ca3fb3da357ac275df86b54bb159fb265e677d72421ead46b22db35c3737051" + +[[tasks]] +name = "clawbench/537-daily-life-food-crumbl" +digest = "sha256:472ceed808c87b30be9fdb6266f643e52eaaf894f34ff228636d9ee248118c7f" + +[[tasks]] +name = "clawbench/539-daily-life-health-jefit" +digest = "sha256:cde3ce41116c693aacff64ebdc9718694041f2e2bc7e8ebe1ed8f1fe4b6c2ccd" + +[[tasks]] +name = "clawbench/542-daily-life-pets-wag" +digest = "sha256:6cc5e89584fb98c9e17de10a4dee1ce98633fbff7d86a21e6f1d73dbe60640f9" + +[[tasks]] +name = "clawbench/551-finance-investment-crypto-wallet-trezor" +digest = "sha256:b11c537d212411aba8605e571797fbf5001ae1cf90de5f790e9345ab7972bc25" + +[[tasks]] +name = "clawbench/552-finance-investment-business-payment-plooto" +digest = "sha256:37f4a9d29799c4a22b0a6b9e33f9666b3f9149a03b5656a071e8b50811d55df1" + +[[tasks]] +name = "clawbench/555-finance-investment-insurance-insureon" +digest = "sha256:441a134068271854d4c738172e134c9f4aefba43ed53674911facb96b774c42c" + +[[tasks]] +name = "clawbench/559-finance-investment-crowdfunding-frontfundr" +digest = "sha256:ae830b1d3be9ac766ea54ef240e1d1095ff9593337f48a0d9d9d0686bb07d969" + +[[tasks]] +name = "clawbench/564-daily-life-event-registration-race-roster" +digest = "sha256:37d6359d56dcef8c833d60efe1a2ebe21be6a1f7820a39129f52bac12d6e8223" + +[[tasks]] +name = "clawbench/565-job-search-hr-job-search-jopwell" +digest = "sha256:31cb6b5738d2cf90c7003c61c5ddc94fb6f00528361be507e5ad2c1739136824" + +[[tasks]] +name = "clawbench/566-job-search-hr-job-search-ziprecruiter" +digest = "sha256:b0014dd1c3c9b029f9bcce625a7678b1ce652c0d04820b91953fedfb614afdad" + +[[tasks]] +name = "clawbench/569-job-search-hr-job-search-careerbuilder" +digest = "sha256:98716ee2777717aaf798022a8208a927bf2a8344c03a2dfccd490def72fd7228" + +[[tasks]] +name = "clawbench/570-job-search-hr-job-search-hired" +digest = "sha256:73b257f6ad33e64b60c069e57074502f254a5895091cbea4af1ce6172ac1746a" + +[[tasks]] +name = "clawbench/571-job-search-hr-recruitment-mgmt-workable" +digest = "sha256:ef9492fb034b2b517a8f2dbf7388b6f2aff197ed3ba7aa55d10aaf15b209e1bf" + +[[tasks]] +name = "clawbench/576-office-secretary-tasks-reports-ftc-reportfraud" +digest = "sha256:4850f188d44ac8dee3d0afc3961bce1a68c3aa07eb6289b20a5bc68844bb5dae" + +[[tasks]] +name = "clawbench/583-office-secretary-tasks-support-tickets-freshdesk" +digest = "sha256:a023b8825a187ef320ccd33557d109c60ca77608385574e832c4162472666deb" + +[[tasks]] +name = "clawbench/598-academia-research-legal-docs-formswift" +digest = "sha256:5b0f15865bf91d9122d1bbad27c09bd399d40adc672c3b10c8a9004a41aa1447" + +[[tasks]] +name = "clawbench/606-education-learning-kids-courses-outschool" +digest = "sha256:718a18938ae4c04867498be44a21f441f8ab4419900b5a1b920721cf8a9b1a46" + +[[tasks]] +name = "clawbench/607-education-learning-art-courses-creativebug" +digest = "sha256:076c3eeae74d4e374e2f9f5320783c47d41aef5b27066336c2c7e6b828b5f9a8" + +[[tasks]] +name = "clawbench/609-education-learning-meditation-spirit-rock-meditation-center" +digest = "sha256:3da639ccc9723f5320836b9e2eee1eac681b450e7f8a78bc55b8c598ac0c03b7" + +[[tasks]] +name = "clawbench/615-travel-flights-spirit-airlines" +digest = "sha256:7ab36ba598c3f57f5ec7ea3404a94e341fd2e4d2edcc5115cc1c2e88115c4717" + +[[tasks]] +name = "clawbench/618-travel-train-bus-12go-asia" +digest = "sha256:4fbd2bc882a7c7c975c86091c0a4483ac05b4325e778f3271407ef136b81bfe7" + +[[tasks]] +name = "clawbench/625-travel-camping-outdoor-parks-canada-reservations" +digest = "sha256:154b1307046b90f49adf35fa9fd8bcfe7a3ed377164c001d12ac66a860036def" + +[[tasks]] +name = "clawbench/626-travel-bus-flixbus" +digest = "sha256:b47824d1449dec98c32023a4570192b76368a24870b4aad765784cb5e7a9696c" + +[[tasks]] +name = "clawbench/627-travel-flights-momondo" +digest = "sha256:c0070048fe767dc88182be0895ac5a79cf7dc769e9372d81805bb5afe34c0d0e" + +[[tasks]] +name = "clawbench/632-shopping-commerce-beauty-care-olaplex" +digest = "sha256:c2e7179345bd68d486062de6744c3620491fa5274b70e20e4b4d68d4fcd91190" + +[[tasks]] +name = "clawbench/634-shopping-commerce-apparel-dooney-bourke" +digest = "sha256:8f9601201bc29461fded4c8d154e56f2554afd1e0cfda4e652bd443fbc0b7925" + +[[tasks]] +name = "clawbench/635-shopping-commerce-gifts-uncommon-goods" +digest = "sha256:39bbf7db4ec5a4b4cbbc46f7e0d542c7e9c66c3aaa3338faf5d4181a1130bcfe" + +[[tasks]] +name = "clawbench/636-shopping-commerce-auto-parts-rockauto" +digest = "sha256:95a5092e2ce9b710e6e86e9e2687106d5f91d9c8c7661fcf8e4a1b4734af288a" + +[[tasks]] +name = "clawbench/638-shopping-commerce-print-custom-vistaprint" +digest = "sha256:34c7c5a6204f77569c1206e2c6ce91e08d9decf9b1fc700fc5679497ef34bd4e" + +[[tasks]] +name = "clawbench/639-shopping-commerce-luxury-mansur-gavriel" +digest = "sha256:2ebfc4612a9a57d83b833f093014cc1dc90c71e359dd728826d6a057e46af5ab" + +[[tasks]] +name = "clawbench/671-entertainment-gaming-humble-bundle" +digest = "sha256:7f4a977218a46609cd236db890b7bdc20e81053c7cd4a9328331d93e78cd8b39" + +[[tasks]] +name = "clawbench/672-entertainment-hobbies-anime-streaming-crunchyroll" +digest = "sha256:1324196f8c46cbe4d4283ed30eb8ee1a2aee152abaf098002fa1ce79bac55f47" + +[[tasks]] +name = "clawbench/674-entertainment-hobbies-masterclass-masterclass" +digest = "sha256:382b82c0820a7297101f80e34bd3ef06805ac12f876278eb827cee9727e00419" + +[[tasks]] +name = "clawbench/676-government-civic-legal-docs-legalnature" +digest = "sha256:67eabb754fe4ba9719d6d17786d3dba3f896213d40f7917728b48c1d17d11dbb" + +[[tasks]] +name = "clawbench/685-personal-management-budget-mgmt-everydollar" +digest = "sha256:18747e44cd69e9e89fa3b069d732ae4908100d40515377752062a767d1b8b6f8" + +[[tasks]] +name = "clawbench/687-personal-management-vpn-subscription-ipvanish" +digest = "sha256:28701b54241691f5bf23b76b2732d1e4cacdba9d6bab26a485512165954f90ae" + +[[tasks]] +name = "clawbench/688-personal-management-insurance-compare-insurify" +digest = "sha256:9a0fd7a83834ae8e0209fd943fbf3bc0e1cdad7b0a3ccad4e7358e796df781af" + +[[tasks]] +name = "clawbench/695-automation-workflows-recurring-order-stumptown-coffee" +digest = "sha256:73405896060eb43dec3824c8320eb5dc6461cd8412f7679097f504604888f40d" + +[[tasks]] +name = "clawbench/697-automation-workflows-recurring-order-bean-box" +digest = "sha256:b86097b39f2eaa5f68c807ae89c5a0ec6bfdee3c7434af934694985cfc5960a5" + +[[tasks]] +name = "clawbench/699-automation-workflows-recurring-order-mistobox" +digest = "sha256:401acf6af87572780a5400ee6670078c2d408bc113e3ffc6299b4adce59aada0" + +[[tasks]] +name = "clawbench/700-deletion-revocation-data-deletion-deleteme" +digest = "sha256:769ccc7ac4c2d48bfa6ab912998f47880d2ac35bbdd20d670db9995562163362" + +[[tasks]] +name = "clawbench/705-rating-voting-wine-review-vivino" +digest = "sha256:47ff9da578ffd0e32732f8a88c4d4aa2ee2eec8ab20a5d89c41c3efd4f782bab" + +[[tasks]] +name = "clawbench/706-rating-voting-beer-review-beeradvocate" +digest = "sha256:05795c4ff4f14f9751a3cc50681254fa676ac7fc0b669b8a87e0b900ec14a133" + +[[tasks]] +name = "clawbench/707-rating-voting-social-wine-untappd" +digest = "sha256:39375702aa988ccae8e4c20756dd34f39a8065d0f6dfcecee8529138e1c0990d" + +[[tasks]] +name = "clawbench/708-rating-voting-professor-review-ratemyprofessors" +digest = "sha256:20ae060c81fadbc3d7bcc51b75b1213b0d8f86e9dbb680ba2fe7913412d2fe03" + +[[tasks]] +name = "clawbench/709-rating-voting-service-review-angi" +digest = "sha256:35624e3f3f368db11d4c72c446c177bd7b70270b7cb2e1be40781d8141368fb2" + +[[tasks]] +name = "clawbench/710-creation-init-interior-design-roomsketcher" +digest = "sha256:3a005d5e4649a8ee52902bbcccbc2da66aa916aaa1de145222c14c845df4cd8c" + +[[tasks]] +name = "clawbench/711-creation-init-color-design-coolors" +digest = "sha256:972a74c48ab18f13490fbb2774926ab34b7172b798dc0b117bb76974dcf95938" + +[[tasks]] +name = "clawbench/712-creation-init-website-create-squarespace" +digest = "sha256:ac206506e855b1c4b0efec8c402bdd88ace1c3437fa5c50a256a106fed8d8838" + +[[tasks]] +name = "clawbench/713-creation-init-website-build-wix" +digest = "sha256:3bcc7ce0205cf659c55a11a998818d4ffb40fe3544c983d15bff8417654e6378" + +[[tasks]] +name = "clawbench/735-home-services-maintenance-house-cleaning-bark" +digest = "sha256:706b0dadf37d113f48b3bd3349dc8d4c5d1bdced904a184f47c0c818e193e484" + +[[tasks]] +name = "clawbench/736-home-services-maintenance-plumbing-ace-hardware" +digest = "sha256:52f096d8802cafef0393514b53f38d7da92a65372a1db9010627e3d0852e0d23" + +[[tasks]] +name = "clawbench/737-home-services-maintenance-kitchen-remodel-lowes" +digest = "sha256:f610e97239eee3144ab98cf4da2cbf370ae80044ea03fee5c4fb11ea0b4ce75c" + +[[tasks]] +name = "clawbench/738-home-services-maintenance-equipment-install-amazon-home-services" +digest = "sha256:d57feabf95533b435e06bc8517e22945432ecba408500bcb19ad6317bf4ddb6d" + +[[tasks]] +name = "clawbench/750-automotive-vehicle-services-car-insurance-compare-kanetix" +digest = "sha256:28747e8c877626c41443be29c1e088ee20218c9fcc40564b7444765220c5cdf2" + +[[tasks]] +name = "clawbench/751-automotive-vehicle-services-car-lease-sixt" +digest = "sha256:40ab005ba0d852599945f16a9f2bab0d5b2509dcb27b347e577d13d6de05d54d" + +[[tasks]] +name = "clawbench/754-automotive-vehicle-services-used-car-listing-autotrader" +digest = "sha256:eac262b2db42e835e7b7088b2e3e0d7674951e27a7fe4490a193879fe5962ae9" + +[[tasks]] +name = "clawbench/763-automotive-vehicle-services-car-lease-autoslash" +digest = "sha256:bc44ec2795f65ba646dde7e520a32be3e533b2f64cd26ea65d33fc30670119ed" + +[[tasks]] +name = "clawbench/766-nonprofit-charity-donation-doctors-without-borders-msf" +digest = "sha256:271f8ea0f2854fa492a2a82bdf3f92f46cc47c8f0116dc93931a668c6d86cd38" + +[[tasks]] +name = "clawbench/768-nonprofit-charity-community-crowdfund-ioby" +digest = "sha256:8607b8e53c83ab4a56016031558d6357616dca88563b4f27dbff73c0c7acbf25" + +[[tasks]] +name = "clawbench/770-nonprofit-charity-volunteer-apply-on-make-a-wish-foundation-website-complete-and-submit-a-volunteer-application-form-selecting-the-wish-granter-role-and-entering-city-phoenix-az" +digest = "sha256:2b30c197517eb2182169547423ab16af04f46ba36faf986a2fa77422006e17a7" + +[[tasks]] +name = "clawbench/774-nonprofit-charity-nonprofit-job-apply-charity-village" +digest = "sha256:f28b3bb6a3af7b4cf1095e35b3fcadea826c6d444690b4069dc8d4a9c392fe87" + +[[tasks]] +name = "clawbench/776-nonprofit-charity-volunteer-signup-idealist" +digest = "sha256:44b42a0178f254750aebb39998dce6e0f09d446856e24d580fe5749520ae0c10" + +[[tasks]] +name = "clawbench/778-nonprofit-charity-donation-globalgiving" +digest = "sha256:095fef113fcf96f2c9d56ff79b2ddd8b9aa8efd450698d406d9704d8541acbf9" + +[[tasks]] +name = "clawbench/780-beauty-personal-care-skincare-purchase-soko-glam" +digest = "sha256:acd80e59422b75f8df5f2dcb1224b0acf3de7d616124641a22725e19bb051faa" + +[[tasks]] +name = "clawbench/781-beauty-personal-care-beauty-booking-bluemercury" +digest = "sha256:69c5161c7baf9f1b7e8aedb7528c925d0e8e0fed58d0f4bbd306f8913ea69532" + +[[tasks]] +name = "clawbench/782-beauty-personal-care-skincare-purchase-paulas-choice" +digest = "sha256:32f5f899dcac9299601b0995325728274680846c16627da7658dc0327d52016b" + +[[tasks]] +name = "clawbench/783-beauty-personal-care-beauty-booking-ulta-beauty" +digest = "sha256:322806a4e4d09cfa2dd2c392fca0a7cddf508b906f656a864f9244ee57f675fb" + +[[tasks]] +name = "clawbench/785-beauty-personal-care-skincare-curology" +digest = "sha256:275727c942bfbdb6fadfde751a1449b47171fa813c4603132e272508b66eeebf" + +[[tasks]] +name = "clawbench/788-beauty-personal-care-makeup-the-ordinary" +digest = "sha256:f437c6503ac3cd704b32e68c03d1767d803086742b122cbe397f459a28ee1ffd" + +[[tasks]] +name = "clawbench/789-beauty-personal-care-makeup-fenty-beauty" +digest = "sha256:47e0f82c83d8def77af11ad163dc29ecabeee361cb398f6fa2181002a4c3cb44" + +[[tasks]] +name = "clawbench/793-beauty-personal-care-beauty-retail-mac-cosmetics" +digest = "sha256:a2421641c8e8af1afc4a02adb43ad0718aefd911e4ea8f7019c4b563126fce58" + +[[tasks]] +name = "clawbench/794-beauty-personal-care-salon-booking-styleseat" +digest = "sha256:189abe44408de71c0165bb683c1a41bc43d52ce2326afc68073391d3d571e65e" + +[[tasks]] +name = "clawbench/795-pet-animal-care-pet-adoption-aspca" +digest = "sha256:846482151844cfddbcf9dd97bf3b2caf6014ab4f9f7c71415b851b9974ff831e" + +[[tasks]] +name = "clawbench/796-pet-animal-care-pet-supplies-grooming-petsmart" +digest = "sha256:f9386bed29d8acbe8b88559df760c199cc665f6eaa93f0cc32719938692b3788" + +[[tasks]] +name = "clawbench/801-pet-animal-care-pet-friendly-travel-bringfido" +digest = "sha256:c9bfea9c4a7b149dbb0f185b9c1d7b1c4cc13bd3691e9a60960b2ba965e08382" + +[[tasks]] +name = "clawbench/803-pet-animal-care-pet-medical-pawp" +digest = "sha256:716191e5ce2294116e17932cfdb3e17fb4c6020dbb73f22a8794e73e248f0bef" + +[[tasks]] +name = "clawbench/807-pet-animal-care-pet-dna-embark" +digest = "sha256:3c1926daadb27491d5be97eefe9cae3e402b3a0ea42672132645793ef32b309a" + +[[tasks]] +name = "clawbench/809-pet-animal-care-pet-adopt-petfinder" +digest = "sha256:9abedc9ebab2cfc5e3722d92bbafd6b30fa4073af815a9dbf6e2d34bc3691290" + +[[tasks]] +name = "clawbench/812-pet-animal-care-pet-subscription-ollie" +digest = "sha256:c0ae50cb395ccb00781d1ae4031189e54cdacff056f76761c0a6d11724769c6d" + +[[tasks]] +name = "clawbench/815-personal-management-records-mgmt-myheritage" +digest = "sha256:78186c0d60ded83fd0d895c8d908f8ea2c4bac68643bce7918820e077822068e" + +[[tasks]] +name = "clawbench/821-education-learning-reading-self-study-blinkist" +digest = "sha256:4374713504ebd7cc9663aad29cc06085a78fb139b95e50bd80041cc59436d6a8" + +[[tasks]] +name = "clawbench/861-entertainment-hobbies-movies-cineplex" +digest = "sha256:c669998c404740a06000b7d68132f04380c89a2402c4597ca795ecd0df8be492" + +[[tasks]] +name = "clawbench/862-entertainment-hobbies-movies-amc-theatres" +digest = "sha256:74b0944b746aaf2b3f90d9b226dd2a84fbdb9a54744c741e4ea035c0b9a1a8fa" + +[[tasks]] +name = "clawbench/864-entertainment-hobbies-show-tickets-ticketmaster" +digest = "sha256:214fe9251e753bf2b81c99e8067e0901bbb6e1edb3f0bf59891f69bb4775add6" + +[[tasks]] +name = "clawbench/865-travel-outdoor-hipcamp" +digest = "sha256:619710f2d78e00e5c115b0f6116cd055a92d6183ece36a360549d6988a38718a" + +[[tasks]] +name = "clawbench/867-entertainment-hobbies-movies-fandango" +digest = "sha256:34b1bf8e1884937f7829f8fa98ecb037d6d305869bc0d82d7eaa290f8f0db02e" + +[[tasks]] +name = "clawbench/872-daily-life-food-opentable" +digest = "sha256:cccf51e67421d55ccbedfac742be20e0bd68d154fbe5c1fb9bc2bff614dfc8a6" + +[[tasks]] +name = "clawbench/873-daily-life-food-resy" +digest = "sha256:dc4209361202dc7a75d8a34e571221c3567db1656fa3d9cc129edbef17d851c9" + +[[tasks]] +name = "clawbench/876-entertainment-hobbies-show-tickets-vivid-seats" +digest = "sha256:8f4602b54e39334214ae304152ac8d40720f8a1e3c3950c4b8aa230713801b76" + +[[tasks]] +name = "clawbench/877-entertainment-hobbies-show-tickets-stubhub" +digest = "sha256:686c2358f4a48a048a5cd44478ad6c93bb27f24bd993f6a3688e0f9f1c2aac41" + +[[tasks]] +name = "clawbench/878-travel-outdoor-ontario-parks" +digest = "sha256:c7a5cf8dfe95e0667b59bb1505f38edf9be9f16bbedb8c809c0486045543c064" + +[[tasks]] +name = "clawbench/883-education-learning-hobby-class-sur-la-table" +digest = "sha256:2f8bb7f7b06805e71b135bad061ae6576c2d64acd8f68d0132ab56f41cf85f6b" + +[[tasks]] +name = "clawbench/884-entertainment-hobbies-experience-breakout-games" +digest = "sha256:ce7751c98fc13c7347676ab569893dd43d4cb722f52d55678fe707a629bd4480" + +[[tasks]] +name = "clawbench/885-entertainment-hobbies-experience-bowlero" +digest = "sha256:bccf6ceb082c51348b93154779b0780dcb56ce733a9f85bcb9ae9fb86f6ae6bc" + +[[tasks]] +name = "clawbench/886-entertainment-hobbies-experience-topgolf" +digest = "sha256:3615cd0ee90ff7c1cba907156ef4cd6be75ce86e1ad18f3d9547ce22862faf16" diff --git a/harbor/dataset.toml b/harbor/dataset.toml new file mode 100644 index 00000000..d9b163b0 --- /dev/null +++ b/harbor/dataset.toml @@ -0,0 +1,528 @@ +# Dataset manifest for tiger-ai-lab/clawbench-v2 +# Add tasks using: harbor add / +# Publish using: harbor publish + +[dataset] +name = "tiger-ai-lab/clawbench-v2" +version = "1.0.0" +description = "ClawBench V2: 129 everyday tasks on live consumer websites, scored by request interception + LLM judge. https://claw-bench.com" +keywords = [] +[[dataset.authors]] +name = "Yuxuan Zhang" +email = "mtrxcop@gmail.com" + +[[tasks]] +name = "clawbench/v2-047-daily-life-personal-care-taskrabbit" +digest = "sha256:dade41a43cf36dbbc352fc55102153f0e431d093bfc00bfb32abccf54b5b3637" + +[[tasks]] +name = "clawbench/v2-086-job-search-hr-cv-autofill-greenhouse-meta" +digest = "sha256:62f0698d3285980d88b0207975890a3fcb2a428a52cdea1f2209d5f5c62e2414" + +[[tasks]] +name = "clawbench/v2-089-job-search-hr-cv-autofill-simplify-jobs" +digest = "sha256:26f45c750670248c2d334e0a039823c687f63d1c8457b52477a688b9390c0b1e" + +[[tasks]] +name = "clawbench/v2-091-job-search-hr-job-apply-indeed" +digest = "sha256:35ed852d1cb4d4d6f8dadd2cb2d8e07cc26bd0d1fd741663c52a6fb8f3420dff" + +[[tasks]] +name = "clawbench/v2-1010-rating-voting-review-myrecipes" +digest = "sha256:4e8b6715291d77dd5666ac1ff83b964fe0f76eca9da483a4ee2256061f4ee65d" + +[[tasks]] +name = "clawbench/v2-1033-education-learning-registration-khanacademy" +digest = "sha256:7ff0721e00d19d56f2d6bbb6c3dd1340de51d649ba9892ed421fbe65faf01ff7" + +[[tasks]] +name = "clawbench/v2-1035-education-learning-enrollment-edx" +digest = "sha256:7b3b5c2dee9fd3a021f943233918eca4d83f30254391eb752b1655d1597e2221" + +[[tasks]] +name = "clawbench/v2-1045-weworkremotely" +digest = "sha256:473b7e2fb80a40ff0b4dd9f4d74891c8cded2caf053cc942ffe230e2c392e819" + +[[tasks]] +name = "clawbench/v2-1065b-daily-life-home-services-handy" +digest = "sha256:59bf3f72f5e534d64a195e8aee77f153f4a7082acb5779829416e6c628500b67" + +[[tasks]] +name = "clawbench/v2-1088-nonprofit-charity-petition-change" +digest = "sha256:f20f2cfd751155b230c8a51af6112042aff367a3027d7bbcf313c8a69f4136b7" + +[[tasks]] +name = "clawbench/v2-1093-personal-management-travel-tripit" +digest = "sha256:525217bdba405e2d11b4bce2beaadb3ca79f96b74c0186bf1aff86a97db79237" + +[[tasks]] +name = "clawbench/v2-1095-health-mealplan-eatthismuch" +digest = "sha256:4bef7e79d38c5fe20f3fbce75334cab66c3405d026b69b7f3b83a7a4b1f2acb6" + +[[tasks]] +name = "clawbench/v2-1097-beauty-personal-care-shopping-theordinary" +digest = "sha256:66dc0d7ab8d8a3e6200f65ba71c60299d8ff48caa6116269ec3e28a956ac66f8" + +[[tasks]] +name = "clawbench/v2-1100-food-cooking-collection-myrecipes" +digest = "sha256:912a4e887502cae5115092ba674bc7139e363ff871fe1705ca3f9f0b395c02f2" + +[[tasks]] +name = "clawbench/v2-1101-hobbies-gaming-discussion-ravelry" +digest = "sha256:7a6db063100ae762a581bb4cfe892c4a91d90a905027bd3690ad6af3a9e448b0" + +[[tasks]] +name = "clawbench/v2-1102-fitness-social-post-strava" +digest = "sha256:fe7ba8f7c6bba4e69e51b463afcc9ab55ea31d3e7788b37d4ae9e505692c8390" + +[[tasks]] +name = "clawbench/v2-1103-fitness-club-strava" +digest = "sha256:081d8bec60f8f857faacb9153d7f666120be473bac8aa0ccecedb9283f1d6070" + +[[tasks]] +name = "clawbench/v2-1104-fitness-edit-activity-strava" +digest = "sha256:f3622d9b740596fe0c83efa0f2fd89bdf53a0a6bde79daa1e10e8d5dc57b0eff" + +[[tasks]] +name = "clawbench/v2-1107-hobbies-gaming-forum-boardgamegeek" +digest = "sha256:c143b105af94824d2dab09d752f407917d5006c373350a255d2da65f2a88a852" + +[[tasks]] +name = "clawbench/v2-1108-civic-petition-change" +digest = "sha256:21fb51471422195a75d622284299dbf6d71a74a210c8e8f691561228ba3c8ed7" + +[[tasks]] +name = "clawbench/v2-1111-civic-petition-change" +digest = "sha256:aa99c6afd31c2c2f9a630f5b96070b1c23ab033e75ea47948f0491507f24e0bf" + +[[tasks]] +name = "clawbench/v2-1112-nonprofit-petition-update-change" +digest = "sha256:70a9f99c0dd2b67081c31af9a46950e26438984a8010ce739d4f6188461e468d" + +[[tasks]] +name = "clawbench/v2-1113-health-mealplan-eatthismuch" +digest = "sha256:d732374209692d37ecb8028847885e9051a1d5e97cb190f4bd127eca2db0a16b" + +[[tasks]] +name = "clawbench/v2-1114-education-enrollment-edx" +digest = "sha256:692b85b03da597160ad96f053d490b1a536089dc1f1a5b9448aa9d70d4802628" + +[[tasks]] +name = "clawbench/v2-1115-education-enrollment-edx" +digest = "sha256:a776ac7923dbafdde12eb987b49b4e4b83f148b1add313f93a203b857b8793a0" + +[[tasks]] +name = "clawbench/v2-1116-education-enrollment-edx" +digest = "sha256:939484d731d4a15f24b5ff5639da1ffe7309d22df644d0e17540935a04e62be4" + +[[tasks]] +name = "clawbench/v2-1117" +digest = "sha256:9d206fabf25facd5b730de3f2eda95941344ace1f54af79f17ffb4a336ae9977" + +[[tasks]] +name = "clawbench/v2-1118" +digest = "sha256:cd83dae20bcf6f9060255e7c0a091515161dd5591eb77facdafe87910992c83b" + +[[tasks]] +name = "clawbench/v2-1120b-services-booking-handy" +digest = "sha256:d3baaa7008a5ad5ba67beb343f32b214db81204232a282591e37c01e22001a2e" + +[[tasks]] +name = "clawbench/v2-1121b-services-booking-handy" +digest = "sha256:d3f1ff9495f397dee293ed8d6a7b6e5dd4281ade9ea0099b6c00638e17f24dc5" + +[[tasks]] +name = "clawbench/v2-1122-education-course-khanacademy" +digest = "sha256:92db0dfb025fdf34dac4b3e7519d507135ad4ee91f19d5c06ea8f032b4f543df" + +[[tasks]] +name = "clawbench/v2-1123-education-course-khanacademy" +digest = "sha256:dba78d6b8dfab2b481c3a252e818a90ae224ed59ef2fcf8c51f353b9a5f5a086" + +[[tasks]] +name = "clawbench/v2-1124-education-course-khanacademy" +digest = "sha256:c0744abc2a0b917375f5a0bc29f65324b62736e8748645517717b854037f6c92" + +[[tasks]] +name = "clawbench/v2-1125-education-course-khanacademy" +digest = "sha256:f57411fc200c8ce3d2d76de21ad8805536f61a4acc7b1ed362c6d99665353c5b" + +[[tasks]] +name = "clawbench/v2-1126-food-cooking-collection-myrecipes" +digest = "sha256:f43996ef121d9a1a061052b5ff8fc297f0886b5038c876775dde42237ba59fdf" + +[[tasks]] +name = "clawbench/v2-1130-training-class-search-redcross" +digest = "sha256:738c849a4c316ae9f4923bb74b71723bf48ccf83bf5e86018cd0e2ca78ed7f62" + +[[tasks]] +name = "clawbench/v2-1131-training-add-to-cart-redcross" +digest = "sha256:dfcade40ea0cab4936086f548dc6fd13b60655fae661ae50f9a6630205e4785a" + +[[tasks]] +name = "clawbench/v2-1132-training-enroll-course-redcross" +digest = "sha256:646f408d6a3adb6c6dc8e334266503da1750667d88a9f4cf804ff2030708929f" + +[[tasks]] +name = "clawbench/v2-1133-newsletter-signup-redcross" +digest = "sha256:642ccbe2aa393f6b91893994189d1475bb90f644ee31fff4ea4b8bed724e9dce" + +[[tasks]] +name = "clawbench/v2-1134-chapter-finder-redcross" +digest = "sha256:5e0582cb36ffb0c801e368b104dd8d6b250fedd4b6028491d3d6ea27e6a635a1" + +[[tasks]] +name = "clawbench/v2-1135-events-tickets-ticketmaster" +digest = "sha256:13b7e40a6e4734553b5b17383aaa2077b31dc98a6e3de8a9bc5d20b7f5a67357" + +[[tasks]] +name = "clawbench/v2-1136-events-tickets-ticketmaster" +digest = "sha256:c9ae3e0764ba5f1ec554ca5467e2f860fb4d6827b6976388f978ee2d5a6e31ba" + +[[tasks]] +name = "clawbench/v2-1137-entertainment-watchlist-trakt" +digest = "sha256:404860a11bd1a96bb0d5609a770a035ee0751f7ea1fa1c15f703d527d34d6f33" + +[[tasks]] +name = "clawbench/v2-1138-entertainment-watchlist-trakt" +digest = "sha256:61cd10f87817c265448a280cd6da84e762798d5325e4591d29d3b583cb9ccc3e" + +[[tasks]] +name = "clawbench/v2-1139-travel-itinerary-tripit" +digest = "sha256:5d736f263a029319b78d2bec886da4dfe4bd3093294edb4e6b53973f3d5f158b" + +[[tasks]] +name = "clawbench/v2-1140-travel-itinerary-tripit" +digest = "sha256:ca704a8635c5e3c20f093986a1bef03b3ddfeff06e94b84fef6204732372d000" + +[[tasks]] +name = "clawbench/v2-1141-travel-itinerary-tripit" +digest = "sha256:009a7c149d1bd88deb7ce0557bcd9a6fcdae9280b5cb71e6e972f71cc546caed" + +[[tasks]] +name = "clawbench/v2-1144-weworkremotely" +digest = "sha256:2531e6298110975468af897adff6069bcc681be38bd2accb5a81749de8918f28" + +[[tasks]] +name = "clawbench/v2-1145-weworkremotely" +digest = "sha256:848fbfcff7f3756b7e2269c2bc5ca370b4e4baaecba13b4b7d49c2df1934f91e" + +[[tasks]] +name = "clawbench/v2-1146-weworkremotely" +digest = "sha256:0031d805d96cb43ea16cc989bfe77ce029e99857d6efd440e507e0a7de3498b1" + +[[tasks]] +name = "clawbench/v2-1201-beauty-cart-theordinary" +digest = "sha256:9a349d9c6ab22edcd3ca9620e3937945acb67954b5bf2093f423f89e58c6af71" + +[[tasks]] +name = "clawbench/v2-1202-beauty-cart-quantity-theordinary" +digest = "sha256:a25c978a6821eddc78ed4d84bfd4a8f319e4b4c0bf0afe934f28a8390e974604" + +[[tasks]] +name = "clawbench/v2-179-dev-tech-github-ops-github" +digest = "sha256:eb43db272ad8aba595164b63a4233665cc0a710bc30197725d6302d28b1e5901" + +[[tasks]] +name = "clawbench/v2-180-dev-tech-github-ops-github" +digest = "sha256:aa6d6e1c40c54cc2872b35cb0e8749288ba6f72f826b3a4737f2412cb1d1d9c1" + +[[tasks]] +name = "clawbench/v2-215-academia-research-paper-tables-overleaf" +digest = "sha256:e8ade6600ad3ad878265d83a660f3e98096a501bfec80191e6cca3206f28c3be" + +[[tasks]] +name = "clawbench/v2-234-daily-life-events-ticketmaster" +digest = "sha256:68d58b663583b96e526ae9c90024828ec5bbae805235459b6a6577e776d1a800" + +[[tasks]] +name = "clawbench/v2-242-academia-research-research-tools-overleaf" +digest = "sha256:f59ed59623ca21db9ceb1ab696cb53a8cd7895a1036c106f40195499c0f6e994" + +[[tasks]] +name = "clawbench/v2-247-academia-research-research-tools-semantic-scholar" +digest = "sha256:6d3e81483ba85f914b1d25be78d945742806b42848914cf5a621a37754ee93ae" + +[[tasks]] +name = "clawbench/v2-265-education-learning-general-coursera" +digest = "sha256:e6e98e7b923915c88ed9c5cb0f04c114dee4e292e49b3f147994b3368eee3f44" + +[[tasks]] +name = "clawbench/v2-266-education-learning-general-leetcode" +digest = "sha256:7e50de3efd7bc9f606bdee93284885de776d31ed0d58fd4bb16b25d05cfe4b08" + +[[tasks]] +name = "clawbench/v2-273-education-learning-general-edx" +digest = "sha256:70a89937bd6bbaca340cb2a24235d1e4dddc077814859b8a4434f8cc8030b1f7" + +[[tasks]] +name = "clawbench/v2-274-office-secretary-tasks-calendar-when2meet" +digest = "sha256:6bc5474d28d0b8d9a462e5c98806e13ecfda9ad8c024b041ab2922280617d862" + +[[tasks]] +name = "clawbench/v2-284-office-secretary-tasks-calendar-when2meet" +digest = "sha256:5a81bfee58357e7c20903776acc3a1c31190cb40be44ca1d5d92c9ecd3bd4ea6" + +[[tasks]] +name = "clawbench/v2-369-entertainment-hobbies-general-goodreads" +digest = "sha256:505d25db1483eeeb7488127898dddcc61de06bb9ffcbd68edbafa39283c8e7aa" + +[[tasks]] +name = "clawbench/v2-372-entertainment-hobbies-general-eventbrite" +digest = "sha256:5b9592e837ad5152ede1653ac8c70876845283a7403fa6d8c83c8f32e6ad590d" + +[[tasks]] +name = "clawbench/v2-413-personal-management-personal-tools-todoist" +digest = "sha256:5df7d5609978a5db9080924868b72e32cb3282d26bc742213a0e7667c9022b9f" + +[[tasks]] +name = "clawbench/v2-468-rating-voting-general-glassdoor" +digest = "sha256:3b0c0143bcf4a59e99ad2492a778ef7c5c27f9692ef95abe947aae94d29abe94" + +[[tasks]] +name = "clawbench/v2-469-rating-voting-general-tripadvisor" +digest = "sha256:ead827ed55846d0bf95ba6365b83337093c1938716d5e7d84393f4a97a2b67a6" + +[[tasks]] +name = "clawbench/v2-470-rating-voting-general-trustpilot" +digest = "sha256:0e98e10ef4482fb77137003a1624d53a2f7297adc67d21de523b54d0a8f6de24" + +[[tasks]] +name = "clawbench/v2-474-rating-voting-general-capterra" +digest = "sha256:d75a5bb0c6ce7a7c74455e5b79e640cd23f264baac002443c3d932a39f4e7894" + +[[tasks]] +name = "clawbench/v2-475-rating-voting-general-g2" +digest = "sha256:e6770e7de339488a1323ea066e6f8df45e7c17ee9d75686bffbd25489faa4657" + +[[tasks]] +name = "clawbench/v2-483-creation-init-general-airtable" +digest = "sha256:d7b00a9d7f31c61e69659deed124b90c23f6fb99167fa526c4e1004fb5bd4733" + +[[tasks]] +name = "clawbench/v2-485-creation-init-general-webflow" +digest = "sha256:b9c1b03ed0e1978bbc39772246bfd2f3b2d8660d0916fddd1d840c368f122d63" + +[[tasks]] +name = "clawbench/v2-486-creation-init-general-mailchimp" +digest = "sha256:5a17170c63138eb5f52fd693d731deb75ec7c7249a31c6cadb60236d29663755" + +[[tasks]] +name = "clawbench/v2-487-creation-init-general-typeform" +digest = "sha256:7014b4251848df51688411efe1965b7caef8da539d7f4a68173b3ea4c23f5221" + +[[tasks]] +name = "clawbench/v2-488-creation-init-general-substack" +digest = "sha256:60de88af02b8a1631ebea74c9524fe706b97bb3ffd7539076fa28cdae63ab62b" + +[[tasks]] +name = "clawbench/v2-500-office-secretary-tasks-signup-doodle" +digest = "sha256:b877d0be7bcbf86333ac8ecc22c4577d1e16ae51b626020a6a95a4f04ea3916f" + +[[tasks]] +name = "clawbench/v2-501-office-secretary-tasks-signup-doodle" +digest = "sha256:908f2f2bfbd903f45da9ee7b595a26c2835735380693861871f038886f63292d" + +[[tasks]] +name = "clawbench/v2-502-office-secretary-tasks-calendar-doodle" +digest = "sha256:0f8c48001316ca297353cc4abcc5b659a94db88afefcba92457dbf233d82355f" + +[[tasks]] +name = "clawbench/v2-503-office-secretary-tasks-calendar-doodle" +digest = "sha256:ebad6e7119db4f608f67f8a685fc4d4aeaae28b07c87806fe4f0e297cceea713" + +[[tasks]] +name = "clawbench/v2-520-shopping-retail-target" +digest = "sha256:7eab94e56e5dd7f157f27cd6323753528c7fd210f523b740ce92907302c9aa92" + +[[tasks]] +name = "clawbench/v2-521-shopping-retail-target" +digest = "sha256:020653727c1fcddef316d91f1a18be3cb9056240fe0f77dc89733fb5a1946355" + +[[tasks]] +name = "clawbench/v2-522-shopping-retail-target" +digest = "sha256:aa67f9c13578c3559565b9795d4b840ab2c2496a4f5f893f2229aa90ae02105d" + +[[tasks]] +name = "clawbench/v2-523-shopping-retail-target" +digest = "sha256:a1804c352edfc7ae34c8b7c0afcc812c5b826c895a31801c7756f6596b816be6" + +[[tasks]] +name = "clawbench/v2-530-shopping-marketplace-etsy" +digest = "sha256:b8c7e2097327ec55447312c6330a6414f00ebdaabe9df8a60214c18dbdfb708f" + +[[tasks]] +name = "clawbench/v2-531-shopping-marketplace-etsy" +digest = "sha256:5b721d4ec1cc13f28ef2f0ec0909601a114e585622de1a9ceba7548879c03e8d" + +[[tasks]] +name = "clawbench/v2-533-daily-life-utilities-inmyarea" +digest = "sha256:efd87257842e3038adf63cf83ce451e8f22460877ba56dfa527aebeaa766e3cb" + +[[tasks]] +name = "clawbench/v2-535-daily-life-shopping-etsy" +digest = "sha256:e642752221d537bc52b3b6629c5f9ce4ce826a8a5fd5b7fae560b78c79075256" + +[[tasks]] +name = "clawbench/v2-536-daily-life-shopping-etsy" +digest = "sha256:c59486d6072565494c4caceb8ae3d70eba4667d8637ef87cdc2f21aff3392b5f" + +[[tasks]] +name = "clawbench/v2-560" +digest = "sha256:34cf7c447f1f9997719296949326bbae264df9e007947263ef3937997761fe2d" + +[[tasks]] +name = "clawbench/v2-564" +digest = "sha256:28a74b9f788b5e7c02d8d37dcddd11bf0d7f4e9b1fe769a3ba0e14f42741752d" + +[[tasks]] +name = "clawbench/v2-571-job-search-hr-recruitment-mgmt-workable" +digest = "sha256:0960d3ee80f7e50912395c077f045fa0033ec3035444b63a837c3c027018fd84" + +[[tasks]] +name = "clawbench/v2-583-office-secretary-tasks-support-tickets-freshdesk" +digest = "sha256:9928cd6967cbf93baf0d97023f17d83538e5b3d332cec9296fa9e7df8af7108b" + +[[tasks]] +name = "clawbench/v2-596-daily-life-fitness-strava" +digest = "sha256:860cd3f9fdc68f33aa7192307b085a967207687ae215665e1671a3833d1e1481" + +[[tasks]] +name = "clawbench/v2-597-daily-life-fitness-strava" +digest = "sha256:6e82073b6cdfc131c2d51ec804dd84cb1e5b07c29f783eed6f1d6037626e5936" + +[[tasks]] +name = "clawbench/v2-598-academia-research-legal-docs-formswift" +digest = "sha256:cc4b190988d12a73d1c5b6c6ba7dda809492fd491646cdc17e8aae08e1ab09c3" + +[[tasks]] +name = "clawbench/v2-600-hobbies-crafts-community-ravelry" +digest = "sha256:62919bcec1c8b6393d0322af20b9868891088e9faf3ee5e4bcffe8856f339d2e" + +[[tasks]] +name = "clawbench/v2-601-hobbies-gaming-reviews-ravelry" +digest = "sha256:7c64bf38e6c8ded1776c942290984439a01ceefe756e5a583a3ee0c594a30074" + +[[tasks]] +name = "clawbench/v2-602-hobbies-crafts-community-ravelry" +digest = "sha256:269e7cbea6a95193de080e6eb39cb3b4a6864facbe653a269bed06eb9dbc9c7f" + +[[tasks]] +name = "clawbench/v2-603-hobbies-crafts-community-ravelry" +digest = "sha256:9b2837912e1cd166acd8936e8d27e568302fe46234e9e9f35c3a2c6256d32401" + +[[tasks]] +name = "clawbench/v2-607-hobbies-gaming-reviews-boardgamegeek" +digest = "sha256:65e7814fbf89537c93958fe04748ad6b9027ae08be565dc7827098faa9294f6b" + +[[tasks]] +name = "clawbench/v2-608-education-learning-kids-courses-outschool" +digest = "sha256:7c4cf4d97a71c22c2d0a4dc954a92959bffe5d22fe31f3014fba99506284630d" + +[[tasks]] +name = "clawbench/v2-609-education-learning-meditation-spirit-rock-meditation-center" +digest = "sha256:2c510eb63ad7f8435fffaa30f25b7509470af305021eb6b48337a01c4b947763" + +[[tasks]] +name = "clawbench/v2-610-hobbies-gaming-reviews-boardgamegeek" +digest = "sha256:d42f8921b56a21aa3f6cb21f22adbd66ebc2e28ef2f597875bb93294c6ec0d49" + +[[tasks]] +name = "clawbench/v2-630-daily-life-productivity-habitica" +digest = "sha256:37f2029c583d6e8723f5691e2597825bbb14ec3e7a5e041ad8e0cdf7f123895d" + +[[tasks]] +name = "clawbench/v2-631-daily-life-productivity-habitica" +digest = "sha256:a48ba7e8a2cda090b5f1dcb4c068409082d9ddca2700ccd12a1e78969e2d1f1b" + +[[tasks]] +name = "clawbench/v2-633-daily-life-productivity-habitica" +digest = "sha256:19b92247ea15410e3e2494a422e045bba2e644c62e913d035bf89120db5499ed" + +[[tasks]] +name = "clawbench/v2-634-daily-life-productivity-habitica" +digest = "sha256:4985159d7af309b29bd1d484da731de9e9bd1006abe9849e3bc7b10ec64d59ae" + +[[tasks]] +name = "clawbench/v2-635-entertainment-tracking-trakt" +digest = "sha256:0e33a17ea2e83eea631f8a34345477a0f023ead6258dc1a96644000051fc2036" + +[[tasks]] +name = "clawbench/v2-638-entertainment-tracking-trakt" +digest = "sha256:e69272f6f9fde888d7e944cfac76381cb5b8db7772a91521caa46751a6a5a2ec" + +[[tasks]] +name = "clawbench/v2-705-rating-voting-wine-review-vivino" +digest = "sha256:8e592cc472abdfa71100b76df2f0c7ab91c373c8e544584acf59d8a9e861200b" + +[[tasks]] +name = "clawbench/v2-706-rating-voting-beer-review-beeradvocate" +digest = "sha256:9895b2b2ca5bfc60e2d192e5e6fc7cd7be2c4c916de7992f51cdec11f9c0b77c" + +[[tasks]] +name = "clawbench/v2-707-rating-voting-social-wine-untappd" +digest = "sha256:408a889f0f43ba2ff65e1f2e24ac8130fbe27ba7b8f1eb885ddd5860730d0e60" + +[[tasks]] +name = "clawbench/v2-708-rating-voting-professor-review-ratemyprofessors" +digest = "sha256:d3893062d811dca585dc443d2a70666bad79d81375cf461b6a9e469823d1d9ef" + +[[tasks]] +name = "clawbench/v2-711-creation-init-color-design-coolors" +digest = "sha256:e6893db8ffcc39d23e07d996f362096c06240239ba319efffb25b67dcc6284ee" + +[[tasks]] +name = "clawbench/v2-735-home-services-maintenance-house-cleaning-bark" +digest = "sha256:cf98f85fc360e44be2894793475b7ed2806eeb9a3a0a0b4e99dd26d908ea9dac" + +[[tasks]] +name = "clawbench/v2-737-home-services-maintenance-kitchen-remodel-lowes" +digest = "sha256:079115b0bea57416ab46166b313d55b47c3644b71733eaba3fc2c64ca1832774" + +[[tasks]] +name = "clawbench/v2-763-automotive-vehicle-services-car-lease-autoslash" +digest = "sha256:e6cd6d9db87bd0c1925b9066423d42365f1e4098f3f4325cf87c8b30ee4ad49e" + +[[tasks]] +name = "clawbench/v2-774-nonprofit-charity-nonprofit-job-apply-charity-village" +digest = "sha256:431847fa294cc00751375d34c39d82e7b06e94ba8efa5706cee533480dcbb239" + +[[tasks]] +name = "clawbench/v2-776-nonprofit-charity-volunteer-signup-idealist" +digest = "sha256:14057c31e1b926c011e6993097cd0ea390862178001b8639fa38781d39d04ddf" + +[[tasks]] +name = "clawbench/v2-794-beauty-personal-care-salon-booking-styleseat" +digest = "sha256:a028d46800a45ac77a29da8bdd322aa2ba62bd2b58f7d017555a597de48329f8" + +[[tasks]] +name = "clawbench/v2-815-pet-animal-care-favorite-pet" +digest = "sha256:cf19b287efaf34c2acb94b00f5143329512c239de4669d760701d6c340cba532" + +[[tasks]] +name = "clawbench/v2-816-pet-animal-care-remove-favorite" +digest = "sha256:ee532c54bc3db449f7dd5e026514a4b557b38ed0123ed01b97ba914921724306" + +[[tasks]] +name = "clawbench/v2-817-pet-animal-care-create-adopter-profile" +digest = "sha256:d38d0e2f3e5d7a210ac424c8f5abed40125a087e31732f9ece76bff4124394b3" + +[[tasks]] +name = "clawbench/v2-901-entertainment-imdb-add-to-watchlist" +digest = "sha256:f13d2830714c6a92f42d33a9e6f89e7d3c2bcca7466941f7ae169b449cbe6ac9" + +[[tasks]] +name = "clawbench/v2-902-entertainment-imdb-rate-title" +digest = "sha256:8ba2bd45613a2602b9d0382bead28fd2c40871a27775234aca8c6ffdbe34b5f7" + +[[tasks]] +name = "clawbench/v2-903-entertainment-imdb-remove-rating" +digest = "sha256:3cb68b31204b5c8702fe214b4629f81289cc9615f79461577ac831b585849857" + +[[tasks]] +name = "clawbench/v2-904-entertainment-imdb-create-list" +digest = "sha256:59d0e325617fb518aaf47ed92c42d00d11e77061de858c1479336c26d798e4ad" + +[[tasks]] +name = "clawbench/v2-905-entertainment-imdb-add-to-list" +digest = "sha256:bf0f0245774cf4aae78ca065f7e2cb02454978cd493f0d9510766c11eb77aebe" diff --git a/harbor/datasets/clawbench-v2/v2-047-daily-life-personal-care-taskrabbit/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-047-daily-life-personal-care-taskrabbit/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-047-daily-life-personal-care-taskrabbit/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-047-daily-life-personal-care-taskrabbit/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-047-daily-life-personal-care-taskrabbit/steps/run/instruction.md new file mode 100644 index 00000000..55a03bd5 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-047-daily-life-personal-care-taskrabbit/steps/run/instruction.md @@ -0,0 +1,30 @@ +Find a moving helper on TaskRabbit, next Saturday 9am-1pm, 1 bedroom apartment + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +Additional files are also available under /my-info/ for this task: +- address_info.json: Address information (references alex_green_personal_info.json) + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-047-daily-life-personal-care-taskrabbit/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-047-daily-life-personal-care-taskrabbit/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-047-daily-life-personal-care-taskrabbit/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-047-daily-life-personal-care-taskrabbit/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-047-daily-life-personal-care-taskrabbit/steps/run/tests/task.json new file mode 100644 index 00000000..81740a8d --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-047-daily-life-personal-care-taskrabbit/steps/run/tests/task.json @@ -0,0 +1,30 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 47, + "metaclass": "daily-life", + "class": "personal-care", + "description": "Find a moving helper on TaskRabbit, next Saturday 9am-1pm, 1 bedroom apartment", + "sites_involved": [ + "taskrabbit.com" + ], + "platform": "taskrabbit", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Find a moving helper on TaskRabbit, next Saturday 9am-1pm, 1 bedroom apartment", + "eval_schema": { + "url_pattern": "taskrabbit\\.(com|ca)/(api/v\\d+/jobs|book/\\d+/confirm)", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [ + { + "path": "extra_info/address_info.json", + "description": "Address information (references alex_green_personal_info.json)" + } + ] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-047-daily-life-personal-care-taskrabbit/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-047-daily-life-personal-care-taskrabbit/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-047-daily-life-personal-care-taskrabbit/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-047-daily-life-personal-care-taskrabbit/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-047-daily-life-personal-care-taskrabbit/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..f6d3d8b3 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-047-daily-life-personal-care-taskrabbit/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "taskrabbit\\.(com|ca)/(api/v\\d+/jobs|book/\\d+/confirm)", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-047-daily-life-personal-care-taskrabbit/steps/run/workdir/extra_info/address_info.json b/harbor/datasets/clawbench-v2/v2-047-daily-life-personal-care-taskrabbit/steps/run/workdir/extra_info/address_info.json new file mode 100644 index 00000000..aca6eb63 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-047-daily-life-personal-care-taskrabbit/steps/run/workdir/extra_info/address_info.json @@ -0,0 +1,7 @@ +{ + "note": "Use home address from alex_green_personal_info.json", + "move out address": "Unit 1208, 664 Spadina Ave, Toronto, ON M5S 2H7, Canada", + "move in address": "450 Front St W, Toronto, ON M5V 0V7", + "Task size": "Large", + "Need Vehicle": "Yes, a Car" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-047-daily-life-personal-care-taskrabbit/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-047-daily-life-personal-care-taskrabbit/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-047-daily-life-personal-care-taskrabbit/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-047-daily-life-personal-care-taskrabbit/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-047-daily-life-personal-care-taskrabbit/steps/run/workdir/task.json new file mode 100644 index 00000000..81740a8d --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-047-daily-life-personal-care-taskrabbit/steps/run/workdir/task.json @@ -0,0 +1,30 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 47, + "metaclass": "daily-life", + "class": "personal-care", + "description": "Find a moving helper on TaskRabbit, next Saturday 9am-1pm, 1 bedroom apartment", + "sites_involved": [ + "taskrabbit.com" + ], + "platform": "taskrabbit", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Find a moving helper on TaskRabbit, next Saturday 9am-1pm, 1 bedroom apartment", + "eval_schema": { + "url_pattern": "taskrabbit\\.(com|ca)/(api/v\\d+/jobs|book/\\d+/confirm)", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [ + { + "path": "extra_info/address_info.json", + "description": "Address information (references alex_green_personal_info.json)" + } + ] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-047-daily-life-personal-care-taskrabbit/task.toml b/harbor/datasets/clawbench-v2/v2-047-daily-life-personal-care-taskrabbit/task.toml new file mode 100644 index 00000000..6d5ce3a3 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-047-daily-life-personal-care-taskrabbit/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-047-daily-life-personal-care-taskrabbit" +description = "Find a moving helper on TaskRabbit, next Saturday 9am-1pm, 1 bedroom apartment" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-047-daily-life-personal-care-taskrabbit" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-086-job-search-hr-cv-autofill-greenhouse-meta/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-086-job-search-hr-cv-autofill-greenhouse-meta/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-086-job-search-hr-cv-autofill-greenhouse-meta/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-086-job-search-hr-cv-autofill-greenhouse-meta/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-086-job-search-hr-cv-autofill-greenhouse-meta/steps/run/instruction.md new file mode 100644 index 00000000..0dcda7de --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-086-job-search-hr-cv-autofill-greenhouse-meta/steps/run/instruction.md @@ -0,0 +1,30 @@ +Extract information from resume.pdf and fill out the CodePath Senior Software Engineer application on Greenhouse + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +Additional files are also available under /my-info/ for this task: +- job_links.json: Job posting URL(s) to apply to + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-086-job-search-hr-cv-autofill-greenhouse-meta/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-086-job-search-hr-cv-autofill-greenhouse-meta/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-086-job-search-hr-cv-autofill-greenhouse-meta/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-086-job-search-hr-cv-autofill-greenhouse-meta/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-086-job-search-hr-cv-autofill-greenhouse-meta/steps/run/tests/task.json new file mode 100644 index 00000000..e6a8fcef --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-086-job-search-hr-cv-autofill-greenhouse-meta/steps/run/tests/task.json @@ -0,0 +1,30 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 86, + "metaclass": "job-search-hr", + "class": "cv-autofill", + "description": "Extract information from resume.pdf and fill out the CodePath Senior Software Engineer application on Greenhouse", + "sites_involved": [ + "greenhouse.com" + ], + "platform": "greenhouse-codepath", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Extract information from resume.pdf and fill out the CodePath Senior Software Engineer application on Greenhouse", + "eval_schema": { + "url_pattern": "boards-api\\.greenhouse\\.io/v1/boards/.+/jobs/\\d+|job-boards\\.greenhouse\\.io/.+/jobs/\\d+", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [ + { + "path": "extra_info/job_links.json", + "description": "Job posting URL(s) to apply to" + } + ] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-086-job-search-hr-cv-autofill-greenhouse-meta/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-086-job-search-hr-cv-autofill-greenhouse-meta/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-086-job-search-hr-cv-autofill-greenhouse-meta/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-086-job-search-hr-cv-autofill-greenhouse-meta/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-086-job-search-hr-cv-autofill-greenhouse-meta/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..2f18323b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-086-job-search-hr-cv-autofill-greenhouse-meta/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "boards-api\\.greenhouse\\.io/v1/boards/.+/jobs/\\d+|job-boards\\.greenhouse\\.io/.+/jobs/\\d+", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-086-job-search-hr-cv-autofill-greenhouse-meta/steps/run/workdir/extra_info/job_links.json b/harbor/datasets/clawbench-v2/v2-086-job-search-hr-cv-autofill-greenhouse-meta/steps/run/workdir/extra_info/job_links.json new file mode 100644 index 00000000..1baf5dbf --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-086-job-search-hr-cv-autofill-greenhouse-meta/steps/run/workdir/extra_info/job_links.json @@ -0,0 +1,5 @@ +{ + "job_url": "https://job-boards.greenhouse.io/codepath/jobs/4526154007", + "job_title": "Senior Software Engineer", + "company": "CodePath" +} diff --git a/harbor/datasets/clawbench-v2/v2-086-job-search-hr-cv-autofill-greenhouse-meta/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-086-job-search-hr-cv-autofill-greenhouse-meta/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-086-job-search-hr-cv-autofill-greenhouse-meta/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-086-job-search-hr-cv-autofill-greenhouse-meta/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-086-job-search-hr-cv-autofill-greenhouse-meta/steps/run/workdir/task.json new file mode 100644 index 00000000..e6a8fcef --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-086-job-search-hr-cv-autofill-greenhouse-meta/steps/run/workdir/task.json @@ -0,0 +1,30 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 86, + "metaclass": "job-search-hr", + "class": "cv-autofill", + "description": "Extract information from resume.pdf and fill out the CodePath Senior Software Engineer application on Greenhouse", + "sites_involved": [ + "greenhouse.com" + ], + "platform": "greenhouse-codepath", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Extract information from resume.pdf and fill out the CodePath Senior Software Engineer application on Greenhouse", + "eval_schema": { + "url_pattern": "boards-api\\.greenhouse\\.io/v1/boards/.+/jobs/\\d+|job-boards\\.greenhouse\\.io/.+/jobs/\\d+", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [ + { + "path": "extra_info/job_links.json", + "description": "Job posting URL(s) to apply to" + } + ] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-086-job-search-hr-cv-autofill-greenhouse-meta/task.toml b/harbor/datasets/clawbench-v2/v2-086-job-search-hr-cv-autofill-greenhouse-meta/task.toml new file mode 100644 index 00000000..01f625d7 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-086-job-search-hr-cv-autofill-greenhouse-meta/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-086-job-search-hr-cv-autofill-greenhouse-meta" +description = "Extract information from resume.pdf and fill out the CodePath Senior Software Engineer application on Greenhouse" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-086-job-search-hr-cv-autofill-greenhouse-meta" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-089-job-search-hr-cv-autofill-simplify-jobs/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-089-job-search-hr-cv-autofill-simplify-jobs/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-089-job-search-hr-cv-autofill-simplify-jobs/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-089-job-search-hr-cv-autofill-simplify-jobs/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-089-job-search-hr-cv-autofill-simplify-jobs/steps/run/instruction.md new file mode 100644 index 00000000..f9e419fa --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-089-job-search-hr-cv-autofill-simplify-jobs/steps/run/instruction.md @@ -0,0 +1,30 @@ +Use Simplify Jobs to one-click auto-fill the Amazon Applied Scientist application + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +Additional files are also available under /my-info/ for this task: +- job_links.json: Job posting URL(s) to apply to + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-089-job-search-hr-cv-autofill-simplify-jobs/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-089-job-search-hr-cv-autofill-simplify-jobs/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-089-job-search-hr-cv-autofill-simplify-jobs/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-089-job-search-hr-cv-autofill-simplify-jobs/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-089-job-search-hr-cv-autofill-simplify-jobs/steps/run/tests/task.json new file mode 100644 index 00000000..64e0e47c --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-089-job-search-hr-cv-autofill-simplify-jobs/steps/run/tests/task.json @@ -0,0 +1,30 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 89, + "metaclass": "job-search-hr", + "class": "cv-autofill", + "description": "Use Simplify Jobs to one-click auto-fill the Amazon Applied Scientist application", + "sites_involved": [ + "simplify.jobs" + ], + "platform": "simplify-jobs", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Use Simplify Jobs to one-click auto-fill the Amazon Applied Scientist application", + "eval_schema": { + "url_pattern": "api\\.simplify\\.jobs/v2/candidate/me/application", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [ + { + "path": "extra_info/job_links.json", + "description": "Job posting URL(s) to apply to" + } + ] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-089-job-search-hr-cv-autofill-simplify-jobs/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-089-job-search-hr-cv-autofill-simplify-jobs/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-089-job-search-hr-cv-autofill-simplify-jobs/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-089-job-search-hr-cv-autofill-simplify-jobs/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-089-job-search-hr-cv-autofill-simplify-jobs/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..8cb3cab4 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-089-job-search-hr-cv-autofill-simplify-jobs/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "api\\.simplify\\.jobs/v2/candidate/me/application", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-089-job-search-hr-cv-autofill-simplify-jobs/steps/run/workdir/extra_info/job_links.json b/harbor/datasets/clawbench-v2/v2-089-job-search-hr-cv-autofill-simplify-jobs/steps/run/workdir/extra_info/job_links.json new file mode 100644 index 00000000..9169efd1 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-089-job-search-hr-cv-autofill-simplify-jobs/steps/run/workdir/extra_info/job_links.json @@ -0,0 +1,5 @@ +{ + "job_url": "https://simplify.jobs/p/d4eaf15e-63f7-4024-931c-f8cb52184dc4/Applied-Scientist", + "job_title": "Applied Scientist", + "company": "Amazon" +} diff --git a/harbor/datasets/clawbench-v2/v2-089-job-search-hr-cv-autofill-simplify-jobs/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-089-job-search-hr-cv-autofill-simplify-jobs/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-089-job-search-hr-cv-autofill-simplify-jobs/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-089-job-search-hr-cv-autofill-simplify-jobs/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-089-job-search-hr-cv-autofill-simplify-jobs/steps/run/workdir/task.json new file mode 100644 index 00000000..64e0e47c --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-089-job-search-hr-cv-autofill-simplify-jobs/steps/run/workdir/task.json @@ -0,0 +1,30 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 89, + "metaclass": "job-search-hr", + "class": "cv-autofill", + "description": "Use Simplify Jobs to one-click auto-fill the Amazon Applied Scientist application", + "sites_involved": [ + "simplify.jobs" + ], + "platform": "simplify-jobs", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Use Simplify Jobs to one-click auto-fill the Amazon Applied Scientist application", + "eval_schema": { + "url_pattern": "api\\.simplify\\.jobs/v2/candidate/me/application", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [ + { + "path": "extra_info/job_links.json", + "description": "Job posting URL(s) to apply to" + } + ] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-089-job-search-hr-cv-autofill-simplify-jobs/task.toml b/harbor/datasets/clawbench-v2/v2-089-job-search-hr-cv-autofill-simplify-jobs/task.toml new file mode 100644 index 00000000..1f9bc43e --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-089-job-search-hr-cv-autofill-simplify-jobs/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-089-job-search-hr-cv-autofill-simplify-jobs" +description = "Use Simplify Jobs to one-click auto-fill the Amazon Applied Scientist application" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-089-job-search-hr-cv-autofill-simplify-jobs" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-091-job-search-hr-job-apply-indeed/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-091-job-search-hr-job-apply-indeed/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-091-job-search-hr-job-apply-indeed/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-091-job-search-hr-job-apply-indeed/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-091-job-search-hr-job-apply-indeed/steps/run/instruction.md new file mode 100644 index 00000000..6f55c628 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-091-job-search-hr-job-apply-indeed/steps/run/instruction.md @@ -0,0 +1,27 @@ +Search "Senior Software Engineer" (Toronto) on Indeed, apply to the top-ranked listing + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-091-job-search-hr-job-apply-indeed/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-091-job-search-hr-job-apply-indeed/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-091-job-search-hr-job-apply-indeed/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-091-job-search-hr-job-apply-indeed/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-091-job-search-hr-job-apply-indeed/steps/run/tests/task.json new file mode 100644 index 00000000..1b93b0f8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-091-job-search-hr-job-apply-indeed/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 91, + "metaclass": "job-search-hr", + "class": "job-apply", + "description": "Search \"Senior Software Engineer\" (Toronto) on Indeed, apply to the top-ranked listing", + "sites_involved": [ + "indeed.com" + ], + "platform": "indeed", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Search \"Senior Software Engineer\" (Toronto) on Indeed, apply to the top-ranked listing", + "eval_schema": { + "url_pattern": "smartapply\\.indeed\\.com/beta/indeedapply/resumeapply|apply\\.indeed\\.com/indeedapply/postresumeapply|smartapply\\.indeed\\.com/beta/indeedapply/submit", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-091-job-search-hr-job-apply-indeed/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-091-job-search-hr-job-apply-indeed/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-091-job-search-hr-job-apply-indeed/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-091-job-search-hr-job-apply-indeed/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-091-job-search-hr-job-apply-indeed/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..eb4016fe --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-091-job-search-hr-job-apply-indeed/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "smartapply\\.indeed\\.com/beta/indeedapply/resumeapply|apply\\.indeed\\.com/indeedapply/postresumeapply|smartapply\\.indeed\\.com/beta/indeedapply/submit", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-091-job-search-hr-job-apply-indeed/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-091-job-search-hr-job-apply-indeed/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-091-job-search-hr-job-apply-indeed/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-091-job-search-hr-job-apply-indeed/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-091-job-search-hr-job-apply-indeed/steps/run/workdir/task.json new file mode 100644 index 00000000..1b93b0f8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-091-job-search-hr-job-apply-indeed/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 91, + "metaclass": "job-search-hr", + "class": "job-apply", + "description": "Search \"Senior Software Engineer\" (Toronto) on Indeed, apply to the top-ranked listing", + "sites_involved": [ + "indeed.com" + ], + "platform": "indeed", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Search \"Senior Software Engineer\" (Toronto) on Indeed, apply to the top-ranked listing", + "eval_schema": { + "url_pattern": "smartapply\\.indeed\\.com/beta/indeedapply/resumeapply|apply\\.indeed\\.com/indeedapply/postresumeapply|smartapply\\.indeed\\.com/beta/indeedapply/submit", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-091-job-search-hr-job-apply-indeed/task.toml b/harbor/datasets/clawbench-v2/v2-091-job-search-hr-job-apply-indeed/task.toml new file mode 100644 index 00000000..4257c0be --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-091-job-search-hr-job-apply-indeed/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-091-job-search-hr-job-apply-indeed" +description = "Search \"Senior Software Engineer\" (Toronto) on Indeed, apply to the top-ranked listing" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-091-job-search-hr-job-apply-indeed" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-1010-rating-voting-review-myrecipes/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-1010-rating-voting-review-myrecipes/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1010-rating-voting-review-myrecipes/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-1010-rating-voting-review-myrecipes/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-1010-rating-voting-review-myrecipes/steps/run/instruction.md new file mode 100644 index 00000000..95978511 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1010-rating-voting-review-myrecipes/steps/run/instruction.md @@ -0,0 +1,27 @@ +Rate a Vegan Chocolate Chip Cookies Recipe on MyRecipes with 4 stars and add a helpful tip + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-1010-rating-voting-review-myrecipes/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-1010-rating-voting-review-myrecipes/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1010-rating-voting-review-myrecipes/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-1010-rating-voting-review-myrecipes/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-1010-rating-voting-review-myrecipes/steps/run/tests/task.json new file mode 100644 index 00000000..65d9935b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1010-rating-voting-review-myrecipes/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1010, + "metaclass": "rating-voting", + "class": "rating", + "description": "Rate a Vegan Chocolate Chip Cookies Recipe on MyRecipes with 4 stars and add a helpful tip", + "sites_involved": [ + "myrecipes.com" + ], + "platform": "myrecipes", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Rate a Vegan Chocolate Chip Cookies Recipe on MyRecipes with 4 stars and add a helpful tip", + "eval_schema": { + "url_pattern": "myrecipes\\.com/api/v\\d+/review/save", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1010-rating-voting-review-myrecipes/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-1010-rating-voting-review-myrecipes/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1010-rating-voting-review-myrecipes/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-1010-rating-voting-review-myrecipes/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-1010-rating-voting-review-myrecipes/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..436c0a9e --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1010-rating-voting-review-myrecipes/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "myrecipes\\.com/api/v\\d+/review/save", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1010-rating-voting-review-myrecipes/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-1010-rating-voting-review-myrecipes/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1010-rating-voting-review-myrecipes/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-1010-rating-voting-review-myrecipes/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-1010-rating-voting-review-myrecipes/steps/run/workdir/task.json new file mode 100644 index 00000000..65d9935b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1010-rating-voting-review-myrecipes/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1010, + "metaclass": "rating-voting", + "class": "rating", + "description": "Rate a Vegan Chocolate Chip Cookies Recipe on MyRecipes with 4 stars and add a helpful tip", + "sites_involved": [ + "myrecipes.com" + ], + "platform": "myrecipes", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Rate a Vegan Chocolate Chip Cookies Recipe on MyRecipes with 4 stars and add a helpful tip", + "eval_schema": { + "url_pattern": "myrecipes\\.com/api/v\\d+/review/save", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1010-rating-voting-review-myrecipes/task.toml b/harbor/datasets/clawbench-v2/v2-1010-rating-voting-review-myrecipes/task.toml new file mode 100644 index 00000000..64678096 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1010-rating-voting-review-myrecipes/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-1010-rating-voting-review-myrecipes" +description = "Rate a Vegan Chocolate Chip Cookies Recipe on MyRecipes with 4 stars and add a helpful tip" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-1010-rating-voting-review-myrecipes" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-1033-education-learning-registration-khanacademy/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-1033-education-learning-registration-khanacademy/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1033-education-learning-registration-khanacademy/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-1033-education-learning-registration-khanacademy/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-1033-education-learning-registration-khanacademy/steps/run/instruction.md new file mode 100644 index 00000000..ab3522a3 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1033-education-learning-registration-khanacademy/steps/run/instruction.md @@ -0,0 +1,27 @@ +Navigate to the 'World War I: military & diplomacy' lesson on Khan Academy + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-1033-education-learning-registration-khanacademy/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-1033-education-learning-registration-khanacademy/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1033-education-learning-registration-khanacademy/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-1033-education-learning-registration-khanacademy/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-1033-education-learning-registration-khanacademy/steps/run/tests/task.json new file mode 100644 index 00000000..4a68e3df --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1033-education-learning-registration-khanacademy/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1033, + "metaclass": "education-learning", + "class": "registration", + "description": "Navigate to the 'World War I: military & diplomacy' lesson on Khan Academy", + "sites_involved": [ + "khanacademy.org" + ], + "platform": "khanacademy", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Navigate to the 'World War I: military & diplomacy' lesson on Khan Academy", + "eval_schema": { + "url_pattern": "khanacademy\\.org/humanities/ap-us-history/period-7/apush-world-war-i-military-and-diplomacy-lesson", + "method": "GET" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1033-education-learning-registration-khanacademy/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-1033-education-learning-registration-khanacademy/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1033-education-learning-registration-khanacademy/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-1033-education-learning-registration-khanacademy/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-1033-education-learning-registration-khanacademy/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..e2f61d0a --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1033-education-learning-registration-khanacademy/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "khanacademy\\.org/humanities/ap-us-history/period-7/apush-world-war-i-military-and-diplomacy-lesson", + "method": "GET" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1033-education-learning-registration-khanacademy/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-1033-education-learning-registration-khanacademy/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1033-education-learning-registration-khanacademy/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-1033-education-learning-registration-khanacademy/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-1033-education-learning-registration-khanacademy/steps/run/workdir/task.json new file mode 100644 index 00000000..4a68e3df --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1033-education-learning-registration-khanacademy/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1033, + "metaclass": "education-learning", + "class": "registration", + "description": "Navigate to the 'World War I: military & diplomacy' lesson on Khan Academy", + "sites_involved": [ + "khanacademy.org" + ], + "platform": "khanacademy", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Navigate to the 'World War I: military & diplomacy' lesson on Khan Academy", + "eval_schema": { + "url_pattern": "khanacademy\\.org/humanities/ap-us-history/period-7/apush-world-war-i-military-and-diplomacy-lesson", + "method": "GET" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1033-education-learning-registration-khanacademy/task.toml b/harbor/datasets/clawbench-v2/v2-1033-education-learning-registration-khanacademy/task.toml new file mode 100644 index 00000000..c417954e --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1033-education-learning-registration-khanacademy/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-1033-education-learning-registration-khanacademy" +description = "Navigate to the 'World War I: military & diplomacy' lesson on Khan Academy" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-1033-education-learning-registration-khanacademy" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-1035-education-learning-enrollment-edx/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-1035-education-learning-enrollment-edx/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1035-education-learning-enrollment-edx/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-1035-education-learning-enrollment-edx/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-1035-education-learning-enrollment-edx/steps/run/instruction.md new file mode 100644 index 00000000..5d34d7f4 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1035-education-learning-enrollment-edx/steps/run/instruction.md @@ -0,0 +1,27 @@ +Log into edX and enroll in the free 'Machine Learning with Python: from Linear Models to Deep Learning' course by MIT on edX + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-1035-education-learning-enrollment-edx/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-1035-education-learning-enrollment-edx/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1035-education-learning-enrollment-edx/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-1035-education-learning-enrollment-edx/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-1035-education-learning-enrollment-edx/steps/run/tests/task.json new file mode 100644 index 00000000..2a172dbe --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1035-education-learning-enrollment-edx/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1035, + "metaclass": "education-learning", + "class": "enrollment", + "description": "Enroll in the free 'Machine Learning with Python: from Linear Models to Deep Learning' course by MIT on edX", + "sites_involved": [ + "edx.org" + ], + "platform": "edx", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Log into edX and enroll in the free 'Machine Learning with Python: from Linear Models to Deep Learning' course by MIT on edX", + "eval_schema": { + "url_pattern": "www\\.edx\\.org/(track-select/|learn/)", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1035-education-learning-enrollment-edx/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-1035-education-learning-enrollment-edx/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1035-education-learning-enrollment-edx/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-1035-education-learning-enrollment-edx/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-1035-education-learning-enrollment-edx/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..08fbfe5c --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1035-education-learning-enrollment-edx/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "www\\.edx\\.org/(track-select/|learn/)", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1035-education-learning-enrollment-edx/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-1035-education-learning-enrollment-edx/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1035-education-learning-enrollment-edx/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-1035-education-learning-enrollment-edx/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-1035-education-learning-enrollment-edx/steps/run/workdir/task.json new file mode 100644 index 00000000..2a172dbe --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1035-education-learning-enrollment-edx/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1035, + "metaclass": "education-learning", + "class": "enrollment", + "description": "Enroll in the free 'Machine Learning with Python: from Linear Models to Deep Learning' course by MIT on edX", + "sites_involved": [ + "edx.org" + ], + "platform": "edx", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Log into edX and enroll in the free 'Machine Learning with Python: from Linear Models to Deep Learning' course by MIT on edX", + "eval_schema": { + "url_pattern": "www\\.edx\\.org/(track-select/|learn/)", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1035-education-learning-enrollment-edx/task.toml b/harbor/datasets/clawbench-v2/v2-1035-education-learning-enrollment-edx/task.toml new file mode 100644 index 00000000..fa3b989d --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1035-education-learning-enrollment-edx/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-1035-education-learning-enrollment-edx" +description = "Enroll in the free 'Machine Learning with Python: from Linear Models to Deep Learning' course by MIT on edX" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-1035-education-learning-enrollment-edx" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-1045-weworkremotely/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-1045-weworkremotely/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1045-weworkremotely/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-1045-weworkremotely/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-1045-weworkremotely/steps/run/instruction.md new file mode 100644 index 00000000..7837a8a0 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1045-weworkremotely/steps/run/instruction.md @@ -0,0 +1,27 @@ +Go to We Work Remotely (https://weworkremotely.com/remote-jobs/new) and complete the 4-step job posting form to post a remote job listing. Fill in all required fields across all steps and place the order. + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-1045-weworkremotely/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-1045-weworkremotely/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1045-weworkremotely/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-1045-weworkremotely/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-1045-weworkremotely/steps/run/tests/task.json new file mode 100644 index 00000000..cea24600 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1045-weworkremotely/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1045, + "metaclass": "jobs", + "class": "posting", + "description": "Go to We Work Remotely (https://weworkremotely.com/remote-jobs/new) and complete the 4-step job posting form to post a remote job listing. Fill in all required fields across all steps and place the order.", + "sites_involved": [ + "weworkremotely.com" + ], + "platform": "weworkremotely", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Go to We Work Remotely (https://weworkremotely.com/remote-jobs/new) and complete the 4-step job posting form to post a remote job listing. Fill in all required fields across all steps and place the order.", + "eval_schema": { + "url_pattern": "weworkremotely\\.com/listings", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1045-weworkremotely/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-1045-weworkremotely/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1045-weworkremotely/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-1045-weworkremotely/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-1045-weworkremotely/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..a0db4258 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1045-weworkremotely/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "weworkremotely\\.com/listings", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1045-weworkremotely/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-1045-weworkremotely/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1045-weworkremotely/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-1045-weworkremotely/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-1045-weworkremotely/steps/run/workdir/task.json new file mode 100644 index 00000000..cea24600 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1045-weworkremotely/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1045, + "metaclass": "jobs", + "class": "posting", + "description": "Go to We Work Remotely (https://weworkremotely.com/remote-jobs/new) and complete the 4-step job posting form to post a remote job listing. Fill in all required fields across all steps and place the order.", + "sites_involved": [ + "weworkremotely.com" + ], + "platform": "weworkremotely", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Go to We Work Remotely (https://weworkremotely.com/remote-jobs/new) and complete the 4-step job posting form to post a remote job listing. Fill in all required fields across all steps and place the order.", + "eval_schema": { + "url_pattern": "weworkremotely\\.com/listings", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1045-weworkremotely/task.toml b/harbor/datasets/clawbench-v2/v2-1045-weworkremotely/task.toml new file mode 100644 index 00000000..ebbf9055 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1045-weworkremotely/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-1045-weworkremotely" +description = "Go to We Work Remotely (https://weworkremotely.com/remote-jobs/new) and complete the 4-step job posting form to post a remote job listing. Fill in all required fields across all steps and place the order." +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-1045-weworkremotely" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-1065b-daily-life-home-services-handy/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-1065b-daily-life-home-services-handy/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1065b-daily-life-home-services-handy/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-1065b-daily-life-home-services-handy/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-1065b-daily-life-home-services-handy/steps/run/instruction.md new file mode 100644 index 00000000..36e40e76 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1065b-daily-life-home-services-handy/steps/run/instruction.md @@ -0,0 +1,27 @@ +Get a price quote on Handy for a house cleaning service for next Saturday at 9am for a 3-bedroom, 2-bathroom apartment in Brooklyn (ZIP 11201). Select 3 hours duration. + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-1065b-daily-life-home-services-handy/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-1065b-daily-life-home-services-handy/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1065b-daily-life-home-services-handy/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-1065b-daily-life-home-services-handy/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-1065b-daily-life-home-services-handy/steps/run/tests/task.json new file mode 100644 index 00000000..f5723c36 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1065b-daily-life-home-services-handy/steps/run/tests/task.json @@ -0,0 +1,26 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1065, + "metaclass": "daily-life", + "class": "home-services", + "description": "Navigate to handy.com/services/home-cleaning, fill in ZIP 11201, 3 beds, 2 baths, next Saturday at 9am, 3 hours, then click 'Get a Price'.", + "sites_involved": [ + "handy.com" + ], + "platform": "handy", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + }, + "task_key": "1065b" + }, + "instruction": "Get a price quote on Handy for a house cleaning service for next Saturday at 9am for a 3-bedroom, 2-bathroom apartment in Brooklyn (ZIP 11201). Select 3 hours duration.", + "eval_schema": { + "url_pattern": "handy\\.com/quotes/\\d+/finalize", + "method": "GET" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1065b-daily-life-home-services-handy/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-1065b-daily-life-home-services-handy/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1065b-daily-life-home-services-handy/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-1065b-daily-life-home-services-handy/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-1065b-daily-life-home-services-handy/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..312be356 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1065b-daily-life-home-services-handy/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "handy\\.com/quotes/\\d+/finalize", + "method": "GET" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1065b-daily-life-home-services-handy/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-1065b-daily-life-home-services-handy/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1065b-daily-life-home-services-handy/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-1065b-daily-life-home-services-handy/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-1065b-daily-life-home-services-handy/steps/run/workdir/task.json new file mode 100644 index 00000000..f5723c36 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1065b-daily-life-home-services-handy/steps/run/workdir/task.json @@ -0,0 +1,26 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1065, + "metaclass": "daily-life", + "class": "home-services", + "description": "Navigate to handy.com/services/home-cleaning, fill in ZIP 11201, 3 beds, 2 baths, next Saturday at 9am, 3 hours, then click 'Get a Price'.", + "sites_involved": [ + "handy.com" + ], + "platform": "handy", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + }, + "task_key": "1065b" + }, + "instruction": "Get a price quote on Handy for a house cleaning service for next Saturday at 9am for a 3-bedroom, 2-bathroom apartment in Brooklyn (ZIP 11201). Select 3 hours duration.", + "eval_schema": { + "url_pattern": "handy\\.com/quotes/\\d+/finalize", + "method": "GET" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1065b-daily-life-home-services-handy/task.toml b/harbor/datasets/clawbench-v2/v2-1065b-daily-life-home-services-handy/task.toml new file mode 100644 index 00000000..8ea92fc1 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1065b-daily-life-home-services-handy/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-1065b-daily-life-home-services-handy" +description = "Navigate to handy.com/services/home-cleaning, fill in ZIP 11201, 3 beds, 2 baths, next Saturday at 9am, 3 hours, then click 'Get a Price'." +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-1065b-daily-life-home-services-handy" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-1088-nonprofit-charity-petition-change/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-1088-nonprofit-charity-petition-change/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1088-nonprofit-charity-petition-change/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-1088-nonprofit-charity-petition-change/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-1088-nonprofit-charity-petition-change/steps/run/instruction.md new file mode 100644 index 00000000..34a6ef8d --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1088-nonprofit-charity-petition-change/steps/run/instruction.md @@ -0,0 +1,27 @@ +Start a petition on Change.org to add more late-night bus routes in downtown Toronto + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-1088-nonprofit-charity-petition-change/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-1088-nonprofit-charity-petition-change/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1088-nonprofit-charity-petition-change/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-1088-nonprofit-charity-petition-change/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-1088-nonprofit-charity-petition-change/steps/run/tests/task.json new file mode 100644 index 00000000..08388dfa --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1088-nonprofit-charity-petition-change/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1088, + "metaclass": "nonprofit-charity", + "class": "petition", + "description": "Start a petition on Change.org to add more late-night bus routes in downtown Toronto", + "sites_involved": [ + "change.org" + ], + "platform": "change", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Start a petition on Change.org to add more late-night bus routes in downtown Toronto", + "eval_schema": { + "url_pattern": "change\\.org/api-proxy/graphql.*op=GenerateAiDraft|change\\.org/api-proxy/graphql.*op=CreatePetition|change\\.org/api-proxy/graphql.*op=SAPCreatePetition", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1088-nonprofit-charity-petition-change/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-1088-nonprofit-charity-petition-change/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1088-nonprofit-charity-petition-change/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-1088-nonprofit-charity-petition-change/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-1088-nonprofit-charity-petition-change/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..992db3bb --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1088-nonprofit-charity-petition-change/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "change\\.org/api-proxy/graphql.*op=GenerateAiDraft|change\\.org/api-proxy/graphql.*op=CreatePetition|change\\.org/api-proxy/graphql.*op=SAPCreatePetition", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1088-nonprofit-charity-petition-change/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-1088-nonprofit-charity-petition-change/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1088-nonprofit-charity-petition-change/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-1088-nonprofit-charity-petition-change/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-1088-nonprofit-charity-petition-change/steps/run/workdir/task.json new file mode 100644 index 00000000..08388dfa --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1088-nonprofit-charity-petition-change/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1088, + "metaclass": "nonprofit-charity", + "class": "petition", + "description": "Start a petition on Change.org to add more late-night bus routes in downtown Toronto", + "sites_involved": [ + "change.org" + ], + "platform": "change", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Start a petition on Change.org to add more late-night bus routes in downtown Toronto", + "eval_schema": { + "url_pattern": "change\\.org/api-proxy/graphql.*op=GenerateAiDraft|change\\.org/api-proxy/graphql.*op=CreatePetition|change\\.org/api-proxy/graphql.*op=SAPCreatePetition", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1088-nonprofit-charity-petition-change/task.toml b/harbor/datasets/clawbench-v2/v2-1088-nonprofit-charity-petition-change/task.toml new file mode 100644 index 00000000..3ab18ff4 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1088-nonprofit-charity-petition-change/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-1088-nonprofit-charity-petition-change" +description = "Start a petition on Change.org to add more late-night bus routes in downtown Toronto" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-1088-nonprofit-charity-petition-change" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-1093-personal-management-travel-tripit/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-1093-personal-management-travel-tripit/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1093-personal-management-travel-tripit/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-1093-personal-management-travel-tripit/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-1093-personal-management-travel-tripit/steps/run/instruction.md new file mode 100644 index 00000000..0c0497e4 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1093-personal-management-travel-tripit/steps/run/instruction.md @@ -0,0 +1,27 @@ +Create a travel itinerary on TripIt for a 7-day trip to Vancouver from June 15 to June 22 with a flight from SFO + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-1093-personal-management-travel-tripit/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-1093-personal-management-travel-tripit/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1093-personal-management-travel-tripit/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-1093-personal-management-travel-tripit/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-1093-personal-management-travel-tripit/steps/run/tests/task.json new file mode 100644 index 00000000..42876d11 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1093-personal-management-travel-tripit/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1093, + "metaclass": "personal-management", + "class": "travel", + "description": "Create a travel itinerary on TripIt for a 7-day trip to Vancouver from June 15 to June 22 with a flight from SFO", + "sites_involved": [ + "tripit.com" + ], + "platform": "tripit", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Create a travel itinerary on TripIt for a 7-day trip to Vancouver from June 15 to June 22 with a flight from SFO", + "eval_schema": { + "url_pattern": "tripit\\.com/api/v\\d+/create", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1093-personal-management-travel-tripit/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-1093-personal-management-travel-tripit/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1093-personal-management-travel-tripit/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-1093-personal-management-travel-tripit/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-1093-personal-management-travel-tripit/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..f2199052 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1093-personal-management-travel-tripit/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "tripit\\.com/api/v\\d+/create", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1093-personal-management-travel-tripit/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-1093-personal-management-travel-tripit/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1093-personal-management-travel-tripit/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-1093-personal-management-travel-tripit/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-1093-personal-management-travel-tripit/steps/run/workdir/task.json new file mode 100644 index 00000000..42876d11 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1093-personal-management-travel-tripit/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1093, + "metaclass": "personal-management", + "class": "travel", + "description": "Create a travel itinerary on TripIt for a 7-day trip to Vancouver from June 15 to June 22 with a flight from SFO", + "sites_involved": [ + "tripit.com" + ], + "platform": "tripit", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Create a travel itinerary on TripIt for a 7-day trip to Vancouver from June 15 to June 22 with a flight from SFO", + "eval_schema": { + "url_pattern": "tripit\\.com/api/v\\d+/create", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1093-personal-management-travel-tripit/task.toml b/harbor/datasets/clawbench-v2/v2-1093-personal-management-travel-tripit/task.toml new file mode 100644 index 00000000..34c203bc --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1093-personal-management-travel-tripit/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-1093-personal-management-travel-tripit" +description = "Create a travel itinerary on TripIt for a 7-day trip to Vancouver from June 15 to June 22 with a flight from SFO" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-1093-personal-management-travel-tripit" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-1095-health-mealplan-eatthismuch/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-1095-health-mealplan-eatthismuch/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1095-health-mealplan-eatthismuch/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-1095-health-mealplan-eatthismuch/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-1095-health-mealplan-eatthismuch/steps/run/instruction.md new file mode 100644 index 00000000..96c60401 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1095-health-mealplan-eatthismuch/steps/run/instruction.md @@ -0,0 +1,27 @@ +Add a new custom meal type called 'Morning Snack' to your Eat This Much meal layout. Set the meal size to 'Tiny' and the cooking preference to 'No Cooking', then save it. + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-1095-health-mealplan-eatthismuch/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-1095-health-mealplan-eatthismuch/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1095-health-mealplan-eatthismuch/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-1095-health-mealplan-eatthismuch/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-1095-health-mealplan-eatthismuch/steps/run/tests/task.json new file mode 100644 index 00000000..b74f97db --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1095-health-mealplan-eatthismuch/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1095, + "metaclass": "health-fitness", + "class": "mealplan", + "description": "Add a new custom meal type called 'Morning Snack' to your Eat This Much meal layout. Set the meal size to 'Tiny' and the cooking preference to 'No Cooking', then save it.", + "sites_involved": [ + "eatthismuch.com" + ], + "platform": "eatthismuch", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Add a new custom meal type called 'Morning Snack' to your Eat This Much meal layout. Set the meal size to 'Tiny' and the cooking preference to 'No Cooking', then save it.", + "eval_schema": { + "url_pattern": "eatthismuch\\.com/api/v\\d+/mealtype/", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1095-health-mealplan-eatthismuch/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-1095-health-mealplan-eatthismuch/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1095-health-mealplan-eatthismuch/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-1095-health-mealplan-eatthismuch/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-1095-health-mealplan-eatthismuch/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..dcd33e60 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1095-health-mealplan-eatthismuch/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "eatthismuch\\.com/api/v\\d+/mealtype/", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1095-health-mealplan-eatthismuch/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-1095-health-mealplan-eatthismuch/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1095-health-mealplan-eatthismuch/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-1095-health-mealplan-eatthismuch/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-1095-health-mealplan-eatthismuch/steps/run/workdir/task.json new file mode 100644 index 00000000..b74f97db --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1095-health-mealplan-eatthismuch/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1095, + "metaclass": "health-fitness", + "class": "mealplan", + "description": "Add a new custom meal type called 'Morning Snack' to your Eat This Much meal layout. Set the meal size to 'Tiny' and the cooking preference to 'No Cooking', then save it.", + "sites_involved": [ + "eatthismuch.com" + ], + "platform": "eatthismuch", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Add a new custom meal type called 'Morning Snack' to your Eat This Much meal layout. Set the meal size to 'Tiny' and the cooking preference to 'No Cooking', then save it.", + "eval_schema": { + "url_pattern": "eatthismuch\\.com/api/v\\d+/mealtype/", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1095-health-mealplan-eatthismuch/task.toml b/harbor/datasets/clawbench-v2/v2-1095-health-mealplan-eatthismuch/task.toml new file mode 100644 index 00000000..9b85cc83 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1095-health-mealplan-eatthismuch/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-1095-health-mealplan-eatthismuch" +description = "Add a new custom meal type called 'Morning Snack' to your Eat This Much meal layout. Set the meal size to 'Tiny' and the cooking preference to 'No Cooking', then save it." +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-1095-health-mealplan-eatthismuch" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-1097-beauty-personal-care-shopping-theordinary/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-1097-beauty-personal-care-shopping-theordinary/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1097-beauty-personal-care-shopping-theordinary/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-1097-beauty-personal-care-shopping-theordinary/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-1097-beauty-personal-care-shopping-theordinary/steps/run/instruction.md new file mode 100644 index 00000000..b9d76b55 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1097-beauty-personal-care-shopping-theordinary/steps/run/instruction.md @@ -0,0 +1,27 @@ +Add The Ordinary Niacinamide 10% + Zinc 1% and Hyaluronic Acid 2% + B5 (with Ceramides) to cart on The Ordinary website + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-1097-beauty-personal-care-shopping-theordinary/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-1097-beauty-personal-care-shopping-theordinary/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1097-beauty-personal-care-shopping-theordinary/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-1097-beauty-personal-care-shopping-theordinary/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-1097-beauty-personal-care-shopping-theordinary/steps/run/tests/task.json new file mode 100644 index 00000000..d486e20a --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1097-beauty-personal-care-shopping-theordinary/steps/run/tests/task.json @@ -0,0 +1,29 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1097, + "metaclass": "beauty-personal-care", + "class": "shopping", + "description": "Add The Ordinary Niacinamide 10% + Zinc 1% and Hyaluronic Acid 2% + B5 (with Ceramides) to cart on The Ordinary website", + "sites_involved": [ + "theordinary.com" + ], + "platform": "theordinary", + "product_urls": [ + "https://theordinary.com/en-us/niacinamide-10-zinc-1-serum-100436.html", + "https://theordinary.com/en-us/hyaluronic-acid-2-b5-serum-with-ceramides-100637.html" + ], + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Add The Ordinary Niacinamide 10% + Zinc 1% and Hyaluronic Acid 2% + B5 (with Ceramides) to cart on The Ordinary website", + "eval_schema": { + "url_pattern": "theordinary\\.com.*Cart-AddProduct", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1097-beauty-personal-care-shopping-theordinary/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-1097-beauty-personal-care-shopping-theordinary/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1097-beauty-personal-care-shopping-theordinary/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-1097-beauty-personal-care-shopping-theordinary/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-1097-beauty-personal-care-shopping-theordinary/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..d429958b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1097-beauty-personal-care-shopping-theordinary/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "theordinary\\.com.*Cart-AddProduct", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1097-beauty-personal-care-shopping-theordinary/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-1097-beauty-personal-care-shopping-theordinary/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1097-beauty-personal-care-shopping-theordinary/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-1097-beauty-personal-care-shopping-theordinary/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-1097-beauty-personal-care-shopping-theordinary/steps/run/workdir/task.json new file mode 100644 index 00000000..d486e20a --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1097-beauty-personal-care-shopping-theordinary/steps/run/workdir/task.json @@ -0,0 +1,29 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1097, + "metaclass": "beauty-personal-care", + "class": "shopping", + "description": "Add The Ordinary Niacinamide 10% + Zinc 1% and Hyaluronic Acid 2% + B5 (with Ceramides) to cart on The Ordinary website", + "sites_involved": [ + "theordinary.com" + ], + "platform": "theordinary", + "product_urls": [ + "https://theordinary.com/en-us/niacinamide-10-zinc-1-serum-100436.html", + "https://theordinary.com/en-us/hyaluronic-acid-2-b5-serum-with-ceramides-100637.html" + ], + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Add The Ordinary Niacinamide 10% + Zinc 1% and Hyaluronic Acid 2% + B5 (with Ceramides) to cart on The Ordinary website", + "eval_schema": { + "url_pattern": "theordinary\\.com.*Cart-AddProduct", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1097-beauty-personal-care-shopping-theordinary/task.toml b/harbor/datasets/clawbench-v2/v2-1097-beauty-personal-care-shopping-theordinary/task.toml new file mode 100644 index 00000000..eb794f77 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1097-beauty-personal-care-shopping-theordinary/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-1097-beauty-personal-care-shopping-theordinary" +description = "Add The Ordinary Niacinamide 10% + Zinc 1% and Hyaluronic Acid 2% + B5 (with Ceramides) to cart on The Ordinary website" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-1097-beauty-personal-care-shopping-theordinary" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-1100-food-cooking-collection-myrecipes/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-1100-food-cooking-collection-myrecipes/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1100-food-cooking-collection-myrecipes/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-1100-food-cooking-collection-myrecipes/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-1100-food-cooking-collection-myrecipes/steps/run/instruction.md new file mode 100644 index 00000000..0085425a --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1100-food-cooking-collection-myrecipes/steps/run/instruction.md @@ -0,0 +1,27 @@ +Add Baked Ziti to Want to Try collection on MyRecipes + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-1100-food-cooking-collection-myrecipes/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-1100-food-cooking-collection-myrecipes/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1100-food-cooking-collection-myrecipes/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-1100-food-cooking-collection-myrecipes/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-1100-food-cooking-collection-myrecipes/steps/run/tests/task.json new file mode 100644 index 00000000..7f430600 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1100-food-cooking-collection-myrecipes/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1100, + "metaclass": "food-cooking", + "class": "collection", + "description": "Add Baked Ziti to Want to Try collection on MyRecipes", + "sites_involved": [ + "myrecipes.com" + ], + "platform": "myrecipes", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Add Baked Ziti to Want to Try collection on MyRecipes", + "eval_schema": { + "url_pattern": "myrecipes\\.com/collections/bookmarks/save", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1100-food-cooking-collection-myrecipes/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-1100-food-cooking-collection-myrecipes/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1100-food-cooking-collection-myrecipes/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-1100-food-cooking-collection-myrecipes/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-1100-food-cooking-collection-myrecipes/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..739c9319 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1100-food-cooking-collection-myrecipes/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "myrecipes\\.com/collections/bookmarks/save", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1100-food-cooking-collection-myrecipes/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-1100-food-cooking-collection-myrecipes/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1100-food-cooking-collection-myrecipes/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-1100-food-cooking-collection-myrecipes/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-1100-food-cooking-collection-myrecipes/steps/run/workdir/task.json new file mode 100644 index 00000000..7f430600 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1100-food-cooking-collection-myrecipes/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1100, + "metaclass": "food-cooking", + "class": "collection", + "description": "Add Baked Ziti to Want to Try collection on MyRecipes", + "sites_involved": [ + "myrecipes.com" + ], + "platform": "myrecipes", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Add Baked Ziti to Want to Try collection on MyRecipes", + "eval_schema": { + "url_pattern": "myrecipes\\.com/collections/bookmarks/save", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1100-food-cooking-collection-myrecipes/task.toml b/harbor/datasets/clawbench-v2/v2-1100-food-cooking-collection-myrecipes/task.toml new file mode 100644 index 00000000..43292bd2 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1100-food-cooking-collection-myrecipes/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-1100-food-cooking-collection-myrecipes" +description = "Add Baked Ziti to Want to Try collection on MyRecipes" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-1100-food-cooking-collection-myrecipes" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-1101-hobbies-gaming-discussion-ravelry/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-1101-hobbies-gaming-discussion-ravelry/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1101-hobbies-gaming-discussion-ravelry/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-1101-hobbies-gaming-discussion-ravelry/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-1101-hobbies-gaming-discussion-ravelry/steps/run/instruction.md new file mode 100644 index 00000000..f013eb37 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1101-hobbies-gaming-discussion-ravelry/steps/run/instruction.md @@ -0,0 +1,27 @@ +Post a fun discussion topic in 'Sock Knitters' group on Ravelry + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-1101-hobbies-gaming-discussion-ravelry/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-1101-hobbies-gaming-discussion-ravelry/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1101-hobbies-gaming-discussion-ravelry/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-1101-hobbies-gaming-discussion-ravelry/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-1101-hobbies-gaming-discussion-ravelry/steps/run/tests/task.json new file mode 100644 index 00000000..14993281 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1101-hobbies-gaming-discussion-ravelry/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1101, + "metaclass": "hobbies-gaming", + "class": "discussion", + "description": "Post a fun discussion topic in 'Sock Knitters' group on Ravelry", + "sites_involved": [ + "ravelry.com" + ], + "platform": "ravelry", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Post a fun discussion topic in 'Sock Knitters' group on Ravelry", + "eval_schema": { + "url_pattern": "ravelry\\.com/discuss/[^/]+/topics", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1101-hobbies-gaming-discussion-ravelry/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-1101-hobbies-gaming-discussion-ravelry/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1101-hobbies-gaming-discussion-ravelry/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-1101-hobbies-gaming-discussion-ravelry/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-1101-hobbies-gaming-discussion-ravelry/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..616cfb47 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1101-hobbies-gaming-discussion-ravelry/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "ravelry\\.com/discuss/[^/]+/topics", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1101-hobbies-gaming-discussion-ravelry/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-1101-hobbies-gaming-discussion-ravelry/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1101-hobbies-gaming-discussion-ravelry/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-1101-hobbies-gaming-discussion-ravelry/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-1101-hobbies-gaming-discussion-ravelry/steps/run/workdir/task.json new file mode 100644 index 00000000..14993281 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1101-hobbies-gaming-discussion-ravelry/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1101, + "metaclass": "hobbies-gaming", + "class": "discussion", + "description": "Post a fun discussion topic in 'Sock Knitters' group on Ravelry", + "sites_involved": [ + "ravelry.com" + ], + "platform": "ravelry", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Post a fun discussion topic in 'Sock Knitters' group on Ravelry", + "eval_schema": { + "url_pattern": "ravelry\\.com/discuss/[^/]+/topics", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1101-hobbies-gaming-discussion-ravelry/task.toml b/harbor/datasets/clawbench-v2/v2-1101-hobbies-gaming-discussion-ravelry/task.toml new file mode 100644 index 00000000..efecf850 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1101-hobbies-gaming-discussion-ravelry/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-1101-hobbies-gaming-discussion-ravelry" +description = "Post a fun discussion topic in 'Sock Knitters' group on Ravelry" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-1101-hobbies-gaming-discussion-ravelry" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-1102-fitness-social-post-strava/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-1102-fitness-social-post-strava/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1102-fitness-social-post-strava/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-1102-fitness-social-post-strava/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-1102-fitness-social-post-strava/steps/run/instruction.md new file mode 100644 index 00000000..87a7e476 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1102-fitness-social-post-strava/steps/run/instruction.md @@ -0,0 +1,27 @@ +Publish a post on Strava with the caption 'Great morning ride!' + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-1102-fitness-social-post-strava/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-1102-fitness-social-post-strava/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1102-fitness-social-post-strava/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-1102-fitness-social-post-strava/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-1102-fitness-social-post-strava/steps/run/tests/task.json new file mode 100644 index 00000000..4568dbfb --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1102-fitness-social-post-strava/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1102, + "metaclass": "fitness", + "class": "social", + "description": "Publish a post on Strava with the caption 'Great morning ride!' via the athlete Posts page", + "sites_involved": [ + "strava.com" + ], + "platform": "strava", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Publish a post on Strava with the caption 'Great morning ride!'", + "eval_schema": { + "url_pattern": "strava\\.com\\/athletes\\/\\d+\\/posts", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1102-fitness-social-post-strava/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-1102-fitness-social-post-strava/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1102-fitness-social-post-strava/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-1102-fitness-social-post-strava/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-1102-fitness-social-post-strava/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..531eaee6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1102-fitness-social-post-strava/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "strava\\.com\\/athletes\\/\\d+\\/posts", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1102-fitness-social-post-strava/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-1102-fitness-social-post-strava/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1102-fitness-social-post-strava/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-1102-fitness-social-post-strava/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-1102-fitness-social-post-strava/steps/run/workdir/task.json new file mode 100644 index 00000000..4568dbfb --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1102-fitness-social-post-strava/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1102, + "metaclass": "fitness", + "class": "social", + "description": "Publish a post on Strava with the caption 'Great morning ride!' via the athlete Posts page", + "sites_involved": [ + "strava.com" + ], + "platform": "strava", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Publish a post on Strava with the caption 'Great morning ride!'", + "eval_schema": { + "url_pattern": "strava\\.com\\/athletes\\/\\d+\\/posts", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1102-fitness-social-post-strava/task.toml b/harbor/datasets/clawbench-v2/v2-1102-fitness-social-post-strava/task.toml new file mode 100644 index 00000000..dcf2dde0 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1102-fitness-social-post-strava/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-1102-fitness-social-post-strava" +description = "Publish a post on Strava with the caption 'Great morning ride!' via the athlete Posts page" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-1102-fitness-social-post-strava" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-1103-fitness-club-strava/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-1103-fitness-club-strava/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1103-fitness-club-strava/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-1103-fitness-club-strava/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-1103-fitness-club-strava/steps/run/instruction.md new file mode 100644 index 00000000..744e63d4 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1103-fitness-club-strava/steps/run/instruction.md @@ -0,0 +1,27 @@ +Create a cycling club on Strava called 'Portland Weekend Riders' for casual group rides every Saturday morning in Portland, OR + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-1103-fitness-club-strava/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-1103-fitness-club-strava/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1103-fitness-club-strava/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-1103-fitness-club-strava/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-1103-fitness-club-strava/steps/run/tests/task.json new file mode 100644 index 00000000..893e789c --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1103-fitness-club-strava/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1103, + "metaclass": "fitness", + "class": "club", + "description": "Create a cycling club on Strava called 'Portland Weekend Riders' for casual group rides every Saturday morning in Portland, OR", + "sites_involved": [ + "strava.com" + ], + "platform": "strava", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Create a cycling club on Strava called 'Portland Weekend Riders' for casual group rides every Saturday morning in Portland, OR", + "eval_schema": { + "url_pattern": "strava\\.com\\/clubs", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1103-fitness-club-strava/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-1103-fitness-club-strava/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1103-fitness-club-strava/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-1103-fitness-club-strava/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-1103-fitness-club-strava/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..f448d35f --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1103-fitness-club-strava/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "strava\\.com\\/clubs", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1103-fitness-club-strava/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-1103-fitness-club-strava/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1103-fitness-club-strava/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-1103-fitness-club-strava/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-1103-fitness-club-strava/steps/run/workdir/task.json new file mode 100644 index 00000000..893e789c --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1103-fitness-club-strava/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1103, + "metaclass": "fitness", + "class": "club", + "description": "Create a cycling club on Strava called 'Portland Weekend Riders' for casual group rides every Saturday morning in Portland, OR", + "sites_involved": [ + "strava.com" + ], + "platform": "strava", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Create a cycling club on Strava called 'Portland Weekend Riders' for casual group rides every Saturday morning in Portland, OR", + "eval_schema": { + "url_pattern": "strava\\.com\\/clubs", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1103-fitness-club-strava/task.toml b/harbor/datasets/clawbench-v2/v2-1103-fitness-club-strava/task.toml new file mode 100644 index 00000000..5bc41179 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1103-fitness-club-strava/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-1103-fitness-club-strava" +description = "Create a cycling club on Strava called 'Portland Weekend Riders' for casual group rides every Saturday morning in Portland, OR" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-1103-fitness-club-strava" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-1104-fitness-edit-activity-strava/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-1104-fitness-edit-activity-strava/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1104-fitness-edit-activity-strava/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-1104-fitness-edit-activity-strava/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-1104-fitness-edit-activity-strava/steps/run/instruction.md new file mode 100644 index 00000000..9b3979e0 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1104-fitness-edit-activity-strava/steps/run/instruction.md @@ -0,0 +1,27 @@ +Edit the most recent activity on Strava and add the description 'Felt strong today — great tempo throughout.' + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-1104-fitness-edit-activity-strava/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-1104-fitness-edit-activity-strava/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1104-fitness-edit-activity-strava/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-1104-fitness-edit-activity-strava/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-1104-fitness-edit-activity-strava/steps/run/tests/task.json new file mode 100644 index 00000000..a51dbe8b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1104-fitness-edit-activity-strava/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1104, + "metaclass": "fitness", + "class": "activity-edit", + "description": "Navigate to the most recent Strava activity, open the edit form, and save a new description", + "sites_involved": [ + "strava.com" + ], + "platform": "strava", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Edit the most recent activity on Strava and add the description 'Felt strong today — great tempo throughout.'", + "eval_schema": { + "url_pattern": "strava\\.com\\/activities\\/\\d+", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1104-fitness-edit-activity-strava/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-1104-fitness-edit-activity-strava/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1104-fitness-edit-activity-strava/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-1104-fitness-edit-activity-strava/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-1104-fitness-edit-activity-strava/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..c9ef0378 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1104-fitness-edit-activity-strava/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "strava\\.com\\/activities\\/\\d+", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1104-fitness-edit-activity-strava/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-1104-fitness-edit-activity-strava/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1104-fitness-edit-activity-strava/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-1104-fitness-edit-activity-strava/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-1104-fitness-edit-activity-strava/steps/run/workdir/task.json new file mode 100644 index 00000000..a51dbe8b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1104-fitness-edit-activity-strava/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1104, + "metaclass": "fitness", + "class": "activity-edit", + "description": "Navigate to the most recent Strava activity, open the edit form, and save a new description", + "sites_involved": [ + "strava.com" + ], + "platform": "strava", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Edit the most recent activity on Strava and add the description 'Felt strong today — great tempo throughout.'", + "eval_schema": { + "url_pattern": "strava\\.com\\/activities\\/\\d+", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1104-fitness-edit-activity-strava/task.toml b/harbor/datasets/clawbench-v2/v2-1104-fitness-edit-activity-strava/task.toml new file mode 100644 index 00000000..96b55007 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1104-fitness-edit-activity-strava/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-1104-fitness-edit-activity-strava" +description = "Navigate to the most recent Strava activity, open the edit form, and save a new description" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-1104-fitness-edit-activity-strava" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-1107-hobbies-gaming-forum-boardgamegeek/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-1107-hobbies-gaming-forum-boardgamegeek/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1107-hobbies-gaming-forum-boardgamegeek/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-1107-hobbies-gaming-forum-boardgamegeek/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-1107-hobbies-gaming-forum-boardgamegeek/steps/run/instruction.md new file mode 100644 index 00000000..3b61d6b4 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1107-hobbies-gaming-forum-boardgamegeek/steps/run/instruction.md @@ -0,0 +1,27 @@ +Post a question in the 'Ticket to Ride' forum on BoardGameGeek asking about the best strategy for long routes + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-1107-hobbies-gaming-forum-boardgamegeek/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-1107-hobbies-gaming-forum-boardgamegeek/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1107-hobbies-gaming-forum-boardgamegeek/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-1107-hobbies-gaming-forum-boardgamegeek/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-1107-hobbies-gaming-forum-boardgamegeek/steps/run/tests/task.json new file mode 100644 index 00000000..9930ed63 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1107-hobbies-gaming-forum-boardgamegeek/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1107, + "metaclass": "hobbies-gaming", + "class": "forum", + "description": "Post a question in the 'Ticket to Ride' forum on BoardGameGeek asking about the best strategy for long routes", + "sites_involved": [ + "boardgamegeek.com" + ], + "platform": "boardgamegeek", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Post a question in the 'Ticket to Ride' forum on BoardGameGeek asking about the best strategy for long routes", + "eval_schema": { + "url_pattern": "geekdo\\.com/api/thread", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1107-hobbies-gaming-forum-boardgamegeek/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-1107-hobbies-gaming-forum-boardgamegeek/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1107-hobbies-gaming-forum-boardgamegeek/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-1107-hobbies-gaming-forum-boardgamegeek/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-1107-hobbies-gaming-forum-boardgamegeek/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..a6afbacf --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1107-hobbies-gaming-forum-boardgamegeek/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "geekdo\\.com/api/thread", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1107-hobbies-gaming-forum-boardgamegeek/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-1107-hobbies-gaming-forum-boardgamegeek/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1107-hobbies-gaming-forum-boardgamegeek/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-1107-hobbies-gaming-forum-boardgamegeek/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-1107-hobbies-gaming-forum-boardgamegeek/steps/run/workdir/task.json new file mode 100644 index 00000000..9930ed63 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1107-hobbies-gaming-forum-boardgamegeek/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1107, + "metaclass": "hobbies-gaming", + "class": "forum", + "description": "Post a question in the 'Ticket to Ride' forum on BoardGameGeek asking about the best strategy for long routes", + "sites_involved": [ + "boardgamegeek.com" + ], + "platform": "boardgamegeek", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Post a question in the 'Ticket to Ride' forum on BoardGameGeek asking about the best strategy for long routes", + "eval_schema": { + "url_pattern": "geekdo\\.com/api/thread", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1107-hobbies-gaming-forum-boardgamegeek/task.toml b/harbor/datasets/clawbench-v2/v2-1107-hobbies-gaming-forum-boardgamegeek/task.toml new file mode 100644 index 00000000..7ba60473 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1107-hobbies-gaming-forum-boardgamegeek/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-1107-hobbies-gaming-forum-boardgamegeek" +description = "Post a question in the 'Ticket to Ride' forum on BoardGameGeek asking about the best strategy for long routes" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-1107-hobbies-gaming-forum-boardgamegeek" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-1108-civic-petition-change/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-1108-civic-petition-change/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1108-civic-petition-change/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-1108-civic-petition-change/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-1108-civic-petition-change/steps/run/instruction.md new file mode 100644 index 00000000..f0d4656b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1108-civic-petition-change/steps/run/instruction.md @@ -0,0 +1,27 @@ +Start a petition on Change.org to install more bike lanes on Market Street in San Francisco + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-1108-civic-petition-change/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-1108-civic-petition-change/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1108-civic-petition-change/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-1108-civic-petition-change/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-1108-civic-petition-change/steps/run/tests/task.json new file mode 100644 index 00000000..e03dbd3d --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1108-civic-petition-change/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1108, + "metaclass": "civic-engagement", + "class": "petition", + "description": "Start a petition on Change.org to install more bike lanes on Market Street in San Francisco", + "sites_involved": [ + "change.org" + ], + "platform": "change", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Start a petition on Change.org to install more bike lanes on Market Street in San Francisco", + "eval_schema": { + "url_pattern": "change\\.org/api-proxy/graphql.*op=GenerateAiDraft|change\\.org/api-proxy/graphql.*op=CreatePetition|change\\.org/api-proxy/graphql.*op=SAPCreatePetition", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1108-civic-petition-change/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-1108-civic-petition-change/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1108-civic-petition-change/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-1108-civic-petition-change/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-1108-civic-petition-change/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..992db3bb --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1108-civic-petition-change/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "change\\.org/api-proxy/graphql.*op=GenerateAiDraft|change\\.org/api-proxy/graphql.*op=CreatePetition|change\\.org/api-proxy/graphql.*op=SAPCreatePetition", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1108-civic-petition-change/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-1108-civic-petition-change/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1108-civic-petition-change/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-1108-civic-petition-change/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-1108-civic-petition-change/steps/run/workdir/task.json new file mode 100644 index 00000000..e03dbd3d --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1108-civic-petition-change/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1108, + "metaclass": "civic-engagement", + "class": "petition", + "description": "Start a petition on Change.org to install more bike lanes on Market Street in San Francisco", + "sites_involved": [ + "change.org" + ], + "platform": "change", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Start a petition on Change.org to install more bike lanes on Market Street in San Francisco", + "eval_schema": { + "url_pattern": "change\\.org/api-proxy/graphql.*op=GenerateAiDraft|change\\.org/api-proxy/graphql.*op=CreatePetition|change\\.org/api-proxy/graphql.*op=SAPCreatePetition", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1108-civic-petition-change/task.toml b/harbor/datasets/clawbench-v2/v2-1108-civic-petition-change/task.toml new file mode 100644 index 00000000..eeaadc29 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1108-civic-petition-change/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-1108-civic-petition-change" +description = "Start a petition on Change.org to install more bike lanes on Market Street in San Francisco" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-1108-civic-petition-change" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-1111-civic-petition-change/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-1111-civic-petition-change/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1111-civic-petition-change/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-1111-civic-petition-change/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-1111-civic-petition-change/steps/run/instruction.md new file mode 100644 index 00000000..4db4ad90 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1111-civic-petition-change/steps/run/instruction.md @@ -0,0 +1,27 @@ +On Change.org, write and publish a petition from scratch (without AI assistance) calling for free public Wi-Fi in all New York City public libraries + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-1111-civic-petition-change/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-1111-civic-petition-change/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1111-civic-petition-change/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-1111-civic-petition-change/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-1111-civic-petition-change/steps/run/tests/task.json new file mode 100644 index 00000000..eb8b8c63 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1111-civic-petition-change/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1111, + "metaclass": "civic-engagement", + "class": "petition", + "description": "On Change.org, write and publish a petition from scratch (without AI assistance) calling for free public Wi-Fi in all New York City public libraries", + "sites_involved": [ + "change.org" + ], + "platform": "change", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "On Change.org, write and publish a petition from scratch (without AI assistance) calling for free public Wi-Fi in all New York City public libraries", + "eval_schema": { + "url_pattern": "change\\.org/api-proxy/graphql.*op=SAPCreatePetition", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1111-civic-petition-change/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-1111-civic-petition-change/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1111-civic-petition-change/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-1111-civic-petition-change/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-1111-civic-petition-change/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..423bf8ec --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1111-civic-petition-change/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "change\\.org/api-proxy/graphql.*op=SAPCreatePetition", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1111-civic-petition-change/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-1111-civic-petition-change/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1111-civic-petition-change/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-1111-civic-petition-change/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-1111-civic-petition-change/steps/run/workdir/task.json new file mode 100644 index 00000000..eb8b8c63 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1111-civic-petition-change/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1111, + "metaclass": "civic-engagement", + "class": "petition", + "description": "On Change.org, write and publish a petition from scratch (without AI assistance) calling for free public Wi-Fi in all New York City public libraries", + "sites_involved": [ + "change.org" + ], + "platform": "change", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "On Change.org, write and publish a petition from scratch (without AI assistance) calling for free public Wi-Fi in all New York City public libraries", + "eval_schema": { + "url_pattern": "change\\.org/api-proxy/graphql.*op=SAPCreatePetition", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1111-civic-petition-change/task.toml b/harbor/datasets/clawbench-v2/v2-1111-civic-petition-change/task.toml new file mode 100644 index 00000000..78f11b2f --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1111-civic-petition-change/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-1111-civic-petition-change" +description = "On Change.org, write and publish a petition from scratch (without AI assistance) calling for free public Wi-Fi in all New York City public libraries" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-1111-civic-petition-change" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-1112-nonprofit-petition-update-change/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-1112-nonprofit-petition-update-change/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1112-nonprofit-petition-update-change/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-1112-nonprofit-petition-update-change/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-1112-nonprofit-petition-update-change/steps/run/instruction.md new file mode 100644 index 00000000..082f3ef9 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1112-nonprofit-petition-update-change/steps/run/instruction.md @@ -0,0 +1,27 @@ +On Change.org, find your most recently created petition and post a supporter update announcing that the petition has reached a new milestone and urging followers to keep sharing + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-1112-nonprofit-petition-update-change/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-1112-nonprofit-petition-update-change/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1112-nonprofit-petition-update-change/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-1112-nonprofit-petition-update-change/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-1112-nonprofit-petition-update-change/steps/run/tests/task.json new file mode 100644 index 00000000..9e88a0eb --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1112-nonprofit-petition-update-change/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1112, + "metaclass": "nonprofit-charity", + "class": "petition-management", + "description": "On Change.org, find your most recently created petition and post a supporter update announcing that the petition has reached a new milestone and urging followers to keep sharing", + "sites_involved": [ + "change.org" + ], + "platform": "change", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "On Change.org, find your most recently created petition and post a supporter update announcing that the petition has reached a new milestone and urging followers to keep sharing", + "eval_schema": { + "url_pattern": "change\\.org/api-proxy/graphql.*op=CreatePetitionUpdate", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1112-nonprofit-petition-update-change/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-1112-nonprofit-petition-update-change/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1112-nonprofit-petition-update-change/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-1112-nonprofit-petition-update-change/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-1112-nonprofit-petition-update-change/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..ba5dfaaf --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1112-nonprofit-petition-update-change/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "change\\.org/api-proxy/graphql.*op=CreatePetitionUpdate", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1112-nonprofit-petition-update-change/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-1112-nonprofit-petition-update-change/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1112-nonprofit-petition-update-change/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-1112-nonprofit-petition-update-change/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-1112-nonprofit-petition-update-change/steps/run/workdir/task.json new file mode 100644 index 00000000..9e88a0eb --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1112-nonprofit-petition-update-change/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1112, + "metaclass": "nonprofit-charity", + "class": "petition-management", + "description": "On Change.org, find your most recently created petition and post a supporter update announcing that the petition has reached a new milestone and urging followers to keep sharing", + "sites_involved": [ + "change.org" + ], + "platform": "change", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "On Change.org, find your most recently created petition and post a supporter update announcing that the petition has reached a new milestone and urging followers to keep sharing", + "eval_schema": { + "url_pattern": "change\\.org/api-proxy/graphql.*op=CreatePetitionUpdate", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1112-nonprofit-petition-update-change/task.toml b/harbor/datasets/clawbench-v2/v2-1112-nonprofit-petition-update-change/task.toml new file mode 100644 index 00000000..6f706c0f --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1112-nonprofit-petition-update-change/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-1112-nonprofit-petition-update-change" +description = "On Change.org, find your most recently created petition and post a supporter update announcing that the petition has reached a new milestone and urging followers to keep sharing" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-1112-nonprofit-petition-update-change" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-1113-health-mealplan-eatthismuch/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-1113-health-mealplan-eatthismuch/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1113-health-mealplan-eatthismuch/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-1113-health-mealplan-eatthismuch/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-1113-health-mealplan-eatthismuch/steps/run/instruction.md new file mode 100644 index 00000000..c6098cb9 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1113-health-mealplan-eatthismuch/steps/run/instruction.md @@ -0,0 +1,27 @@ +Set your dietary preferences on Eat This Much to keto with a daily budget of $15, then navigate to the planner and regenerate today's meals. + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-1113-health-mealplan-eatthismuch/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-1113-health-mealplan-eatthismuch/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1113-health-mealplan-eatthismuch/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-1113-health-mealplan-eatthismuch/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-1113-health-mealplan-eatthismuch/steps/run/tests/task.json new file mode 100644 index 00000000..409d0484 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1113-health-mealplan-eatthismuch/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1113, + "metaclass": "health-fitness", + "class": "mealplan", + "description": "Set your dietary preferences on Eat This Much to keto with a daily budget of $15, then navigate to the planner and regenerate today's meals.", + "sites_involved": [ + "eatthismuch.com" + ], + "platform": "eatthismuch", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Set your dietary preferences on Eat This Much to keto with a daily budget of $15, then navigate to the planner and regenerate today's meals.", + "eval_schema": { + "url_pattern": "eatthismuch\\.com/g/generate/day", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1113-health-mealplan-eatthismuch/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-1113-health-mealplan-eatthismuch/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1113-health-mealplan-eatthismuch/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-1113-health-mealplan-eatthismuch/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-1113-health-mealplan-eatthismuch/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..79b140bb --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1113-health-mealplan-eatthismuch/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "eatthismuch\\.com/g/generate/day", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1113-health-mealplan-eatthismuch/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-1113-health-mealplan-eatthismuch/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1113-health-mealplan-eatthismuch/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-1113-health-mealplan-eatthismuch/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-1113-health-mealplan-eatthismuch/steps/run/workdir/task.json new file mode 100644 index 00000000..409d0484 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1113-health-mealplan-eatthismuch/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1113, + "metaclass": "health-fitness", + "class": "mealplan", + "description": "Set your dietary preferences on Eat This Much to keto with a daily budget of $15, then navigate to the planner and regenerate today's meals.", + "sites_involved": [ + "eatthismuch.com" + ], + "platform": "eatthismuch", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Set your dietary preferences on Eat This Much to keto with a daily budget of $15, then navigate to the planner and regenerate today's meals.", + "eval_schema": { + "url_pattern": "eatthismuch\\.com/g/generate/day", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1113-health-mealplan-eatthismuch/task.toml b/harbor/datasets/clawbench-v2/v2-1113-health-mealplan-eatthismuch/task.toml new file mode 100644 index 00000000..e22d9cf9 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1113-health-mealplan-eatthismuch/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-1113-health-mealplan-eatthismuch" +description = "Set your dietary preferences on Eat This Much to keto with a daily budget of $15, then navigate to the planner and regenerate today's meals." +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-1113-health-mealplan-eatthismuch" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-1114-education-enrollment-edx/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-1114-education-enrollment-edx/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1114-education-enrollment-edx/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-1114-education-enrollment-edx/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-1114-education-enrollment-edx/steps/run/instruction.md new file mode 100644 index 00000000..9cb21f5d --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1114-education-enrollment-edx/steps/run/instruction.md @@ -0,0 +1,27 @@ +Log into edX and enroll in the free 'Introduction to Computer Science' course by Harvard (CS50) on edX + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-1114-education-enrollment-edx/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-1114-education-enrollment-edx/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1114-education-enrollment-edx/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-1114-education-enrollment-edx/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-1114-education-enrollment-edx/steps/run/tests/task.json new file mode 100644 index 00000000..4e992dac --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1114-education-enrollment-edx/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1114, + "metaclass": "education", + "class": "enrollment", + "description": "Enroll in the free 'Introduction to Computer Science' course by Harvard (CS50) on edX", + "sites_involved": [ + "edx.org" + ], + "platform": "edx", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Log into edX and enroll in the free 'Introduction to Computer Science' course by Harvard (CS50) on edX", + "eval_schema": { + "url_pattern": "www\\.edx\\.org/(track-select/|learn/)", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1114-education-enrollment-edx/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-1114-education-enrollment-edx/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1114-education-enrollment-edx/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-1114-education-enrollment-edx/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-1114-education-enrollment-edx/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..08fbfe5c --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1114-education-enrollment-edx/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "www\\.edx\\.org/(track-select/|learn/)", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1114-education-enrollment-edx/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-1114-education-enrollment-edx/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1114-education-enrollment-edx/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-1114-education-enrollment-edx/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-1114-education-enrollment-edx/steps/run/workdir/task.json new file mode 100644 index 00000000..4e992dac --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1114-education-enrollment-edx/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1114, + "metaclass": "education", + "class": "enrollment", + "description": "Enroll in the free 'Introduction to Computer Science' course by Harvard (CS50) on edX", + "sites_involved": [ + "edx.org" + ], + "platform": "edx", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Log into edX and enroll in the free 'Introduction to Computer Science' course by Harvard (CS50) on edX", + "eval_schema": { + "url_pattern": "www\\.edx\\.org/(track-select/|learn/)", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1114-education-enrollment-edx/task.toml b/harbor/datasets/clawbench-v2/v2-1114-education-enrollment-edx/task.toml new file mode 100644 index 00000000..74f8bd51 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1114-education-enrollment-edx/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-1114-education-enrollment-edx" +description = "Enroll in the free 'Introduction to Computer Science' course by Harvard (CS50) on edX" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-1114-education-enrollment-edx" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-1115-education-enrollment-edx/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-1115-education-enrollment-edx/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1115-education-enrollment-edx/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-1115-education-enrollment-edx/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-1115-education-enrollment-edx/steps/run/instruction.md new file mode 100644 index 00000000..3c914966 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1115-education-enrollment-edx/steps/run/instruction.md @@ -0,0 +1,27 @@ +Log into edX and enroll in the free 'Algorithmic Design and Techniques' course by UC San Diego on edX + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-1115-education-enrollment-edx/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-1115-education-enrollment-edx/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1115-education-enrollment-edx/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-1115-education-enrollment-edx/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-1115-education-enrollment-edx/steps/run/tests/task.json new file mode 100644 index 00000000..7f41fc96 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1115-education-enrollment-edx/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1115, + "metaclass": "education", + "class": "enrollment", + "description": "Enroll in the free 'Algorithmic Design and Techniques' course by UC San Diego on edX", + "sites_involved": [ + "edx.org" + ], + "platform": "edx", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Log into edX and enroll in the free 'Algorithmic Design and Techniques' course by UC San Diego on edX", + "eval_schema": { + "url_pattern": "www\\.edx\\.org/(track-select/|learn/)", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1115-education-enrollment-edx/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-1115-education-enrollment-edx/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1115-education-enrollment-edx/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-1115-education-enrollment-edx/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-1115-education-enrollment-edx/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..08fbfe5c --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1115-education-enrollment-edx/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "www\\.edx\\.org/(track-select/|learn/)", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1115-education-enrollment-edx/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-1115-education-enrollment-edx/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1115-education-enrollment-edx/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-1115-education-enrollment-edx/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-1115-education-enrollment-edx/steps/run/workdir/task.json new file mode 100644 index 00000000..7f41fc96 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1115-education-enrollment-edx/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1115, + "metaclass": "education", + "class": "enrollment", + "description": "Enroll in the free 'Algorithmic Design and Techniques' course by UC San Diego on edX", + "sites_involved": [ + "edx.org" + ], + "platform": "edx", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Log into edX and enroll in the free 'Algorithmic Design and Techniques' course by UC San Diego on edX", + "eval_schema": { + "url_pattern": "www\\.edx\\.org/(track-select/|learn/)", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1115-education-enrollment-edx/task.toml b/harbor/datasets/clawbench-v2/v2-1115-education-enrollment-edx/task.toml new file mode 100644 index 00000000..fae07286 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1115-education-enrollment-edx/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-1115-education-enrollment-edx" +description = "Enroll in the free 'Algorithmic Design and Techniques' course by UC San Diego on edX" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-1115-education-enrollment-edx" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-1116-education-enrollment-edx/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-1116-education-enrollment-edx/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1116-education-enrollment-edx/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-1116-education-enrollment-edx/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-1116-education-enrollment-edx/steps/run/instruction.md new file mode 100644 index 00000000..894589f7 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1116-education-enrollment-edx/steps/run/instruction.md @@ -0,0 +1,27 @@ +Log into edX and enroll in the free 'Python Basics for Data Science' course by IBM on edX + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-1116-education-enrollment-edx/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-1116-education-enrollment-edx/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1116-education-enrollment-edx/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-1116-education-enrollment-edx/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-1116-education-enrollment-edx/steps/run/tests/task.json new file mode 100644 index 00000000..174b2f00 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1116-education-enrollment-edx/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1116, + "metaclass": "education", + "class": "enrollment", + "description": "Enroll in the free 'Python Basics for Data Science' course by IBM on edX", + "sites_involved": [ + "edx.org" + ], + "platform": "edx", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Log into edX and enroll in the free 'Python Basics for Data Science' course by IBM on edX", + "eval_schema": { + "url_pattern": "www\\.edx\\.org/(track-select/|learn/)", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1116-education-enrollment-edx/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-1116-education-enrollment-edx/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1116-education-enrollment-edx/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-1116-education-enrollment-edx/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-1116-education-enrollment-edx/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..08fbfe5c --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1116-education-enrollment-edx/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "www\\.edx\\.org/(track-select/|learn/)", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1116-education-enrollment-edx/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-1116-education-enrollment-edx/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1116-education-enrollment-edx/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-1116-education-enrollment-edx/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-1116-education-enrollment-edx/steps/run/workdir/task.json new file mode 100644 index 00000000..174b2f00 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1116-education-enrollment-edx/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1116, + "metaclass": "education", + "class": "enrollment", + "description": "Enroll in the free 'Python Basics for Data Science' course by IBM on edX", + "sites_involved": [ + "edx.org" + ], + "platform": "edx", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Log into edX and enroll in the free 'Python Basics for Data Science' course by IBM on edX", + "eval_schema": { + "url_pattern": "www\\.edx\\.org/(track-select/|learn/)", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1116-education-enrollment-edx/task.toml b/harbor/datasets/clawbench-v2/v2-1116-education-enrollment-edx/task.toml new file mode 100644 index 00000000..7156d7f3 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1116-education-enrollment-edx/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-1116-education-enrollment-edx" +description = "Enroll in the free 'Python Basics for Data Science' course by IBM on edX" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-1116-education-enrollment-edx" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-1117/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-1117/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1117/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-1117/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-1117/steps/run/instruction.md new file mode 100644 index 00000000..524a92e9 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1117/steps/run/instruction.md @@ -0,0 +1,29 @@ +Search for the 'Hearts2Heal Monthly Wellness Meetup' event in Austin, TX on Eventbrite and register for the next available date. + +Note: The event is free and held on the 4th Tuesday of each month at Capital Factory, Austin TX. Event URL: https://www.eventbrite.com/e/hearts2heal-monthly-wellness-meetup-tickets-1983102574183 + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-1117/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-1117/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1117/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-1117/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-1117/steps/run/tests/task.json new file mode 100644 index 00000000..2f756939 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1117/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1117, + "metaclass": "web_task", + "class": "registration", + "description": "Register for a free recurring wellness meetup event in Austin on Eventbrite", + "sites_involved": [ + "https://www.eventbrite.com" + ], + "platform": "Eventbrite", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Search for the 'Hearts2Heal Monthly Wellness Meetup' event in Austin, TX on Eventbrite and register for the next available date.\n\nNote: The event is free and held on the 4th Tuesday of each month at Capital Factory, Austin TX. Event URL: https://www.eventbrite.com/e/hearts2heal-monthly-wellness-meetup-tickets-1983102574183", + "eval_schema": { + "url_pattern": "eventbrite\\.com/purchase-flow-webapp/api/v\\d+/orders/create", + "method": "POST" + }, + "time_limit": 120, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1117/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-1117/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1117/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-1117/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-1117/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..96c60f42 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1117/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "eventbrite\\.com/purchase-flow-webapp/api/v\\d+/orders/create", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1117/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-1117/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1117/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-1117/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-1117/steps/run/workdir/task.json new file mode 100644 index 00000000..2f756939 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1117/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1117, + "metaclass": "web_task", + "class": "registration", + "description": "Register for a free recurring wellness meetup event in Austin on Eventbrite", + "sites_involved": [ + "https://www.eventbrite.com" + ], + "platform": "Eventbrite", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Search for the 'Hearts2Heal Monthly Wellness Meetup' event in Austin, TX on Eventbrite and register for the next available date.\n\nNote: The event is free and held on the 4th Tuesday of each month at Capital Factory, Austin TX. Event URL: https://www.eventbrite.com/e/hearts2heal-monthly-wellness-meetup-tickets-1983102574183", + "eval_schema": { + "url_pattern": "eventbrite\\.com/purchase-flow-webapp/api/v\\d+/orders/create", + "method": "POST" + }, + "time_limit": 120, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1117/task.toml b/harbor/datasets/clawbench-v2/v2-1117/task.toml new file mode 100644 index 00000000..444ad848 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1117/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-1117" +description = "Register for a free recurring wellness meetup event in Austin on Eventbrite" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-1117" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 7200.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-1118/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-1118/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1118/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-1118/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-1118/steps/run/instruction.md new file mode 100644 index 00000000..ebc9b58f --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1118/steps/run/instruction.md @@ -0,0 +1,29 @@ +Find the 'Sazón Latin Food Festival 2026: Chicago's Taste of Latin American Food' on Eventbrite and register for 2 General Admission tickets. + +Note: The event is free and takes place on Sunday, June 14, 2026, 12 PM – 5 PM at KOVAL Store, Chicago, IL. Event URL: https://www.eventbrite.com/e/sazon-latin-food-festival-2026-chicagos-taste-of-latin-american-food-tickets-1980904085444 + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-1118/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-1118/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1118/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-1118/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-1118/steps/run/tests/task.json new file mode 100644 index 00000000..9bff16f2 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1118/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1118, + "metaclass": "web_task", + "class": "registration", + "description": "Register for 2 free tickets to a Latin food festival in Chicago on Eventbrite", + "sites_involved": [ + "https://www.eventbrite.com" + ], + "platform": "Eventbrite", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Find the 'Sazón Latin Food Festival 2026: Chicago's Taste of Latin American Food' on Eventbrite and register for 2 General Admission tickets.\n\nNote: The event is free and takes place on Sunday, June 14, 2026, 12 PM – 5 PM at KOVAL Store, Chicago, IL. Event URL: https://www.eventbrite.com/e/sazon-latin-food-festival-2026-chicagos-taste-of-latin-american-food-tickets-1980904085444", + "eval_schema": { + "url_pattern": "eventbrite\\.com/purchase-flow-webapp/api/v\\d+/orders/create", + "method": "POST" + }, + "time_limit": 120, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1118/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-1118/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1118/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-1118/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-1118/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..96c60f42 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1118/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "eventbrite\\.com/purchase-flow-webapp/api/v\\d+/orders/create", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1118/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-1118/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1118/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-1118/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-1118/steps/run/workdir/task.json new file mode 100644 index 00000000..9bff16f2 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1118/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1118, + "metaclass": "web_task", + "class": "registration", + "description": "Register for 2 free tickets to a Latin food festival in Chicago on Eventbrite", + "sites_involved": [ + "https://www.eventbrite.com" + ], + "platform": "Eventbrite", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Find the 'Sazón Latin Food Festival 2026: Chicago's Taste of Latin American Food' on Eventbrite and register for 2 General Admission tickets.\n\nNote: The event is free and takes place on Sunday, June 14, 2026, 12 PM – 5 PM at KOVAL Store, Chicago, IL. Event URL: https://www.eventbrite.com/e/sazon-latin-food-festival-2026-chicagos-taste-of-latin-american-food-tickets-1980904085444", + "eval_schema": { + "url_pattern": "eventbrite\\.com/purchase-flow-webapp/api/v\\d+/orders/create", + "method": "POST" + }, + "time_limit": 120, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1118/task.toml b/harbor/datasets/clawbench-v2/v2-1118/task.toml new file mode 100644 index 00000000..fac20fb9 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1118/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-1118" +description = "Register for 2 free tickets to a Latin food festival in Chicago on Eventbrite" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-1118" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 7200.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-1120b-services-booking-handy/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-1120b-services-booking-handy/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1120b-services-booking-handy/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-1120b-services-booking-handy/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-1120b-services-booking-handy/steps/run/instruction.md new file mode 100644 index 00000000..cae7182c --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1120b-services-booking-handy/steps/run/instruction.md @@ -0,0 +1,27 @@ +Get a price quote on Handy for a furniture assembly service for next Friday at 2pm in Brooklyn (ZIP 11201) for 1 medium furniture item. Add the description 'IKEA KALLAX bookshelf assembly' in the job description field. + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-1120b-services-booking-handy/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-1120b-services-booking-handy/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1120b-services-booking-handy/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-1120b-services-booking-handy/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-1120b-services-booking-handy/steps/run/tests/task.json new file mode 100644 index 00000000..2c96fe8f --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1120b-services-booking-handy/steps/run/tests/task.json @@ -0,0 +1,26 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1120, + "metaclass": "home-services", + "class": "booking", + "description": "Navigate to handy.com/services/furniture-assembly, enter ZIP 11201, set 1 medium furniture item, add the job description, set date to next Friday at 2pm, then click 'Get a Price'.", + "sites_involved": [ + "handy.com" + ], + "platform": "handy", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + }, + "task_key": "1120b" + }, + "instruction": "Get a price quote on Handy for a furniture assembly service for next Friday at 2pm in Brooklyn (ZIP 11201) for 1 medium furniture item. Add the description 'IKEA KALLAX bookshelf assembly' in the job description field.", + "eval_schema": { + "url_pattern": "handy\\.com/quotes/\\d+/finalize", + "method": "GET" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1120b-services-booking-handy/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-1120b-services-booking-handy/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1120b-services-booking-handy/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-1120b-services-booking-handy/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-1120b-services-booking-handy/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..312be356 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1120b-services-booking-handy/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "handy\\.com/quotes/\\d+/finalize", + "method": "GET" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1120b-services-booking-handy/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-1120b-services-booking-handy/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1120b-services-booking-handy/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-1120b-services-booking-handy/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-1120b-services-booking-handy/steps/run/workdir/task.json new file mode 100644 index 00000000..2c96fe8f --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1120b-services-booking-handy/steps/run/workdir/task.json @@ -0,0 +1,26 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1120, + "metaclass": "home-services", + "class": "booking", + "description": "Navigate to handy.com/services/furniture-assembly, enter ZIP 11201, set 1 medium furniture item, add the job description, set date to next Friday at 2pm, then click 'Get a Price'.", + "sites_involved": [ + "handy.com" + ], + "platform": "handy", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + }, + "task_key": "1120b" + }, + "instruction": "Get a price quote on Handy for a furniture assembly service for next Friday at 2pm in Brooklyn (ZIP 11201) for 1 medium furniture item. Add the description 'IKEA KALLAX bookshelf assembly' in the job description field.", + "eval_schema": { + "url_pattern": "handy\\.com/quotes/\\d+/finalize", + "method": "GET" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1120b-services-booking-handy/task.toml b/harbor/datasets/clawbench-v2/v2-1120b-services-booking-handy/task.toml new file mode 100644 index 00000000..471470a8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1120b-services-booking-handy/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-1120b-services-booking-handy" +description = "Navigate to handy.com/services/furniture-assembly, enter ZIP 11201, set 1 medium furniture item, add the job description, set date to next Friday at 2pm, then click 'Get a Price'." +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-1120b-services-booking-handy" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-1121b-services-booking-handy/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-1121b-services-booking-handy/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1121b-services-booking-handy/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-1121b-services-booking-handy/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-1121b-services-booking-handy/steps/run/instruction.md new file mode 100644 index 00000000..ba02e7fc --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1121b-services-booking-handy/steps/run/instruction.md @@ -0,0 +1,27 @@ +Get a price quote on Handy for a TV mounting service for next Wednesday at 10am in Brooklyn (ZIP 11201) for 1 TV. + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-1121b-services-booking-handy/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-1121b-services-booking-handy/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1121b-services-booking-handy/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-1121b-services-booking-handy/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-1121b-services-booking-handy/steps/run/tests/task.json new file mode 100644 index 00000000..f70e6d49 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1121b-services-booking-handy/steps/run/tests/task.json @@ -0,0 +1,26 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1121, + "metaclass": "home-services", + "class": "booking", + "description": "Navigate to handy.com/services/tv-mounting, enter ZIP 11201, set quantity to 1 TV, set date to next Wednesday at 10am, then click 'Get a Price'.", + "sites_involved": [ + "handy.com" + ], + "platform": "handy", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + }, + "task_key": "1121b" + }, + "instruction": "Get a price quote on Handy for a TV mounting service for next Wednesday at 10am in Brooklyn (ZIP 11201) for 1 TV.", + "eval_schema": { + "url_pattern": "handy\\.com/quotes/\\d+/finalize", + "method": "GET" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1121b-services-booking-handy/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-1121b-services-booking-handy/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1121b-services-booking-handy/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-1121b-services-booking-handy/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-1121b-services-booking-handy/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..312be356 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1121b-services-booking-handy/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "handy\\.com/quotes/\\d+/finalize", + "method": "GET" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1121b-services-booking-handy/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-1121b-services-booking-handy/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1121b-services-booking-handy/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-1121b-services-booking-handy/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-1121b-services-booking-handy/steps/run/workdir/task.json new file mode 100644 index 00000000..f70e6d49 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1121b-services-booking-handy/steps/run/workdir/task.json @@ -0,0 +1,26 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1121, + "metaclass": "home-services", + "class": "booking", + "description": "Navigate to handy.com/services/tv-mounting, enter ZIP 11201, set quantity to 1 TV, set date to next Wednesday at 10am, then click 'Get a Price'.", + "sites_involved": [ + "handy.com" + ], + "platform": "handy", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + }, + "task_key": "1121b" + }, + "instruction": "Get a price quote on Handy for a TV mounting service for next Wednesday at 10am in Brooklyn (ZIP 11201) for 1 TV.", + "eval_schema": { + "url_pattern": "handy\\.com/quotes/\\d+/finalize", + "method": "GET" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1121b-services-booking-handy/task.toml b/harbor/datasets/clawbench-v2/v2-1121b-services-booking-handy/task.toml new file mode 100644 index 00000000..51396cee --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1121b-services-booking-handy/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-1121b-services-booking-handy" +description = "Navigate to handy.com/services/tv-mounting, enter ZIP 11201, set quantity to 1 TV, set date to next Wednesday at 10am, then click 'Get a Price'." +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-1121b-services-booking-handy" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-1122-education-course-khanacademy/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-1122-education-course-khanacademy/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1122-education-course-khanacademy/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-1122-education-course-khanacademy/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-1122-education-course-khanacademy/steps/run/instruction.md new file mode 100644 index 00000000..322d1feb --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1122-education-course-khanacademy/steps/run/instruction.md @@ -0,0 +1,27 @@ +Navigate to the 'Algebra 1' course on Khan Academy, go to the 'Linear equations & graphs' unit, and answer the first question of the 'Solutions to 2-variable equations' practice exercise + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-1122-education-course-khanacademy/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-1122-education-course-khanacademy/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1122-education-course-khanacademy/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-1122-education-course-khanacademy/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-1122-education-course-khanacademy/steps/run/tests/task.json new file mode 100644 index 00000000..50d36019 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1122-education-course-khanacademy/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1122, + "metaclass": "education", + "class": "course", + "description": "Navigate to the Algebra 1 linear equations unit on Khan Academy and submit an answer to the first practice exercise question", + "sites_involved": [ + "khanacademy.org" + ], + "platform": "khanacademy", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Navigate to the 'Algebra 1' course on Khan Academy, go to the 'Linear equations & graphs' unit, and answer the first question of the 'Solutions to 2-variable equations' practice exercise", + "eval_schema": { + "url_pattern": "khanacademy\\.org/api/internal/graphql/attemptProblem", + "method": "POST" + }, + "time_limit": 60, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1122-education-course-khanacademy/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-1122-education-course-khanacademy/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1122-education-course-khanacademy/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-1122-education-course-khanacademy/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-1122-education-course-khanacademy/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..a9d234d5 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1122-education-course-khanacademy/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "khanacademy\\.org/api/internal/graphql/attemptProblem", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1122-education-course-khanacademy/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-1122-education-course-khanacademy/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1122-education-course-khanacademy/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-1122-education-course-khanacademy/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-1122-education-course-khanacademy/steps/run/workdir/task.json new file mode 100644 index 00000000..50d36019 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1122-education-course-khanacademy/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1122, + "metaclass": "education", + "class": "course", + "description": "Navigate to the Algebra 1 linear equations unit on Khan Academy and submit an answer to the first practice exercise question", + "sites_involved": [ + "khanacademy.org" + ], + "platform": "khanacademy", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Navigate to the 'Algebra 1' course on Khan Academy, go to the 'Linear equations & graphs' unit, and answer the first question of the 'Solutions to 2-variable equations' practice exercise", + "eval_schema": { + "url_pattern": "khanacademy\\.org/api/internal/graphql/attemptProblem", + "method": "POST" + }, + "time_limit": 60, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1122-education-course-khanacademy/task.toml b/harbor/datasets/clawbench-v2/v2-1122-education-course-khanacademy/task.toml new file mode 100644 index 00000000..9b6007a6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1122-education-course-khanacademy/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-1122-education-course-khanacademy" +description = "Navigate to the Algebra 1 linear equations unit on Khan Academy and submit an answer to the first practice exercise question" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-1122-education-course-khanacademy" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 3600.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-1123-education-course-khanacademy/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-1123-education-course-khanacademy/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1123-education-course-khanacademy/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-1123-education-course-khanacademy/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-1123-education-course-khanacademy/steps/run/instruction.md new file mode 100644 index 00000000..b846363d --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1123-education-course-khanacademy/steps/run/instruction.md @@ -0,0 +1,27 @@ +Navigate to the AP Biology 'Cell Communication and Cell Cycle' unit on Khan Academy, go to the 'Cell cycle' lesson, and answer the first question of the Cell cycle practice exercise + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-1123-education-course-khanacademy/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-1123-education-course-khanacademy/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1123-education-course-khanacademy/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-1123-education-course-khanacademy/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-1123-education-course-khanacademy/steps/run/tests/task.json new file mode 100644 index 00000000..a1823376 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1123-education-course-khanacademy/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1123, + "metaclass": "education", + "class": "course", + "description": "Navigate to the AP Biology Cell cycle lesson on Khan Academy and submit an answer to the practice exercise", + "sites_involved": [ + "khanacademy.org" + ], + "platform": "khanacademy", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Navigate to the AP Biology 'Cell Communication and Cell Cycle' unit on Khan Academy, go to the 'Cell cycle' lesson, and answer the first question of the Cell cycle practice exercise", + "eval_schema": { + "url_pattern": "khanacademy\\.org/api/internal/graphql/attemptProblem", + "method": "POST" + }, + "time_limit": 60, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1123-education-course-khanacademy/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-1123-education-course-khanacademy/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1123-education-course-khanacademy/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-1123-education-course-khanacademy/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-1123-education-course-khanacademy/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..a9d234d5 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1123-education-course-khanacademy/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "khanacademy\\.org/api/internal/graphql/attemptProblem", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1123-education-course-khanacademy/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-1123-education-course-khanacademy/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1123-education-course-khanacademy/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-1123-education-course-khanacademy/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-1123-education-course-khanacademy/steps/run/workdir/task.json new file mode 100644 index 00000000..a1823376 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1123-education-course-khanacademy/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1123, + "metaclass": "education", + "class": "course", + "description": "Navigate to the AP Biology Cell cycle lesson on Khan Academy and submit an answer to the practice exercise", + "sites_involved": [ + "khanacademy.org" + ], + "platform": "khanacademy", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Navigate to the AP Biology 'Cell Communication and Cell Cycle' unit on Khan Academy, go to the 'Cell cycle' lesson, and answer the first question of the Cell cycle practice exercise", + "eval_schema": { + "url_pattern": "khanacademy\\.org/api/internal/graphql/attemptProblem", + "method": "POST" + }, + "time_limit": 60, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1123-education-course-khanacademy/task.toml b/harbor/datasets/clawbench-v2/v2-1123-education-course-khanacademy/task.toml new file mode 100644 index 00000000..f07947d9 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1123-education-course-khanacademy/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-1123-education-course-khanacademy" +description = "Navigate to the AP Biology Cell cycle lesson on Khan Academy and submit an answer to the practice exercise" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-1123-education-course-khanacademy" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 3600.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-1124-education-course-khanacademy/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-1124-education-course-khanacademy/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1124-education-course-khanacademy/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-1124-education-course-khanacademy/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-1124-education-course-khanacademy/steps/run/instruction.md new file mode 100644 index 00000000..9866eb85 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1124-education-course-khanacademy/steps/run/instruction.md @@ -0,0 +1,27 @@ +Search for 'Organic Chemistry' on Khan Academy and add the course to your learning list + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-1124-education-course-khanacademy/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-1124-education-course-khanacademy/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1124-education-course-khanacademy/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-1124-education-course-khanacademy/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-1124-education-course-khanacademy/steps/run/tests/task.json new file mode 100644 index 00000000..49f4489c --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1124-education-course-khanacademy/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1124, + "metaclass": "education", + "class": "course", + "description": "Search for 'Organic Chemistry' on Khan Academy and add the course to your learning list", + "sites_involved": [ + "khanacademy.org" + ], + "platform": "khanacademy", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Search for 'Organic Chemistry' on Khan Academy and add the course to your learning list", + "eval_schema": { + "url_pattern": "khanacademy\\.org/api/internal/graphql/setUserCourses", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1124-education-course-khanacademy/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-1124-education-course-khanacademy/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1124-education-course-khanacademy/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-1124-education-course-khanacademy/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-1124-education-course-khanacademy/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..eea1b947 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1124-education-course-khanacademy/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "khanacademy\\.org/api/internal/graphql/setUserCourses", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1124-education-course-khanacademy/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-1124-education-course-khanacademy/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1124-education-course-khanacademy/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-1124-education-course-khanacademy/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-1124-education-course-khanacademy/steps/run/workdir/task.json new file mode 100644 index 00000000..49f4489c --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1124-education-course-khanacademy/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1124, + "metaclass": "education", + "class": "course", + "description": "Search for 'Organic Chemistry' on Khan Academy and add the course to your learning list", + "sites_involved": [ + "khanacademy.org" + ], + "platform": "khanacademy", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Search for 'Organic Chemistry' on Khan Academy and add the course to your learning list", + "eval_schema": { + "url_pattern": "khanacademy\\.org/api/internal/graphql/setUserCourses", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1124-education-course-khanacademy/task.toml b/harbor/datasets/clawbench-v2/v2-1124-education-course-khanacademy/task.toml new file mode 100644 index 00000000..b23c8e1f --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1124-education-course-khanacademy/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-1124-education-course-khanacademy" +description = "Search for 'Organic Chemistry' on Khan Academy and add the course to your learning list" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-1124-education-course-khanacademy" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-1125-education-course-khanacademy/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-1125-education-course-khanacademy/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1125-education-course-khanacademy/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-1125-education-course-khanacademy/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-1125-education-course-khanacademy/steps/run/instruction.md new file mode 100644 index 00000000..ea3e2726 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1125-education-course-khanacademy/steps/run/instruction.md @@ -0,0 +1,27 @@ +On Khan Academy, navigate to the AP US History course, go to the 'World War I: military and diplomacy' lesson, and answer the first question of the practice exercise + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-1125-education-course-khanacademy/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-1125-education-course-khanacademy/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1125-education-course-khanacademy/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-1125-education-course-khanacademy/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-1125-education-course-khanacademy/steps/run/tests/task.json new file mode 100644 index 00000000..d957781a --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1125-education-course-khanacademy/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1125, + "metaclass": "education", + "class": "course", + "description": "Navigate through AP US History to the WWI military and diplomacy practice exercise and submit an answer", + "sites_involved": [ + "khanacademy.org" + ], + "platform": "khanacademy", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "On Khan Academy, navigate to the AP US History course, go to the 'World War I: military and diplomacy' lesson, and answer the first question of the practice exercise", + "eval_schema": { + "url_pattern": "khanacademy\\.org/api/internal/graphql/attemptProblem", + "method": "POST" + }, + "time_limit": 60, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1125-education-course-khanacademy/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-1125-education-course-khanacademy/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1125-education-course-khanacademy/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-1125-education-course-khanacademy/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-1125-education-course-khanacademy/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..a9d234d5 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1125-education-course-khanacademy/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "khanacademy\\.org/api/internal/graphql/attemptProblem", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1125-education-course-khanacademy/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-1125-education-course-khanacademy/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1125-education-course-khanacademy/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-1125-education-course-khanacademy/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-1125-education-course-khanacademy/steps/run/workdir/task.json new file mode 100644 index 00000000..d957781a --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1125-education-course-khanacademy/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1125, + "metaclass": "education", + "class": "course", + "description": "Navigate through AP US History to the WWI military and diplomacy practice exercise and submit an answer", + "sites_involved": [ + "khanacademy.org" + ], + "platform": "khanacademy", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "On Khan Academy, navigate to the AP US History course, go to the 'World War I: military and diplomacy' lesson, and answer the first question of the practice exercise", + "eval_schema": { + "url_pattern": "khanacademy\\.org/api/internal/graphql/attemptProblem", + "method": "POST" + }, + "time_limit": 60, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1125-education-course-khanacademy/task.toml b/harbor/datasets/clawbench-v2/v2-1125-education-course-khanacademy/task.toml new file mode 100644 index 00000000..0e275d09 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1125-education-course-khanacademy/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-1125-education-course-khanacademy" +description = "Navigate through AP US History to the WWI military and diplomacy practice exercise and submit an answer" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-1125-education-course-khanacademy" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 3600.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-1126-food-cooking-collection-myrecipes/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-1126-food-cooking-collection-myrecipes/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1126-food-cooking-collection-myrecipes/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-1126-food-cooking-collection-myrecipes/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-1126-food-cooking-collection-myrecipes/steps/run/instruction.md new file mode 100644 index 00000000..85f852f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1126-food-cooking-collection-myrecipes/steps/run/instruction.md @@ -0,0 +1,27 @@ +Search for 'chicken soup' on MyRecipes and save the top result to your Want to Try collection + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-1126-food-cooking-collection-myrecipes/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-1126-food-cooking-collection-myrecipes/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1126-food-cooking-collection-myrecipes/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-1126-food-cooking-collection-myrecipes/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-1126-food-cooking-collection-myrecipes/steps/run/tests/task.json new file mode 100644 index 00000000..9724cab6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1126-food-cooking-collection-myrecipes/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1126, + "metaclass": "food-cooking", + "class": "collection", + "description": "Search for 'chicken soup' on MyRecipes and save the top result to your Want to Try collection", + "sites_involved": [ + "myrecipes.com" + ], + "platform": "myrecipes", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Search for 'chicken soup' on MyRecipes and save the top result to your Want to Try collection", + "eval_schema": { + "url_pattern": "myrecipes\\.com/collections/bookmarks/save", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1126-food-cooking-collection-myrecipes/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-1126-food-cooking-collection-myrecipes/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1126-food-cooking-collection-myrecipes/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-1126-food-cooking-collection-myrecipes/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-1126-food-cooking-collection-myrecipes/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..739c9319 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1126-food-cooking-collection-myrecipes/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "myrecipes\\.com/collections/bookmarks/save", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1126-food-cooking-collection-myrecipes/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-1126-food-cooking-collection-myrecipes/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1126-food-cooking-collection-myrecipes/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-1126-food-cooking-collection-myrecipes/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-1126-food-cooking-collection-myrecipes/steps/run/workdir/task.json new file mode 100644 index 00000000..9724cab6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1126-food-cooking-collection-myrecipes/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1126, + "metaclass": "food-cooking", + "class": "collection", + "description": "Search for 'chicken soup' on MyRecipes and save the top result to your Want to Try collection", + "sites_involved": [ + "myrecipes.com" + ], + "platform": "myrecipes", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Search for 'chicken soup' on MyRecipes and save the top result to your Want to Try collection", + "eval_schema": { + "url_pattern": "myrecipes\\.com/collections/bookmarks/save", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1126-food-cooking-collection-myrecipes/task.toml b/harbor/datasets/clawbench-v2/v2-1126-food-cooking-collection-myrecipes/task.toml new file mode 100644 index 00000000..f92b06f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1126-food-cooking-collection-myrecipes/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-1126-food-cooking-collection-myrecipes" +description = "Search for 'chicken soup' on MyRecipes and save the top result to your Want to Try collection" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-1126-food-cooking-collection-myrecipes" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-1130-training-class-search-redcross/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-1130-training-class-search-redcross/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1130-training-class-search-redcross/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-1130-training-class-search-redcross/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-1130-training-class-search-redcross/steps/run/instruction.md new file mode 100644 index 00000000..0a09784a --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1130-training-class-search-redcross/steps/run/instruction.md @@ -0,0 +1,27 @@ +Search for CPR certification classes in zip code 10001 on the Red Cross training website and find the first available in-person class + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-1130-training-class-search-redcross/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-1130-training-class-search-redcross/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1130-training-class-search-redcross/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-1130-training-class-search-redcross/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-1130-training-class-search-redcross/steps/run/tests/task.json new file mode 100644 index 00000000..bd7e4fc7 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1130-training-class-search-redcross/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1130, + "metaclass": "charity", + "class": "search", + "description": "Search for CPR certification classes by zip code on the Red Cross training site", + "sites_involved": [ + "redcross.org" + ], + "platform": "redcross", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Search for CPR certification classes in zip code 10001 on the Red Cross training website and find the first available in-person class", + "eval_schema": { + "url_pattern": "redcross\\.org\\/take-a-class\\/search.*(?:q|query|zip|searchtype|location)=", + "method": "GET" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1130-training-class-search-redcross/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-1130-training-class-search-redcross/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1130-training-class-search-redcross/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-1130-training-class-search-redcross/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-1130-training-class-search-redcross/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..3dbd5d38 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1130-training-class-search-redcross/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "redcross\\.org\\/take-a-class\\/search.*(?:q|query|zip|searchtype|location)=", + "method": "GET" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1130-training-class-search-redcross/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-1130-training-class-search-redcross/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1130-training-class-search-redcross/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-1130-training-class-search-redcross/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-1130-training-class-search-redcross/steps/run/workdir/task.json new file mode 100644 index 00000000..bd7e4fc7 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1130-training-class-search-redcross/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1130, + "metaclass": "charity", + "class": "search", + "description": "Search for CPR certification classes by zip code on the Red Cross training site", + "sites_involved": [ + "redcross.org" + ], + "platform": "redcross", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Search for CPR certification classes in zip code 10001 on the Red Cross training website and find the first available in-person class", + "eval_schema": { + "url_pattern": "redcross\\.org\\/take-a-class\\/search.*(?:q|query|zip|searchtype|location)=", + "method": "GET" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1130-training-class-search-redcross/task.toml b/harbor/datasets/clawbench-v2/v2-1130-training-class-search-redcross/task.toml new file mode 100644 index 00000000..9e90aaa3 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1130-training-class-search-redcross/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-1130-training-class-search-redcross" +description = "Search for CPR certification classes by zip code on the Red Cross training site" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-1130-training-class-search-redcross" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-1131-training-add-to-cart-redcross/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-1131-training-add-to-cart-redcross/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1131-training-add-to-cart-redcross/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-1131-training-add-to-cart-redcross/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-1131-training-add-to-cart-redcross/steps/run/instruction.md new file mode 100644 index 00000000..1487a146 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1131-training-add-to-cart-redcross/steps/run/instruction.md @@ -0,0 +1,27 @@ +Add the 'Adult First Aid/CPR/AED Online' course to the cart on the Red Cross training website + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-1131-training-add-to-cart-redcross/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-1131-training-add-to-cart-redcross/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1131-training-add-to-cart-redcross/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-1131-training-add-to-cart-redcross/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-1131-training-add-to-cart-redcross/steps/run/tests/task.json new file mode 100644 index 00000000..8dbd61cf --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1131-training-add-to-cart-redcross/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1131, + "metaclass": "charity", + "class": "training", + "description": "Add the Adult First Aid/CPR/AED Online course to the shopping cart without completing purchase", + "sites_involved": [ + "redcross.org" + ], + "platform": "redcross", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Add the 'Adult First Aid/CPR/AED Online' course to the cart on the Red Cross training website", + "eval_schema": { + "url_pattern": "redcross\\.org\\/on\\/demandware\\.store\\/Sites-RedCross-Site\\/default\\/Product-Detail", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1131-training-add-to-cart-redcross/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-1131-training-add-to-cart-redcross/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1131-training-add-to-cart-redcross/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-1131-training-add-to-cart-redcross/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-1131-training-add-to-cart-redcross/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..2028f61d --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1131-training-add-to-cart-redcross/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "redcross\\.org\\/on\\/demandware\\.store\\/Sites-RedCross-Site\\/default\\/Product-Detail", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1131-training-add-to-cart-redcross/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-1131-training-add-to-cart-redcross/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1131-training-add-to-cart-redcross/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-1131-training-add-to-cart-redcross/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-1131-training-add-to-cart-redcross/steps/run/workdir/task.json new file mode 100644 index 00000000..8dbd61cf --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1131-training-add-to-cart-redcross/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1131, + "metaclass": "charity", + "class": "training", + "description": "Add the Adult First Aid/CPR/AED Online course to the shopping cart without completing purchase", + "sites_involved": [ + "redcross.org" + ], + "platform": "redcross", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Add the 'Adult First Aid/CPR/AED Online' course to the cart on the Red Cross training website", + "eval_schema": { + "url_pattern": "redcross\\.org\\/on\\/demandware\\.store\\/Sites-RedCross-Site\\/default\\/Product-Detail", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1131-training-add-to-cart-redcross/task.toml b/harbor/datasets/clawbench-v2/v2-1131-training-add-to-cart-redcross/task.toml new file mode 100644 index 00000000..850e8f21 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1131-training-add-to-cart-redcross/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-1131-training-add-to-cart-redcross" +description = "Add the Adult First Aid/CPR/AED Online course to the shopping cart without completing purchase" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-1131-training-add-to-cart-redcross" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-1132-training-enroll-course-redcross/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-1132-training-enroll-course-redcross/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1132-training-enroll-course-redcross/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-1132-training-enroll-course-redcross/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-1132-training-enroll-course-redcross/steps/run/instruction.md new file mode 100644 index 00000000..d4a2fb3a --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1132-training-enroll-course-redcross/steps/run/instruction.md @@ -0,0 +1,27 @@ +Enroll in an online First Aid/CPR certification course on the Red Cross website + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-1132-training-enroll-course-redcross/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-1132-training-enroll-course-redcross/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1132-training-enroll-course-redcross/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-1132-training-enroll-course-redcross/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-1132-training-enroll-course-redcross/steps/run/tests/task.json new file mode 100644 index 00000000..6b903eb8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1132-training-enroll-course-redcross/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1132, + "metaclass": "charity", + "class": "training", + "description": "Enroll in an online First Aid/CPR certification course on the Red Cross website (cart checkout submission)", + "sites_involved": [ + "redcross.org" + ], + "platform": "redcross", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Enroll in an online First Aid/CPR certification course on the Red Cross website", + "eval_schema": { + "url_pattern": "redcross\\.org\\/take-a-class\\/cart\\/checkout-confirmation", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1132-training-enroll-course-redcross/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-1132-training-enroll-course-redcross/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1132-training-enroll-course-redcross/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-1132-training-enroll-course-redcross/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-1132-training-enroll-course-redcross/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..bc4baf2f --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1132-training-enroll-course-redcross/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "redcross\\.org\\/take-a-class\\/cart\\/checkout-confirmation", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1132-training-enroll-course-redcross/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-1132-training-enroll-course-redcross/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1132-training-enroll-course-redcross/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-1132-training-enroll-course-redcross/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-1132-training-enroll-course-redcross/steps/run/workdir/task.json new file mode 100644 index 00000000..6b903eb8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1132-training-enroll-course-redcross/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1132, + "metaclass": "charity", + "class": "training", + "description": "Enroll in an online First Aid/CPR certification course on the Red Cross website (cart checkout submission)", + "sites_involved": [ + "redcross.org" + ], + "platform": "redcross", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Enroll in an online First Aid/CPR certification course on the Red Cross website", + "eval_schema": { + "url_pattern": "redcross\\.org\\/take-a-class\\/cart\\/checkout-confirmation", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1132-training-enroll-course-redcross/task.toml b/harbor/datasets/clawbench-v2/v2-1132-training-enroll-course-redcross/task.toml new file mode 100644 index 00000000..d0278287 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1132-training-enroll-course-redcross/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-1132-training-enroll-course-redcross" +description = "Enroll in an online First Aid/CPR certification course on the Red Cross website (cart checkout submission)" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-1132-training-enroll-course-redcross" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-1133-newsletter-signup-redcross/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-1133-newsletter-signup-redcross/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1133-newsletter-signup-redcross/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-1133-newsletter-signup-redcross/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-1133-newsletter-signup-redcross/steps/run/instruction.md new file mode 100644 index 00000000..66b11af4 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1133-newsletter-signup-redcross/steps/run/instruction.md @@ -0,0 +1,27 @@ +Sign up for the Red Cross training newsletter using the email signup form on their training website + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-1133-newsletter-signup-redcross/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-1133-newsletter-signup-redcross/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1133-newsletter-signup-redcross/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-1133-newsletter-signup-redcross/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-1133-newsletter-signup-redcross/steps/run/tests/task.json new file mode 100644 index 00000000..1e0d011b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1133-newsletter-signup-redcross/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1133, + "metaclass": "charity", + "class": "newsletter", + "description": "Sign up for Red Cross training & safety email newsletter", + "sites_involved": [ + "redcross.org" + ], + "platform": "redcross", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Sign up for the Red Cross training newsletter using the email signup form on their training website", + "eval_schema": { + "url_pattern": "redcross\\.org\\/on\\/demandware\\.store\\/Sites-RedCross-Site\\/default\\/ExactTargetEmailSignUp", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1133-newsletter-signup-redcross/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-1133-newsletter-signup-redcross/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1133-newsletter-signup-redcross/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-1133-newsletter-signup-redcross/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-1133-newsletter-signup-redcross/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..0d396960 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1133-newsletter-signup-redcross/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "redcross\\.org\\/on\\/demandware\\.store\\/Sites-RedCross-Site\\/default\\/ExactTargetEmailSignUp", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1133-newsletter-signup-redcross/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-1133-newsletter-signup-redcross/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1133-newsletter-signup-redcross/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-1133-newsletter-signup-redcross/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-1133-newsletter-signup-redcross/steps/run/workdir/task.json new file mode 100644 index 00000000..1e0d011b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1133-newsletter-signup-redcross/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1133, + "metaclass": "charity", + "class": "newsletter", + "description": "Sign up for Red Cross training & safety email newsletter", + "sites_involved": [ + "redcross.org" + ], + "platform": "redcross", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Sign up for the Red Cross training newsletter using the email signup form on their training website", + "eval_schema": { + "url_pattern": "redcross\\.org\\/on\\/demandware\\.store\\/Sites-RedCross-Site\\/default\\/ExactTargetEmailSignUp", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1133-newsletter-signup-redcross/task.toml b/harbor/datasets/clawbench-v2/v2-1133-newsletter-signup-redcross/task.toml new file mode 100644 index 00000000..c2a83d40 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1133-newsletter-signup-redcross/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-1133-newsletter-signup-redcross" +description = "Sign up for Red Cross training & safety email newsletter" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-1133-newsletter-signup-redcross" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-1134-chapter-finder-redcross/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-1134-chapter-finder-redcross/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1134-chapter-finder-redcross/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-1134-chapter-finder-redcross/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-1134-chapter-finder-redcross/steps/run/instruction.md new file mode 100644 index 00000000..787ca997 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1134-chapter-finder-redcross/steps/run/instruction.md @@ -0,0 +1,27 @@ +Find the nearest Red Cross chapter to zip code 90210 using the local chapter finder on redcross.org + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-1134-chapter-finder-redcross/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-1134-chapter-finder-redcross/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1134-chapter-finder-redcross/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-1134-chapter-finder-redcross/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-1134-chapter-finder-redcross/steps/run/tests/task.json new file mode 100644 index 00000000..57acd1b8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1134-chapter-finder-redcross/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1134, + "metaclass": "charity", + "class": "search", + "description": "Find the nearest Red Cross local chapter by zip code using the chapter finder tool", + "sites_involved": [ + "redcross.org" + ], + "platform": "redcross", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Find the nearest Red Cross chapter to zip code 90210 using the local chapter finder on redcross.org", + "eval_schema": { + "url_pattern": "redcross\\.org\\/find-your-local-chapter", + "method": "GET" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1134-chapter-finder-redcross/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-1134-chapter-finder-redcross/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1134-chapter-finder-redcross/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-1134-chapter-finder-redcross/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-1134-chapter-finder-redcross/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..fae08989 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1134-chapter-finder-redcross/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "redcross\\.org\\/find-your-local-chapter", + "method": "GET" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1134-chapter-finder-redcross/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-1134-chapter-finder-redcross/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1134-chapter-finder-redcross/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-1134-chapter-finder-redcross/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-1134-chapter-finder-redcross/steps/run/workdir/task.json new file mode 100644 index 00000000..57acd1b8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1134-chapter-finder-redcross/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1134, + "metaclass": "charity", + "class": "search", + "description": "Find the nearest Red Cross local chapter by zip code using the chapter finder tool", + "sites_involved": [ + "redcross.org" + ], + "platform": "redcross", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Find the nearest Red Cross chapter to zip code 90210 using the local chapter finder on redcross.org", + "eval_schema": { + "url_pattern": "redcross\\.org\\/find-your-local-chapter", + "method": "GET" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1134-chapter-finder-redcross/task.toml b/harbor/datasets/clawbench-v2/v2-1134-chapter-finder-redcross/task.toml new file mode 100644 index 00000000..40ad0963 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1134-chapter-finder-redcross/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-1134-chapter-finder-redcross" +description = "Find the nearest Red Cross local chapter by zip code using the chapter finder tool" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-1134-chapter-finder-redcross" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-1135-events-tickets-ticketmaster/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-1135-events-tickets-ticketmaster/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1135-events-tickets-ticketmaster/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-1135-events-tickets-ticketmaster/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-1135-events-tickets-ticketmaster/steps/run/instruction.md new file mode 100644 index 00000000..89b54484 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1135-events-tickets-ticketmaster/steps/run/instruction.md @@ -0,0 +1,27 @@ +Search for upcoming NBA basketball games on Ticketmaster in a major US city and view available ticket pricing for 2 tickets + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-1135-events-tickets-ticketmaster/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-1135-events-tickets-ticketmaster/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1135-events-tickets-ticketmaster/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-1135-events-tickets-ticketmaster/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-1135-events-tickets-ticketmaster/steps/run/tests/task.json new file mode 100644 index 00000000..e519eedd --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1135-events-tickets-ticketmaster/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1135, + "metaclass": "events", + "class": "tickets", + "description": "Navigate to Ticketmaster, search for NBA basketball games in a major US city, open an event page, select 2 tickets, and view pricing.", + "sites_involved": [ + "ticketmaster.com" + ], + "platform": "ticketmaster", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Search for upcoming NBA basketball games on Ticketmaster in a major US city and view available ticket pricing for 2 tickets", + "eval_schema": { + "url_pattern": "ticketmaster\\.com\\/[^/]+-tickets-[^/]+\\/event\\/[A-Z0-9]+", + "method": "GET" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1135-events-tickets-ticketmaster/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-1135-events-tickets-ticketmaster/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1135-events-tickets-ticketmaster/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-1135-events-tickets-ticketmaster/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-1135-events-tickets-ticketmaster/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..1564c019 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1135-events-tickets-ticketmaster/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "ticketmaster\\.com\\/[^/]+-tickets-[^/]+\\/event\\/[A-Z0-9]+", + "method": "GET" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1135-events-tickets-ticketmaster/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-1135-events-tickets-ticketmaster/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1135-events-tickets-ticketmaster/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-1135-events-tickets-ticketmaster/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-1135-events-tickets-ticketmaster/steps/run/workdir/task.json new file mode 100644 index 00000000..e519eedd --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1135-events-tickets-ticketmaster/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1135, + "metaclass": "events", + "class": "tickets", + "description": "Navigate to Ticketmaster, search for NBA basketball games in a major US city, open an event page, select 2 tickets, and view pricing.", + "sites_involved": [ + "ticketmaster.com" + ], + "platform": "ticketmaster", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Search for upcoming NBA basketball games on Ticketmaster in a major US city and view available ticket pricing for 2 tickets", + "eval_schema": { + "url_pattern": "ticketmaster\\.com\\/[^/]+-tickets-[^/]+\\/event\\/[A-Z0-9]+", + "method": "GET" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1135-events-tickets-ticketmaster/task.toml b/harbor/datasets/clawbench-v2/v2-1135-events-tickets-ticketmaster/task.toml new file mode 100644 index 00000000..637e2ee0 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1135-events-tickets-ticketmaster/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-1135-events-tickets-ticketmaster" +description = "Navigate to Ticketmaster, search for NBA basketball games in a major US city, open an event page, select 2 tickets, and view pricing." +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-1135-events-tickets-ticketmaster" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-1136-events-tickets-ticketmaster/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-1136-events-tickets-ticketmaster/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1136-events-tickets-ticketmaster/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-1136-events-tickets-ticketmaster/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-1136-events-tickets-ticketmaster/steps/run/instruction.md new file mode 100644 index 00000000..1eefc63c --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1136-events-tickets-ticketmaster/steps/run/instruction.md @@ -0,0 +1,27 @@ +Find a Broadway or theater show in New York City on Ticketmaster, open the event page for one of the results, and use the 'Best Seats' option to view the top recommended seats for 4 tickets + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-1136-events-tickets-ticketmaster/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-1136-events-tickets-ticketmaster/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1136-events-tickets-ticketmaster/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-1136-events-tickets-ticketmaster/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-1136-events-tickets-ticketmaster/steps/run/tests/task.json new file mode 100644 index 00000000..165da118 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1136-events-tickets-ticketmaster/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1136, + "metaclass": "events", + "class": "tickets", + "description": "Find a Broadway or theater show in New York City on Ticketmaster, open the event page for one of the results, and use the 'Best Seats' option to view the top recommended seats for 4 tickets", + "sites_involved": [ + "ticketmaster.com" + ], + "platform": "ticketmaster", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Find a Broadway or theater show in New York City on Ticketmaster, open the event page for one of the results, and use the 'Best Seats' option to view the top recommended seats for 4 tickets", + "eval_schema": { + "url_pattern": "services\\.ticketmaster\\.com/api/ismds/event/[A-Z0-9]+/quickpicks.*qty=4.*sort=-quality", + "method": "GET" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1136-events-tickets-ticketmaster/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-1136-events-tickets-ticketmaster/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1136-events-tickets-ticketmaster/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-1136-events-tickets-ticketmaster/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-1136-events-tickets-ticketmaster/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..2f5b028c --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1136-events-tickets-ticketmaster/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "services\\.ticketmaster\\.com/api/ismds/event/[A-Z0-9]+/quickpicks.*qty=4.*sort=-quality", + "method": "GET" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1136-events-tickets-ticketmaster/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-1136-events-tickets-ticketmaster/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1136-events-tickets-ticketmaster/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-1136-events-tickets-ticketmaster/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-1136-events-tickets-ticketmaster/steps/run/workdir/task.json new file mode 100644 index 00000000..165da118 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1136-events-tickets-ticketmaster/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1136, + "metaclass": "events", + "class": "tickets", + "description": "Find a Broadway or theater show in New York City on Ticketmaster, open the event page for one of the results, and use the 'Best Seats' option to view the top recommended seats for 4 tickets", + "sites_involved": [ + "ticketmaster.com" + ], + "platform": "ticketmaster", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Find a Broadway or theater show in New York City on Ticketmaster, open the event page for one of the results, and use the 'Best Seats' option to view the top recommended seats for 4 tickets", + "eval_schema": { + "url_pattern": "services\\.ticketmaster\\.com/api/ismds/event/[A-Z0-9]+/quickpicks.*qty=4.*sort=-quality", + "method": "GET" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1136-events-tickets-ticketmaster/task.toml b/harbor/datasets/clawbench-v2/v2-1136-events-tickets-ticketmaster/task.toml new file mode 100644 index 00000000..05208803 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1136-events-tickets-ticketmaster/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-1136-events-tickets-ticketmaster" +description = "Find a Broadway or theater show in New York City on Ticketmaster, open the event page for one of the results, and use the 'Best Seats' option to view the top recommended seats for 4 tickets" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-1136-events-tickets-ticketmaster" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-1137-entertainment-watchlist-trakt/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-1137-entertainment-watchlist-trakt/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1137-entertainment-watchlist-trakt/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-1137-entertainment-watchlist-trakt/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-1137-entertainment-watchlist-trakt/steps/run/instruction.md new file mode 100644 index 00000000..24b7871b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1137-entertainment-watchlist-trakt/steps/run/instruction.md @@ -0,0 +1,27 @@ +Search for 'The Office' on Trakt.tv and rate it 10 out of 10 stars + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-1137-entertainment-watchlist-trakt/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-1137-entertainment-watchlist-trakt/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1137-entertainment-watchlist-trakt/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-1137-entertainment-watchlist-trakt/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-1137-entertainment-watchlist-trakt/steps/run/tests/task.json new file mode 100644 index 00000000..de8317a2 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1137-entertainment-watchlist-trakt/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1137, + "metaclass": "entertainment", + "class": "rating", + "description": "Search for 'The Office' on Trakt.tv and rate it 10 out of 10 stars", + "sites_involved": [ + "trakt.tv" + ], + "platform": "trakt", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Search for 'The Office' on Trakt.tv and rate it 10 out of 10 stars", + "eval_schema": { + "url_pattern": "trakt\\.tv.*(sync/ratings)", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1137-entertainment-watchlist-trakt/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-1137-entertainment-watchlist-trakt/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1137-entertainment-watchlist-trakt/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-1137-entertainment-watchlist-trakt/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-1137-entertainment-watchlist-trakt/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..94fe7efa --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1137-entertainment-watchlist-trakt/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "trakt\\.tv.*(sync/ratings)", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1137-entertainment-watchlist-trakt/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-1137-entertainment-watchlist-trakt/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1137-entertainment-watchlist-trakt/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-1137-entertainment-watchlist-trakt/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-1137-entertainment-watchlist-trakt/steps/run/workdir/task.json new file mode 100644 index 00000000..de8317a2 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1137-entertainment-watchlist-trakt/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1137, + "metaclass": "entertainment", + "class": "rating", + "description": "Search for 'The Office' on Trakt.tv and rate it 10 out of 10 stars", + "sites_involved": [ + "trakt.tv" + ], + "platform": "trakt", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Search for 'The Office' on Trakt.tv and rate it 10 out of 10 stars", + "eval_schema": { + "url_pattern": "trakt\\.tv.*(sync/ratings)", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1137-entertainment-watchlist-trakt/task.toml b/harbor/datasets/clawbench-v2/v2-1137-entertainment-watchlist-trakt/task.toml new file mode 100644 index 00000000..8baeb95c --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1137-entertainment-watchlist-trakt/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-1137-entertainment-watchlist-trakt" +description = "Search for 'The Office' on Trakt.tv and rate it 10 out of 10 stars" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-1137-entertainment-watchlist-trakt" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-1138-entertainment-watchlist-trakt/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-1138-entertainment-watchlist-trakt/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1138-entertainment-watchlist-trakt/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-1138-entertainment-watchlist-trakt/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-1138-entertainment-watchlist-trakt/steps/run/instruction.md new file mode 100644 index 00000000..e1ad0dc5 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1138-entertainment-watchlist-trakt/steps/run/instruction.md @@ -0,0 +1,27 @@ +Mark the first 3 episodes of 'Stranger Things' Season 1 as watched on Trakt.tv + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-1138-entertainment-watchlist-trakt/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-1138-entertainment-watchlist-trakt/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1138-entertainment-watchlist-trakt/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-1138-entertainment-watchlist-trakt/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-1138-entertainment-watchlist-trakt/steps/run/tests/task.json new file mode 100644 index 00000000..01c413c8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1138-entertainment-watchlist-trakt/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1138, + "metaclass": "entertainment", + "class": "history", + "description": "Mark the first 3 episodes of 'Stranger Things' Season 1 as watched on Trakt.tv", + "sites_involved": [ + "trakt.tv" + ], + "platform": "trakt", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Mark the first 3 episodes of 'Stranger Things' Season 1 as watched on Trakt.tv", + "eval_schema": { + "url_pattern": "trakt\\.tv.*(sync/history)", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1138-entertainment-watchlist-trakt/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-1138-entertainment-watchlist-trakt/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1138-entertainment-watchlist-trakt/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-1138-entertainment-watchlist-trakt/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-1138-entertainment-watchlist-trakt/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..9cba6569 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1138-entertainment-watchlist-trakt/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "trakt\\.tv.*(sync/history)", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1138-entertainment-watchlist-trakt/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-1138-entertainment-watchlist-trakt/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1138-entertainment-watchlist-trakt/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-1138-entertainment-watchlist-trakt/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-1138-entertainment-watchlist-trakt/steps/run/workdir/task.json new file mode 100644 index 00000000..01c413c8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1138-entertainment-watchlist-trakt/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1138, + "metaclass": "entertainment", + "class": "history", + "description": "Mark the first 3 episodes of 'Stranger Things' Season 1 as watched on Trakt.tv", + "sites_involved": [ + "trakt.tv" + ], + "platform": "trakt", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Mark the first 3 episodes of 'Stranger Things' Season 1 as watched on Trakt.tv", + "eval_schema": { + "url_pattern": "trakt\\.tv.*(sync/history)", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1138-entertainment-watchlist-trakt/task.toml b/harbor/datasets/clawbench-v2/v2-1138-entertainment-watchlist-trakt/task.toml new file mode 100644 index 00000000..8bab4b90 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1138-entertainment-watchlist-trakt/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-1138-entertainment-watchlist-trakt" +description = "Mark the first 3 episodes of 'Stranger Things' Season 1 as watched on Trakt.tv" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-1138-entertainment-watchlist-trakt" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-1139-travel-itinerary-tripit/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-1139-travel-itinerary-tripit/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1139-travel-itinerary-tripit/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-1139-travel-itinerary-tripit/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-1139-travel-itinerary-tripit/steps/run/instruction.md new file mode 100644 index 00000000..59c8c6b3 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1139-travel-itinerary-tripit/steps/run/instruction.md @@ -0,0 +1,27 @@ +Create a travel itinerary on TripIt for a 4-day weekend trip to New York from May 23 to May 26 flying from LAX + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-1139-travel-itinerary-tripit/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-1139-travel-itinerary-tripit/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1139-travel-itinerary-tripit/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-1139-travel-itinerary-tripit/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-1139-travel-itinerary-tripit/steps/run/tests/task.json new file mode 100644 index 00000000..64738c54 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1139-travel-itinerary-tripit/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1139, + "metaclass": "travel", + "class": "itinerary", + "description": "Create a travel itinerary on TripIt for a 4-day weekend trip to New York from May 23 to May 26 flying from LAX", + "sites_involved": [ + "tripit.com" + ], + "platform": "tripit", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Create a travel itinerary on TripIt for a 4-day weekend trip to New York from May 23 to May 26 flying from LAX", + "eval_schema": { + "url_pattern": "tripit\\.com/api/v\\d+/create", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1139-travel-itinerary-tripit/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-1139-travel-itinerary-tripit/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1139-travel-itinerary-tripit/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-1139-travel-itinerary-tripit/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-1139-travel-itinerary-tripit/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..f2199052 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1139-travel-itinerary-tripit/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "tripit\\.com/api/v\\d+/create", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1139-travel-itinerary-tripit/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-1139-travel-itinerary-tripit/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1139-travel-itinerary-tripit/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-1139-travel-itinerary-tripit/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-1139-travel-itinerary-tripit/steps/run/workdir/task.json new file mode 100644 index 00000000..64738c54 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1139-travel-itinerary-tripit/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1139, + "metaclass": "travel", + "class": "itinerary", + "description": "Create a travel itinerary on TripIt for a 4-day weekend trip to New York from May 23 to May 26 flying from LAX", + "sites_involved": [ + "tripit.com" + ], + "platform": "tripit", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Create a travel itinerary on TripIt for a 4-day weekend trip to New York from May 23 to May 26 flying from LAX", + "eval_schema": { + "url_pattern": "tripit\\.com/api/v\\d+/create", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1139-travel-itinerary-tripit/task.toml b/harbor/datasets/clawbench-v2/v2-1139-travel-itinerary-tripit/task.toml new file mode 100644 index 00000000..b40b9f2a --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1139-travel-itinerary-tripit/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-1139-travel-itinerary-tripit" +description = "Create a travel itinerary on TripIt for a 4-day weekend trip to New York from May 23 to May 26 flying from LAX" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-1139-travel-itinerary-tripit" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-1140-travel-itinerary-tripit/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-1140-travel-itinerary-tripit/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1140-travel-itinerary-tripit/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-1140-travel-itinerary-tripit/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-1140-travel-itinerary-tripit/steps/run/instruction.md new file mode 100644 index 00000000..865bc29d --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1140-travel-itinerary-tripit/steps/run/instruction.md @@ -0,0 +1,27 @@ +Add a hotel reservation at the Hilton Midtown Manhattan from May 23-26 to your existing New York trip on TripIt + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-1140-travel-itinerary-tripit/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-1140-travel-itinerary-tripit/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1140-travel-itinerary-tripit/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-1140-travel-itinerary-tripit/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-1140-travel-itinerary-tripit/steps/run/tests/task.json new file mode 100644 index 00000000..29edae73 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1140-travel-itinerary-tripit/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1140, + "metaclass": "travel", + "class": "reservation", + "description": "Add a hotel reservation at the Hilton Midtown Manhattan from May 23-26 to your existing New York trip on TripIt", + "sites_involved": [ + "tripit.com" + ], + "platform": "tripit", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Add a hotel reservation at the Hilton Midtown Manhattan from May 23-26 to your existing New York trip on TripIt", + "eval_schema": { + "url_pattern": "tripit\\.com/api/v\\d+/create", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1140-travel-itinerary-tripit/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-1140-travel-itinerary-tripit/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1140-travel-itinerary-tripit/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-1140-travel-itinerary-tripit/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-1140-travel-itinerary-tripit/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..f2199052 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1140-travel-itinerary-tripit/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "tripit\\.com/api/v\\d+/create", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1140-travel-itinerary-tripit/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-1140-travel-itinerary-tripit/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1140-travel-itinerary-tripit/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-1140-travel-itinerary-tripit/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-1140-travel-itinerary-tripit/steps/run/workdir/task.json new file mode 100644 index 00000000..29edae73 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1140-travel-itinerary-tripit/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1140, + "metaclass": "travel", + "class": "reservation", + "description": "Add a hotel reservation at the Hilton Midtown Manhattan from May 23-26 to your existing New York trip on TripIt", + "sites_involved": [ + "tripit.com" + ], + "platform": "tripit", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Add a hotel reservation at the Hilton Midtown Manhattan from May 23-26 to your existing New York trip on TripIt", + "eval_schema": { + "url_pattern": "tripit\\.com/api/v\\d+/create", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1140-travel-itinerary-tripit/task.toml b/harbor/datasets/clawbench-v2/v2-1140-travel-itinerary-tripit/task.toml new file mode 100644 index 00000000..9d5b14f8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1140-travel-itinerary-tripit/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-1140-travel-itinerary-tripit" +description = "Add a hotel reservation at the Hilton Midtown Manhattan from May 23-26 to your existing New York trip on TripIt" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-1140-travel-itinerary-tripit" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-1141-travel-itinerary-tripit/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-1141-travel-itinerary-tripit/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1141-travel-itinerary-tripit/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-1141-travel-itinerary-tripit/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-1141-travel-itinerary-tripit/steps/run/instruction.md new file mode 100644 index 00000000..b3efca9c --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1141-travel-itinerary-tripit/steps/run/instruction.md @@ -0,0 +1,27 @@ +Create a trip on TripIt for a business conference in Seattle from August 10 to August 13 with a flight from SFO + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-1141-travel-itinerary-tripit/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-1141-travel-itinerary-tripit/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1141-travel-itinerary-tripit/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-1141-travel-itinerary-tripit/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-1141-travel-itinerary-tripit/steps/run/tests/task.json new file mode 100644 index 00000000..a70d518c --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1141-travel-itinerary-tripit/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1141, + "metaclass": "travel", + "class": "itinerary", + "description": "Create a trip on TripIt for a business conference in Seattle from August 10 to August 13 with a flight from SFO", + "sites_involved": [ + "tripit.com" + ], + "platform": "tripit", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Create a trip on TripIt for a business conference in Seattle from August 10 to August 13 with a flight from SFO", + "eval_schema": { + "url_pattern": "tripit\\.com/api/v\\d+/create", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1141-travel-itinerary-tripit/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-1141-travel-itinerary-tripit/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1141-travel-itinerary-tripit/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-1141-travel-itinerary-tripit/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-1141-travel-itinerary-tripit/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..f2199052 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1141-travel-itinerary-tripit/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "tripit\\.com/api/v\\d+/create", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1141-travel-itinerary-tripit/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-1141-travel-itinerary-tripit/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1141-travel-itinerary-tripit/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-1141-travel-itinerary-tripit/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-1141-travel-itinerary-tripit/steps/run/workdir/task.json new file mode 100644 index 00000000..a70d518c --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1141-travel-itinerary-tripit/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1141, + "metaclass": "travel", + "class": "itinerary", + "description": "Create a trip on TripIt for a business conference in Seattle from August 10 to August 13 with a flight from SFO", + "sites_involved": [ + "tripit.com" + ], + "platform": "tripit", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Create a trip on TripIt for a business conference in Seattle from August 10 to August 13 with a flight from SFO", + "eval_schema": { + "url_pattern": "tripit\\.com/api/v\\d+/create", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1141-travel-itinerary-tripit/task.toml b/harbor/datasets/clawbench-v2/v2-1141-travel-itinerary-tripit/task.toml new file mode 100644 index 00000000..ec547d67 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1141-travel-itinerary-tripit/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-1141-travel-itinerary-tripit" +description = "Create a trip on TripIt for a business conference in Seattle from August 10 to August 13 with a flight from SFO" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-1141-travel-itinerary-tripit" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-1144-weworkremotely/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-1144-weworkremotely/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1144-weworkremotely/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-1144-weworkremotely/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-1144-weworkremotely/steps/run/instruction.md new file mode 100644 index 00000000..edbd9b37 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1144-weworkremotely/steps/run/instruction.md @@ -0,0 +1,27 @@ +Go to We Work Remotely (https://weworkremotely.com/remote-jobs/new) and post a remote job listing for a 'Senior Backend Developer' position in the 'Back-End Programming' category. Fill in all required fields across all steps and place the order. + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-1144-weworkremotely/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-1144-weworkremotely/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1144-weworkremotely/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-1144-weworkremotely/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-1144-weworkremotely/steps/run/tests/task.json new file mode 100644 index 00000000..5c7fbdc1 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1144-weworkremotely/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1144, + "metaclass": "jobs", + "class": "posting", + "description": "Go to We Work Remotely (https://weworkremotely.com/remote-jobs/new) and post a remote job listing for a 'Senior Backend Developer' position in the 'Back-End Programming' category. Fill in all required fields across all steps and place the order.", + "sites_involved": [ + "weworkremotely.com" + ], + "platform": "weworkremotely", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Go to We Work Remotely (https://weworkremotely.com/remote-jobs/new) and post a remote job listing for a 'Senior Backend Developer' position in the 'Back-End Programming' category. Fill in all required fields across all steps and place the order.", + "eval_schema": { + "url_pattern": "weworkremotely\\.com/listings", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1144-weworkremotely/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-1144-weworkremotely/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1144-weworkremotely/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-1144-weworkremotely/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-1144-weworkremotely/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..a0db4258 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1144-weworkremotely/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "weworkremotely\\.com/listings", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1144-weworkremotely/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-1144-weworkremotely/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1144-weworkremotely/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-1144-weworkremotely/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-1144-weworkremotely/steps/run/workdir/task.json new file mode 100644 index 00000000..5c7fbdc1 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1144-weworkremotely/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1144, + "metaclass": "jobs", + "class": "posting", + "description": "Go to We Work Remotely (https://weworkremotely.com/remote-jobs/new) and post a remote job listing for a 'Senior Backend Developer' position in the 'Back-End Programming' category. Fill in all required fields across all steps and place the order.", + "sites_involved": [ + "weworkremotely.com" + ], + "platform": "weworkremotely", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Go to We Work Remotely (https://weworkremotely.com/remote-jobs/new) and post a remote job listing for a 'Senior Backend Developer' position in the 'Back-End Programming' category. Fill in all required fields across all steps and place the order.", + "eval_schema": { + "url_pattern": "weworkremotely\\.com/listings", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1144-weworkremotely/task.toml b/harbor/datasets/clawbench-v2/v2-1144-weworkremotely/task.toml new file mode 100644 index 00000000..f8379ece --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1144-weworkremotely/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-1144-weworkremotely" +description = "Go to We Work Remotely (https://weworkremotely.com/remote-jobs/new) and post a remote job listing for a 'Senior Backend Developer' position in the 'Back-End Programming' category. Fill in all required fields across all steps and place the order." +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-1144-weworkremotely" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-1145-weworkremotely/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-1145-weworkremotely/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1145-weworkremotely/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-1145-weworkremotely/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-1145-weworkremotely/steps/run/instruction.md new file mode 100644 index 00000000..edf6bc9b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1145-weworkremotely/steps/run/instruction.md @@ -0,0 +1,27 @@ +Go to We Work Remotely (https://weworkremotely.com/remote-jobs/new) and post a remote job listing for a 'Senior UI/UX Designer' position in the 'Design' category. Fill in all required fields across all steps and place the order. + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-1145-weworkremotely/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-1145-weworkremotely/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1145-weworkremotely/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-1145-weworkremotely/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-1145-weworkremotely/steps/run/tests/task.json new file mode 100644 index 00000000..311643d0 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1145-weworkremotely/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1145, + "metaclass": "jobs", + "class": "posting", + "description": "Go to We Work Remotely (https://weworkremotely.com/remote-jobs/new) and post a remote job listing for a 'Senior UI/UX Designer' position in the 'Design' category. Fill in all required fields across all steps and place the order.", + "sites_involved": [ + "weworkremotely.com" + ], + "platform": "weworkremotely", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Go to We Work Remotely (https://weworkremotely.com/remote-jobs/new) and post a remote job listing for a 'Senior UI/UX Designer' position in the 'Design' category. Fill in all required fields across all steps and place the order.", + "eval_schema": { + "url_pattern": "weworkremotely\\.com/listings", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1145-weworkremotely/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-1145-weworkremotely/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1145-weworkremotely/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-1145-weworkremotely/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-1145-weworkremotely/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..a0db4258 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1145-weworkremotely/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "weworkremotely\\.com/listings", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1145-weworkremotely/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-1145-weworkremotely/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1145-weworkremotely/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-1145-weworkremotely/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-1145-weworkremotely/steps/run/workdir/task.json new file mode 100644 index 00000000..311643d0 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1145-weworkremotely/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1145, + "metaclass": "jobs", + "class": "posting", + "description": "Go to We Work Remotely (https://weworkremotely.com/remote-jobs/new) and post a remote job listing for a 'Senior UI/UX Designer' position in the 'Design' category. Fill in all required fields across all steps and place the order.", + "sites_involved": [ + "weworkremotely.com" + ], + "platform": "weworkremotely", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Go to We Work Remotely (https://weworkremotely.com/remote-jobs/new) and post a remote job listing for a 'Senior UI/UX Designer' position in the 'Design' category. Fill in all required fields across all steps and place the order.", + "eval_schema": { + "url_pattern": "weworkremotely\\.com/listings", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1145-weworkremotely/task.toml b/harbor/datasets/clawbench-v2/v2-1145-weworkremotely/task.toml new file mode 100644 index 00000000..d3625fcb --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1145-weworkremotely/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-1145-weworkremotely" +description = "Go to We Work Remotely (https://weworkremotely.com/remote-jobs/new) and post a remote job listing for a 'Senior UI/UX Designer' position in the 'Design' category. Fill in all required fields across all steps and place the order." +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-1145-weworkremotely" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-1146-weworkremotely/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-1146-weworkremotely/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1146-weworkremotely/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-1146-weworkremotely/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-1146-weworkremotely/steps/run/instruction.md new file mode 100644 index 00000000..0592364a --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1146-weworkremotely/steps/run/instruction.md @@ -0,0 +1,29 @@ +Go to We Work Remotely (https://weworkremotely.com). Find any remote job listing in the 'Back-End Programming' category and save it by clicking the 'Save Job' button on the job detail page. + +Note: The user is already logged in. Any available Back-End Programming job listing works. The 'Save Job' button appears on the individual job detail page. After clicking, the button changes to 'Remove Job' to confirm success. + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-1146-weworkremotely/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-1146-weworkremotely/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1146-weworkremotely/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-1146-weworkremotely/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-1146-weworkremotely/steps/run/tests/task.json new file mode 100644 index 00000000..2001dc75 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1146-weworkremotely/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1146, + "metaclass": "jobs", + "class": "posting", + "description": "Go to We Work Remotely (https://weworkremotely.com). Find any remote job listing in the 'Back-End Programming' category and save it by clicking the 'Save Job' button on the job detail page.", + "sites_involved": [ + "weworkremotely.com" + ], + "platform": "weworkremotely", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Go to We Work Remotely (https://weworkremotely.com). Find any remote job listing in the 'Back-End Programming' category and save it by clicking the 'Save Job' button on the job detail page.\n\nNote: The user is already logged in. Any available Back-End Programming job listing works. The 'Save Job' button appears on the individual job detail page. After clicking, the button changes to 'Remove Job' to confirm success.", + "eval_schema": { + "url_pattern": "weworkremotely\\.com/listings/[^/]+/toggle_save", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1146-weworkremotely/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-1146-weworkremotely/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1146-weworkremotely/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-1146-weworkremotely/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-1146-weworkremotely/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..ebd67e79 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1146-weworkremotely/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "weworkremotely\\.com/listings/[^/]+/toggle_save", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1146-weworkremotely/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-1146-weworkremotely/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1146-weworkremotely/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-1146-weworkremotely/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-1146-weworkremotely/steps/run/workdir/task.json new file mode 100644 index 00000000..2001dc75 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1146-weworkremotely/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1146, + "metaclass": "jobs", + "class": "posting", + "description": "Go to We Work Remotely (https://weworkremotely.com). Find any remote job listing in the 'Back-End Programming' category and save it by clicking the 'Save Job' button on the job detail page.", + "sites_involved": [ + "weworkremotely.com" + ], + "platform": "weworkremotely", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Go to We Work Remotely (https://weworkremotely.com). Find any remote job listing in the 'Back-End Programming' category and save it by clicking the 'Save Job' button on the job detail page.\n\nNote: The user is already logged in. Any available Back-End Programming job listing works. The 'Save Job' button appears on the individual job detail page. After clicking, the button changes to 'Remove Job' to confirm success.", + "eval_schema": { + "url_pattern": "weworkremotely\\.com/listings/[^/]+/toggle_save", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1146-weworkremotely/task.toml b/harbor/datasets/clawbench-v2/v2-1146-weworkremotely/task.toml new file mode 100644 index 00000000..17bce1b3 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1146-weworkremotely/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-1146-weworkremotely" +description = "Go to We Work Remotely (https://weworkremotely.com). Find any remote job listing in the 'Back-End Programming' category and save it by clicking the 'Save Job' button on the job detail page." +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-1146-weworkremotely" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-1201-beauty-cart-theordinary/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-1201-beauty-cart-theordinary/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1201-beauty-cart-theordinary/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-1201-beauty-cart-theordinary/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-1201-beauty-cart-theordinary/steps/run/instruction.md new file mode 100644 index 00000000..dcd09530 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1201-beauty-cart-theordinary/steps/run/instruction.md @@ -0,0 +1,27 @@ +Search for 'retinol' on The Ordinary website, then add the Granactive Retinoid 2% Emulsion to the cart + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-1201-beauty-cart-theordinary/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-1201-beauty-cart-theordinary/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1201-beauty-cart-theordinary/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-1201-beauty-cart-theordinary/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-1201-beauty-cart-theordinary/steps/run/tests/task.json new file mode 100644 index 00000000..788d0ade --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1201-beauty-cart-theordinary/steps/run/tests/task.json @@ -0,0 +1,28 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1201, + "metaclass": "beauty-personal-care", + "class": "cart", + "description": "Use the site search to find a retinoid product and add it to cart, testing both the search flow and cart functionality", + "sites_involved": [ + "theordinary.com" + ], + "platform": "theordinary", + "product_urls": [ + "https://theordinary.com/en-us/granactive-retinoid-2-emulsion-100458.html" + ], + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Search for 'retinol' on The Ordinary website, then add the Granactive Retinoid 2% Emulsion to the cart", + "eval_schema": { + "url_pattern": "theordinary\\.com.*Cart-AddProduct", + "method": "POST" + }, + "time_limit": 45, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1201-beauty-cart-theordinary/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-1201-beauty-cart-theordinary/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1201-beauty-cart-theordinary/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-1201-beauty-cart-theordinary/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-1201-beauty-cart-theordinary/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..d429958b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1201-beauty-cart-theordinary/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "theordinary\\.com.*Cart-AddProduct", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1201-beauty-cart-theordinary/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-1201-beauty-cart-theordinary/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1201-beauty-cart-theordinary/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-1201-beauty-cart-theordinary/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-1201-beauty-cart-theordinary/steps/run/workdir/task.json new file mode 100644 index 00000000..788d0ade --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1201-beauty-cart-theordinary/steps/run/workdir/task.json @@ -0,0 +1,28 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1201, + "metaclass": "beauty-personal-care", + "class": "cart", + "description": "Use the site search to find a retinoid product and add it to cart, testing both the search flow and cart functionality", + "sites_involved": [ + "theordinary.com" + ], + "platform": "theordinary", + "product_urls": [ + "https://theordinary.com/en-us/granactive-retinoid-2-emulsion-100458.html" + ], + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Search for 'retinol' on The Ordinary website, then add the Granactive Retinoid 2% Emulsion to the cart", + "eval_schema": { + "url_pattern": "theordinary\\.com.*Cart-AddProduct", + "method": "POST" + }, + "time_limit": 45, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1201-beauty-cart-theordinary/task.toml b/harbor/datasets/clawbench-v2/v2-1201-beauty-cart-theordinary/task.toml new file mode 100644 index 00000000..0adf81d3 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1201-beauty-cart-theordinary/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-1201-beauty-cart-theordinary" +description = "Use the site search to find a retinoid product and add it to cart, testing both the search flow and cart functionality" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-1201-beauty-cart-theordinary" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 2700.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-1202-beauty-cart-quantity-theordinary/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-1202-beauty-cart-quantity-theordinary/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1202-beauty-cart-quantity-theordinary/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-1202-beauty-cart-quantity-theordinary/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-1202-beauty-cart-quantity-theordinary/steps/run/instruction.md new file mode 100644 index 00000000..91adadb7 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1202-beauty-cart-quantity-theordinary/steps/run/instruction.md @@ -0,0 +1,27 @@ +Add 2 units of The Ordinary Multi-Peptide + HA Serum to the cart on The Ordinary website + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-1202-beauty-cart-quantity-theordinary/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-1202-beauty-cart-quantity-theordinary/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1202-beauty-cart-quantity-theordinary/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-1202-beauty-cart-quantity-theordinary/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-1202-beauty-cart-quantity-theordinary/steps/run/tests/task.json new file mode 100644 index 00000000..e9f49753 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1202-beauty-cart-quantity-theordinary/steps/run/tests/task.json @@ -0,0 +1,28 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1202, + "metaclass": "beauty-personal-care", + "class": "cart", + "description": "Navigate to a product page, change the quantity to 2, and add to cart — tests the quantity selector interaction before adding to cart", + "sites_involved": [ + "theordinary.com" + ], + "platform": "theordinary", + "product_urls": [ + "https://theordinary.com/en-us/multi-peptide-ha-serum-100444.html" + ], + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Add 2 units of The Ordinary Multi-Peptide + HA Serum to the cart on The Ordinary website", + "eval_schema": { + "url_pattern": "theordinary\\.com.*Cart-AddProduct", + "method": "POST" + }, + "time_limit": 45, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1202-beauty-cart-quantity-theordinary/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-1202-beauty-cart-quantity-theordinary/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1202-beauty-cart-quantity-theordinary/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-1202-beauty-cart-quantity-theordinary/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-1202-beauty-cart-quantity-theordinary/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..d429958b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1202-beauty-cart-quantity-theordinary/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "theordinary\\.com.*Cart-AddProduct", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1202-beauty-cart-quantity-theordinary/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-1202-beauty-cart-quantity-theordinary/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1202-beauty-cart-quantity-theordinary/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-1202-beauty-cart-quantity-theordinary/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-1202-beauty-cart-quantity-theordinary/steps/run/workdir/task.json new file mode 100644 index 00000000..e9f49753 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1202-beauty-cart-quantity-theordinary/steps/run/workdir/task.json @@ -0,0 +1,28 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 1202, + "metaclass": "beauty-personal-care", + "class": "cart", + "description": "Navigate to a product page, change the quantity to 2, and add to cart — tests the quantity selector interaction before adding to cart", + "sites_involved": [ + "theordinary.com" + ], + "platform": "theordinary", + "product_urls": [ + "https://theordinary.com/en-us/multi-peptide-ha-serum-100444.html" + ], + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Add 2 units of The Ordinary Multi-Peptide + HA Serum to the cart on The Ordinary website", + "eval_schema": { + "url_pattern": "theordinary\\.com.*Cart-AddProduct", + "method": "POST" + }, + "time_limit": 45, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-1202-beauty-cart-quantity-theordinary/task.toml b/harbor/datasets/clawbench-v2/v2-1202-beauty-cart-quantity-theordinary/task.toml new file mode 100644 index 00000000..e9ab619c --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-1202-beauty-cart-quantity-theordinary/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-1202-beauty-cart-quantity-theordinary" +description = "Navigate to a product page, change the quantity to 2, and add to cart \u2014 tests the quantity selector interaction before adding to cart" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-1202-beauty-cart-quantity-theordinary" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 2700.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-179-dev-tech-github-ops-github/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-179-dev-tech-github-ops-github/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-179-dev-tech-github-ops-github/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-179-dev-tech-github-ops-github/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-179-dev-tech-github-ops-github/steps/run/instruction.md new file mode 100644 index 00000000..dbb1a374 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-179-dev-tech-github-ops-github/steps/run/instruction.md @@ -0,0 +1,30 @@ +Create repo "openclaw-agent-benchmark" on GitHub + README + MIT LICENSE + .gitignore + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +Additional files are also available under /my-info/ for this task: +- config.json: Configuration details for the task + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-179-dev-tech-github-ops-github/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-179-dev-tech-github-ops-github/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-179-dev-tech-github-ops-github/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-179-dev-tech-github-ops-github/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-179-dev-tech-github-ops-github/steps/run/tests/task.json new file mode 100644 index 00000000..5340f5c7 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-179-dev-tech-github-ops-github/steps/run/tests/task.json @@ -0,0 +1,30 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 179, + "metaclass": "dev-tech", + "class": "github-ops", + "description": "Create repo \"openclaw-agent-benchmark\" on GitHub + README + MIT LICENSE + .gitignore", + "sites_involved": [ + "github.com" + ], + "platform": "github", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Create repo \"openclaw-agent-benchmark\" on GitHub + README + MIT LICENSE + .gitignore", + "eval_schema": { + "url_pattern": "github\\.com/repositories$", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [ + { + "path": "extra_info/config.json", + "description": "Configuration details for the task" + } + ] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-179-dev-tech-github-ops-github/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-179-dev-tech-github-ops-github/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-179-dev-tech-github-ops-github/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-179-dev-tech-github-ops-github/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-179-dev-tech-github-ops-github/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..2b7fe562 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-179-dev-tech-github-ops-github/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "github\\.com/repositories$", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-179-dev-tech-github-ops-github/steps/run/workdir/extra_info/config.json b/harbor/datasets/clawbench-v2/v2-179-dev-tech-github-ops-github/steps/run/workdir/extra_info/config.json new file mode 100644 index 00000000..835ecf1e --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-179-dev-tech-github-ops-github/steps/run/workdir/extra_info/config.json @@ -0,0 +1,13 @@ +{ + "repo_name": "openclaw-agent-benchmark", + "visibility": "public", + "license": "MIT", + "description": "A benchmark for evaluating browser agents on real-world web tasks", + "topics": [ + "benchmark", + "browser-agent", + "evaluation" + ], + "has_wiki": false, + "has_issues": true +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-179-dev-tech-github-ops-github/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-179-dev-tech-github-ops-github/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-179-dev-tech-github-ops-github/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-179-dev-tech-github-ops-github/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-179-dev-tech-github-ops-github/steps/run/workdir/task.json new file mode 100644 index 00000000..5340f5c7 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-179-dev-tech-github-ops-github/steps/run/workdir/task.json @@ -0,0 +1,30 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 179, + "metaclass": "dev-tech", + "class": "github-ops", + "description": "Create repo \"openclaw-agent-benchmark\" on GitHub + README + MIT LICENSE + .gitignore", + "sites_involved": [ + "github.com" + ], + "platform": "github", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Create repo \"openclaw-agent-benchmark\" on GitHub + README + MIT LICENSE + .gitignore", + "eval_schema": { + "url_pattern": "github\\.com/repositories$", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [ + { + "path": "extra_info/config.json", + "description": "Configuration details for the task" + } + ] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-179-dev-tech-github-ops-github/task.toml b/harbor/datasets/clawbench-v2/v2-179-dev-tech-github-ops-github/task.toml new file mode 100644 index 00000000..014ccceb --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-179-dev-tech-github-ops-github/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-179-dev-tech-github-ops-github" +description = "Create repo \"openclaw-agent-benchmark\" on GitHub + README + MIT LICENSE + .gitignore" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-179-dev-tech-github-ops-github" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-180-dev-tech-github-ops-github/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-180-dev-tech-github-ops-github/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-180-dev-tech-github-ops-github/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-180-dev-tech-github-ops-github/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-180-dev-tech-github-ops-github/steps/run/instruction.md new file mode 100644 index 00000000..32a09d7f --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-180-dev-tech-github-ops-github/steps/run/instruction.md @@ -0,0 +1,27 @@ +Fork huggingface/transformers on GitHub and create branch "fix-tokenizer-bug" + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-180-dev-tech-github-ops-github/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-180-dev-tech-github-ops-github/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-180-dev-tech-github-ops-github/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-180-dev-tech-github-ops-github/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-180-dev-tech-github-ops-github/steps/run/tests/task.json new file mode 100644 index 00000000..2b578fbc --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-180-dev-tech-github-ops-github/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 180, + "metaclass": "dev-tech", + "class": "github-ops", + "description": "Fork huggingface/transformers on GitHub and create branch \"fix-tokenizer-bug\"", + "sites_involved": [ + "github.com" + ], + "platform": "github", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Fork huggingface/transformers on GitHub and create branch \"fix-tokenizer-bug\"", + "eval_schema": { + "url_pattern": "github\\.com/[^/]+/transformers/branches$", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-180-dev-tech-github-ops-github/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-180-dev-tech-github-ops-github/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-180-dev-tech-github-ops-github/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-180-dev-tech-github-ops-github/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-180-dev-tech-github-ops-github/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..b38a1cf1 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-180-dev-tech-github-ops-github/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "github\\.com/[^/]+/transformers/branches$", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-180-dev-tech-github-ops-github/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-180-dev-tech-github-ops-github/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-180-dev-tech-github-ops-github/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-180-dev-tech-github-ops-github/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-180-dev-tech-github-ops-github/steps/run/workdir/task.json new file mode 100644 index 00000000..2b578fbc --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-180-dev-tech-github-ops-github/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 180, + "metaclass": "dev-tech", + "class": "github-ops", + "description": "Fork huggingface/transformers on GitHub and create branch \"fix-tokenizer-bug\"", + "sites_involved": [ + "github.com" + ], + "platform": "github", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Fork huggingface/transformers on GitHub and create branch \"fix-tokenizer-bug\"", + "eval_schema": { + "url_pattern": "github\\.com/[^/]+/transformers/branches$", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-180-dev-tech-github-ops-github/task.toml b/harbor/datasets/clawbench-v2/v2-180-dev-tech-github-ops-github/task.toml new file mode 100644 index 00000000..9099f402 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-180-dev-tech-github-ops-github/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-180-dev-tech-github-ops-github" +description = "Fork huggingface/transformers on GitHub and create branch \"fix-tokenizer-bug\"" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-180-dev-tech-github-ops-github" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-215-academia-research-paper-tables-overleaf/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-215-academia-research-paper-tables-overleaf/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-215-academia-research-paper-tables-overleaf/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-215-academia-research-paper-tables-overleaf/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-215-academia-research-paper-tables-overleaf/steps/run/instruction.md new file mode 100644 index 00000000..19806aef --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-215-academia-research-paper-tables-overleaf/steps/run/instruction.md @@ -0,0 +1,30 @@ +On Overleaf, create a new project and write a LaTeX booktabs table with 5 models x 4 metrics + Average + Delta row. The project must compile without errors and display the formatted table. + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +Additional files are also available under /my-info/ for this task: +- raw_results.json: Raw benchmark results: 5 models × 4 metrics + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-215-academia-research-paper-tables-overleaf/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-215-academia-research-paper-tables-overleaf/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-215-academia-research-paper-tables-overleaf/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-215-academia-research-paper-tables-overleaf/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-215-academia-research-paper-tables-overleaf/steps/run/tests/task.json new file mode 100644 index 00000000..23b32d69 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-215-academia-research-paper-tables-overleaf/steps/run/tests/task.json @@ -0,0 +1,30 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 215, + "metaclass": "academia-research", + "class": "paper-tables", + "description": "On Overleaf, create a new project and write a LaTeX booktabs table with 5 models x 4 metrics + Average + Delta row. The project must compile without errors and display the formatted table.", + "sites_involved": [ + "overleaf.com" + ], + "platform": "overleaf", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "On Overleaf, create a new project and write a LaTeX booktabs table with 5 models x 4 metrics + Average + Delta row. The project must compile without errors and display the formatted table.", + "eval_schema": { + "url_pattern": "overleaf\\.com/project/[a-f0-9]+/compile", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [ + { + "path": "extra_info/raw_results.json", + "description": "Raw benchmark results: 5 models × 4 metrics" + } + ] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-215-academia-research-paper-tables-overleaf/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-215-academia-research-paper-tables-overleaf/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-215-academia-research-paper-tables-overleaf/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-215-academia-research-paper-tables-overleaf/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-215-academia-research-paper-tables-overleaf/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..2a70b644 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-215-academia-research-paper-tables-overleaf/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "overleaf\\.com/project/[a-f0-9]+/compile", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-215-academia-research-paper-tables-overleaf/steps/run/workdir/extra_info/raw_results.json b/harbor/datasets/clawbench-v2/v2-215-academia-research-paper-tables-overleaf/steps/run/workdir/extra_info/raw_results.json new file mode 100644 index 00000000..44eebd72 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-215-academia-research-paper-tables-overleaf/steps/run/workdir/extra_info/raw_results.json @@ -0,0 +1,47 @@ +{ + "models": [ + "GPT-4o", + "Claude 3.5 Sonnet", + "Gemini 1.5 Pro", + "Llama 3.1 70B", + "Mistral Large" + ], + "metrics": [ + "Accuracy", + "F1-Score", + "Latency (ms)", + "Cost ($/1K tokens)" + ], + "results": { + "GPT-4o": [ + 0.923, + 0.918, + 342, + 0.015 + ], + "Claude 3.5 Sonnet": [ + 0.931, + 0.926, + 298, + 0.018 + ], + "Gemini 1.5 Pro": [ + 0.908, + 0.902, + 410, + 0.007 + ], + "Llama 3.1 70B": [ + 0.887, + 0.881, + 520, + 0.001 + ], + "Mistral Large": [ + 0.901, + 0.895, + 380, + 0.008 + ] + } +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-215-academia-research-paper-tables-overleaf/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-215-academia-research-paper-tables-overleaf/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-215-academia-research-paper-tables-overleaf/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-215-academia-research-paper-tables-overleaf/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-215-academia-research-paper-tables-overleaf/steps/run/workdir/task.json new file mode 100644 index 00000000..23b32d69 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-215-academia-research-paper-tables-overleaf/steps/run/workdir/task.json @@ -0,0 +1,30 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 215, + "metaclass": "academia-research", + "class": "paper-tables", + "description": "On Overleaf, create a new project and write a LaTeX booktabs table with 5 models x 4 metrics + Average + Delta row. The project must compile without errors and display the formatted table.", + "sites_involved": [ + "overleaf.com" + ], + "platform": "overleaf", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "On Overleaf, create a new project and write a LaTeX booktabs table with 5 models x 4 metrics + Average + Delta row. The project must compile without errors and display the formatted table.", + "eval_schema": { + "url_pattern": "overleaf\\.com/project/[a-f0-9]+/compile", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [ + { + "path": "extra_info/raw_results.json", + "description": "Raw benchmark results: 5 models × 4 metrics" + } + ] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-215-academia-research-paper-tables-overleaf/task.toml b/harbor/datasets/clawbench-v2/v2-215-academia-research-paper-tables-overleaf/task.toml new file mode 100644 index 00000000..3e3ff797 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-215-academia-research-paper-tables-overleaf/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-215-academia-research-paper-tables-overleaf" +description = "On Overleaf, create a new project and write a LaTeX booktabs table with 5 models x 4 metrics + Average + Delta row. The project must compile without errors and display the formatted table." +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-215-academia-research-paper-tables-overleaf" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-234-daily-life-events-ticketmaster/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-234-daily-life-events-ticketmaster/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-234-daily-life-events-ticketmaster/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-234-daily-life-events-ticketmaster/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-234-daily-life-events-ticketmaster/steps/run/instruction.md new file mode 100644 index 00000000..1d871eda --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-234-daily-life-events-ticketmaster/steps/run/instruction.md @@ -0,0 +1,27 @@ +Find tickets for a live comedy show on Ticketmaster in Los Angeles this Saturday evening + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-234-daily-life-events-ticketmaster/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-234-daily-life-events-ticketmaster/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-234-daily-life-events-ticketmaster/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-234-daily-life-events-ticketmaster/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-234-daily-life-events-ticketmaster/steps/run/tests/task.json new file mode 100644 index 00000000..9e2e08fe --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-234-daily-life-events-ticketmaster/steps/run/tests/task.json @@ -0,0 +1,26 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 234, + "metaclass": "daily-life", + "class": "events", + "description": "Find tickets for a live comedy show on Ticketmaster in Los Angeles this Saturday evening", + "sites_involved": [ + "ticketmaster.com", + "checkout.ticketmaster.com" + ], + "platform": "ticketmaster", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Find tickets for a live comedy show on Ticketmaster in Los Angeles this Saturday evening", + "eval_schema": { + "url_pattern": "checkout\\.ticketmaster\\.com\\/[a-f0-9]+", + "method": "GET" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-234-daily-life-events-ticketmaster/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-234-daily-life-events-ticketmaster/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-234-daily-life-events-ticketmaster/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-234-daily-life-events-ticketmaster/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-234-daily-life-events-ticketmaster/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..f2a42a47 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-234-daily-life-events-ticketmaster/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "checkout\\.ticketmaster\\.com\\/[a-f0-9]+", + "method": "GET" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-234-daily-life-events-ticketmaster/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-234-daily-life-events-ticketmaster/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-234-daily-life-events-ticketmaster/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-234-daily-life-events-ticketmaster/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-234-daily-life-events-ticketmaster/steps/run/workdir/task.json new file mode 100644 index 00000000..9e2e08fe --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-234-daily-life-events-ticketmaster/steps/run/workdir/task.json @@ -0,0 +1,26 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 234, + "metaclass": "daily-life", + "class": "events", + "description": "Find tickets for a live comedy show on Ticketmaster in Los Angeles this Saturday evening", + "sites_involved": [ + "ticketmaster.com", + "checkout.ticketmaster.com" + ], + "platform": "ticketmaster", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Find tickets for a live comedy show on Ticketmaster in Los Angeles this Saturday evening", + "eval_schema": { + "url_pattern": "checkout\\.ticketmaster\\.com\\/[a-f0-9]+", + "method": "GET" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-234-daily-life-events-ticketmaster/task.toml b/harbor/datasets/clawbench-v2/v2-234-daily-life-events-ticketmaster/task.toml new file mode 100644 index 00000000..4f10ed62 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-234-daily-life-events-ticketmaster/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-234-daily-life-events-ticketmaster" +description = "Find tickets for a live comedy show on Ticketmaster in Los Angeles this Saturday evening" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-234-daily-life-events-ticketmaster" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-242-academia-research-research-tools-overleaf/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-242-academia-research-research-tools-overleaf/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-242-academia-research-research-tools-overleaf/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-242-academia-research-research-tools-overleaf/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-242-academia-research-research-tools-overleaf/steps/run/instruction.md new file mode 100644 index 00000000..3968f661 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-242-academia-research-research-tools-overleaf/steps/run/instruction.md @@ -0,0 +1,27 @@ +Overleaf: Create project "CVPR2026_Submission" using CVPR 2026 template + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-242-academia-research-research-tools-overleaf/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-242-academia-research-research-tools-overleaf/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-242-academia-research-research-tools-overleaf/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-242-academia-research-research-tools-overleaf/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-242-academia-research-research-tools-overleaf/steps/run/tests/task.json new file mode 100644 index 00000000..9263a06f --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-242-academia-research-research-tools-overleaf/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 242, + "metaclass": "academia-research", + "class": "research-tools", + "description": "Overleaf: Create project \"CVPR2026_Submission\" using CVPR 2026 template", + "sites_involved": [ + "overleaf.com" + ], + "platform": "overleaf", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Overleaf: Create project \"CVPR2026_Submission\" using CVPR 2026 template", + "eval_schema": { + "url_pattern": "overleaf\\.com/project/[a-f0-9]+/settings", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-242-academia-research-research-tools-overleaf/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-242-academia-research-research-tools-overleaf/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-242-academia-research-research-tools-overleaf/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-242-academia-research-research-tools-overleaf/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-242-academia-research-research-tools-overleaf/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..79500081 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-242-academia-research-research-tools-overleaf/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "overleaf\\.com/project/[a-f0-9]+/settings", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-242-academia-research-research-tools-overleaf/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-242-academia-research-research-tools-overleaf/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-242-academia-research-research-tools-overleaf/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-242-academia-research-research-tools-overleaf/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-242-academia-research-research-tools-overleaf/steps/run/workdir/task.json new file mode 100644 index 00000000..9263a06f --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-242-academia-research-research-tools-overleaf/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 242, + "metaclass": "academia-research", + "class": "research-tools", + "description": "Overleaf: Create project \"CVPR2026_Submission\" using CVPR 2026 template", + "sites_involved": [ + "overleaf.com" + ], + "platform": "overleaf", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Overleaf: Create project \"CVPR2026_Submission\" using CVPR 2026 template", + "eval_schema": { + "url_pattern": "overleaf\\.com/project/[a-f0-9]+/settings", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-242-academia-research-research-tools-overleaf/task.toml b/harbor/datasets/clawbench-v2/v2-242-academia-research-research-tools-overleaf/task.toml new file mode 100644 index 00000000..7fc2a801 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-242-academia-research-research-tools-overleaf/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-242-academia-research-research-tools-overleaf" +description = "Overleaf: Create project \"CVPR2026_Submission\" using CVPR 2026 template" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-242-academia-research-research-tools-overleaf" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-247-academia-research-research-tools-semantic-scholar/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-247-academia-research-research-tools-semantic-scholar/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-247-academia-research-research-tools-semantic-scholar/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-247-academia-research-research-tools-semantic-scholar/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-247-academia-research-research-tools-semantic-scholar/steps/run/instruction.md new file mode 100644 index 00000000..e7f14537 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-247-academia-research-research-tools-semantic-scholar/steps/run/instruction.md @@ -0,0 +1,27 @@ +Semantic Scholar: Create feed for "LLMs" + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-247-academia-research-research-tools-semantic-scholar/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-247-academia-research-research-tools-semantic-scholar/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-247-academia-research-research-tools-semantic-scholar/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-247-academia-research-research-tools-semantic-scholar/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-247-academia-research-research-tools-semantic-scholar/steps/run/tests/task.json new file mode 100644 index 00000000..b1415e02 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-247-academia-research-research-tools-semantic-scholar/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 247, + "metaclass": "academia-research", + "class": "research-tools", + "description": "Semantic Scholar: Create feed for \"LLMs\"", + "sites_involved": [ + "semanticscholar.org" + ], + "platform": "semantic-scholar", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Semantic Scholar: Create feed for \"LLMs\"", + "eval_schema": { + "url_pattern": "semanticscholar\\.org/api/1/alert", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-247-academia-research-research-tools-semantic-scholar/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-247-academia-research-research-tools-semantic-scholar/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-247-academia-research-research-tools-semantic-scholar/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-247-academia-research-research-tools-semantic-scholar/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-247-academia-research-research-tools-semantic-scholar/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..b9c9e61d --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-247-academia-research-research-tools-semantic-scholar/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "semanticscholar\\.org/api/1/alert", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-247-academia-research-research-tools-semantic-scholar/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-247-academia-research-research-tools-semantic-scholar/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-247-academia-research-research-tools-semantic-scholar/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-247-academia-research-research-tools-semantic-scholar/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-247-academia-research-research-tools-semantic-scholar/steps/run/workdir/task.json new file mode 100644 index 00000000..b1415e02 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-247-academia-research-research-tools-semantic-scholar/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 247, + "metaclass": "academia-research", + "class": "research-tools", + "description": "Semantic Scholar: Create feed for \"LLMs\"", + "sites_involved": [ + "semanticscholar.org" + ], + "platform": "semantic-scholar", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Semantic Scholar: Create feed for \"LLMs\"", + "eval_schema": { + "url_pattern": "semanticscholar\\.org/api/1/alert", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-247-academia-research-research-tools-semantic-scholar/task.toml b/harbor/datasets/clawbench-v2/v2-247-academia-research-research-tools-semantic-scholar/task.toml new file mode 100644 index 00000000..d84ddd49 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-247-academia-research-research-tools-semantic-scholar/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-247-academia-research-research-tools-semantic-scholar" +description = "Semantic Scholar: Create feed for \"LLMs\"" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-247-academia-research-research-tools-semantic-scholar" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-265-education-learning-general-coursera/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-265-education-learning-general-coursera/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-265-education-learning-general-coursera/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-265-education-learning-general-coursera/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-265-education-learning-general-coursera/steps/run/instruction.md new file mode 100644 index 00000000..3a263ddc --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-265-education-learning-general-coursera/steps/run/instruction.md @@ -0,0 +1,27 @@ +On Coursera, enroll in the "Deep Learning Specialization" by reaching the checkout or payment page with enrollment details filled in. + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-265-education-learning-general-coursera/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-265-education-learning-general-coursera/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-265-education-learning-general-coursera/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-265-education-learning-general-coursera/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-265-education-learning-general-coursera/steps/run/tests/task.json new file mode 100644 index 00000000..0867b34c --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-265-education-learning-general-coursera/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 265, + "metaclass": "education-learning", + "class": "general", + "description": "On Coursera, enroll in the \"Deep Learning Specialization\" by reaching the checkout or payment page with enrollment details filled in.", + "sites_involved": [ + "coursera.org" + ], + "platform": "coursera", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "On Coursera, enroll in the \"Deep Learning Specialization\" by reaching the checkout or payment page with enrollment details filled in.", + "eval_schema": { + "url_pattern": "coursera\\.org/api/subscriptions\\.v1/\\?action=createCart", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-265-education-learning-general-coursera/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-265-education-learning-general-coursera/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-265-education-learning-general-coursera/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-265-education-learning-general-coursera/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-265-education-learning-general-coursera/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..bcabd10e --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-265-education-learning-general-coursera/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "coursera\\.org/api/subscriptions\\.v1/\\?action=createCart", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-265-education-learning-general-coursera/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-265-education-learning-general-coursera/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-265-education-learning-general-coursera/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-265-education-learning-general-coursera/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-265-education-learning-general-coursera/steps/run/workdir/task.json new file mode 100644 index 00000000..0867b34c --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-265-education-learning-general-coursera/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 265, + "metaclass": "education-learning", + "class": "general", + "description": "On Coursera, enroll in the \"Deep Learning Specialization\" by reaching the checkout or payment page with enrollment details filled in.", + "sites_involved": [ + "coursera.org" + ], + "platform": "coursera", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "On Coursera, enroll in the \"Deep Learning Specialization\" by reaching the checkout or payment page with enrollment details filled in.", + "eval_schema": { + "url_pattern": "coursera\\.org/api/subscriptions\\.v1/\\?action=createCart", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-265-education-learning-general-coursera/task.toml b/harbor/datasets/clawbench-v2/v2-265-education-learning-general-coursera/task.toml new file mode 100644 index 00000000..637836cf --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-265-education-learning-general-coursera/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-265-education-learning-general-coursera" +description = "On Coursera, enroll in the \"Deep Learning Specialization\" by reaching the checkout or payment page with enrollment details filled in." +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-265-education-learning-general-coursera" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-266-education-learning-general-leetcode/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-266-education-learning-general-leetcode/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-266-education-learning-general-leetcode/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-266-education-learning-general-leetcode/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-266-education-learning-general-leetcode/steps/run/instruction.md new file mode 100644 index 00000000..8f755a1f --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-266-education-learning-general-leetcode/steps/run/instruction.md @@ -0,0 +1,30 @@ +On LeetCode, open the "Two Sum" problem, write a correct Python solution in the code editor, and click the Submit button. + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +Additional files are also available under /my-info/ for this task: +- solution_code.py: Python solution for the Two Sum problem + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-266-education-learning-general-leetcode/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-266-education-learning-general-leetcode/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-266-education-learning-general-leetcode/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-266-education-learning-general-leetcode/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-266-education-learning-general-leetcode/steps/run/tests/task.json new file mode 100644 index 00000000..dff8e84d --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-266-education-learning-general-leetcode/steps/run/tests/task.json @@ -0,0 +1,30 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 266, + "metaclass": "education-learning", + "class": "general", + "description": "On LeetCode, open the \"Two Sum\" problem, write a correct Python solution in the code editor, and click the Submit button.", + "sites_involved": [ + "leetcode.com" + ], + "platform": "leetcode", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "On LeetCode, open the \"Two Sum\" problem, write a correct Python solution in the code editor, and click the Submit button.", + "eval_schema": { + "url_pattern": "leetcode\\.com/problems/two-sum/submit", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [ + { + "path": "extra_info/solution_code.py", + "description": "Python solution for the Two Sum problem" + } + ] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-266-education-learning-general-leetcode/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-266-education-learning-general-leetcode/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-266-education-learning-general-leetcode/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-266-education-learning-general-leetcode/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-266-education-learning-general-leetcode/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..3aa11bd3 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-266-education-learning-general-leetcode/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "leetcode\\.com/problems/two-sum/submit", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-266-education-learning-general-leetcode/steps/run/workdir/extra_info/solution_code.py b/harbor/datasets/clawbench-v2/v2-266-education-learning-general-leetcode/steps/run/workdir/extra_info/solution_code.py new file mode 100644 index 00000000..ce9d33ba --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-266-education-learning-general-leetcode/steps/run/workdir/extra_info/solution_code.py @@ -0,0 +1,9 @@ +class Solution: + def twoSum(self, nums: list[int], target: int) -> list[int]: + seen = {} + for i, num in enumerate(nums): + complement = target - num + if complement in seen: + return [seen[complement], i] + seen[num] = i + return [] diff --git a/harbor/datasets/clawbench-v2/v2-266-education-learning-general-leetcode/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-266-education-learning-general-leetcode/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-266-education-learning-general-leetcode/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-266-education-learning-general-leetcode/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-266-education-learning-general-leetcode/steps/run/workdir/task.json new file mode 100644 index 00000000..dff8e84d --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-266-education-learning-general-leetcode/steps/run/workdir/task.json @@ -0,0 +1,30 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 266, + "metaclass": "education-learning", + "class": "general", + "description": "On LeetCode, open the \"Two Sum\" problem, write a correct Python solution in the code editor, and click the Submit button.", + "sites_involved": [ + "leetcode.com" + ], + "platform": "leetcode", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "On LeetCode, open the \"Two Sum\" problem, write a correct Python solution in the code editor, and click the Submit button.", + "eval_schema": { + "url_pattern": "leetcode\\.com/problems/two-sum/submit", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [ + { + "path": "extra_info/solution_code.py", + "description": "Python solution for the Two Sum problem" + } + ] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-266-education-learning-general-leetcode/task.toml b/harbor/datasets/clawbench-v2/v2-266-education-learning-general-leetcode/task.toml new file mode 100644 index 00000000..fe04ac08 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-266-education-learning-general-leetcode/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-266-education-learning-general-leetcode" +description = "On LeetCode, open the \"Two Sum\" problem, write a correct Python solution in the code editor, and click the Submit button." +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-266-education-learning-general-leetcode" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-273-education-learning-general-edx/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-273-education-learning-general-edx/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-273-education-learning-general-edx/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-273-education-learning-general-edx/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-273-education-learning-general-edx/steps/run/instruction.md new file mode 100644 index 00000000..e47d9b2a --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-273-education-learning-general-edx/steps/run/instruction.md @@ -0,0 +1,27 @@ +Log into edX and enroll in CS50 by selecting the free audit track and completing the enrollment process until the course content or dashboard is accessible. + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-273-education-learning-general-edx/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-273-education-learning-general-edx/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-273-education-learning-general-edx/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-273-education-learning-general-edx/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-273-education-learning-general-edx/steps/run/tests/task.json new file mode 100644 index 00000000..cdfdcacb --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-273-education-learning-general-edx/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 273, + "metaclass": "education-learning", + "class": "general", + "description": "On edX, enroll in CS50 by selecting the free audit track and completing the enrollment process until the course content or dashboard is accessible.", + "sites_involved": [ + "edx.org" + ], + "platform": "edx", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Log into edX and enroll in CS50 by selecting the free audit track and completing the enrollment process until the course content or dashboard is accessible.", + "eval_schema": { + "url_pattern": "www\\.edx\\.org/(track-select/|learn/)", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-273-education-learning-general-edx/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-273-education-learning-general-edx/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-273-education-learning-general-edx/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-273-education-learning-general-edx/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-273-education-learning-general-edx/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..08fbfe5c --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-273-education-learning-general-edx/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "www\\.edx\\.org/(track-select/|learn/)", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-273-education-learning-general-edx/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-273-education-learning-general-edx/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-273-education-learning-general-edx/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-273-education-learning-general-edx/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-273-education-learning-general-edx/steps/run/workdir/task.json new file mode 100644 index 00000000..cdfdcacb --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-273-education-learning-general-edx/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 273, + "metaclass": "education-learning", + "class": "general", + "description": "On edX, enroll in CS50 by selecting the free audit track and completing the enrollment process until the course content or dashboard is accessible.", + "sites_involved": [ + "edx.org" + ], + "platform": "edx", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Log into edX and enroll in CS50 by selecting the free audit track and completing the enrollment process until the course content or dashboard is accessible.", + "eval_schema": { + "url_pattern": "www\\.edx\\.org/(track-select/|learn/)", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-273-education-learning-general-edx/task.toml b/harbor/datasets/clawbench-v2/v2-273-education-learning-general-edx/task.toml new file mode 100644 index 00000000..886ffaa1 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-273-education-learning-general-edx/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-273-education-learning-general-edx" +description = "On edX, enroll in CS50 by selecting the free audit track and completing the enrollment process until the course content or dashboard is accessible." +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-273-education-learning-general-edx" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-274-office-secretary-tasks-calendar-when2meet/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-274-office-secretary-tasks-calendar-when2meet/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-274-office-secretary-tasks-calendar-when2meet/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-274-office-secretary-tasks-calendar-when2meet/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-274-office-secretary-tasks-calendar-when2meet/steps/run/instruction.md new file mode 100644 index 00000000..2f8877bf --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-274-office-secretary-tasks-calendar-when2meet/steps/run/instruction.md @@ -0,0 +1,27 @@ +Create a When2meet event for a 3-person team meeting next week + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-274-office-secretary-tasks-calendar-when2meet/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-274-office-secretary-tasks-calendar-when2meet/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-274-office-secretary-tasks-calendar-when2meet/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-274-office-secretary-tasks-calendar-when2meet/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-274-office-secretary-tasks-calendar-when2meet/steps/run/tests/task.json new file mode 100644 index 00000000..f5ff4fbb --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-274-office-secretary-tasks-calendar-when2meet/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 274, + "metaclass": "office-secretary-tasks", + "class": "calendar", + "description": "Create a When2meet event for a 3-person team meeting next week", + "sites_involved": [ + "when2meet.com" + ], + "platform": "when2meet", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Create a When2meet event for a 3-person team meeting next week", + "eval_schema": { + "url_pattern": "when2meet\\.com/(SaveNewEvent|SaveTimes|api)", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-274-office-secretary-tasks-calendar-when2meet/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-274-office-secretary-tasks-calendar-when2meet/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-274-office-secretary-tasks-calendar-when2meet/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-274-office-secretary-tasks-calendar-when2meet/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-274-office-secretary-tasks-calendar-when2meet/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..2f9e3540 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-274-office-secretary-tasks-calendar-when2meet/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "when2meet\\.com/(SaveNewEvent|SaveTimes|api)", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-274-office-secretary-tasks-calendar-when2meet/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-274-office-secretary-tasks-calendar-when2meet/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-274-office-secretary-tasks-calendar-when2meet/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-274-office-secretary-tasks-calendar-when2meet/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-274-office-secretary-tasks-calendar-when2meet/steps/run/workdir/task.json new file mode 100644 index 00000000..f5ff4fbb --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-274-office-secretary-tasks-calendar-when2meet/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 274, + "metaclass": "office-secretary-tasks", + "class": "calendar", + "description": "Create a When2meet event for a 3-person team meeting next week", + "sites_involved": [ + "when2meet.com" + ], + "platform": "when2meet", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Create a When2meet event for a 3-person team meeting next week", + "eval_schema": { + "url_pattern": "when2meet\\.com/(SaveNewEvent|SaveTimes|api)", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-274-office-secretary-tasks-calendar-when2meet/task.toml b/harbor/datasets/clawbench-v2/v2-274-office-secretary-tasks-calendar-when2meet/task.toml new file mode 100644 index 00000000..fccdbdc0 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-274-office-secretary-tasks-calendar-when2meet/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-274-office-secretary-tasks-calendar-when2meet" +description = "Create a When2meet event for a 3-person team meeting next week" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-274-office-secretary-tasks-calendar-when2meet" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-284-office-secretary-tasks-calendar-when2meet/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-284-office-secretary-tasks-calendar-when2meet/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-284-office-secretary-tasks-calendar-when2meet/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-284-office-secretary-tasks-calendar-when2meet/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-284-office-secretary-tasks-calendar-when2meet/steps/run/instruction.md new file mode 100644 index 00000000..f75c772b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-284-office-secretary-tasks-calendar-when2meet/steps/run/instruction.md @@ -0,0 +1,27 @@ +Use When2meet to schedule a 90-minute Strategy Session with 3 attendees + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-284-office-secretary-tasks-calendar-when2meet/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-284-office-secretary-tasks-calendar-when2meet/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-284-office-secretary-tasks-calendar-when2meet/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-284-office-secretary-tasks-calendar-when2meet/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-284-office-secretary-tasks-calendar-when2meet/steps/run/tests/task.json new file mode 100644 index 00000000..7560135c --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-284-office-secretary-tasks-calendar-when2meet/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 284, + "metaclass": "office-secretary-tasks", + "class": "calendar", + "description": "Use When2meet to schedule a 90-minute Strategy Session with 3 attendees", + "sites_involved": [ + "when2meet.com" + ], + "platform": "when2meet", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Use When2meet to schedule a 90-minute Strategy Session with 3 attendees", + "eval_schema": { + "url_pattern": "when2meet\\.com/(SaveNewEvent|SaveTimes|api)", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-284-office-secretary-tasks-calendar-when2meet/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-284-office-secretary-tasks-calendar-when2meet/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-284-office-secretary-tasks-calendar-when2meet/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-284-office-secretary-tasks-calendar-when2meet/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-284-office-secretary-tasks-calendar-when2meet/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..2f9e3540 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-284-office-secretary-tasks-calendar-when2meet/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "when2meet\\.com/(SaveNewEvent|SaveTimes|api)", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-284-office-secretary-tasks-calendar-when2meet/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-284-office-secretary-tasks-calendar-when2meet/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-284-office-secretary-tasks-calendar-when2meet/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-284-office-secretary-tasks-calendar-when2meet/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-284-office-secretary-tasks-calendar-when2meet/steps/run/workdir/task.json new file mode 100644 index 00000000..7560135c --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-284-office-secretary-tasks-calendar-when2meet/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 284, + "metaclass": "office-secretary-tasks", + "class": "calendar", + "description": "Use When2meet to schedule a 90-minute Strategy Session with 3 attendees", + "sites_involved": [ + "when2meet.com" + ], + "platform": "when2meet", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Use When2meet to schedule a 90-minute Strategy Session with 3 attendees", + "eval_schema": { + "url_pattern": "when2meet\\.com/(SaveNewEvent|SaveTimes|api)", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-284-office-secretary-tasks-calendar-when2meet/task.toml b/harbor/datasets/clawbench-v2/v2-284-office-secretary-tasks-calendar-when2meet/task.toml new file mode 100644 index 00000000..b919fc90 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-284-office-secretary-tasks-calendar-when2meet/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-284-office-secretary-tasks-calendar-when2meet" +description = "Use When2meet to schedule a 90-minute Strategy Session with 3 attendees" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-284-office-secretary-tasks-calendar-when2meet" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-369-entertainment-hobbies-general-goodreads/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-369-entertainment-hobbies-general-goodreads/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-369-entertainment-hobbies-general-goodreads/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-369-entertainment-hobbies-general-goodreads/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-369-entertainment-hobbies-general-goodreads/steps/run/instruction.md new file mode 100644 index 00000000..48867ed8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-369-entertainment-hobbies-general-goodreads/steps/run/instruction.md @@ -0,0 +1,30 @@ +Goodreads: Create a shelf "ML Must-Reads" and add 10 books + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +Additional files are also available under /my-info/ for this task: +- book_list.json: List of 10 ML books to add to shelf + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-369-entertainment-hobbies-general-goodreads/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-369-entertainment-hobbies-general-goodreads/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-369-entertainment-hobbies-general-goodreads/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-369-entertainment-hobbies-general-goodreads/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-369-entertainment-hobbies-general-goodreads/steps/run/tests/task.json new file mode 100644 index 00000000..082e475f --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-369-entertainment-hobbies-general-goodreads/steps/run/tests/task.json @@ -0,0 +1,30 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 369, + "metaclass": "entertainment-hobbies", + "class": "general", + "description": "Goodreads: Create a shelf \"ML Must-Reads\" and add 10 books", + "sites_involved": [ + "goodreads.com" + ], + "platform": "goodreads", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Goodreads: Create a shelf \"ML Must-Reads\" and add 10 books", + "eval_schema": { + "url_pattern": "goodreads\\.com/shelf/add_to_shelf", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [ + { + "path": "extra_info/book_list.json", + "description": "List of 10 ML books to add to shelf" + } + ] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-369-entertainment-hobbies-general-goodreads/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-369-entertainment-hobbies-general-goodreads/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-369-entertainment-hobbies-general-goodreads/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-369-entertainment-hobbies-general-goodreads/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-369-entertainment-hobbies-general-goodreads/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..cbae0e43 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-369-entertainment-hobbies-general-goodreads/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "goodreads\\.com/shelf/add_to_shelf", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-369-entertainment-hobbies-general-goodreads/steps/run/workdir/extra_info/book_list.json b/harbor/datasets/clawbench-v2/v2-369-entertainment-hobbies-general-goodreads/steps/run/workdir/extra_info/book_list.json new file mode 100644 index 00000000..2689760b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-369-entertainment-hobbies-general-goodreads/steps/run/workdir/extra_info/book_list.json @@ -0,0 +1,14 @@ +{ + "books": [ + "Deep Learning by Ian Goodfellow", + "Pattern Recognition and Machine Learning by Christopher Bishop", + "The Hundred-Page Machine Learning Book by Andriy Burkov", + "Hands-On Machine Learning with Scikit-Learn, Keras, and TensorFlow by Aur\u00e9lien G\u00e9ron", + "Machine Learning: A Probabilistic Perspective by Kevin Murphy", + "Reinforcement Learning: An Introduction by Richard Sutton", + "Speech and Language Processing by Dan Jurafsky", + "Information Theory, Inference and Learning Algorithms by David MacKay", + "Probabilistic Graphical Models by Daphne Koller", + "The Elements of Statistical Learning by Trevor Hastie" + ] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-369-entertainment-hobbies-general-goodreads/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-369-entertainment-hobbies-general-goodreads/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-369-entertainment-hobbies-general-goodreads/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-369-entertainment-hobbies-general-goodreads/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-369-entertainment-hobbies-general-goodreads/steps/run/workdir/task.json new file mode 100644 index 00000000..082e475f --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-369-entertainment-hobbies-general-goodreads/steps/run/workdir/task.json @@ -0,0 +1,30 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 369, + "metaclass": "entertainment-hobbies", + "class": "general", + "description": "Goodreads: Create a shelf \"ML Must-Reads\" and add 10 books", + "sites_involved": [ + "goodreads.com" + ], + "platform": "goodreads", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Goodreads: Create a shelf \"ML Must-Reads\" and add 10 books", + "eval_schema": { + "url_pattern": "goodreads\\.com/shelf/add_to_shelf", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [ + { + "path": "extra_info/book_list.json", + "description": "List of 10 ML books to add to shelf" + } + ] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-369-entertainment-hobbies-general-goodreads/task.toml b/harbor/datasets/clawbench-v2/v2-369-entertainment-hobbies-general-goodreads/task.toml new file mode 100644 index 00000000..eff7de4c --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-369-entertainment-hobbies-general-goodreads/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-369-entertainment-hobbies-general-goodreads" +description = "Goodreads: Create a shelf \"ML Must-Reads\" and add 10 books" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-369-entertainment-hobbies-general-goodreads" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-372-entertainment-hobbies-general-eventbrite/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-372-entertainment-hobbies-general-eventbrite/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-372-entertainment-hobbies-general-eventbrite/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-372-entertainment-hobbies-general-eventbrite/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-372-entertainment-hobbies-general-eventbrite/steps/run/instruction.md new file mode 100644 index 00000000..091d35af --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-372-entertainment-hobbies-general-eventbrite/steps/run/instruction.md @@ -0,0 +1,30 @@ +Eventbrite: Create a free event "ML Paper Reading Group" + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +Additional files are also available under /my-info/ for this task: +- event_details.json: Event details including name, date, time, location, and description + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-372-entertainment-hobbies-general-eventbrite/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-372-entertainment-hobbies-general-eventbrite/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-372-entertainment-hobbies-general-eventbrite/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-372-entertainment-hobbies-general-eventbrite/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-372-entertainment-hobbies-general-eventbrite/steps/run/tests/task.json new file mode 100644 index 00000000..59c714fd --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-372-entertainment-hobbies-general-eventbrite/steps/run/tests/task.json @@ -0,0 +1,30 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 372, + "metaclass": "entertainment-hobbies", + "class": "general", + "description": "Eventbrite: Create a free event \"ML Paper Reading Group\"", + "sites_involved": [ + "eventbrite.com" + ], + "platform": "eventbrite", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Eventbrite: Create a free event \"ML Paper Reading Group\"", + "eval_schema": { + "url_pattern": "www\\.eventbrite\\.com/api/v3/organizations/\\d+/events/auto-create", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [ + { + "path": "extra_info/event_details.json", + "description": "Event details including name, date, time, location, and description" + } + ] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-372-entertainment-hobbies-general-eventbrite/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-372-entertainment-hobbies-general-eventbrite/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-372-entertainment-hobbies-general-eventbrite/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-372-entertainment-hobbies-general-eventbrite/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-372-entertainment-hobbies-general-eventbrite/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..ba04a751 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-372-entertainment-hobbies-general-eventbrite/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "www\\.eventbrite\\.com/api/v3/organizations/\\d+/events/auto-create", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-372-entertainment-hobbies-general-eventbrite/steps/run/workdir/extra_info/event_details.json b/harbor/datasets/clawbench-v2/v2-372-entertainment-hobbies-general-eventbrite/steps/run/workdir/extra_info/event_details.json new file mode 100644 index 00000000..9a607cea --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-372-entertainment-hobbies-general-eventbrite/steps/run/workdir/extra_info/event_details.json @@ -0,0 +1,10 @@ +{ + "event_name": "ML Paper Reading Group", + "event_type": "free", + "date": "2026-04-15", + "time": "6:00 PM - 8:00 PM ET", + "location": "Online (Zoom)", + "description": "Weekly paper reading group focusing on recent ML research. This week: attention mechanisms and transformer architectures. All levels welcome.", + "capacity": 50, + "organizer": "Alex Green" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-372-entertainment-hobbies-general-eventbrite/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-372-entertainment-hobbies-general-eventbrite/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-372-entertainment-hobbies-general-eventbrite/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-372-entertainment-hobbies-general-eventbrite/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-372-entertainment-hobbies-general-eventbrite/steps/run/workdir/task.json new file mode 100644 index 00000000..59c714fd --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-372-entertainment-hobbies-general-eventbrite/steps/run/workdir/task.json @@ -0,0 +1,30 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 372, + "metaclass": "entertainment-hobbies", + "class": "general", + "description": "Eventbrite: Create a free event \"ML Paper Reading Group\"", + "sites_involved": [ + "eventbrite.com" + ], + "platform": "eventbrite", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Eventbrite: Create a free event \"ML Paper Reading Group\"", + "eval_schema": { + "url_pattern": "www\\.eventbrite\\.com/api/v3/organizations/\\d+/events/auto-create", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [ + { + "path": "extra_info/event_details.json", + "description": "Event details including name, date, time, location, and description" + } + ] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-372-entertainment-hobbies-general-eventbrite/task.toml b/harbor/datasets/clawbench-v2/v2-372-entertainment-hobbies-general-eventbrite/task.toml new file mode 100644 index 00000000..65f5875a --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-372-entertainment-hobbies-general-eventbrite/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-372-entertainment-hobbies-general-eventbrite" +description = "Eventbrite: Create a free event \"ML Paper Reading Group\"" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-372-entertainment-hobbies-general-eventbrite" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-413-personal-management-personal-tools-todoist/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-413-personal-management-personal-tools-todoist/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-413-personal-management-personal-tools-todoist/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-413-personal-management-personal-tools-todoist/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-413-personal-management-personal-tools-todoist/steps/run/instruction.md new file mode 100644 index 00000000..93bce813 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-413-personal-management-personal-tools-todoist/steps/run/instruction.md @@ -0,0 +1,30 @@ +Create project "Q2 Engineering Goals" with 8 tasks, due dates, and priorities in Todoist + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +Additional files are also available under /my-info/ for this task: +- task_list.json: Task list with names, assignees, due dates, and priorities + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-413-personal-management-personal-tools-todoist/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-413-personal-management-personal-tools-todoist/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-413-personal-management-personal-tools-todoist/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-413-personal-management-personal-tools-todoist/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-413-personal-management-personal-tools-todoist/steps/run/tests/task.json new file mode 100644 index 00000000..f227ea40 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-413-personal-management-personal-tools-todoist/steps/run/tests/task.json @@ -0,0 +1,30 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 413, + "metaclass": "personal-management", + "class": "personal-tools", + "description": "Create project \"Q2 Engineering Goals\" with 8 tasks, due dates, and priorities in Todoist", + "sites_involved": [ + "todoist.com" + ], + "platform": "todoist", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Create project \"Q2 Engineering Goals\" with 8 tasks, due dates, and priorities in Todoist", + "eval_schema": { + "url_pattern": "app\\.todoist\\.com/api/v1/sync", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [ + { + "path": "extra_info/task_list.json", + "description": "Task list with names, assignees, due dates, and priorities" + } + ] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-413-personal-management-personal-tools-todoist/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-413-personal-management-personal-tools-todoist/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-413-personal-management-personal-tools-todoist/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-413-personal-management-personal-tools-todoist/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-413-personal-management-personal-tools-todoist/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..f9dd8af8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-413-personal-management-personal-tools-todoist/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "app\\.todoist\\.com/api/v1/sync", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-413-personal-management-personal-tools-todoist/steps/run/workdir/extra_info/task_list.json b/harbor/datasets/clawbench-v2/v2-413-personal-management-personal-tools-todoist/steps/run/workdir/extra_info/task_list.json new file mode 100644 index 00000000..96a7a414 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-413-personal-management-personal-tools-todoist/steps/run/workdir/extra_info/task_list.json @@ -0,0 +1,52 @@ +{ + "tasks": [ + { + "name": "Define Q3 OKRs", + "assignee": "Alex Green", + "due_date": "2026-04-07", + "priority": "high" + }, + { + "name": "Review architecture RFC", + "assignee": "Jordan Peters", + "due_date": "2026-04-10", + "priority": "high" + }, + { + "name": "Set up CI/CD pipeline", + "assignee": "Alex Green", + "due_date": "2026-04-14", + "priority": "medium" + }, + { + "name": "Write integration tests", + "assignee": "Alex Green", + "due_date": "2026-04-18", + "priority": "medium" + }, + { + "name": "Prepare sprint demo", + "assignee": "Jordan Peters", + "due_date": "2026-04-21", + "priority": "low" + }, + { + "name": "Update documentation", + "assignee": "Alex Green", + "due_date": "2026-04-25", + "priority": "low" + }, + { + "name": "Security audit review", + "assignee": "Alex Green", + "due_date": "2026-04-28", + "priority": "high" + }, + { + "name": "Performance benchmarking", + "assignee": "Alex Green", + "due_date": "2026-04-30", + "priority": "medium" + } + ] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-413-personal-management-personal-tools-todoist/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-413-personal-management-personal-tools-todoist/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-413-personal-management-personal-tools-todoist/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-413-personal-management-personal-tools-todoist/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-413-personal-management-personal-tools-todoist/steps/run/workdir/task.json new file mode 100644 index 00000000..f227ea40 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-413-personal-management-personal-tools-todoist/steps/run/workdir/task.json @@ -0,0 +1,30 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 413, + "metaclass": "personal-management", + "class": "personal-tools", + "description": "Create project \"Q2 Engineering Goals\" with 8 tasks, due dates, and priorities in Todoist", + "sites_involved": [ + "todoist.com" + ], + "platform": "todoist", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Create project \"Q2 Engineering Goals\" with 8 tasks, due dates, and priorities in Todoist", + "eval_schema": { + "url_pattern": "app\\.todoist\\.com/api/v1/sync", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [ + { + "path": "extra_info/task_list.json", + "description": "Task list with names, assignees, due dates, and priorities" + } + ] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-413-personal-management-personal-tools-todoist/task.toml b/harbor/datasets/clawbench-v2/v2-413-personal-management-personal-tools-todoist/task.toml new file mode 100644 index 00000000..984bf4fa --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-413-personal-management-personal-tools-todoist/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-413-personal-management-personal-tools-todoist" +description = "Create project \"Q2 Engineering Goals\" with 8 tasks, due dates, and priorities in Todoist" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-413-personal-management-personal-tools-todoist" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-468-rating-voting-general-glassdoor/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-468-rating-voting-general-glassdoor/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-468-rating-voting-general-glassdoor/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-468-rating-voting-general-glassdoor/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-468-rating-voting-general-glassdoor/steps/run/instruction.md new file mode 100644 index 00000000..dafb670c --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-468-rating-voting-general-glassdoor/steps/run/instruction.md @@ -0,0 +1,30 @@ +Write an interview review on Glassdoor + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +Additional files are also available under /my-info/ for this task: +- interview_experience.json: Interview experience details for review submission + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-468-rating-voting-general-glassdoor/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-468-rating-voting-general-glassdoor/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-468-rating-voting-general-glassdoor/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-468-rating-voting-general-glassdoor/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-468-rating-voting-general-glassdoor/steps/run/tests/task.json new file mode 100644 index 00000000..d8232d5e --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-468-rating-voting-general-glassdoor/steps/run/tests/task.json @@ -0,0 +1,30 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 468, + "metaclass": "rating-voting", + "class": "general", + "description": "Write an interview review on Glassdoor", + "sites_involved": [ + "glassdoor.com" + ], + "platform": "glassdoor", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Write an interview review on Glassdoor", + "eval_schema": { + "url_pattern": "glassdoor\\.com/(graph|api|a/interview)", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [ + { + "path": "extra_info/interview_experience.json", + "description": "Interview experience details for review submission" + } + ] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-468-rating-voting-general-glassdoor/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-468-rating-voting-general-glassdoor/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-468-rating-voting-general-glassdoor/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-468-rating-voting-general-glassdoor/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-468-rating-voting-general-glassdoor/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..01a6899b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-468-rating-voting-general-glassdoor/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "glassdoor\\.com/(graph|api|a/interview)", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-468-rating-voting-general-glassdoor/steps/run/workdir/extra_info/interview_experience.json b/harbor/datasets/clawbench-v2/v2-468-rating-voting-general-glassdoor/steps/run/workdir/extra_info/interview_experience.json new file mode 100644 index 00000000..39c3b981 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-468-rating-voting-general-glassdoor/steps/run/workdir/extra_info/interview_experience.json @@ -0,0 +1,10 @@ +{ + "company": "TechCorp Inc.", + "position": "Senior Software Engineer", + "date": "2026-02-15", + "interview_type": "On-site (4 rounds)", + "difficulty": "Medium-Hard", + "experience": "Positive overall. The team was friendly and the questions were relevant to the role. Had a system design round focusing on distributed caching, two coding rounds (one on graph algorithms, one on API design), and a behavioral round.", + "offer_received": true, + "would_recommend": true +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-468-rating-voting-general-glassdoor/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-468-rating-voting-general-glassdoor/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-468-rating-voting-general-glassdoor/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-468-rating-voting-general-glassdoor/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-468-rating-voting-general-glassdoor/steps/run/workdir/task.json new file mode 100644 index 00000000..d8232d5e --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-468-rating-voting-general-glassdoor/steps/run/workdir/task.json @@ -0,0 +1,30 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 468, + "metaclass": "rating-voting", + "class": "general", + "description": "Write an interview review on Glassdoor", + "sites_involved": [ + "glassdoor.com" + ], + "platform": "glassdoor", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Write an interview review on Glassdoor", + "eval_schema": { + "url_pattern": "glassdoor\\.com/(graph|api|a/interview)", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [ + { + "path": "extra_info/interview_experience.json", + "description": "Interview experience details for review submission" + } + ] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-468-rating-voting-general-glassdoor/task.toml b/harbor/datasets/clawbench-v2/v2-468-rating-voting-general-glassdoor/task.toml new file mode 100644 index 00000000..df03888a --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-468-rating-voting-general-glassdoor/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-468-rating-voting-general-glassdoor" +description = "Write an interview review on Glassdoor" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-468-rating-voting-general-glassdoor" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-469-rating-voting-general-tripadvisor/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-469-rating-voting-general-tripadvisor/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-469-rating-voting-general-tripadvisor/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-469-rating-voting-general-tripadvisor/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-469-rating-voting-general-tripadvisor/steps/run/instruction.md new file mode 100644 index 00000000..2de4af9d --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-469-rating-voting-general-tripadvisor/steps/run/instruction.md @@ -0,0 +1,30 @@ +Write a 4-star review for a tour on TripAdvisor + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +Additional files are also available under /my-info/ for this task: +- review_content.json: Review content for submission + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-469-rating-voting-general-tripadvisor/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-469-rating-voting-general-tripadvisor/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-469-rating-voting-general-tripadvisor/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-469-rating-voting-general-tripadvisor/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-469-rating-voting-general-tripadvisor/steps/run/tests/task.json new file mode 100644 index 00000000..5499f4d3 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-469-rating-voting-general-tripadvisor/steps/run/tests/task.json @@ -0,0 +1,30 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 469, + "metaclass": "rating-voting", + "class": "general", + "description": "Write a 4-star review for a tour on TripAdvisor", + "sites_involved": [ + "tripadvisor.com" + ], + "platform": "tripadvisor", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Write a 4-star review for a tour on TripAdvisor", + "eval_schema": { + "url_pattern": "tripadvisor\\.com/data/graphql/ids", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [ + { + "path": "extra_info/review_content.json", + "description": "Review content for submission" + } + ] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-469-rating-voting-general-tripadvisor/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-469-rating-voting-general-tripadvisor/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-469-rating-voting-general-tripadvisor/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-469-rating-voting-general-tripadvisor/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-469-rating-voting-general-tripadvisor/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..3164ddaf --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-469-rating-voting-general-tripadvisor/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "tripadvisor\\.com/data/graphql/ids", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-469-rating-voting-general-tripadvisor/steps/run/workdir/extra_info/review_content.json b/harbor/datasets/clawbench-v2/v2-469-rating-voting-general-tripadvisor/steps/run/workdir/extra_info/review_content.json new file mode 100644 index 00000000..e58dcacd --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-469-rating-voting-general-tripadvisor/steps/run/workdir/extra_info/review_content.json @@ -0,0 +1,6 @@ +{ + "review_text": "Excellent service and product quality. The user experience is intuitive and well-designed. Would recommend to colleagues.", + "rating": 4, + "pros": "Easy to use, great customer support, reliable service", + "cons": "Pricing could be more competitive for small teams" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-469-rating-voting-general-tripadvisor/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-469-rating-voting-general-tripadvisor/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-469-rating-voting-general-tripadvisor/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-469-rating-voting-general-tripadvisor/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-469-rating-voting-general-tripadvisor/steps/run/workdir/task.json new file mode 100644 index 00000000..5499f4d3 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-469-rating-voting-general-tripadvisor/steps/run/workdir/task.json @@ -0,0 +1,30 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 469, + "metaclass": "rating-voting", + "class": "general", + "description": "Write a 4-star review for a tour on TripAdvisor", + "sites_involved": [ + "tripadvisor.com" + ], + "platform": "tripadvisor", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Write a 4-star review for a tour on TripAdvisor", + "eval_schema": { + "url_pattern": "tripadvisor\\.com/data/graphql/ids", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [ + { + "path": "extra_info/review_content.json", + "description": "Review content for submission" + } + ] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-469-rating-voting-general-tripadvisor/task.toml b/harbor/datasets/clawbench-v2/v2-469-rating-voting-general-tripadvisor/task.toml new file mode 100644 index 00000000..74bf47a5 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-469-rating-voting-general-tripadvisor/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-469-rating-voting-general-tripadvisor" +description = "Write a 4-star review for a tour on TripAdvisor" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-469-rating-voting-general-tripadvisor" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-470-rating-voting-general-trustpilot/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-470-rating-voting-general-trustpilot/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-470-rating-voting-general-trustpilot/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-470-rating-voting-general-trustpilot/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-470-rating-voting-general-trustpilot/steps/run/instruction.md new file mode 100644 index 00000000..1d2769dc --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-470-rating-voting-general-trustpilot/steps/run/instruction.md @@ -0,0 +1,30 @@ +Write a 5-star review for Wealthsimple on Trustpilot + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +Additional files are also available under /my-info/ for this task: +- review_content.json: Review content for submission + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-470-rating-voting-general-trustpilot/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-470-rating-voting-general-trustpilot/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-470-rating-voting-general-trustpilot/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-470-rating-voting-general-trustpilot/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-470-rating-voting-general-trustpilot/steps/run/tests/task.json new file mode 100644 index 00000000..9dd47090 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-470-rating-voting-general-trustpilot/steps/run/tests/task.json @@ -0,0 +1,30 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 470, + "metaclass": "rating-voting", + "class": "general", + "description": "Write a 5-star review for Wealthsimple on Trustpilot", + "sites_involved": [ + "trustpilot.com" + ], + "platform": "trustpilot", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Write a 5-star review for Wealthsimple on Trustpilot", + "eval_schema": { + "url_pattern": "trustpilot\\.com/api/consumer/review", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [ + { + "path": "extra_info/review_content.json", + "description": "Review content for submission" + } + ] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-470-rating-voting-general-trustpilot/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-470-rating-voting-general-trustpilot/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-470-rating-voting-general-trustpilot/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-470-rating-voting-general-trustpilot/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-470-rating-voting-general-trustpilot/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..9376e0d4 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-470-rating-voting-general-trustpilot/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "trustpilot\\.com/api/consumer/review", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-470-rating-voting-general-trustpilot/steps/run/workdir/extra_info/review_content.json b/harbor/datasets/clawbench-v2/v2-470-rating-voting-general-trustpilot/steps/run/workdir/extra_info/review_content.json new file mode 100644 index 00000000..ae06e8f1 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-470-rating-voting-general-trustpilot/steps/run/workdir/extra_info/review_content.json @@ -0,0 +1,6 @@ +{ + "review_text": "Excellent service and product quality. The user experience is intuitive and well-designed. Would recommend to colleagues.", + "rating": 5, + "pros": "Easy to use, great customer support, reliable service", + "cons": "Pricing could be more competitive for small teams" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-470-rating-voting-general-trustpilot/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-470-rating-voting-general-trustpilot/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-470-rating-voting-general-trustpilot/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-470-rating-voting-general-trustpilot/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-470-rating-voting-general-trustpilot/steps/run/workdir/task.json new file mode 100644 index 00000000..9dd47090 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-470-rating-voting-general-trustpilot/steps/run/workdir/task.json @@ -0,0 +1,30 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 470, + "metaclass": "rating-voting", + "class": "general", + "description": "Write a 5-star review for Wealthsimple on Trustpilot", + "sites_involved": [ + "trustpilot.com" + ], + "platform": "trustpilot", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Write a 5-star review for Wealthsimple on Trustpilot", + "eval_schema": { + "url_pattern": "trustpilot\\.com/api/consumer/review", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [ + { + "path": "extra_info/review_content.json", + "description": "Review content for submission" + } + ] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-470-rating-voting-general-trustpilot/task.toml b/harbor/datasets/clawbench-v2/v2-470-rating-voting-general-trustpilot/task.toml new file mode 100644 index 00000000..af724ef1 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-470-rating-voting-general-trustpilot/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-470-rating-voting-general-trustpilot" +description = "Write a 5-star review for Wealthsimple on Trustpilot" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-470-rating-voting-general-trustpilot" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-474-rating-voting-general-capterra/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-474-rating-voting-general-capterra/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-474-rating-voting-general-capterra/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-474-rating-voting-general-capterra/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-474-rating-voting-general-capterra/steps/run/instruction.md new file mode 100644 index 00000000..6689410d --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-474-rating-voting-general-capterra/steps/run/instruction.md @@ -0,0 +1,27 @@ +Write a review for Notion on Capterra with overall rating 4 stars and exactly this text: "Flexible workspace, steep learning curve." + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-474-rating-voting-general-capterra/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-474-rating-voting-general-capterra/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-474-rating-voting-general-capterra/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-474-rating-voting-general-capterra/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-474-rating-voting-general-capterra/steps/run/tests/task.json new file mode 100644 index 00000000..52d8448e --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-474-rating-voting-general-capterra/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 474, + "metaclass": "rating-voting", + "class": "general", + "description": "Write a review for Notion on Capterra with overall rating 4 stars and exactly this text: \"Flexible workspace, steep learning curve.\"", + "sites_involved": [ + "capterra.com" + ], + "platform": "capterra", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Write a review for Notion on Capterra with overall rating 4 stars and exactly this text: \"Flexible workspace, steep learning curve.\"", + "eval_schema": { + "url_pattern": "reviews\\.capterra\\.com/api/v1/products/[^/]+/partial-review", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-474-rating-voting-general-capterra/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-474-rating-voting-general-capterra/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-474-rating-voting-general-capterra/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-474-rating-voting-general-capterra/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-474-rating-voting-general-capterra/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..7ee75022 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-474-rating-voting-general-capterra/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "reviews\\.capterra\\.com/api/v1/products/[^/]+/partial-review", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-474-rating-voting-general-capterra/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-474-rating-voting-general-capterra/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-474-rating-voting-general-capterra/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-474-rating-voting-general-capterra/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-474-rating-voting-general-capterra/steps/run/workdir/task.json new file mode 100644 index 00000000..52d8448e --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-474-rating-voting-general-capterra/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 474, + "metaclass": "rating-voting", + "class": "general", + "description": "Write a review for Notion on Capterra with overall rating 4 stars and exactly this text: \"Flexible workspace, steep learning curve.\"", + "sites_involved": [ + "capterra.com" + ], + "platform": "capterra", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Write a review for Notion on Capterra with overall rating 4 stars and exactly this text: \"Flexible workspace, steep learning curve.\"", + "eval_schema": { + "url_pattern": "reviews\\.capterra\\.com/api/v1/products/[^/]+/partial-review", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-474-rating-voting-general-capterra/task.toml b/harbor/datasets/clawbench-v2/v2-474-rating-voting-general-capterra/task.toml new file mode 100644 index 00000000..92ab07d4 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-474-rating-voting-general-capterra/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-474-rating-voting-general-capterra" +description = "Write a review for Notion on Capterra with overall rating 4 stars and exactly this text: \"Flexible workspace, steep learning curve.\"" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-474-rating-voting-general-capterra" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-475-rating-voting-general-g2/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-475-rating-voting-general-g2/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-475-rating-voting-general-g2/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-475-rating-voting-general-g2/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-475-rating-voting-general-g2/steps/run/instruction.md new file mode 100644 index 00000000..731dcc60 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-475-rating-voting-general-g2/steps/run/instruction.md @@ -0,0 +1,27 @@ +Write a 4-star review for Jira on G2 with exactly this text: "Powerful tracking, complex UI." + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-475-rating-voting-general-g2/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-475-rating-voting-general-g2/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-475-rating-voting-general-g2/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-475-rating-voting-general-g2/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-475-rating-voting-general-g2/steps/run/tests/task.json new file mode 100644 index 00000000..e4f90a2e --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-475-rating-voting-general-g2/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 475, + "metaclass": "rating-voting", + "class": "general", + "description": "Write a 4-star review for Jira on G2 with exactly this text: \"Powerful tracking, complex UI.\"", + "sites_involved": [ + "g2.com" + ], + "platform": "g2", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Write a 4-star review for Jira on G2 with exactly this text: \"Powerful tracking, complex UI.\"", + "eval_schema": { + "url_pattern": "g2\\.com/survey_responses/.+/autosave_answers\\.json", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-475-rating-voting-general-g2/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-475-rating-voting-general-g2/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-475-rating-voting-general-g2/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-475-rating-voting-general-g2/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-475-rating-voting-general-g2/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..6ba82238 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-475-rating-voting-general-g2/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "g2\\.com/survey_responses/.+/autosave_answers\\.json", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-475-rating-voting-general-g2/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-475-rating-voting-general-g2/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-475-rating-voting-general-g2/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-475-rating-voting-general-g2/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-475-rating-voting-general-g2/steps/run/workdir/task.json new file mode 100644 index 00000000..e4f90a2e --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-475-rating-voting-general-g2/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 475, + "metaclass": "rating-voting", + "class": "general", + "description": "Write a 4-star review for Jira on G2 with exactly this text: \"Powerful tracking, complex UI.\"", + "sites_involved": [ + "g2.com" + ], + "platform": "g2", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Write a 4-star review for Jira on G2 with exactly this text: \"Powerful tracking, complex UI.\"", + "eval_schema": { + "url_pattern": "g2\\.com/survey_responses/.+/autosave_answers\\.json", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-475-rating-voting-general-g2/task.toml b/harbor/datasets/clawbench-v2/v2-475-rating-voting-general-g2/task.toml new file mode 100644 index 00000000..f3068e3b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-475-rating-voting-general-g2/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-475-rating-voting-general-g2" +description = "Write a 4-star review for Jira on G2 with exactly this text: \"Powerful tracking, complex UI.\"" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-475-rating-voting-general-g2" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-483-creation-init-general-airtable/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-483-creation-init-general-airtable/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-483-creation-init-general-airtable/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-483-creation-init-general-airtable/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-483-creation-init-general-airtable/steps/run/instruction.md new file mode 100644 index 00000000..5e24edf2 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-483-creation-init-general-airtable/steps/run/instruction.md @@ -0,0 +1,27 @@ +Create an Airtable base "Conference Tracker" with schema + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-483-creation-init-general-airtable/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-483-creation-init-general-airtable/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-483-creation-init-general-airtable/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-483-creation-init-general-airtable/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-483-creation-init-general-airtable/steps/run/tests/task.json new file mode 100644 index 00000000..19b57396 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-483-creation-init-general-airtable/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 483, + "metaclass": "creation-init", + "class": "general", + "description": "Create an Airtable base \"Conference Tracker\" with schema", + "sites_involved": [ + "airtable.com" + ], + "platform": "airtable", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Create an Airtable base \"Conference Tracker\" with schema", + "eval_schema": { + "url_pattern": "airtable\\.com/v0\\.3/application/[^/]+/create", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-483-creation-init-general-airtable/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-483-creation-init-general-airtable/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-483-creation-init-general-airtable/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-483-creation-init-general-airtable/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-483-creation-init-general-airtable/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..24fb4de9 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-483-creation-init-general-airtable/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "airtable\\.com/v0\\.3/application/[^/]+/create", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-483-creation-init-general-airtable/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-483-creation-init-general-airtable/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-483-creation-init-general-airtable/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-483-creation-init-general-airtable/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-483-creation-init-general-airtable/steps/run/workdir/task.json new file mode 100644 index 00000000..19b57396 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-483-creation-init-general-airtable/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 483, + "metaclass": "creation-init", + "class": "general", + "description": "Create an Airtable base \"Conference Tracker\" with schema", + "sites_involved": [ + "airtable.com" + ], + "platform": "airtable", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Create an Airtable base \"Conference Tracker\" with schema", + "eval_schema": { + "url_pattern": "airtable\\.com/v0\\.3/application/[^/]+/create", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-483-creation-init-general-airtable/task.toml b/harbor/datasets/clawbench-v2/v2-483-creation-init-general-airtable/task.toml new file mode 100644 index 00000000..025d2472 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-483-creation-init-general-airtable/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-483-creation-init-general-airtable" +description = "Create an Airtable base \"Conference Tracker\" with schema" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-483-creation-init-general-airtable" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-485-creation-init-general-webflow/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-485-creation-init-general-webflow/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-485-creation-init-general-webflow/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-485-creation-init-general-webflow/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-485-creation-init-general-webflow/steps/run/instruction.md new file mode 100644 index 00000000..a013007d --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-485-creation-init-general-webflow/steps/run/instruction.md @@ -0,0 +1,27 @@ +Create a portfolio website project on Webflow + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-485-creation-init-general-webflow/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-485-creation-init-general-webflow/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-485-creation-init-general-webflow/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-485-creation-init-general-webflow/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-485-creation-init-general-webflow/steps/run/tests/task.json new file mode 100644 index 00000000..850e02f1 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-485-creation-init-general-webflow/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 485, + "metaclass": "creation-init", + "class": "general", + "description": "Create a portfolio website project on Webflow", + "sites_involved": [ + "webflow.com" + ], + "platform": "webflow", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Create a portfolio website project on Webflow", + "eval_schema": { + "url_pattern": "webflow\\.com/api/workspaces/[^/]+/sites", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-485-creation-init-general-webflow/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-485-creation-init-general-webflow/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-485-creation-init-general-webflow/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-485-creation-init-general-webflow/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-485-creation-init-general-webflow/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..59d9c1c2 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-485-creation-init-general-webflow/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "webflow\\.com/api/workspaces/[^/]+/sites", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-485-creation-init-general-webflow/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-485-creation-init-general-webflow/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-485-creation-init-general-webflow/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-485-creation-init-general-webflow/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-485-creation-init-general-webflow/steps/run/workdir/task.json new file mode 100644 index 00000000..850e02f1 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-485-creation-init-general-webflow/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 485, + "metaclass": "creation-init", + "class": "general", + "description": "Create a portfolio website project on Webflow", + "sites_involved": [ + "webflow.com" + ], + "platform": "webflow", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Create a portfolio website project on Webflow", + "eval_schema": { + "url_pattern": "webflow\\.com/api/workspaces/[^/]+/sites", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-485-creation-init-general-webflow/task.toml b/harbor/datasets/clawbench-v2/v2-485-creation-init-general-webflow/task.toml new file mode 100644 index 00000000..3903a4e3 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-485-creation-init-general-webflow/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-485-creation-init-general-webflow" +description = "Create a portfolio website project on Webflow" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-485-creation-init-general-webflow" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-486-creation-init-general-mailchimp/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-486-creation-init-general-mailchimp/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-486-creation-init-general-mailchimp/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-486-creation-init-general-mailchimp/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-486-creation-init-general-mailchimp/steps/run/instruction.md new file mode 100644 index 00000000..0847ed8c --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-486-creation-init-general-mailchimp/steps/run/instruction.md @@ -0,0 +1,30 @@ +Create a Mailchimp email campaign with audience set to research newsletter + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +Additional files are also available under /my-info/ for this task: +- content.json: Content and links for the task + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-486-creation-init-general-mailchimp/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-486-creation-init-general-mailchimp/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-486-creation-init-general-mailchimp/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-486-creation-init-general-mailchimp/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-486-creation-init-general-mailchimp/steps/run/tests/task.json new file mode 100644 index 00000000..cf27e23a --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-486-creation-init-general-mailchimp/steps/run/tests/task.json @@ -0,0 +1,30 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 486, + "metaclass": "creation-init", + "class": "general", + "description": "Create a Mailchimp email campaign with audience set to research newsletter", + "sites_involved": [ + "mailchimp.com" + ], + "platform": "mailchimp", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Create a Mailchimp email campaign with audience set to research newsletter", + "eval_schema": { + "url_pattern": "admin\\.mailchimp\\.com/campaigns/(wizard|send|schedule)", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [ + { + "path": "extra_info/content.json", + "description": "Content and links for the task" + } + ] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-486-creation-init-general-mailchimp/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-486-creation-init-general-mailchimp/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-486-creation-init-general-mailchimp/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-486-creation-init-general-mailchimp/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-486-creation-init-general-mailchimp/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..716e3b30 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-486-creation-init-general-mailchimp/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "admin\\.mailchimp\\.com/campaigns/(wizard|send|schedule)", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-486-creation-init-general-mailchimp/steps/run/workdir/extra_info/content.json b/harbor/datasets/clawbench-v2/v2-486-creation-init-general-mailchimp/steps/run/workdir/extra_info/content.json new file mode 100644 index 00000000..d5d611eb --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-486-creation-init-general-mailchimp/steps/run/workdir/extra_info/content.json @@ -0,0 +1,3 @@ +{ + "content": "Sharing our latest research on scaling language models \u2014 key findings on training dynamics, emergent capabilities, and practical deployment considerations. #MachineLearning #AI #Research" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-486-creation-init-general-mailchimp/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-486-creation-init-general-mailchimp/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-486-creation-init-general-mailchimp/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-486-creation-init-general-mailchimp/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-486-creation-init-general-mailchimp/steps/run/workdir/task.json new file mode 100644 index 00000000..cf27e23a --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-486-creation-init-general-mailchimp/steps/run/workdir/task.json @@ -0,0 +1,30 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 486, + "metaclass": "creation-init", + "class": "general", + "description": "Create a Mailchimp email campaign with audience set to research newsletter", + "sites_involved": [ + "mailchimp.com" + ], + "platform": "mailchimp", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Create a Mailchimp email campaign with audience set to research newsletter", + "eval_schema": { + "url_pattern": "admin\\.mailchimp\\.com/campaigns/(wizard|send|schedule)", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [ + { + "path": "extra_info/content.json", + "description": "Content and links for the task" + } + ] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-486-creation-init-general-mailchimp/task.toml b/harbor/datasets/clawbench-v2/v2-486-creation-init-general-mailchimp/task.toml new file mode 100644 index 00000000..787662b7 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-486-creation-init-general-mailchimp/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-486-creation-init-general-mailchimp" +description = "Create a Mailchimp email campaign with audience set to research newsletter" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-486-creation-init-general-mailchimp" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-487-creation-init-general-typeform/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-487-creation-init-general-typeform/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-487-creation-init-general-typeform/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-487-creation-init-general-typeform/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-487-creation-init-general-typeform/steps/run/instruction.md new file mode 100644 index 00000000..bc1e13ec --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-487-creation-init-general-typeform/steps/run/instruction.md @@ -0,0 +1,30 @@ +Create a Typeform user research survey with 10 questions + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +Additional files are also available under /my-info/ for this task: +- survey_questions.json: Survey questions for user research with 10 questions of various types + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-487-creation-init-general-typeform/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-487-creation-init-general-typeform/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-487-creation-init-general-typeform/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-487-creation-init-general-typeform/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-487-creation-init-general-typeform/steps/run/tests/task.json new file mode 100644 index 00000000..9fcb27c2 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-487-creation-init-general-typeform/steps/run/tests/task.json @@ -0,0 +1,30 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 487, + "metaclass": "creation-init", + "class": "general", + "description": "Create a Typeform user research survey with 10 questions", + "sites_involved": [ + "typeform.com" + ], + "platform": "typeform", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Create a Typeform user research survey with 10 questions", + "eval_schema": { + "url_pattern": "admin\\.typeform\\.com/bff/bob-the-builder/forms/", + "method": "PUT" + }, + "time_limit": 30, + "extra_info": [ + { + "path": "extra_info/survey_questions.json", + "description": "Survey questions for user research with 10 questions of various types" + } + ] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-487-creation-init-general-typeform/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-487-creation-init-general-typeform/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-487-creation-init-general-typeform/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-487-creation-init-general-typeform/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-487-creation-init-general-typeform/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..42f59675 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-487-creation-init-general-typeform/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "admin\\.typeform\\.com/bff/bob-the-builder/forms/", + "method": "PUT" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-487-creation-init-general-typeform/steps/run/workdir/extra_info/survey_questions.json b/harbor/datasets/clawbench-v2/v2-487-creation-init-general-typeform/steps/run/workdir/extra_info/survey_questions.json new file mode 100644 index 00000000..b79816bb --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-487-creation-init-general-typeform/steps/run/workdir/extra_info/survey_questions.json @@ -0,0 +1,85 @@ +{ + "title": "User Research Survey: Developer Productivity", + "questions": [ + { + "id": 1, + "type": "multiple_choice", + "text": "What is your primary programming language?", + "options": [ + "Python", + "JavaScript/TypeScript", + "Go", + "Java", + "Rust", + "Other" + ] + }, + { + "id": 2, + "type": "rating", + "text": "How satisfied are you with your current IDE?", + "scale": "1-5" + }, + { + "id": 3, + "type": "short_text", + "text": "What is the biggest bottleneck in your daily workflow?" + }, + { + "id": 4, + "type": "multiple_choice", + "text": "How many hours per week do you spend on code reviews?", + "options": [ + "0-2", + "3-5", + "6-10", + "10+" + ] + }, + { + "id": 5, + "type": "yes_no", + "text": "Do you use AI-assisted coding tools?" + }, + { + "id": 6, + "type": "long_text", + "text": "Describe your ideal development environment." + }, + { + "id": 7, + "type": "multiple_choice", + "text": "What deployment method do you use most?", + "options": [ + "Docker/K8s", + "Serverless", + "VMs", + "PaaS", + "Other" + ] + }, + { + "id": 8, + "type": "rating", + "text": "How would you rate your team's CI/CD pipeline?", + "scale": "1-5" + }, + { + "id": 9, + "type": "short_text", + "text": "What tool do you wish existed?" + }, + { + "id": 10, + "type": "multiple_choice", + "text": "How do you prefer to learn new technologies?", + "options": [ + "Documentation", + "Video tutorials", + "Hands-on projects", + "Courses", + "Peer learning" + ] + } + ] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-487-creation-init-general-typeform/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-487-creation-init-general-typeform/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-487-creation-init-general-typeform/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-487-creation-init-general-typeform/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-487-creation-init-general-typeform/steps/run/workdir/task.json new file mode 100644 index 00000000..9fcb27c2 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-487-creation-init-general-typeform/steps/run/workdir/task.json @@ -0,0 +1,30 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 487, + "metaclass": "creation-init", + "class": "general", + "description": "Create a Typeform user research survey with 10 questions", + "sites_involved": [ + "typeform.com" + ], + "platform": "typeform", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Create a Typeform user research survey with 10 questions", + "eval_schema": { + "url_pattern": "admin\\.typeform\\.com/bff/bob-the-builder/forms/", + "method": "PUT" + }, + "time_limit": 30, + "extra_info": [ + { + "path": "extra_info/survey_questions.json", + "description": "Survey questions for user research with 10 questions of various types" + } + ] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-487-creation-init-general-typeform/task.toml b/harbor/datasets/clawbench-v2/v2-487-creation-init-general-typeform/task.toml new file mode 100644 index 00000000..d35de80d --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-487-creation-init-general-typeform/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-487-creation-init-general-typeform" +description = "Create a Typeform user research survey with 10 questions" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-487-creation-init-general-typeform" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-488-creation-init-general-substack/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-488-creation-init-general-substack/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-488-creation-init-general-substack/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-488-creation-init-general-substack/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-488-creation-init-general-substack/steps/run/instruction.md new file mode 100644 index 00000000..84060e0e --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-488-creation-init-general-substack/steps/run/instruction.md @@ -0,0 +1,30 @@ +Create a Substack newsletter "ML Research Roundup" and publish the first issue + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +Additional files are also available under /my-info/ for this task: +- content.json: Content and links for the task + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-488-creation-init-general-substack/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-488-creation-init-general-substack/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-488-creation-init-general-substack/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-488-creation-init-general-substack/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-488-creation-init-general-substack/steps/run/tests/task.json new file mode 100644 index 00000000..3b463cf1 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-488-creation-init-general-substack/steps/run/tests/task.json @@ -0,0 +1,30 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 488, + "metaclass": "creation-init", + "class": "general", + "description": "Create a Substack newsletter \"ML Research Roundup\" and publish the first issue", + "sites_involved": [ + "substack.com" + ], + "platform": "substack", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Create a Substack newsletter \"ML Research Roundup\" and publish the first issue", + "eval_schema": { + "url_pattern": "substack\\.com/api/v1/drafts/\\d+/publish", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [ + { + "path": "extra_info/content.json", + "description": "Content and links for the task" + } + ] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-488-creation-init-general-substack/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-488-creation-init-general-substack/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-488-creation-init-general-substack/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-488-creation-init-general-substack/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-488-creation-init-general-substack/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..28be8b31 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-488-creation-init-general-substack/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "substack\\.com/api/v1/drafts/\\d+/publish", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-488-creation-init-general-substack/steps/run/workdir/extra_info/content.json b/harbor/datasets/clawbench-v2/v2-488-creation-init-general-substack/steps/run/workdir/extra_info/content.json new file mode 100644 index 00000000..d5d611eb --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-488-creation-init-general-substack/steps/run/workdir/extra_info/content.json @@ -0,0 +1,3 @@ +{ + "content": "Sharing our latest research on scaling language models \u2014 key findings on training dynamics, emergent capabilities, and practical deployment considerations. #MachineLearning #AI #Research" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-488-creation-init-general-substack/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-488-creation-init-general-substack/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-488-creation-init-general-substack/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-488-creation-init-general-substack/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-488-creation-init-general-substack/steps/run/workdir/task.json new file mode 100644 index 00000000..3b463cf1 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-488-creation-init-general-substack/steps/run/workdir/task.json @@ -0,0 +1,30 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 488, + "metaclass": "creation-init", + "class": "general", + "description": "Create a Substack newsletter \"ML Research Roundup\" and publish the first issue", + "sites_involved": [ + "substack.com" + ], + "platform": "substack", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Create a Substack newsletter \"ML Research Roundup\" and publish the first issue", + "eval_schema": { + "url_pattern": "substack\\.com/api/v1/drafts/\\d+/publish", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [ + { + "path": "extra_info/content.json", + "description": "Content and links for the task" + } + ] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-488-creation-init-general-substack/task.toml b/harbor/datasets/clawbench-v2/v2-488-creation-init-general-substack/task.toml new file mode 100644 index 00000000..461d76a3 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-488-creation-init-general-substack/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-488-creation-init-general-substack" +description = "Create a Substack newsletter \"ML Research Roundup\" and publish the first issue" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-488-creation-init-general-substack" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-500-office-secretary-tasks-signup-doodle/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-500-office-secretary-tasks-signup-doodle/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-500-office-secretary-tasks-signup-doodle/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-500-office-secretary-tasks-signup-doodle/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-500-office-secretary-tasks-signup-doodle/steps/run/instruction.md new file mode 100644 index 00000000..78ff3130 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-500-office-secretary-tasks-signup-doodle/steps/run/instruction.md @@ -0,0 +1,27 @@ +Create a sign-up sheet on Doodle for volunteers to bring snacks to the office Friday party + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-500-office-secretary-tasks-signup-doodle/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-500-office-secretary-tasks-signup-doodle/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-500-office-secretary-tasks-signup-doodle/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-500-office-secretary-tasks-signup-doodle/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-500-office-secretary-tasks-signup-doodle/steps/run/tests/task.json new file mode 100644 index 00000000..04a6a741 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-500-office-secretary-tasks-signup-doodle/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 500, + "metaclass": "office-secretary-tasks", + "class": "signup", + "description": "Create a sign-up sheet on Doodle for volunteers to bring snacks to the office Friday party", + "sites_involved": [ + "doodle.com" + ], + "platform": "doodle", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Create a sign-up sheet on Doodle for volunteers to bring snacks to the office Friday party", + "eval_schema": { + "url_pattern": "api\\.doodle\\.com/svc-sign-up-sheet/v[\\d.]+/sheets", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-500-office-secretary-tasks-signup-doodle/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-500-office-secretary-tasks-signup-doodle/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-500-office-secretary-tasks-signup-doodle/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-500-office-secretary-tasks-signup-doodle/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-500-office-secretary-tasks-signup-doodle/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..1daece90 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-500-office-secretary-tasks-signup-doodle/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "api\\.doodle\\.com/svc-sign-up-sheet/v[\\d.]+/sheets", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-500-office-secretary-tasks-signup-doodle/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-500-office-secretary-tasks-signup-doodle/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-500-office-secretary-tasks-signup-doodle/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-500-office-secretary-tasks-signup-doodle/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-500-office-secretary-tasks-signup-doodle/steps/run/workdir/task.json new file mode 100644 index 00000000..04a6a741 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-500-office-secretary-tasks-signup-doodle/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 500, + "metaclass": "office-secretary-tasks", + "class": "signup", + "description": "Create a sign-up sheet on Doodle for volunteers to bring snacks to the office Friday party", + "sites_involved": [ + "doodle.com" + ], + "platform": "doodle", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Create a sign-up sheet on Doodle for volunteers to bring snacks to the office Friday party", + "eval_schema": { + "url_pattern": "api\\.doodle\\.com/svc-sign-up-sheet/v[\\d.]+/sheets", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-500-office-secretary-tasks-signup-doodle/task.toml b/harbor/datasets/clawbench-v2/v2-500-office-secretary-tasks-signup-doodle/task.toml new file mode 100644 index 00000000..e78d90c6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-500-office-secretary-tasks-signup-doodle/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-500-office-secretary-tasks-signup-doodle" +description = "Create a sign-up sheet on Doodle for volunteers to bring snacks to the office Friday party" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-500-office-secretary-tasks-signup-doodle" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-501-office-secretary-tasks-signup-doodle/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-501-office-secretary-tasks-signup-doodle/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-501-office-secretary-tasks-signup-doodle/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-501-office-secretary-tasks-signup-doodle/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-501-office-secretary-tasks-signup-doodle/steps/run/instruction.md new file mode 100644 index 00000000..a7e38d54 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-501-office-secretary-tasks-signup-doodle/steps/run/instruction.md @@ -0,0 +1,27 @@ +Create a sign-up sheet on Doodle for team members to pick presentation slots at the quarterly all-hands + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-501-office-secretary-tasks-signup-doodle/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-501-office-secretary-tasks-signup-doodle/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-501-office-secretary-tasks-signup-doodle/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-501-office-secretary-tasks-signup-doodle/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-501-office-secretary-tasks-signup-doodle/steps/run/tests/task.json new file mode 100644 index 00000000..3214bf55 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-501-office-secretary-tasks-signup-doodle/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 501, + "metaclass": "office-secretary-tasks", + "class": "signup", + "description": "Create a sign-up sheet on Doodle for team members to pick presentation slots at the quarterly all-hands", + "sites_involved": [ + "doodle.com" + ], + "platform": "doodle", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Create a sign-up sheet on Doodle for team members to pick presentation slots at the quarterly all-hands", + "eval_schema": { + "url_pattern": "api\\.doodle\\.com/svc-sign-up-sheet/v[\\d.]+/sheets", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-501-office-secretary-tasks-signup-doodle/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-501-office-secretary-tasks-signup-doodle/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-501-office-secretary-tasks-signup-doodle/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-501-office-secretary-tasks-signup-doodle/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-501-office-secretary-tasks-signup-doodle/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..1daece90 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-501-office-secretary-tasks-signup-doodle/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "api\\.doodle\\.com/svc-sign-up-sheet/v[\\d.]+/sheets", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-501-office-secretary-tasks-signup-doodle/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-501-office-secretary-tasks-signup-doodle/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-501-office-secretary-tasks-signup-doodle/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-501-office-secretary-tasks-signup-doodle/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-501-office-secretary-tasks-signup-doodle/steps/run/workdir/task.json new file mode 100644 index 00000000..3214bf55 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-501-office-secretary-tasks-signup-doodle/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 501, + "metaclass": "office-secretary-tasks", + "class": "signup", + "description": "Create a sign-up sheet on Doodle for team members to pick presentation slots at the quarterly all-hands", + "sites_involved": [ + "doodle.com" + ], + "platform": "doodle", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Create a sign-up sheet on Doodle for team members to pick presentation slots at the quarterly all-hands", + "eval_schema": { + "url_pattern": "api\\.doodle\\.com/svc-sign-up-sheet/v[\\d.]+/sheets", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-501-office-secretary-tasks-signup-doodle/task.toml b/harbor/datasets/clawbench-v2/v2-501-office-secretary-tasks-signup-doodle/task.toml new file mode 100644 index 00000000..c707f770 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-501-office-secretary-tasks-signup-doodle/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-501-office-secretary-tasks-signup-doodle" +description = "Create a sign-up sheet on Doodle for team members to pick presentation slots at the quarterly all-hands" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-501-office-secretary-tasks-signup-doodle" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-502-office-secretary-tasks-calendar-doodle/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-502-office-secretary-tasks-calendar-doodle/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-502-office-secretary-tasks-calendar-doodle/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-502-office-secretary-tasks-calendar-doodle/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-502-office-secretary-tasks-calendar-doodle/steps/run/instruction.md new file mode 100644 index 00000000..cd859c30 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-502-office-secretary-tasks-calendar-doodle/steps/run/instruction.md @@ -0,0 +1,27 @@ +Create a Doodle group poll for a 1-hour Client Sync on Thursday + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-502-office-secretary-tasks-calendar-doodle/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-502-office-secretary-tasks-calendar-doodle/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-502-office-secretary-tasks-calendar-doodle/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-502-office-secretary-tasks-calendar-doodle/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-502-office-secretary-tasks-calendar-doodle/steps/run/tests/task.json new file mode 100644 index 00000000..240869d4 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-502-office-secretary-tasks-calendar-doodle/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 502, + "metaclass": "office-secretary-tasks", + "class": "calendar", + "description": "Create a Doodle group poll for a 1-hour Client Sync on Thursday", + "sites_involved": [ + "doodle.com" + ], + "platform": "doodle", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Create a Doodle group poll for a 1-hour Client Sync on Thursday", + "eval_schema": { + "url_pattern": "api\\.doodle\\.com/scheduling/scheduling-attempts", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-502-office-secretary-tasks-calendar-doodle/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-502-office-secretary-tasks-calendar-doodle/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-502-office-secretary-tasks-calendar-doodle/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-502-office-secretary-tasks-calendar-doodle/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-502-office-secretary-tasks-calendar-doodle/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..b126d723 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-502-office-secretary-tasks-calendar-doodle/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "api\\.doodle\\.com/scheduling/scheduling-attempts", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-502-office-secretary-tasks-calendar-doodle/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-502-office-secretary-tasks-calendar-doodle/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-502-office-secretary-tasks-calendar-doodle/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-502-office-secretary-tasks-calendar-doodle/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-502-office-secretary-tasks-calendar-doodle/steps/run/workdir/task.json new file mode 100644 index 00000000..240869d4 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-502-office-secretary-tasks-calendar-doodle/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 502, + "metaclass": "office-secretary-tasks", + "class": "calendar", + "description": "Create a Doodle group poll for a 1-hour Client Sync on Thursday", + "sites_involved": [ + "doodle.com" + ], + "platform": "doodle", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Create a Doodle group poll for a 1-hour Client Sync on Thursday", + "eval_schema": { + "url_pattern": "api\\.doodle\\.com/scheduling/scheduling-attempts", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-502-office-secretary-tasks-calendar-doodle/task.toml b/harbor/datasets/clawbench-v2/v2-502-office-secretary-tasks-calendar-doodle/task.toml new file mode 100644 index 00000000..12e71a7e --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-502-office-secretary-tasks-calendar-doodle/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-502-office-secretary-tasks-calendar-doodle" +description = "Create a Doodle group poll for a 1-hour Client Sync on Thursday" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-502-office-secretary-tasks-calendar-doodle" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-503-office-secretary-tasks-calendar-doodle/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-503-office-secretary-tasks-calendar-doodle/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-503-office-secretary-tasks-calendar-doodle/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-503-office-secretary-tasks-calendar-doodle/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-503-office-secretary-tasks-calendar-doodle/steps/run/instruction.md new file mode 100644 index 00000000..a0e71c03 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-503-office-secretary-tasks-calendar-doodle/steps/run/instruction.md @@ -0,0 +1,27 @@ +Use Doodle to find a time tomorrow for a 30-minute design review in SF + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-503-office-secretary-tasks-calendar-doodle/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-503-office-secretary-tasks-calendar-doodle/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-503-office-secretary-tasks-calendar-doodle/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-503-office-secretary-tasks-calendar-doodle/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-503-office-secretary-tasks-calendar-doodle/steps/run/tests/task.json new file mode 100644 index 00000000..71433141 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-503-office-secretary-tasks-calendar-doodle/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 503, + "metaclass": "office-secretary-tasks", + "class": "calendar", + "description": "Use Doodle to find a time tomorrow for a 30-minute design review in SF", + "sites_involved": [ + "doodle.com" + ], + "platform": "doodle", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Use Doodle to find a time tomorrow for a 30-minute design review in SF", + "eval_schema": { + "url_pattern": "api\\.doodle\\.com/scheduling/scheduling-attempts", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-503-office-secretary-tasks-calendar-doodle/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-503-office-secretary-tasks-calendar-doodle/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-503-office-secretary-tasks-calendar-doodle/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-503-office-secretary-tasks-calendar-doodle/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-503-office-secretary-tasks-calendar-doodle/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..b126d723 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-503-office-secretary-tasks-calendar-doodle/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "api\\.doodle\\.com/scheduling/scheduling-attempts", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-503-office-secretary-tasks-calendar-doodle/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-503-office-secretary-tasks-calendar-doodle/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-503-office-secretary-tasks-calendar-doodle/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-503-office-secretary-tasks-calendar-doodle/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-503-office-secretary-tasks-calendar-doodle/steps/run/workdir/task.json new file mode 100644 index 00000000..71433141 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-503-office-secretary-tasks-calendar-doodle/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 503, + "metaclass": "office-secretary-tasks", + "class": "calendar", + "description": "Use Doodle to find a time tomorrow for a 30-minute design review in SF", + "sites_involved": [ + "doodle.com" + ], + "platform": "doodle", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Use Doodle to find a time tomorrow for a 30-minute design review in SF", + "eval_schema": { + "url_pattern": "api\\.doodle\\.com/scheduling/scheduling-attempts", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-503-office-secretary-tasks-calendar-doodle/task.toml b/harbor/datasets/clawbench-v2/v2-503-office-secretary-tasks-calendar-doodle/task.toml new file mode 100644 index 00000000..a067c879 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-503-office-secretary-tasks-calendar-doodle/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-503-office-secretary-tasks-calendar-doodle" +description = "Use Doodle to find a time tomorrow for a 30-minute design review in SF" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-503-office-secretary-tasks-calendar-doodle" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-520-shopping-retail-target/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-520-shopping-retail-target/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-520-shopping-retail-target/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-520-shopping-retail-target/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-520-shopping-retail-target/steps/run/instruction.md new file mode 100644 index 00000000..051fd7e8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-520-shopping-retail-target/steps/run/instruction.md @@ -0,0 +1,27 @@ +Search Target for a 6-piece cotton bath towel set in white and add the highest rated one to cart + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-520-shopping-retail-target/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-520-shopping-retail-target/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-520-shopping-retail-target/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-520-shopping-retail-target/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-520-shopping-retail-target/steps/run/tests/task.json new file mode 100644 index 00000000..4768bd0f --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-520-shopping-retail-target/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 520, + "metaclass": "shopping", + "class": "retail", + "description": "Search Target for a 6-piece cotton bath towel set in white and add the highest rated to cart", + "sites_involved": [ + "target.com" + ], + "platform": "target", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Search Target for a 6-piece cotton bath towel set in white and add the highest rated one to cart", + "eval_schema": { + "url_pattern": "carts\\.target\\.com/web_checkouts/v\\d+/cart_items", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-520-shopping-retail-target/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-520-shopping-retail-target/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-520-shopping-retail-target/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-520-shopping-retail-target/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-520-shopping-retail-target/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..50481bf4 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-520-shopping-retail-target/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "carts\\.target\\.com/web_checkouts/v\\d+/cart_items", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-520-shopping-retail-target/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-520-shopping-retail-target/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-520-shopping-retail-target/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-520-shopping-retail-target/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-520-shopping-retail-target/steps/run/workdir/task.json new file mode 100644 index 00000000..4768bd0f --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-520-shopping-retail-target/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 520, + "metaclass": "shopping", + "class": "retail", + "description": "Search Target for a 6-piece cotton bath towel set in white and add the highest rated to cart", + "sites_involved": [ + "target.com" + ], + "platform": "target", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Search Target for a 6-piece cotton bath towel set in white and add the highest rated one to cart", + "eval_schema": { + "url_pattern": "carts\\.target\\.com/web_checkouts/v\\d+/cart_items", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-520-shopping-retail-target/task.toml b/harbor/datasets/clawbench-v2/v2-520-shopping-retail-target/task.toml new file mode 100644 index 00000000..06a93e06 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-520-shopping-retail-target/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-520-shopping-retail-target" +description = "Search Target for a 6-piece cotton bath towel set in white and add the highest rated to cart" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-520-shopping-retail-target" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-521-shopping-retail-target/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-521-shopping-retail-target/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-521-shopping-retail-target/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-521-shopping-retail-target/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-521-shopping-retail-target/steps/run/instruction.md new file mode 100644 index 00000000..64199573 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-521-shopping-retail-target/steps/run/instruction.md @@ -0,0 +1,27 @@ +Search Target for a stainless steel insulated water bottle and add the best-selling one to cart + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-521-shopping-retail-target/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-521-shopping-retail-target/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-521-shopping-retail-target/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-521-shopping-retail-target/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-521-shopping-retail-target/steps/run/tests/task.json new file mode 100644 index 00000000..bd782533 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-521-shopping-retail-target/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 521, + "metaclass": "shopping", + "class": "retail", + "description": "Search Target for a stainless steel insulated water bottle, sort by best-selling, and add the top result to cart", + "sites_involved": [ + "target.com" + ], + "platform": "target", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Search Target for a stainless steel insulated water bottle and add the best-selling one to cart", + "eval_schema": { + "url_pattern": "carts\\.target\\.com/web_checkouts/v\\d+/cart_items", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-521-shopping-retail-target/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-521-shopping-retail-target/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-521-shopping-retail-target/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-521-shopping-retail-target/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-521-shopping-retail-target/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..50481bf4 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-521-shopping-retail-target/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "carts\\.target\\.com/web_checkouts/v\\d+/cart_items", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-521-shopping-retail-target/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-521-shopping-retail-target/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-521-shopping-retail-target/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-521-shopping-retail-target/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-521-shopping-retail-target/steps/run/workdir/task.json new file mode 100644 index 00000000..bd782533 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-521-shopping-retail-target/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 521, + "metaclass": "shopping", + "class": "retail", + "description": "Search Target for a stainless steel insulated water bottle, sort by best-selling, and add the top result to cart", + "sites_involved": [ + "target.com" + ], + "platform": "target", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Search Target for a stainless steel insulated water bottle and add the best-selling one to cart", + "eval_schema": { + "url_pattern": "carts\\.target\\.com/web_checkouts/v\\d+/cart_items", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-521-shopping-retail-target/task.toml b/harbor/datasets/clawbench-v2/v2-521-shopping-retail-target/task.toml new file mode 100644 index 00000000..337895a0 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-521-shopping-retail-target/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-521-shopping-retail-target" +description = "Search Target for a stainless steel insulated water bottle, sort by best-selling, and add the top result to cart" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-521-shopping-retail-target" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-522-shopping-retail-target/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-522-shopping-retail-target/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-522-shopping-retail-target/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-522-shopping-retail-target/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-522-shopping-retail-target/steps/run/instruction.md new file mode 100644 index 00000000..1d6bd1fd --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-522-shopping-retail-target/steps/run/instruction.md @@ -0,0 +1,27 @@ +Search Target for a non-stick frying pan 12 inch and add the highest rated one to cart + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-522-shopping-retail-target/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-522-shopping-retail-target/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-522-shopping-retail-target/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-522-shopping-retail-target/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-522-shopping-retail-target/steps/run/tests/task.json new file mode 100644 index 00000000..7d92f1ad --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-522-shopping-retail-target/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 522, + "metaclass": "shopping", + "class": "retail", + "description": "Search Target for a 12-inch non-stick frying pan, find the highest rated one, and add it to cart", + "sites_involved": [ + "target.com" + ], + "platform": "target", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Search Target for a non-stick frying pan 12 inch and add the highest rated one to cart", + "eval_schema": { + "url_pattern": "carts\\.target\\.com/web_checkouts/v\\d+/cart_items", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-522-shopping-retail-target/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-522-shopping-retail-target/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-522-shopping-retail-target/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-522-shopping-retail-target/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-522-shopping-retail-target/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..50481bf4 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-522-shopping-retail-target/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "carts\\.target\\.com/web_checkouts/v\\d+/cart_items", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-522-shopping-retail-target/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-522-shopping-retail-target/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-522-shopping-retail-target/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-522-shopping-retail-target/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-522-shopping-retail-target/steps/run/workdir/task.json new file mode 100644 index 00000000..7d92f1ad --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-522-shopping-retail-target/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 522, + "metaclass": "shopping", + "class": "retail", + "description": "Search Target for a 12-inch non-stick frying pan, find the highest rated one, and add it to cart", + "sites_involved": [ + "target.com" + ], + "platform": "target", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Search Target for a non-stick frying pan 12 inch and add the highest rated one to cart", + "eval_schema": { + "url_pattern": "carts\\.target\\.com/web_checkouts/v\\d+/cart_items", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-522-shopping-retail-target/task.toml b/harbor/datasets/clawbench-v2/v2-522-shopping-retail-target/task.toml new file mode 100644 index 00000000..62d40eb0 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-522-shopping-retail-target/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-522-shopping-retail-target" +description = "Search Target for a 12-inch non-stick frying pan, find the highest rated one, and add it to cart" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-522-shopping-retail-target" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-523-shopping-retail-target/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-523-shopping-retail-target/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-523-shopping-retail-target/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-523-shopping-retail-target/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-523-shopping-retail-target/steps/run/instruction.md new file mode 100644 index 00000000..a35cc682 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-523-shopping-retail-target/steps/run/instruction.md @@ -0,0 +1,27 @@ +Search Target for a queen size microfiber bed sheet set in gray and add the lowest priced one to cart + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-523-shopping-retail-target/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-523-shopping-retail-target/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-523-shopping-retail-target/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-523-shopping-retail-target/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-523-shopping-retail-target/steps/run/tests/task.json new file mode 100644 index 00000000..ea244ccb --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-523-shopping-retail-target/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 523, + "metaclass": "shopping", + "class": "retail", + "description": "Search Target for a queen size microfiber bed sheet set in gray, sort by lowest price, and add the top result to cart", + "sites_involved": [ + "target.com" + ], + "platform": "target", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Search Target for a queen size microfiber bed sheet set in gray and add the lowest priced one to cart", + "eval_schema": { + "url_pattern": "carts\\.target\\.com/web_checkouts/v\\d+/cart_items", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-523-shopping-retail-target/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-523-shopping-retail-target/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-523-shopping-retail-target/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-523-shopping-retail-target/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-523-shopping-retail-target/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..50481bf4 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-523-shopping-retail-target/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "carts\\.target\\.com/web_checkouts/v\\d+/cart_items", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-523-shopping-retail-target/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-523-shopping-retail-target/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-523-shopping-retail-target/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-523-shopping-retail-target/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-523-shopping-retail-target/steps/run/workdir/task.json new file mode 100644 index 00000000..ea244ccb --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-523-shopping-retail-target/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 523, + "metaclass": "shopping", + "class": "retail", + "description": "Search Target for a queen size microfiber bed sheet set in gray, sort by lowest price, and add the top result to cart", + "sites_involved": [ + "target.com" + ], + "platform": "target", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Search Target for a queen size microfiber bed sheet set in gray and add the lowest priced one to cart", + "eval_schema": { + "url_pattern": "carts\\.target\\.com/web_checkouts/v\\d+/cart_items", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-523-shopping-retail-target/task.toml b/harbor/datasets/clawbench-v2/v2-523-shopping-retail-target/task.toml new file mode 100644 index 00000000..8205a0bd --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-523-shopping-retail-target/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-523-shopping-retail-target" +description = "Search Target for a queen size microfiber bed sheet set in gray, sort by lowest price, and add the top result to cart" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-523-shopping-retail-target" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-530-shopping-marketplace-etsy/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-530-shopping-marketplace-etsy/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-530-shopping-marketplace-etsy/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-530-shopping-marketplace-etsy/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-530-shopping-marketplace-etsy/steps/run/instruction.md new file mode 100644 index 00000000..c4939fb6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-530-shopping-marketplace-etsy/steps/run/instruction.md @@ -0,0 +1,27 @@ +Search Etsy for a personalized leather bifold wallet with initials engraving and add it to cart + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-530-shopping-marketplace-etsy/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-530-shopping-marketplace-etsy/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-530-shopping-marketplace-etsy/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-530-shopping-marketplace-etsy/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-530-shopping-marketplace-etsy/steps/run/tests/task.json new file mode 100644 index 00000000..6cd8906d --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-530-shopping-marketplace-etsy/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 530, + "metaclass": "shopping", + "class": "marketplace", + "description": "Search Etsy for a personalized leather bifold wallet with initials engraving and add it to cart", + "sites_involved": [ + "etsy.com" + ], + "platform": "etsy", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Search Etsy for a personalized leather bifold wallet with initials engraving and add it to cart", + "eval_schema": { + "url_pattern": "etsy\\.com/cart/listing", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-530-shopping-marketplace-etsy/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-530-shopping-marketplace-etsy/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-530-shopping-marketplace-etsy/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-530-shopping-marketplace-etsy/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-530-shopping-marketplace-etsy/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..111ed380 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-530-shopping-marketplace-etsy/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "etsy\\.com/cart/listing", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-530-shopping-marketplace-etsy/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-530-shopping-marketplace-etsy/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-530-shopping-marketplace-etsy/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-530-shopping-marketplace-etsy/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-530-shopping-marketplace-etsy/steps/run/workdir/task.json new file mode 100644 index 00000000..6cd8906d --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-530-shopping-marketplace-etsy/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 530, + "metaclass": "shopping", + "class": "marketplace", + "description": "Search Etsy for a personalized leather bifold wallet with initials engraving and add it to cart", + "sites_involved": [ + "etsy.com" + ], + "platform": "etsy", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Search Etsy for a personalized leather bifold wallet with initials engraving and add it to cart", + "eval_schema": { + "url_pattern": "etsy\\.com/cart/listing", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-530-shopping-marketplace-etsy/task.toml b/harbor/datasets/clawbench-v2/v2-530-shopping-marketplace-etsy/task.toml new file mode 100644 index 00000000..5b722a65 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-530-shopping-marketplace-etsy/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-530-shopping-marketplace-etsy" +description = "Search Etsy for a personalized leather bifold wallet with initials engraving and add it to cart" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-530-shopping-marketplace-etsy" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-531-shopping-marketplace-etsy/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-531-shopping-marketplace-etsy/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-531-shopping-marketplace-etsy/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-531-shopping-marketplace-etsy/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-531-shopping-marketplace-etsy/steps/run/instruction.md new file mode 100644 index 00000000..380ebde1 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-531-shopping-marketplace-etsy/steps/run/instruction.md @@ -0,0 +1,27 @@ +Find a leather dog collar with custom name embroidery for a medium-sized dog on Etsy and add to cart + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-531-shopping-marketplace-etsy/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-531-shopping-marketplace-etsy/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-531-shopping-marketplace-etsy/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-531-shopping-marketplace-etsy/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-531-shopping-marketplace-etsy/steps/run/tests/task.json new file mode 100644 index 00000000..ffabcc01 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-531-shopping-marketplace-etsy/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 531, + "metaclass": "shopping", + "class": "marketplace", + "description": "Find a leather dog collar with custom name embroidery for a medium-sized dog on Etsy and add to cart", + "sites_involved": [ + "etsy.com" + ], + "platform": "etsy", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Find a leather dog collar with custom name embroidery for a medium-sized dog on Etsy and add to cart", + "eval_schema": { + "url_pattern": "etsy\\.com/cart/listing", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-531-shopping-marketplace-etsy/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-531-shopping-marketplace-etsy/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-531-shopping-marketplace-etsy/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-531-shopping-marketplace-etsy/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-531-shopping-marketplace-etsy/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..111ed380 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-531-shopping-marketplace-etsy/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "etsy\\.com/cart/listing", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-531-shopping-marketplace-etsy/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-531-shopping-marketplace-etsy/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-531-shopping-marketplace-etsy/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-531-shopping-marketplace-etsy/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-531-shopping-marketplace-etsy/steps/run/workdir/task.json new file mode 100644 index 00000000..ffabcc01 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-531-shopping-marketplace-etsy/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 531, + "metaclass": "shopping", + "class": "marketplace", + "description": "Find a leather dog collar with custom name embroidery for a medium-sized dog on Etsy and add to cart", + "sites_involved": [ + "etsy.com" + ], + "platform": "etsy", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Find a leather dog collar with custom name embroidery for a medium-sized dog on Etsy and add to cart", + "eval_schema": { + "url_pattern": "etsy\\.com/cart/listing", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-531-shopping-marketplace-etsy/task.toml b/harbor/datasets/clawbench-v2/v2-531-shopping-marketplace-etsy/task.toml new file mode 100644 index 00000000..0b7f234e --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-531-shopping-marketplace-etsy/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-531-shopping-marketplace-etsy" +description = "Find a leather dog collar with custom name embroidery for a medium-sized dog on Etsy and add to cart" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-531-shopping-marketplace-etsy" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-533-daily-life-utilities-inmyarea/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-533-daily-life-utilities-inmyarea/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-533-daily-life-utilities-inmyarea/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-533-daily-life-utilities-inmyarea/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-533-daily-life-utilities-inmyarea/steps/run/instruction.md new file mode 100644 index 00000000..fc2755d0 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-533-daily-life-utilities-inmyarea/steps/run/instruction.md @@ -0,0 +1,27 @@ +On InMyArea, enter home address (ZIP 90210), select an internet service plan, click Order Now, then on the provider's signup page fill in personal information and complete the subscription + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-533-daily-life-utilities-inmyarea/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-533-daily-life-utilities-inmyarea/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-533-daily-life-utilities-inmyarea/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-533-daily-life-utilities-inmyarea/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-533-daily-life-utilities-inmyarea/steps/run/tests/task.json new file mode 100644 index 00000000..47a718a0 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-533-daily-life-utilities-inmyarea/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 533, + "metaclass": "daily-life", + "class": "utilities", + "description": "On InMyArea, enter home address (ZIP 90210), select an internet service plan, click Order Now, then on the provider's signup page fill in personal information and complete the subscription", + "sites_involved": [ + "inmyarea.com" + ], + "platform": "inmyarea", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "On InMyArea, enter home address (ZIP 90210), select an internet service plan, click Order Now, then on the provider's signup page fill in personal information and complete the subscription", + "eval_schema": { + "url_pattern": "att\\.com/msapi/salesapi/wireless-sales-eapi/v2/addtocart", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-533-daily-life-utilities-inmyarea/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-533-daily-life-utilities-inmyarea/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-533-daily-life-utilities-inmyarea/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-533-daily-life-utilities-inmyarea/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-533-daily-life-utilities-inmyarea/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..18327e58 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-533-daily-life-utilities-inmyarea/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "att\\.com/msapi/salesapi/wireless-sales-eapi/v2/addtocart", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-533-daily-life-utilities-inmyarea/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-533-daily-life-utilities-inmyarea/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-533-daily-life-utilities-inmyarea/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-533-daily-life-utilities-inmyarea/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-533-daily-life-utilities-inmyarea/steps/run/workdir/task.json new file mode 100644 index 00000000..47a718a0 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-533-daily-life-utilities-inmyarea/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 533, + "metaclass": "daily-life", + "class": "utilities", + "description": "On InMyArea, enter home address (ZIP 90210), select an internet service plan, click Order Now, then on the provider's signup page fill in personal information and complete the subscription", + "sites_involved": [ + "inmyarea.com" + ], + "platform": "inmyarea", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "On InMyArea, enter home address (ZIP 90210), select an internet service plan, click Order Now, then on the provider's signup page fill in personal information and complete the subscription", + "eval_schema": { + "url_pattern": "att\\.com/msapi/salesapi/wireless-sales-eapi/v2/addtocart", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-533-daily-life-utilities-inmyarea/task.toml b/harbor/datasets/clawbench-v2/v2-533-daily-life-utilities-inmyarea/task.toml new file mode 100644 index 00000000..98582a2c --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-533-daily-life-utilities-inmyarea/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-533-daily-life-utilities-inmyarea" +description = "On InMyArea, enter home address (ZIP 90210), select an internet service plan, click Order Now, then on the provider's signup page fill in personal information and complete the subscription" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-533-daily-life-utilities-inmyarea" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-535-daily-life-shopping-etsy/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-535-daily-life-shopping-etsy/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-535-daily-life-shopping-etsy/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-535-daily-life-shopping-etsy/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-535-daily-life-shopping-etsy/steps/run/instruction.md new file mode 100644 index 00000000..6ce639be --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-535-daily-life-shopping-etsy/steps/run/instruction.md @@ -0,0 +1,27 @@ +Search Etsy for a handmade blue ceramic flower vase under $50 and add it to your favorites + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-535-daily-life-shopping-etsy/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-535-daily-life-shopping-etsy/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-535-daily-life-shopping-etsy/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-535-daily-life-shopping-etsy/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-535-daily-life-shopping-etsy/steps/run/tests/task.json new file mode 100644 index 00000000..6cb370dd --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-535-daily-life-shopping-etsy/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 535, + "metaclass": "daily-life", + "class": "shopping", + "description": "Search Etsy for a handmade blue ceramic flower vase under $50 and add it to your favorites", + "sites_involved": [ + "etsy.com" + ], + "platform": "etsy", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Search Etsy for a handmade blue ceramic flower vase under $50 and add it to your favorites", + "eval_schema": { + "url_pattern": "etsy\\.com/api/v\\d+/ajax/bespoke/member/neu/specs/favoriteConfirmationToast", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-535-daily-life-shopping-etsy/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-535-daily-life-shopping-etsy/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-535-daily-life-shopping-etsy/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-535-daily-life-shopping-etsy/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-535-daily-life-shopping-etsy/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..96b0fa9c --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-535-daily-life-shopping-etsy/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "etsy\\.com/api/v\\d+/ajax/bespoke/member/neu/specs/favoriteConfirmationToast", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-535-daily-life-shopping-etsy/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-535-daily-life-shopping-etsy/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-535-daily-life-shopping-etsy/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-535-daily-life-shopping-etsy/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-535-daily-life-shopping-etsy/steps/run/workdir/task.json new file mode 100644 index 00000000..6cb370dd --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-535-daily-life-shopping-etsy/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 535, + "metaclass": "daily-life", + "class": "shopping", + "description": "Search Etsy for a handmade blue ceramic flower vase under $50 and add it to your favorites", + "sites_involved": [ + "etsy.com" + ], + "platform": "etsy", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Search Etsy for a handmade blue ceramic flower vase under $50 and add it to your favorites", + "eval_schema": { + "url_pattern": "etsy\\.com/api/v\\d+/ajax/bespoke/member/neu/specs/favoriteConfirmationToast", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-535-daily-life-shopping-etsy/task.toml b/harbor/datasets/clawbench-v2/v2-535-daily-life-shopping-etsy/task.toml new file mode 100644 index 00000000..ea76f864 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-535-daily-life-shopping-etsy/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-535-daily-life-shopping-etsy" +description = "Search Etsy for a handmade blue ceramic flower vase under $50 and add it to your favorites" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-535-daily-life-shopping-etsy" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-536-daily-life-shopping-etsy/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-536-daily-life-shopping-etsy/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-536-daily-life-shopping-etsy/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-536-daily-life-shopping-etsy/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-536-daily-life-shopping-etsy/steps/run/instruction.md new file mode 100644 index 00000000..71777c40 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-536-daily-life-shopping-etsy/steps/run/instruction.md @@ -0,0 +1,27 @@ +Find a mid-century modern wooden bookshelf on Etsy priced under $300 and add it to your favorites + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-536-daily-life-shopping-etsy/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-536-daily-life-shopping-etsy/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-536-daily-life-shopping-etsy/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-536-daily-life-shopping-etsy/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-536-daily-life-shopping-etsy/steps/run/tests/task.json new file mode 100644 index 00000000..bbd3ce63 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-536-daily-life-shopping-etsy/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 536, + "metaclass": "daily-life", + "class": "shopping", + "description": "Find a mid-century modern wooden bookshelf on Etsy priced under $300 and add it to your favorites", + "sites_involved": [ + "etsy.com" + ], + "platform": "etsy", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Find a mid-century modern wooden bookshelf on Etsy priced under $300 and add it to your favorites", + "eval_schema": { + "url_pattern": "etsy\\.com/api/v\\d+/ajax/bespoke/member/neu/specs/favoriteConfirmationToast", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-536-daily-life-shopping-etsy/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-536-daily-life-shopping-etsy/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-536-daily-life-shopping-etsy/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-536-daily-life-shopping-etsy/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-536-daily-life-shopping-etsy/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..96b0fa9c --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-536-daily-life-shopping-etsy/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "etsy\\.com/api/v\\d+/ajax/bespoke/member/neu/specs/favoriteConfirmationToast", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-536-daily-life-shopping-etsy/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-536-daily-life-shopping-etsy/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-536-daily-life-shopping-etsy/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-536-daily-life-shopping-etsy/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-536-daily-life-shopping-etsy/steps/run/workdir/task.json new file mode 100644 index 00000000..bbd3ce63 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-536-daily-life-shopping-etsy/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 536, + "metaclass": "daily-life", + "class": "shopping", + "description": "Find a mid-century modern wooden bookshelf on Etsy priced under $300 and add it to your favorites", + "sites_involved": [ + "etsy.com" + ], + "platform": "etsy", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Find a mid-century modern wooden bookshelf on Etsy priced under $300 and add it to your favorites", + "eval_schema": { + "url_pattern": "etsy\\.com/api/v\\d+/ajax/bespoke/member/neu/specs/favoriteConfirmationToast", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-536-daily-life-shopping-etsy/task.toml b/harbor/datasets/clawbench-v2/v2-536-daily-life-shopping-etsy/task.toml new file mode 100644 index 00000000..837eee78 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-536-daily-life-shopping-etsy/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-536-daily-life-shopping-etsy" +description = "Find a mid-century modern wooden bookshelf on Etsy priced under $300 and add it to your favorites" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-536-daily-life-shopping-etsy" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-560/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-560/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-560/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-560/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-560/steps/run/instruction.md new file mode 100644 index 00000000..027f5974 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-560/steps/run/instruction.md @@ -0,0 +1,29 @@ +Search for a free tech networking event in San Francisco on Eventbrite and register for it. + +Note: The event is free. Look for a tech networking event in San Francisco and complete the registration flow. + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-560/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-560/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-560/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-560/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-560/steps/run/tests/task.json new file mode 100644 index 00000000..8dcb4e3e --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-560/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 560, + "metaclass": "web_task", + "class": "registration", + "description": "Register for a free tech networking event in San Francisco on Eventbrite", + "sites_involved": [ + "https://www.eventbrite.com" + ], + "platform": "Eventbrite", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Search for a free tech networking event in San Francisco on Eventbrite and register for it.\n\nNote: The event is free. Look for a tech networking event in San Francisco and complete the registration flow.", + "eval_schema": { + "url_pattern": "eventbrite\\.com/purchase-flow-webapp/api/v\\d+/orders/create", + "method": "POST" + }, + "time_limit": 120, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-560/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-560/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-560/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-560/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-560/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..96c60f42 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-560/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "eventbrite\\.com/purchase-flow-webapp/api/v\\d+/orders/create", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-560/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-560/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-560/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-560/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-560/steps/run/workdir/task.json new file mode 100644 index 00000000..8dcb4e3e --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-560/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 560, + "metaclass": "web_task", + "class": "registration", + "description": "Register for a free tech networking event in San Francisco on Eventbrite", + "sites_involved": [ + "https://www.eventbrite.com" + ], + "platform": "Eventbrite", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Search for a free tech networking event in San Francisco on Eventbrite and register for it.\n\nNote: The event is free. Look for a tech networking event in San Francisco and complete the registration flow.", + "eval_schema": { + "url_pattern": "eventbrite\\.com/purchase-flow-webapp/api/v\\d+/orders/create", + "method": "POST" + }, + "time_limit": 120, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-560/task.toml b/harbor/datasets/clawbench-v2/v2-560/task.toml new file mode 100644 index 00000000..bdf39ef2 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-560/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-560" +description = "Register for a free tech networking event in San Francisco on Eventbrite" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-560" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 7200.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-564/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-564/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-564/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-564/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-564/steps/run/instruction.md new file mode 100644 index 00000000..e94ae6c1 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-564/steps/run/instruction.md @@ -0,0 +1,29 @@ +Search for a free startup pitch night event in New York on Eventbrite and register for 2 tickets. + +Note: The event is free. Look for a startup pitch night in New York and register for 2 tickets. + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-564/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-564/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-564/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-564/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-564/steps/run/tests/task.json new file mode 100644 index 00000000..98791cce --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-564/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 564, + "metaclass": "web_task", + "class": "registration", + "description": "Register for 2 tickets to a free startup pitch night in New York on Eventbrite", + "sites_involved": [ + "https://www.eventbrite.com" + ], + "platform": "Eventbrite", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Search for a free startup pitch night event in New York on Eventbrite and register for 2 tickets.\n\nNote: The event is free. Look for a startup pitch night in New York and register for 2 tickets.", + "eval_schema": { + "url_pattern": "eventbrite\\.com/purchase-flow-webapp/api/v\\d+/orders/create", + "method": "POST" + }, + "time_limit": 120, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-564/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-564/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-564/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-564/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-564/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..96c60f42 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-564/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "eventbrite\\.com/purchase-flow-webapp/api/v\\d+/orders/create", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-564/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-564/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-564/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-564/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-564/steps/run/workdir/task.json new file mode 100644 index 00000000..98791cce --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-564/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 564, + "metaclass": "web_task", + "class": "registration", + "description": "Register for 2 tickets to a free startup pitch night in New York on Eventbrite", + "sites_involved": [ + "https://www.eventbrite.com" + ], + "platform": "Eventbrite", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Search for a free startup pitch night event in New York on Eventbrite and register for 2 tickets.\n\nNote: The event is free. Look for a startup pitch night in New York and register for 2 tickets.", + "eval_schema": { + "url_pattern": "eventbrite\\.com/purchase-flow-webapp/api/v\\d+/orders/create", + "method": "POST" + }, + "time_limit": 120, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-564/task.toml b/harbor/datasets/clawbench-v2/v2-564/task.toml new file mode 100644 index 00000000..8a99af42 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-564/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-564" +description = "Register for 2 tickets to a free startup pitch night in New York on Eventbrite" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-564" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 7200.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-571-job-search-hr-recruitment-mgmt-workable/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-571-job-search-hr-recruitment-mgmt-workable/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-571-job-search-hr-recruitment-mgmt-workable/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-571-job-search-hr-recruitment-mgmt-workable/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-571-job-search-hr-recruitment-mgmt-workable/steps/run/instruction.md new file mode 100644 index 00000000..e5049a79 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-571-job-search-hr-recruitment-mgmt-workable/steps/run/instruction.md @@ -0,0 +1,27 @@ +Sign up for a Workable free trial, create a new job position, and publish it to the Workable careers page + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-571-job-search-hr-recruitment-mgmt-workable/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-571-job-search-hr-recruitment-mgmt-workable/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-571-job-search-hr-recruitment-mgmt-workable/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-571-job-search-hr-recruitment-mgmt-workable/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-571-job-search-hr-recruitment-mgmt-workable/steps/run/tests/task.json new file mode 100644 index 00000000..1f418690 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-571-job-search-hr-recruitment-mgmt-workable/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 571, + "metaclass": "job-search-hr", + "class": "recruitment-mgmt", + "description": "Sign up for a Workable free trial, create a new job position, and publish it to the Workable careers page", + "sites_involved": [ + "workable.com" + ], + "platform": "workable", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Sign up for a Workable free trial, create a new job position, and publish it to the Workable careers page", + "eval_schema": { + "url_pattern": "workable\\.com.*(jobs?/(create|publish|new|draft)|api/(spi|v\\d+)/jobs|careers/publish|/openings)", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-571-job-search-hr-recruitment-mgmt-workable/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-571-job-search-hr-recruitment-mgmt-workable/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-571-job-search-hr-recruitment-mgmt-workable/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-571-job-search-hr-recruitment-mgmt-workable/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-571-job-search-hr-recruitment-mgmt-workable/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..da7c4318 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-571-job-search-hr-recruitment-mgmt-workable/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "workable\\.com.*(jobs?/(create|publish|new|draft)|api/(spi|v\\d+)/jobs|careers/publish|/openings)", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-571-job-search-hr-recruitment-mgmt-workable/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-571-job-search-hr-recruitment-mgmt-workable/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-571-job-search-hr-recruitment-mgmt-workable/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-571-job-search-hr-recruitment-mgmt-workable/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-571-job-search-hr-recruitment-mgmt-workable/steps/run/workdir/task.json new file mode 100644 index 00000000..1f418690 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-571-job-search-hr-recruitment-mgmt-workable/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 571, + "metaclass": "job-search-hr", + "class": "recruitment-mgmt", + "description": "Sign up for a Workable free trial, create a new job position, and publish it to the Workable careers page", + "sites_involved": [ + "workable.com" + ], + "platform": "workable", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Sign up for a Workable free trial, create a new job position, and publish it to the Workable careers page", + "eval_schema": { + "url_pattern": "workable\\.com.*(jobs?/(create|publish|new|draft)|api/(spi|v\\d+)/jobs|careers/publish|/openings)", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-571-job-search-hr-recruitment-mgmt-workable/task.toml b/harbor/datasets/clawbench-v2/v2-571-job-search-hr-recruitment-mgmt-workable/task.toml new file mode 100644 index 00000000..cbee919e --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-571-job-search-hr-recruitment-mgmt-workable/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-571-job-search-hr-recruitment-mgmt-workable" +description = "Sign up for a Workable free trial, create a new job position, and publish it to the Workable careers page" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-571-job-search-hr-recruitment-mgmt-workable" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-583-office-secretary-tasks-support-tickets-freshdesk/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-583-office-secretary-tasks-support-tickets-freshdesk/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-583-office-secretary-tasks-support-tickets-freshdesk/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-583-office-secretary-tasks-support-tickets-freshdesk/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-583-office-secretary-tasks-support-tickets-freshdesk/steps/run/instruction.md new file mode 100644 index 00000000..41ae9a71 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-583-office-secretary-tasks-support-tickets-freshdesk/steps/run/instruction.md @@ -0,0 +1,27 @@ +Register a free Freshdesk Sprout account, create a new support ticket with subject "Invoice Dispute - Order #10042", describe a billing discrepancy, set priority to High, and assign to a test agent + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-583-office-secretary-tasks-support-tickets-freshdesk/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-583-office-secretary-tasks-support-tickets-freshdesk/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-583-office-secretary-tasks-support-tickets-freshdesk/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-583-office-secretary-tasks-support-tickets-freshdesk/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-583-office-secretary-tasks-support-tickets-freshdesk/steps/run/tests/task.json new file mode 100644 index 00000000..5412849b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-583-office-secretary-tasks-support-tickets-freshdesk/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 583, + "metaclass": "office-secretary-tasks", + "class": "customer-support", + "description": "Register a free Freshdesk Sprout account, create a new support ticket with subject \"Invoice Dispute - Order #10042\", describe a billing discrepancy, set priority to High, and assign to a test agent", + "sites_involved": [ + "freshdesk.com" + ], + "platform": "freshdesk", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Register a free Freshdesk Sprout account, create a new support ticket with subject \"Invoice Dispute - Order #10042\", describe a billing discrepancy, set priority to High, and assign to a test agent", + "eval_schema": { + "url_pattern": "freshdesk\\.com/api/_/tickets", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-583-office-secretary-tasks-support-tickets-freshdesk/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-583-office-secretary-tasks-support-tickets-freshdesk/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-583-office-secretary-tasks-support-tickets-freshdesk/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-583-office-secretary-tasks-support-tickets-freshdesk/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-583-office-secretary-tasks-support-tickets-freshdesk/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..018f964d --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-583-office-secretary-tasks-support-tickets-freshdesk/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "freshdesk\\.com/api/_/tickets", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-583-office-secretary-tasks-support-tickets-freshdesk/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-583-office-secretary-tasks-support-tickets-freshdesk/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-583-office-secretary-tasks-support-tickets-freshdesk/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-583-office-secretary-tasks-support-tickets-freshdesk/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-583-office-secretary-tasks-support-tickets-freshdesk/steps/run/workdir/task.json new file mode 100644 index 00000000..5412849b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-583-office-secretary-tasks-support-tickets-freshdesk/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 583, + "metaclass": "office-secretary-tasks", + "class": "customer-support", + "description": "Register a free Freshdesk Sprout account, create a new support ticket with subject \"Invoice Dispute - Order #10042\", describe a billing discrepancy, set priority to High, and assign to a test agent", + "sites_involved": [ + "freshdesk.com" + ], + "platform": "freshdesk", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Register a free Freshdesk Sprout account, create a new support ticket with subject \"Invoice Dispute - Order #10042\", describe a billing discrepancy, set priority to High, and assign to a test agent", + "eval_schema": { + "url_pattern": "freshdesk\\.com/api/_/tickets", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-583-office-secretary-tasks-support-tickets-freshdesk/task.toml b/harbor/datasets/clawbench-v2/v2-583-office-secretary-tasks-support-tickets-freshdesk/task.toml new file mode 100644 index 00000000..23eb1ff7 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-583-office-secretary-tasks-support-tickets-freshdesk/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-583-office-secretary-tasks-support-tickets-freshdesk" +description = "Register a free Freshdesk Sprout account, create a new support ticket with subject \"Invoice Dispute - Order #10042\", describe a billing discrepancy, set priority to High, and assign to a test agent" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-583-office-secretary-tasks-support-tickets-freshdesk" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-596-daily-life-fitness-strava/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-596-daily-life-fitness-strava/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-596-daily-life-fitness-strava/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-596-daily-life-fitness-strava/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-596-daily-life-fitness-strava/steps/run/instruction.md new file mode 100644 index 00000000..2b417a93 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-596-daily-life-fitness-strava/steps/run/instruction.md @@ -0,0 +1,27 @@ +Add a 20-mile cycling activity to Strava titled 'Morning Ride' + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-596-daily-life-fitness-strava/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-596-daily-life-fitness-strava/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-596-daily-life-fitness-strava/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-596-daily-life-fitness-strava/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-596-daily-life-fitness-strava/steps/run/tests/task.json new file mode 100644 index 00000000..0af09427 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-596-daily-life-fitness-strava/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 596, + "metaclass": "daily-life", + "class": "fitness", + "description": "Add a 20-mile cycling activity to Strava titled 'Morning Ride'", + "sites_involved": [ + "strava.com" + ], + "platform": "strava", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Add a 20-mile cycling activity to Strava titled 'Morning Ride'", + "eval_schema": { + "url_pattern": "strava\\.com\\/activities", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-596-daily-life-fitness-strava/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-596-daily-life-fitness-strava/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-596-daily-life-fitness-strava/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-596-daily-life-fitness-strava/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-596-daily-life-fitness-strava/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..76c0411d --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-596-daily-life-fitness-strava/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "strava\\.com\\/activities", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-596-daily-life-fitness-strava/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-596-daily-life-fitness-strava/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-596-daily-life-fitness-strava/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-596-daily-life-fitness-strava/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-596-daily-life-fitness-strava/steps/run/workdir/task.json new file mode 100644 index 00000000..0af09427 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-596-daily-life-fitness-strava/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 596, + "metaclass": "daily-life", + "class": "fitness", + "description": "Add a 20-mile cycling activity to Strava titled 'Morning Ride'", + "sites_involved": [ + "strava.com" + ], + "platform": "strava", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Add a 20-mile cycling activity to Strava titled 'Morning Ride'", + "eval_schema": { + "url_pattern": "strava\\.com\\/activities", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-596-daily-life-fitness-strava/task.toml b/harbor/datasets/clawbench-v2/v2-596-daily-life-fitness-strava/task.toml new file mode 100644 index 00000000..5416fed6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-596-daily-life-fitness-strava/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-596-daily-life-fitness-strava" +description = "Add a 20-mile cycling activity to Strava titled 'Morning Ride'" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-596-daily-life-fitness-strava" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-597-daily-life-fitness-strava/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-597-daily-life-fitness-strava/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-597-daily-life-fitness-strava/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-597-daily-life-fitness-strava/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-597-daily-life-fitness-strava/steps/run/instruction.md new file mode 100644 index 00000000..125cccf6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-597-daily-life-fitness-strava/steps/run/instruction.md @@ -0,0 +1,27 @@ +Log a 45-minute yoga session on Strava from yesterday evening + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-597-daily-life-fitness-strava/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-597-daily-life-fitness-strava/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-597-daily-life-fitness-strava/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-597-daily-life-fitness-strava/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-597-daily-life-fitness-strava/steps/run/tests/task.json new file mode 100644 index 00000000..4eb524e2 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-597-daily-life-fitness-strava/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 597, + "metaclass": "daily-life", + "class": "fitness", + "description": "Log a 45-minute yoga session on Strava from yesterday evening", + "sites_involved": [ + "strava.com" + ], + "platform": "strava", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Log a 45-minute yoga session on Strava from yesterday evening", + "eval_schema": { + "url_pattern": "strava\\.com\\/activities", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-597-daily-life-fitness-strava/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-597-daily-life-fitness-strava/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-597-daily-life-fitness-strava/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-597-daily-life-fitness-strava/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-597-daily-life-fitness-strava/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..76c0411d --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-597-daily-life-fitness-strava/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "strava\\.com\\/activities", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-597-daily-life-fitness-strava/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-597-daily-life-fitness-strava/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-597-daily-life-fitness-strava/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-597-daily-life-fitness-strava/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-597-daily-life-fitness-strava/steps/run/workdir/task.json new file mode 100644 index 00000000..4eb524e2 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-597-daily-life-fitness-strava/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 597, + "metaclass": "daily-life", + "class": "fitness", + "description": "Log a 45-minute yoga session on Strava from yesterday evening", + "sites_involved": [ + "strava.com" + ], + "platform": "strava", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Log a 45-minute yoga session on Strava from yesterday evening", + "eval_schema": { + "url_pattern": "strava\\.com\\/activities", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-597-daily-life-fitness-strava/task.toml b/harbor/datasets/clawbench-v2/v2-597-daily-life-fitness-strava/task.toml new file mode 100644 index 00000000..531f5f92 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-597-daily-life-fitness-strava/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-597-daily-life-fitness-strava" +description = "Log a 45-minute yoga session on Strava from yesterday evening" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-597-daily-life-fitness-strava" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-598-academia-research-legal-docs-formswift/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-598-academia-research-legal-docs-formswift/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-598-academia-research-legal-docs-formswift/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-598-academia-research-legal-docs-formswift/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-598-academia-research-legal-docs-formswift/steps/run/instruction.md new file mode 100644 index 00000000..adcd5e2f --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-598-academia-research-legal-docs-formswift/steps/run/instruction.md @@ -0,0 +1,27 @@ +On FormSwift, search for a "Power of Attorney" template, fill in the principal's name, agent's name, and scope of authority, then download the completed document + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-598-academia-research-legal-docs-formswift/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-598-academia-research-legal-docs-formswift/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-598-academia-research-legal-docs-formswift/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-598-academia-research-legal-docs-formswift/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-598-academia-research-legal-docs-formswift/steps/run/tests/task.json new file mode 100644 index 00000000..e1c97c52 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-598-academia-research-legal-docs-formswift/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 598, + "metaclass": "academia-research", + "class": "legal-docs", + "description": "On FormSwift, search for a \"Power of Attorney\" template, fill in the principal's name, agent's name, and scope of authority, then download the completed document", + "sites_involved": [ + "formswift.com" + ], + "platform": "formswift", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "On FormSwift, search for a \"Power of Attorney\" template, fill in the principal's name, agent's name, and scope of authority, then download the completed document", + "eval_schema": { + "url_pattern": "api\\.formswift\\.com/static/document/save/", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-598-academia-research-legal-docs-formswift/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-598-academia-research-legal-docs-formswift/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-598-academia-research-legal-docs-formswift/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-598-academia-research-legal-docs-formswift/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-598-academia-research-legal-docs-formswift/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..bebebe35 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-598-academia-research-legal-docs-formswift/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "api\\.formswift\\.com/static/document/save/", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-598-academia-research-legal-docs-formswift/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-598-academia-research-legal-docs-formswift/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-598-academia-research-legal-docs-formswift/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-598-academia-research-legal-docs-formswift/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-598-academia-research-legal-docs-formswift/steps/run/workdir/task.json new file mode 100644 index 00000000..e1c97c52 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-598-academia-research-legal-docs-formswift/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 598, + "metaclass": "academia-research", + "class": "legal-docs", + "description": "On FormSwift, search for a \"Power of Attorney\" template, fill in the principal's name, agent's name, and scope of authority, then download the completed document", + "sites_involved": [ + "formswift.com" + ], + "platform": "formswift", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "On FormSwift, search for a \"Power of Attorney\" template, fill in the principal's name, agent's name, and scope of authority, then download the completed document", + "eval_schema": { + "url_pattern": "api\\.formswift\\.com/static/document/save/", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-598-academia-research-legal-docs-formswift/task.toml b/harbor/datasets/clawbench-v2/v2-598-academia-research-legal-docs-formswift/task.toml new file mode 100644 index 00000000..77f78656 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-598-academia-research-legal-docs-formswift/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-598-academia-research-legal-docs-formswift" +description = "On FormSwift, search for a \"Power of Attorney\" template, fill in the principal's name, agent's name, and scope of authority, then download the completed document" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-598-academia-research-legal-docs-formswift" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-600-hobbies-crafts-community-ravelry/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-600-hobbies-crafts-community-ravelry/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-600-hobbies-crafts-community-ravelry/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-600-hobbies-crafts-community-ravelry/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-600-hobbies-crafts-community-ravelry/steps/run/instruction.md new file mode 100644 index 00000000..15ae991a --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-600-hobbies-crafts-community-ravelry/steps/run/instruction.md @@ -0,0 +1,27 @@ +Search for a free beginner knitting pattern for a scarf on Ravelry and add it to your library + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-600-hobbies-crafts-community-ravelry/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-600-hobbies-crafts-community-ravelry/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-600-hobbies-crafts-community-ravelry/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-600-hobbies-crafts-community-ravelry/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-600-hobbies-crafts-community-ravelry/steps/run/tests/task.json new file mode 100644 index 00000000..7d32fcfe --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-600-hobbies-crafts-community-ravelry/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 600, + "metaclass": "hobbies-crafts", + "class": "community", + "description": "Search for a free beginner knitting pattern for a scarf on Ravelry and add it to your library", + "sites_involved": [ + "ravelry.com" + ], + "platform": "ravelry", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Search for a free beginner knitting pattern for a scarf on Ravelry and add it to your library", + "eval_schema": { + "url_pattern": "ravelry\\.com/patterns/library/[^/]+/add_to_library", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-600-hobbies-crafts-community-ravelry/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-600-hobbies-crafts-community-ravelry/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-600-hobbies-crafts-community-ravelry/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-600-hobbies-crafts-community-ravelry/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-600-hobbies-crafts-community-ravelry/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..d6299607 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-600-hobbies-crafts-community-ravelry/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "ravelry\\.com/patterns/library/[^/]+/add_to_library", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-600-hobbies-crafts-community-ravelry/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-600-hobbies-crafts-community-ravelry/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-600-hobbies-crafts-community-ravelry/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-600-hobbies-crafts-community-ravelry/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-600-hobbies-crafts-community-ravelry/steps/run/workdir/task.json new file mode 100644 index 00000000..7d32fcfe --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-600-hobbies-crafts-community-ravelry/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 600, + "metaclass": "hobbies-crafts", + "class": "community", + "description": "Search for a free beginner knitting pattern for a scarf on Ravelry and add it to your library", + "sites_involved": [ + "ravelry.com" + ], + "platform": "ravelry", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Search for a free beginner knitting pattern for a scarf on Ravelry and add it to your library", + "eval_schema": { + "url_pattern": "ravelry\\.com/patterns/library/[^/]+/add_to_library", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-600-hobbies-crafts-community-ravelry/task.toml b/harbor/datasets/clawbench-v2/v2-600-hobbies-crafts-community-ravelry/task.toml new file mode 100644 index 00000000..f172dd0d --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-600-hobbies-crafts-community-ravelry/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-600-hobbies-crafts-community-ravelry" +description = "Search for a free beginner knitting pattern for a scarf on Ravelry and add it to your library" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-600-hobbies-crafts-community-ravelry" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-601-hobbies-gaming-reviews-ravelry/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-601-hobbies-gaming-reviews-ravelry/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-601-hobbies-gaming-reviews-ravelry/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-601-hobbies-gaming-reviews-ravelry/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-601-hobbies-gaming-reviews-ravelry/steps/run/instruction.md new file mode 100644 index 00000000..cadd2267 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-601-hobbies-gaming-reviews-ravelry/steps/run/instruction.md @@ -0,0 +1,27 @@ +Find a highly-rated crochet amigurumi pattern on Ravelry and add it to your favorites + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-601-hobbies-gaming-reviews-ravelry/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-601-hobbies-gaming-reviews-ravelry/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-601-hobbies-gaming-reviews-ravelry/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-601-hobbies-gaming-reviews-ravelry/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-601-hobbies-gaming-reviews-ravelry/steps/run/tests/task.json new file mode 100644 index 00000000..cd4f699b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-601-hobbies-gaming-reviews-ravelry/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 601, + "metaclass": "hobbies-gaming", + "class": "reviews", + "description": "Find a highly-rated crochet amigurumi pattern on Ravelry and add it to your favorites", + "sites_involved": [ + "ravelry.com" + ], + "platform": "ravelry", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Find a highly-rated crochet amigurumi pattern on Ravelry and add it to your favorites", + "eval_schema": { + "url_pattern": "ravelry\\.com/patterns/library/[^/]+/bookmark", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-601-hobbies-gaming-reviews-ravelry/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-601-hobbies-gaming-reviews-ravelry/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-601-hobbies-gaming-reviews-ravelry/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-601-hobbies-gaming-reviews-ravelry/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-601-hobbies-gaming-reviews-ravelry/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..eed69457 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-601-hobbies-gaming-reviews-ravelry/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "ravelry\\.com/patterns/library/[^/]+/bookmark", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-601-hobbies-gaming-reviews-ravelry/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-601-hobbies-gaming-reviews-ravelry/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-601-hobbies-gaming-reviews-ravelry/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-601-hobbies-gaming-reviews-ravelry/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-601-hobbies-gaming-reviews-ravelry/steps/run/workdir/task.json new file mode 100644 index 00000000..cd4f699b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-601-hobbies-gaming-reviews-ravelry/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 601, + "metaclass": "hobbies-gaming", + "class": "reviews", + "description": "Find a highly-rated crochet amigurumi pattern on Ravelry and add it to your favorites", + "sites_involved": [ + "ravelry.com" + ], + "platform": "ravelry", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Find a highly-rated crochet amigurumi pattern on Ravelry and add it to your favorites", + "eval_schema": { + "url_pattern": "ravelry\\.com/patterns/library/[^/]+/bookmark", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-601-hobbies-gaming-reviews-ravelry/task.toml b/harbor/datasets/clawbench-v2/v2-601-hobbies-gaming-reviews-ravelry/task.toml new file mode 100644 index 00000000..4d8d9784 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-601-hobbies-gaming-reviews-ravelry/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-601-hobbies-gaming-reviews-ravelry" +description = "Find a highly-rated crochet amigurumi pattern on Ravelry and add it to your favorites" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-601-hobbies-gaming-reviews-ravelry" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-602-hobbies-crafts-community-ravelry/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-602-hobbies-crafts-community-ravelry/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-602-hobbies-crafts-community-ravelry/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-602-hobbies-crafts-community-ravelry/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-602-hobbies-crafts-community-ravelry/steps/run/instruction.md new file mode 100644 index 00000000..5ad95a08 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-602-hobbies-crafts-community-ravelry/steps/run/instruction.md @@ -0,0 +1,27 @@ +Search for worsted weight yarn on Ravelry and add the most popular result to your stash + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-602-hobbies-crafts-community-ravelry/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-602-hobbies-crafts-community-ravelry/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-602-hobbies-crafts-community-ravelry/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-602-hobbies-crafts-community-ravelry/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-602-hobbies-crafts-community-ravelry/steps/run/tests/task.json new file mode 100644 index 00000000..b3aacba8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-602-hobbies-crafts-community-ravelry/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 602, + "metaclass": "hobbies-crafts", + "class": "community", + "description": "Search for worsted weight yarn on Ravelry and add the most popular brand to your stash", + "sites_involved": [ + "ravelry.com" + ], + "platform": "ravelry", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Search for worsted weight yarn on Ravelry and add the most popular result to your stash", + "eval_schema": { + "url_pattern": "ravelry\\.com/people/[^/]+/stash/[^/]+/quick", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-602-hobbies-crafts-community-ravelry/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-602-hobbies-crafts-community-ravelry/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-602-hobbies-crafts-community-ravelry/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-602-hobbies-crafts-community-ravelry/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-602-hobbies-crafts-community-ravelry/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..70bf353b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-602-hobbies-crafts-community-ravelry/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "ravelry\\.com/people/[^/]+/stash/[^/]+/quick", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-602-hobbies-crafts-community-ravelry/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-602-hobbies-crafts-community-ravelry/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-602-hobbies-crafts-community-ravelry/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-602-hobbies-crafts-community-ravelry/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-602-hobbies-crafts-community-ravelry/steps/run/workdir/task.json new file mode 100644 index 00000000..b3aacba8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-602-hobbies-crafts-community-ravelry/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 602, + "metaclass": "hobbies-crafts", + "class": "community", + "description": "Search for worsted weight yarn on Ravelry and add the most popular brand to your stash", + "sites_involved": [ + "ravelry.com" + ], + "platform": "ravelry", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Search for worsted weight yarn on Ravelry and add the most popular result to your stash", + "eval_schema": { + "url_pattern": "ravelry\\.com/people/[^/]+/stash/[^/]+/quick", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-602-hobbies-crafts-community-ravelry/task.toml b/harbor/datasets/clawbench-v2/v2-602-hobbies-crafts-community-ravelry/task.toml new file mode 100644 index 00000000..2aefff2e --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-602-hobbies-crafts-community-ravelry/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-602-hobbies-crafts-community-ravelry" +description = "Search for worsted weight yarn on Ravelry and add the most popular brand to your stash" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-602-hobbies-crafts-community-ravelry" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-603-hobbies-crafts-community-ravelry/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-603-hobbies-crafts-community-ravelry/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-603-hobbies-crafts-community-ravelry/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-603-hobbies-crafts-community-ravelry/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-603-hobbies-crafts-community-ravelry/steps/run/instruction.md new file mode 100644 index 00000000..8467e7f3 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-603-hobbies-crafts-community-ravelry/steps/run/instruction.md @@ -0,0 +1,27 @@ +Find a knitting group on Ravelry, join it, and post a greeting message in its chatroom + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-603-hobbies-crafts-community-ravelry/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-603-hobbies-crafts-community-ravelry/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-603-hobbies-crafts-community-ravelry/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-603-hobbies-crafts-community-ravelry/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-603-hobbies-crafts-community-ravelry/steps/run/tests/task.json new file mode 100644 index 00000000..4030a077 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-603-hobbies-crafts-community-ravelry/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 603, + "metaclass": "hobbies-crafts", + "class": "community", + "description": "Find a knitting group on Ravelry, join it, and post a greeting message in its chatroom", + "sites_involved": [ + "ravelry.com" + ], + "platform": "ravelry", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Find a knitting group on Ravelry, join it, and post a greeting message in its chatroom", + "eval_schema": { + "url_pattern": "ravelry\\.com/chats/\\d+/talk", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-603-hobbies-crafts-community-ravelry/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-603-hobbies-crafts-community-ravelry/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-603-hobbies-crafts-community-ravelry/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-603-hobbies-crafts-community-ravelry/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-603-hobbies-crafts-community-ravelry/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..0fccd3d2 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-603-hobbies-crafts-community-ravelry/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "ravelry\\.com/chats/\\d+/talk", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-603-hobbies-crafts-community-ravelry/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-603-hobbies-crafts-community-ravelry/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-603-hobbies-crafts-community-ravelry/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-603-hobbies-crafts-community-ravelry/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-603-hobbies-crafts-community-ravelry/steps/run/workdir/task.json new file mode 100644 index 00000000..4030a077 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-603-hobbies-crafts-community-ravelry/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 603, + "metaclass": "hobbies-crafts", + "class": "community", + "description": "Find a knitting group on Ravelry, join it, and post a greeting message in its chatroom", + "sites_involved": [ + "ravelry.com" + ], + "platform": "ravelry", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Find a knitting group on Ravelry, join it, and post a greeting message in its chatroom", + "eval_schema": { + "url_pattern": "ravelry\\.com/chats/\\d+/talk", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-603-hobbies-crafts-community-ravelry/task.toml b/harbor/datasets/clawbench-v2/v2-603-hobbies-crafts-community-ravelry/task.toml new file mode 100644 index 00000000..b6f08167 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-603-hobbies-crafts-community-ravelry/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-603-hobbies-crafts-community-ravelry" +description = "Find a knitting group on Ravelry, join it, and post a greeting message in its chatroom" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-603-hobbies-crafts-community-ravelry" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-607-hobbies-gaming-reviews-boardgamegeek/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-607-hobbies-gaming-reviews-boardgamegeek/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-607-hobbies-gaming-reviews-boardgamegeek/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-607-hobbies-gaming-reviews-boardgamegeek/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-607-hobbies-gaming-reviews-boardgamegeek/steps/run/instruction.md new file mode 100644 index 00000000..e0364004 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-607-hobbies-gaming-reviews-boardgamegeek/steps/run/instruction.md @@ -0,0 +1,27 @@ +Log a play for 'Pandemic' on BoardGameGeek with 4 players and a duration of 60 minutes + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-607-hobbies-gaming-reviews-boardgamegeek/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-607-hobbies-gaming-reviews-boardgamegeek/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-607-hobbies-gaming-reviews-boardgamegeek/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-607-hobbies-gaming-reviews-boardgamegeek/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-607-hobbies-gaming-reviews-boardgamegeek/steps/run/tests/task.json new file mode 100644 index 00000000..c5cab6f9 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-607-hobbies-gaming-reviews-boardgamegeek/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 607, + "metaclass": "hobbies-gaming", + "class": "reviews", + "description": "Log a play for 'Pandemic' on BoardGameGeek with 4 players and a duration of 60 minutes", + "sites_involved": [ + "boardgamegeek.com" + ], + "platform": "boardgamegeek", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Log a play for 'Pandemic' on BoardGameGeek with 4 players and a duration of 60 minutes", + "eval_schema": { + "url_pattern": "boardgamegeek\\.com/(geekplay|api/plays)", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-607-hobbies-gaming-reviews-boardgamegeek/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-607-hobbies-gaming-reviews-boardgamegeek/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-607-hobbies-gaming-reviews-boardgamegeek/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-607-hobbies-gaming-reviews-boardgamegeek/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-607-hobbies-gaming-reviews-boardgamegeek/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..cfc94b2b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-607-hobbies-gaming-reviews-boardgamegeek/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "boardgamegeek\\.com/(geekplay|api/plays)", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-607-hobbies-gaming-reviews-boardgamegeek/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-607-hobbies-gaming-reviews-boardgamegeek/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-607-hobbies-gaming-reviews-boardgamegeek/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-607-hobbies-gaming-reviews-boardgamegeek/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-607-hobbies-gaming-reviews-boardgamegeek/steps/run/workdir/task.json new file mode 100644 index 00000000..c5cab6f9 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-607-hobbies-gaming-reviews-boardgamegeek/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 607, + "metaclass": "hobbies-gaming", + "class": "reviews", + "description": "Log a play for 'Pandemic' on BoardGameGeek with 4 players and a duration of 60 minutes", + "sites_involved": [ + "boardgamegeek.com" + ], + "platform": "boardgamegeek", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Log a play for 'Pandemic' on BoardGameGeek with 4 players and a duration of 60 minutes", + "eval_schema": { + "url_pattern": "boardgamegeek\\.com/(geekplay|api/plays)", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-607-hobbies-gaming-reviews-boardgamegeek/task.toml b/harbor/datasets/clawbench-v2/v2-607-hobbies-gaming-reviews-boardgamegeek/task.toml new file mode 100644 index 00000000..43195a95 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-607-hobbies-gaming-reviews-boardgamegeek/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-607-hobbies-gaming-reviews-boardgamegeek" +description = "Log a play for 'Pandemic' on BoardGameGeek with 4 players and a duration of 60 minutes" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-607-hobbies-gaming-reviews-boardgamegeek" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-608-education-learning-kids-courses-outschool/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-608-education-learning-kids-courses-outschool/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-608-education-learning-kids-courses-outschool/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-608-education-learning-kids-courses-outschool/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-608-education-learning-kids-courses-outschool/steps/run/instruction.md new file mode 100644 index 00000000..6159b181 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-608-education-learning-kids-courses-outschool/steps/run/instruction.md @@ -0,0 +1,27 @@ +Search for a Minecraft coding class for kids on Outschool and open the detail page of one of the results to view its description, schedule, and teacher information + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-608-education-learning-kids-courses-outschool/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-608-education-learning-kids-courses-outschool/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-608-education-learning-kids-courses-outschool/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-608-education-learning-kids-courses-outschool/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-608-education-learning-kids-courses-outschool/steps/run/tests/task.json new file mode 100644 index 00000000..3c52b73f --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-608-education-learning-kids-courses-outschool/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 608, + "metaclass": "education-learning", + "class": "kids-courses", + "description": "Search for a Minecraft coding class for kids on Outschool and open the detail page of one of the results to view its description, schedule, and teacher information", + "sites_involved": [ + "outschool.com" + ], + "platform": "outschool", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Search for a Minecraft coding class for kids on Outschool and open the detail page of one of the results to view its description, schedule, and teacher information", + "eval_schema": { + "url_pattern": "outschool\\.com/classes/", + "method": "GET" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-608-education-learning-kids-courses-outschool/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-608-education-learning-kids-courses-outschool/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-608-education-learning-kids-courses-outschool/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-608-education-learning-kids-courses-outschool/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-608-education-learning-kids-courses-outschool/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..2c61716d --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-608-education-learning-kids-courses-outschool/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "outschool\\.com/classes/", + "method": "GET" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-608-education-learning-kids-courses-outschool/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-608-education-learning-kids-courses-outschool/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-608-education-learning-kids-courses-outschool/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-608-education-learning-kids-courses-outschool/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-608-education-learning-kids-courses-outschool/steps/run/workdir/task.json new file mode 100644 index 00000000..3c52b73f --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-608-education-learning-kids-courses-outschool/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 608, + "metaclass": "education-learning", + "class": "kids-courses", + "description": "Search for a Minecraft coding class for kids on Outschool and open the detail page of one of the results to view its description, schedule, and teacher information", + "sites_involved": [ + "outschool.com" + ], + "platform": "outschool", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Search for a Minecraft coding class for kids on Outschool and open the detail page of one of the results to view its description, schedule, and teacher information", + "eval_schema": { + "url_pattern": "outschool\\.com/classes/", + "method": "GET" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-608-education-learning-kids-courses-outschool/task.toml b/harbor/datasets/clawbench-v2/v2-608-education-learning-kids-courses-outschool/task.toml new file mode 100644 index 00000000..4b4cc3d5 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-608-education-learning-kids-courses-outschool/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-608-education-learning-kids-courses-outschool" +description = "Search for a Minecraft coding class for kids on Outschool and open the detail page of one of the results to view its description, schedule, and teacher information" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-608-education-learning-kids-courses-outschool" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-609-education-learning-meditation-spirit-rock-meditation-center/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-609-education-learning-meditation-spirit-rock-meditation-center/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-609-education-learning-meditation-spirit-rock-meditation-center/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-609-education-learning-meditation-spirit-rock-meditation-center/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-609-education-learning-meditation-spirit-rock-meditation-center/steps/run/instruction.md new file mode 100644 index 00000000..f4144651 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-609-education-learning-meditation-spirit-rock-meditation-center/steps/run/instruction.md @@ -0,0 +1,27 @@ +Browse upcoming weekend retreat offerings on the Spirit Rock Meditation Center website, select one session, and complete the registration + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-609-education-learning-meditation-spirit-rock-meditation-center/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-609-education-learning-meditation-spirit-rock-meditation-center/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-609-education-learning-meditation-spirit-rock-meditation-center/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-609-education-learning-meditation-spirit-rock-meditation-center/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-609-education-learning-meditation-spirit-rock-meditation-center/steps/run/tests/task.json new file mode 100644 index 00000000..4b2d5c87 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-609-education-learning-meditation-spirit-rock-meditation-center/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 609, + "metaclass": "education-learning", + "class": "meditation", + "description": "Browse upcoming weekend retreat offerings on the Spirit Rock Meditation Center website, select one session, and complete the registration", + "sites_involved": [ + "spiritrock.org" + ], + "platform": "spirit-rock", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Browse upcoming weekend retreat offerings on the Spirit Rock Meditation Center website, select one session, and complete the registration", + "eval_schema": { + "url_pattern": "spirit-rock\\.secure\\.retreat\\.guru/program/.*\\?form=1", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-609-education-learning-meditation-spirit-rock-meditation-center/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-609-education-learning-meditation-spirit-rock-meditation-center/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-609-education-learning-meditation-spirit-rock-meditation-center/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-609-education-learning-meditation-spirit-rock-meditation-center/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-609-education-learning-meditation-spirit-rock-meditation-center/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..ed896fe7 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-609-education-learning-meditation-spirit-rock-meditation-center/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "spirit-rock\\.secure\\.retreat\\.guru/program/.*\\?form=1", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-609-education-learning-meditation-spirit-rock-meditation-center/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-609-education-learning-meditation-spirit-rock-meditation-center/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-609-education-learning-meditation-spirit-rock-meditation-center/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-609-education-learning-meditation-spirit-rock-meditation-center/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-609-education-learning-meditation-spirit-rock-meditation-center/steps/run/workdir/task.json new file mode 100644 index 00000000..4b2d5c87 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-609-education-learning-meditation-spirit-rock-meditation-center/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 609, + "metaclass": "education-learning", + "class": "meditation", + "description": "Browse upcoming weekend retreat offerings on the Spirit Rock Meditation Center website, select one session, and complete the registration", + "sites_involved": [ + "spiritrock.org" + ], + "platform": "spirit-rock", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Browse upcoming weekend retreat offerings on the Spirit Rock Meditation Center website, select one session, and complete the registration", + "eval_schema": { + "url_pattern": "spirit-rock\\.secure\\.retreat\\.guru/program/.*\\?form=1", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-609-education-learning-meditation-spirit-rock-meditation-center/task.toml b/harbor/datasets/clawbench-v2/v2-609-education-learning-meditation-spirit-rock-meditation-center/task.toml new file mode 100644 index 00000000..a1347450 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-609-education-learning-meditation-spirit-rock-meditation-center/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-609-education-learning-meditation-spirit-rock-meditation-center" +description = "Browse upcoming weekend retreat offerings on the Spirit Rock Meditation Center website, select one session, and complete the registration" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-609-education-learning-meditation-spirit-rock-meditation-center" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-610-hobbies-gaming-reviews-boardgamegeek/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-610-hobbies-gaming-reviews-boardgamegeek/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-610-hobbies-gaming-reviews-boardgamegeek/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-610-hobbies-gaming-reviews-boardgamegeek/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-610-hobbies-gaming-reviews-boardgamegeek/steps/run/instruction.md new file mode 100644 index 00000000..8b9b38af --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-610-hobbies-gaming-reviews-boardgamegeek/steps/run/instruction.md @@ -0,0 +1,27 @@ +Add yourself as a fan to 'Brass: Birmingham' (2018) on BoardGameGeek + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-610-hobbies-gaming-reviews-boardgamegeek/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-610-hobbies-gaming-reviews-boardgamegeek/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-610-hobbies-gaming-reviews-boardgamegeek/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-610-hobbies-gaming-reviews-boardgamegeek/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-610-hobbies-gaming-reviews-boardgamegeek/steps/run/tests/task.json new file mode 100644 index 00000000..b9ce7845 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-610-hobbies-gaming-reviews-boardgamegeek/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 610, + "metaclass": "entertainment-hobbies", + "class": "gaming-reviews", + "description": "Add yourself as a fan to 'Brass: Birmingham' (2018) on BoardGameGeek", + "sites_involved": [ + "boardgamegeek.com" + ], + "platform": "boardgamegeek", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Add yourself as a fan to 'Brass: Birmingham' (2018) on BoardGameGeek", + "eval_schema": { + "url_pattern": "boardgamegeek\\.com/api/fans", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-610-hobbies-gaming-reviews-boardgamegeek/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-610-hobbies-gaming-reviews-boardgamegeek/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-610-hobbies-gaming-reviews-boardgamegeek/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-610-hobbies-gaming-reviews-boardgamegeek/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-610-hobbies-gaming-reviews-boardgamegeek/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..8ac070ec --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-610-hobbies-gaming-reviews-boardgamegeek/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "boardgamegeek\\.com/api/fans", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-610-hobbies-gaming-reviews-boardgamegeek/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-610-hobbies-gaming-reviews-boardgamegeek/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-610-hobbies-gaming-reviews-boardgamegeek/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-610-hobbies-gaming-reviews-boardgamegeek/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-610-hobbies-gaming-reviews-boardgamegeek/steps/run/workdir/task.json new file mode 100644 index 00000000..b9ce7845 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-610-hobbies-gaming-reviews-boardgamegeek/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 610, + "metaclass": "entertainment-hobbies", + "class": "gaming-reviews", + "description": "Add yourself as a fan to 'Brass: Birmingham' (2018) on BoardGameGeek", + "sites_involved": [ + "boardgamegeek.com" + ], + "platform": "boardgamegeek", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Add yourself as a fan to 'Brass: Birmingham' (2018) on BoardGameGeek", + "eval_schema": { + "url_pattern": "boardgamegeek\\.com/api/fans", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-610-hobbies-gaming-reviews-boardgamegeek/task.toml b/harbor/datasets/clawbench-v2/v2-610-hobbies-gaming-reviews-boardgamegeek/task.toml new file mode 100644 index 00000000..1d2492c2 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-610-hobbies-gaming-reviews-boardgamegeek/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-610-hobbies-gaming-reviews-boardgamegeek" +description = "Add yourself as a fan to 'Brass: Birmingham' (2018) on BoardGameGeek" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-610-hobbies-gaming-reviews-boardgamegeek" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-630-daily-life-productivity-habitica/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-630-daily-life-productivity-habitica/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-630-daily-life-productivity-habitica/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-630-daily-life-productivity-habitica/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-630-daily-life-productivity-habitica/steps/run/instruction.md new file mode 100644 index 00000000..69dd70e2 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-630-daily-life-productivity-habitica/steps/run/instruction.md @@ -0,0 +1,27 @@ +Add a new daily task on Habitica for 'Drink 8 glasses of water' + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-630-daily-life-productivity-habitica/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-630-daily-life-productivity-habitica/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-630-daily-life-productivity-habitica/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-630-daily-life-productivity-habitica/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-630-daily-life-productivity-habitica/steps/run/tests/task.json new file mode 100644 index 00000000..4594b04a --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-630-daily-life-productivity-habitica/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 630, + "metaclass": "daily-life", + "class": "productivity", + "description": "Add a new daily task on Habitica for 'Drink 8 glasses of water'", + "sites_involved": [ + "habitica.com" + ], + "platform": "habitica", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Add a new daily task on Habitica for 'Drink 8 glasses of water'", + "eval_schema": { + "url_pattern": "habitica\\.com/api/v\\d+/tasks/user", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-630-daily-life-productivity-habitica/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-630-daily-life-productivity-habitica/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-630-daily-life-productivity-habitica/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-630-daily-life-productivity-habitica/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-630-daily-life-productivity-habitica/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..9575ca31 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-630-daily-life-productivity-habitica/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "habitica\\.com/api/v\\d+/tasks/user", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-630-daily-life-productivity-habitica/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-630-daily-life-productivity-habitica/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-630-daily-life-productivity-habitica/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-630-daily-life-productivity-habitica/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-630-daily-life-productivity-habitica/steps/run/workdir/task.json new file mode 100644 index 00000000..4594b04a --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-630-daily-life-productivity-habitica/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 630, + "metaclass": "daily-life", + "class": "productivity", + "description": "Add a new daily task on Habitica for 'Drink 8 glasses of water'", + "sites_involved": [ + "habitica.com" + ], + "platform": "habitica", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Add a new daily task on Habitica for 'Drink 8 glasses of water'", + "eval_schema": { + "url_pattern": "habitica\\.com/api/v\\d+/tasks/user", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-630-daily-life-productivity-habitica/task.toml b/harbor/datasets/clawbench-v2/v2-630-daily-life-productivity-habitica/task.toml new file mode 100644 index 00000000..51f78b73 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-630-daily-life-productivity-habitica/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-630-daily-life-productivity-habitica" +description = "Add a new daily task on Habitica for 'Drink 8 glasses of water'" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-630-daily-life-productivity-habitica" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-631-daily-life-productivity-habitica/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-631-daily-life-productivity-habitica/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-631-daily-life-productivity-habitica/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-631-daily-life-productivity-habitica/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-631-daily-life-productivity-habitica/steps/run/instruction.md new file mode 100644 index 00000000..a2a7b01c --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-631-daily-life-productivity-habitica/steps/run/instruction.md @@ -0,0 +1,27 @@ +Create a new habit on Habitica called 'Read 10 pages' and set it to positive only + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-631-daily-life-productivity-habitica/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-631-daily-life-productivity-habitica/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-631-daily-life-productivity-habitica/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-631-daily-life-productivity-habitica/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-631-daily-life-productivity-habitica/steps/run/tests/task.json new file mode 100644 index 00000000..44118e24 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-631-daily-life-productivity-habitica/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 631, + "metaclass": "daily-life", + "class": "productivity", + "description": "Create a new habit on Habitica called 'Read 10 pages' and set it to positive only", + "sites_involved": [ + "habitica.com" + ], + "platform": "habitica", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Create a new habit on Habitica called 'Read 10 pages' and set it to positive only", + "eval_schema": { + "url_pattern": "habitica\\.com/api/v\\d+/tasks/user", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-631-daily-life-productivity-habitica/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-631-daily-life-productivity-habitica/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-631-daily-life-productivity-habitica/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-631-daily-life-productivity-habitica/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-631-daily-life-productivity-habitica/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..9575ca31 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-631-daily-life-productivity-habitica/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "habitica\\.com/api/v\\d+/tasks/user", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-631-daily-life-productivity-habitica/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-631-daily-life-productivity-habitica/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-631-daily-life-productivity-habitica/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-631-daily-life-productivity-habitica/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-631-daily-life-productivity-habitica/steps/run/workdir/task.json new file mode 100644 index 00000000..44118e24 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-631-daily-life-productivity-habitica/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 631, + "metaclass": "daily-life", + "class": "productivity", + "description": "Create a new habit on Habitica called 'Read 10 pages' and set it to positive only", + "sites_involved": [ + "habitica.com" + ], + "platform": "habitica", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Create a new habit on Habitica called 'Read 10 pages' and set it to positive only", + "eval_schema": { + "url_pattern": "habitica\\.com/api/v\\d+/tasks/user", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-631-daily-life-productivity-habitica/task.toml b/harbor/datasets/clawbench-v2/v2-631-daily-life-productivity-habitica/task.toml new file mode 100644 index 00000000..caef16cb --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-631-daily-life-productivity-habitica/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-631-daily-life-productivity-habitica" +description = "Create a new habit on Habitica called 'Read 10 pages' and set it to positive only" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-631-daily-life-productivity-habitica" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-633-daily-life-productivity-habitica/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-633-daily-life-productivity-habitica/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-633-daily-life-productivity-habitica/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-633-daily-life-productivity-habitica/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-633-daily-life-productivity-habitica/steps/run/instruction.md new file mode 100644 index 00000000..abc47ba0 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-633-daily-life-productivity-habitica/steps/run/instruction.md @@ -0,0 +1,27 @@ +Create a daily task called 'Morning Workout' on Habitica and mark it as completed to earn experience points + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-633-daily-life-productivity-habitica/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-633-daily-life-productivity-habitica/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-633-daily-life-productivity-habitica/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-633-daily-life-productivity-habitica/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-633-daily-life-productivity-habitica/steps/run/tests/task.json new file mode 100644 index 00000000..d05eb19e --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-633-daily-life-productivity-habitica/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 633, + "metaclass": "daily-life", + "class": "productivity", + "description": "Create a daily task called 'Morning Workout' on Habitica and mark it as completed to earn experience points", + "sites_involved": [ + "habitica.com" + ], + "platform": "habitica", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Create a daily task called 'Morning Workout' on Habitica and mark it as completed to earn experience points", + "eval_schema": { + "url_pattern": "habitica\\.com/api/v\\d+/tasks/[^/]+/score/up", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-633-daily-life-productivity-habitica/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-633-daily-life-productivity-habitica/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-633-daily-life-productivity-habitica/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-633-daily-life-productivity-habitica/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-633-daily-life-productivity-habitica/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..9112de51 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-633-daily-life-productivity-habitica/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "habitica\\.com/api/v\\d+/tasks/[^/]+/score/up", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-633-daily-life-productivity-habitica/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-633-daily-life-productivity-habitica/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-633-daily-life-productivity-habitica/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-633-daily-life-productivity-habitica/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-633-daily-life-productivity-habitica/steps/run/workdir/task.json new file mode 100644 index 00000000..d05eb19e --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-633-daily-life-productivity-habitica/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 633, + "metaclass": "daily-life", + "class": "productivity", + "description": "Create a daily task called 'Morning Workout' on Habitica and mark it as completed to earn experience points", + "sites_involved": [ + "habitica.com" + ], + "platform": "habitica", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Create a daily task called 'Morning Workout' on Habitica and mark it as completed to earn experience points", + "eval_schema": { + "url_pattern": "habitica\\.com/api/v\\d+/tasks/[^/]+/score/up", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-633-daily-life-productivity-habitica/task.toml b/harbor/datasets/clawbench-v2/v2-633-daily-life-productivity-habitica/task.toml new file mode 100644 index 00000000..21e26ea7 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-633-daily-life-productivity-habitica/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-633-daily-life-productivity-habitica" +description = "Create a daily task called 'Morning Workout' on Habitica and mark it as completed to earn experience points" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-633-daily-life-productivity-habitica" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-634-daily-life-productivity-habitica/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-634-daily-life-productivity-habitica/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-634-daily-life-productivity-habitica/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-634-daily-life-productivity-habitica/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-634-daily-life-productivity-habitica/steps/run/instruction.md new file mode 100644 index 00000000..452f8bf3 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-634-daily-life-productivity-habitica/steps/run/instruction.md @@ -0,0 +1,27 @@ +Add a To-Do item on Habitica called 'Buy a birthday gift for Mom' with medium priority and a due date of next Friday + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-634-daily-life-productivity-habitica/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-634-daily-life-productivity-habitica/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-634-daily-life-productivity-habitica/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-634-daily-life-productivity-habitica/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-634-daily-life-productivity-habitica/steps/run/tests/task.json new file mode 100644 index 00000000..31471828 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-634-daily-life-productivity-habitica/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 634, + "metaclass": "daily-life", + "class": "productivity", + "description": "Add a To-Do item on Habitica called 'Buy a birthday gift for Mom' with medium priority and a due date of next Friday", + "sites_involved": [ + "habitica.com" + ], + "platform": "habitica", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Add a To-Do item on Habitica called 'Buy a birthday gift for Mom' with medium priority and a due date of next Friday", + "eval_schema": { + "url_pattern": "habitica\\.com/api/v\\d+/tasks/user", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-634-daily-life-productivity-habitica/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-634-daily-life-productivity-habitica/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-634-daily-life-productivity-habitica/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-634-daily-life-productivity-habitica/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-634-daily-life-productivity-habitica/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..9575ca31 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-634-daily-life-productivity-habitica/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "habitica\\.com/api/v\\d+/tasks/user", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-634-daily-life-productivity-habitica/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-634-daily-life-productivity-habitica/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-634-daily-life-productivity-habitica/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-634-daily-life-productivity-habitica/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-634-daily-life-productivity-habitica/steps/run/workdir/task.json new file mode 100644 index 00000000..31471828 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-634-daily-life-productivity-habitica/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 634, + "metaclass": "daily-life", + "class": "productivity", + "description": "Add a To-Do item on Habitica called 'Buy a birthday gift for Mom' with medium priority and a due date of next Friday", + "sites_involved": [ + "habitica.com" + ], + "platform": "habitica", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Add a To-Do item on Habitica called 'Buy a birthday gift for Mom' with medium priority and a due date of next Friday", + "eval_schema": { + "url_pattern": "habitica\\.com/api/v\\d+/tasks/user", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-634-daily-life-productivity-habitica/task.toml b/harbor/datasets/clawbench-v2/v2-634-daily-life-productivity-habitica/task.toml new file mode 100644 index 00000000..4b40d262 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-634-daily-life-productivity-habitica/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-634-daily-life-productivity-habitica" +description = "Add a To-Do item on Habitica called 'Buy a birthday gift for Mom' with medium priority and a due date of next Friday" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-634-daily-life-productivity-habitica" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-635-entertainment-tracking-trakt/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-635-entertainment-tracking-trakt/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-635-entertainment-tracking-trakt/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-635-entertainment-tracking-trakt/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-635-entertainment-tracking-trakt/steps/run/instruction.md new file mode 100644 index 00000000..838be4d4 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-635-entertainment-tracking-trakt/steps/run/instruction.md @@ -0,0 +1,27 @@ +Search for 'Breaking Bad' on Trakt.tv and add it to your watchlist + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-635-entertainment-tracking-trakt/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-635-entertainment-tracking-trakt/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-635-entertainment-tracking-trakt/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-635-entertainment-tracking-trakt/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-635-entertainment-tracking-trakt/steps/run/tests/task.json new file mode 100644 index 00000000..990305d4 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-635-entertainment-tracking-trakt/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 635, + "metaclass": "entertainment", + "class": "tracking", + "description": "Search for 'Breaking Bad' on Trakt.tv and add it to your watchlist", + "sites_involved": [ + "trakt.tv" + ], + "platform": "trakt", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Search for 'Breaking Bad' on Trakt.tv and add it to your watchlist", + "eval_schema": { + "url_pattern": "trakt\\.tv.*(sync/watchlist)", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-635-entertainment-tracking-trakt/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-635-entertainment-tracking-trakt/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-635-entertainment-tracking-trakt/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-635-entertainment-tracking-trakt/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-635-entertainment-tracking-trakt/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..0420e88a --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-635-entertainment-tracking-trakt/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "trakt\\.tv.*(sync/watchlist)", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-635-entertainment-tracking-trakt/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-635-entertainment-tracking-trakt/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-635-entertainment-tracking-trakt/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-635-entertainment-tracking-trakt/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-635-entertainment-tracking-trakt/steps/run/workdir/task.json new file mode 100644 index 00000000..990305d4 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-635-entertainment-tracking-trakt/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 635, + "metaclass": "entertainment", + "class": "tracking", + "description": "Search for 'Breaking Bad' on Trakt.tv and add it to your watchlist", + "sites_involved": [ + "trakt.tv" + ], + "platform": "trakt", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Search for 'Breaking Bad' on Trakt.tv and add it to your watchlist", + "eval_schema": { + "url_pattern": "trakt\\.tv.*(sync/watchlist)", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-635-entertainment-tracking-trakt/task.toml b/harbor/datasets/clawbench-v2/v2-635-entertainment-tracking-trakt/task.toml new file mode 100644 index 00000000..0e47941a --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-635-entertainment-tracking-trakt/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-635-entertainment-tracking-trakt" +description = "Search for 'Breaking Bad' on Trakt.tv and add it to your watchlist" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-635-entertainment-tracking-trakt" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-638-entertainment-tracking-trakt/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-638-entertainment-tracking-trakt/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-638-entertainment-tracking-trakt/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-638-entertainment-tracking-trakt/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-638-entertainment-tracking-trakt/steps/run/instruction.md new file mode 100644 index 00000000..69ca5af0 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-638-entertainment-tracking-trakt/steps/run/instruction.md @@ -0,0 +1,27 @@ +Create a new custom list on Trakt.tv named 'Sci-Fi Favorites' + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-638-entertainment-tracking-trakt/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-638-entertainment-tracking-trakt/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-638-entertainment-tracking-trakt/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-638-entertainment-tracking-trakt/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-638-entertainment-tracking-trakt/steps/run/tests/task.json new file mode 100644 index 00000000..5aeb5bca --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-638-entertainment-tracking-trakt/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 638, + "metaclass": "entertainment", + "class": "tracking", + "description": "Create a new custom list on Trakt.tv named 'Sci-Fi Favorites'", + "sites_involved": [ + "trakt.tv" + ], + "platform": "trakt", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Create a new custom list on Trakt.tv named 'Sci-Fi Favorites'", + "eval_schema": { + "url_pattern": "trakt\\.tv.*(users/me/lists)", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-638-entertainment-tracking-trakt/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-638-entertainment-tracking-trakt/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-638-entertainment-tracking-trakt/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-638-entertainment-tracking-trakt/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-638-entertainment-tracking-trakt/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..7fa0991c --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-638-entertainment-tracking-trakt/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "trakt\\.tv.*(users/me/lists)", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-638-entertainment-tracking-trakt/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-638-entertainment-tracking-trakt/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-638-entertainment-tracking-trakt/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-638-entertainment-tracking-trakt/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-638-entertainment-tracking-trakt/steps/run/workdir/task.json new file mode 100644 index 00000000..5aeb5bca --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-638-entertainment-tracking-trakt/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 638, + "metaclass": "entertainment", + "class": "tracking", + "description": "Create a new custom list on Trakt.tv named 'Sci-Fi Favorites'", + "sites_involved": [ + "trakt.tv" + ], + "platform": "trakt", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Create a new custom list on Trakt.tv named 'Sci-Fi Favorites'", + "eval_schema": { + "url_pattern": "trakt\\.tv.*(users/me/lists)", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-638-entertainment-tracking-trakt/task.toml b/harbor/datasets/clawbench-v2/v2-638-entertainment-tracking-trakt/task.toml new file mode 100644 index 00000000..7e082f59 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-638-entertainment-tracking-trakt/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-638-entertainment-tracking-trakt" +description = "Create a new custom list on Trakt.tv named 'Sci-Fi Favorites'" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-638-entertainment-tracking-trakt" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-705-rating-voting-wine-review-vivino/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-705-rating-voting-wine-review-vivino/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-705-rating-voting-wine-review-vivino/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-705-rating-voting-wine-review-vivino/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-705-rating-voting-wine-review-vivino/steps/run/instruction.md new file mode 100644 index 00000000..7d689d20 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-705-rating-voting-wine-review-vivino/steps/run/instruction.md @@ -0,0 +1,27 @@ +Submit a 4-star rating for a Pinot Noir on Vivino with exactly this tasting note: "Light body, cherry notes, smooth finish." + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-705-rating-voting-wine-review-vivino/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-705-rating-voting-wine-review-vivino/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-705-rating-voting-wine-review-vivino/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-705-rating-voting-wine-review-vivino/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-705-rating-voting-wine-review-vivino/steps/run/tests/task.json new file mode 100644 index 00000000..9d0db41c --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-705-rating-voting-wine-review-vivino/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 705, + "metaclass": "rating-voting", + "class": "wine-review", + "description": "Submit a 4-star rating for a Pinot Noir on Vivino with exactly this tasting note: \"Light body, cherry notes, smooth finish.\"", + "sites_involved": [ + "vivino.com" + ], + "platform": "vivino", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Submit a 4-star rating for a Pinot Noir on Vivino with exactly this tasting note: \"Light body, cherry notes, smooth finish.\"", + "eval_schema": { + "url_pattern": "vivino\\.com/api/vintages/\\d+/reviews", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-705-rating-voting-wine-review-vivino/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-705-rating-voting-wine-review-vivino/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-705-rating-voting-wine-review-vivino/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-705-rating-voting-wine-review-vivino/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-705-rating-voting-wine-review-vivino/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..661c5d36 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-705-rating-voting-wine-review-vivino/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "vivino\\.com/api/vintages/\\d+/reviews", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-705-rating-voting-wine-review-vivino/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-705-rating-voting-wine-review-vivino/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-705-rating-voting-wine-review-vivino/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-705-rating-voting-wine-review-vivino/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-705-rating-voting-wine-review-vivino/steps/run/workdir/task.json new file mode 100644 index 00000000..9d0db41c --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-705-rating-voting-wine-review-vivino/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 705, + "metaclass": "rating-voting", + "class": "wine-review", + "description": "Submit a 4-star rating for a Pinot Noir on Vivino with exactly this tasting note: \"Light body, cherry notes, smooth finish.\"", + "sites_involved": [ + "vivino.com" + ], + "platform": "vivino", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Submit a 4-star rating for a Pinot Noir on Vivino with exactly this tasting note: \"Light body, cherry notes, smooth finish.\"", + "eval_schema": { + "url_pattern": "vivino\\.com/api/vintages/\\d+/reviews", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-705-rating-voting-wine-review-vivino/task.toml b/harbor/datasets/clawbench-v2/v2-705-rating-voting-wine-review-vivino/task.toml new file mode 100644 index 00000000..cc026f6e --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-705-rating-voting-wine-review-vivino/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-705-rating-voting-wine-review-vivino" +description = "Submit a 4-star rating for a Pinot Noir on Vivino with exactly this tasting note: \"Light body, cherry notes, smooth finish.\"" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-705-rating-voting-wine-review-vivino" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-706-rating-voting-beer-review-beeradvocate/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-706-rating-voting-beer-review-beeradvocate/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-706-rating-voting-beer-review-beeradvocate/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-706-rating-voting-beer-review-beeradvocate/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-706-rating-voting-beer-review-beeradvocate/steps/run/instruction.md new file mode 100644 index 00000000..ac7839d8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-706-rating-voting-beer-review-beeradvocate/steps/run/instruction.md @@ -0,0 +1,27 @@ +Submit a review for an Imperial Stout on BeerAdvocate with all dimension scores filled and exactly this comment: "Rich malt, roasty aroma, full body." + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-706-rating-voting-beer-review-beeradvocate/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-706-rating-voting-beer-review-beeradvocate/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-706-rating-voting-beer-review-beeradvocate/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-706-rating-voting-beer-review-beeradvocate/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-706-rating-voting-beer-review-beeradvocate/steps/run/tests/task.json new file mode 100644 index 00000000..a487cb90 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-706-rating-voting-beer-review-beeradvocate/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 706, + "metaclass": "rating-voting", + "class": "beer-review", + "description": "Submit a review for an Imperial Stout on BeerAdvocate with all dimension scores filled and exactly this comment: \"Rich malt, roasty aroma, full body.\"", + "sites_involved": [ + "beeradvocate.com" + ], + "platform": "beeradvocate", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Submit a review for an Imperial Stout on BeerAdvocate with all dimension scores filled and exactly this comment: \"Rich malt, roasty aroma, full body.\"", + "eval_schema": { + "url_pattern": "beeradvocate\\.com/beer/rate/\\d+", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-706-rating-voting-beer-review-beeradvocate/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-706-rating-voting-beer-review-beeradvocate/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-706-rating-voting-beer-review-beeradvocate/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-706-rating-voting-beer-review-beeradvocate/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-706-rating-voting-beer-review-beeradvocate/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..f11708d6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-706-rating-voting-beer-review-beeradvocate/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "beeradvocate\\.com/beer/rate/\\d+", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-706-rating-voting-beer-review-beeradvocate/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-706-rating-voting-beer-review-beeradvocate/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-706-rating-voting-beer-review-beeradvocate/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-706-rating-voting-beer-review-beeradvocate/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-706-rating-voting-beer-review-beeradvocate/steps/run/workdir/task.json new file mode 100644 index 00000000..a487cb90 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-706-rating-voting-beer-review-beeradvocate/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 706, + "metaclass": "rating-voting", + "class": "beer-review", + "description": "Submit a review for an Imperial Stout on BeerAdvocate with all dimension scores filled and exactly this comment: \"Rich malt, roasty aroma, full body.\"", + "sites_involved": [ + "beeradvocate.com" + ], + "platform": "beeradvocate", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Submit a review for an Imperial Stout on BeerAdvocate with all dimension scores filled and exactly this comment: \"Rich malt, roasty aroma, full body.\"", + "eval_schema": { + "url_pattern": "beeradvocate\\.com/beer/rate/\\d+", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-706-rating-voting-beer-review-beeradvocate/task.toml b/harbor/datasets/clawbench-v2/v2-706-rating-voting-beer-review-beeradvocate/task.toml new file mode 100644 index 00000000..1283efd0 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-706-rating-voting-beer-review-beeradvocate/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-706-rating-voting-beer-review-beeradvocate" +description = "Submit a review for an Imperial Stout on BeerAdvocate with all dimension scores filled and exactly this comment: \"Rich malt, roasty aroma, full body.\"" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-706-rating-voting-beer-review-beeradvocate" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-707-rating-voting-social-wine-untappd/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-707-rating-voting-social-wine-untappd/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-707-rating-voting-social-wine-untappd/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-707-rating-voting-social-wine-untappd/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-707-rating-voting-social-wine-untappd/steps/run/instruction.md new file mode 100644 index 00000000..19176c55 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-707-rating-voting-social-wine-untappd/steps/run/instruction.md @@ -0,0 +1,27 @@ +Check in an IPA on Untappd with the highest star rating, and check out + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-707-rating-voting-social-wine-untappd/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-707-rating-voting-social-wine-untappd/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-707-rating-voting-social-wine-untappd/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-707-rating-voting-social-wine-untappd/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-707-rating-voting-social-wine-untappd/steps/run/tests/task.json new file mode 100644 index 00000000..ef3bc777 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-707-rating-voting-social-wine-untappd/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 707, + "metaclass": "rating-voting", + "class": "social-wine", + "description": "Check in an IPA on Untappd with the highest star rating, and check out", + "sites_involved": [ + "untappd.com" + ], + "platform": "untappd", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Check in an IPA on Untappd with the highest star rating, and check out", + "eval_schema": { + "url_pattern": "^https://untappd\\.com/api/v2/shop/summaries/\\d+", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-707-rating-voting-social-wine-untappd/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-707-rating-voting-social-wine-untappd/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-707-rating-voting-social-wine-untappd/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-707-rating-voting-social-wine-untappd/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-707-rating-voting-social-wine-untappd/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..19d52b9f --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-707-rating-voting-social-wine-untappd/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "^https://untappd\\.com/api/v2/shop/summaries/\\d+", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-707-rating-voting-social-wine-untappd/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-707-rating-voting-social-wine-untappd/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-707-rating-voting-social-wine-untappd/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-707-rating-voting-social-wine-untappd/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-707-rating-voting-social-wine-untappd/steps/run/workdir/task.json new file mode 100644 index 00000000..ef3bc777 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-707-rating-voting-social-wine-untappd/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 707, + "metaclass": "rating-voting", + "class": "social-wine", + "description": "Check in an IPA on Untappd with the highest star rating, and check out", + "sites_involved": [ + "untappd.com" + ], + "platform": "untappd", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Check in an IPA on Untappd with the highest star rating, and check out", + "eval_schema": { + "url_pattern": "^https://untappd\\.com/api/v2/shop/summaries/\\d+", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-707-rating-voting-social-wine-untappd/task.toml b/harbor/datasets/clawbench-v2/v2-707-rating-voting-social-wine-untappd/task.toml new file mode 100644 index 00000000..a32a6066 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-707-rating-voting-social-wine-untappd/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-707-rating-voting-social-wine-untappd" +description = "Check in an IPA on Untappd with the highest star rating, and check out" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-707-rating-voting-social-wine-untappd" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-708-rating-voting-professor-review-ratemyprofessors/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-708-rating-voting-professor-review-ratemyprofessors/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-708-rating-voting-professor-review-ratemyprofessors/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-708-rating-voting-professor-review-ratemyprofessors/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-708-rating-voting-professor-review-ratemyprofessors/steps/run/instruction.md new file mode 100644 index 00000000..54e6de0c --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-708-rating-voting-professor-review-ratemyprofessors/steps/run/instruction.md @@ -0,0 +1,27 @@ +Submit a rating for a Computer Science professor on RateMyProfessors: quality 4, difficulty 3, would take again Yes, with exactly this comment: "Clear lectures, fair exams." + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-708-rating-voting-professor-review-ratemyprofessors/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-708-rating-voting-professor-review-ratemyprofessors/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-708-rating-voting-professor-review-ratemyprofessors/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-708-rating-voting-professor-review-ratemyprofessors/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-708-rating-voting-professor-review-ratemyprofessors/steps/run/tests/task.json new file mode 100644 index 00000000..6ef049a5 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-708-rating-voting-professor-review-ratemyprofessors/steps/run/tests/task.json @@ -0,0 +1,28 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 708, + "metaclass": "rating-voting", + "class": "professor-review", + "description": "Submit a rating for a Computer Science professor on RateMyProfessors: quality 4, difficulty 3, would take again Yes, with exactly this comment: \"Clear lectures, fair exams.\"", + "sites_involved": [ + "ratemyprofessors.com" + ], + "platform": "ratemyprofessors", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Submit a rating for a Computer Science professor on RateMyProfessors: quality 4, difficulty 3, would take again Yes, with exactly this comment: \"Clear lectures, fair exams.\"", + "eval_schema": { + "url_pattern": "ratemyprofessors\\.com/graphql", + "method": "POST", + "body": { + "operationName": "RateTeacherMutation" + } + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-708-rating-voting-professor-review-ratemyprofessors/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-708-rating-voting-professor-review-ratemyprofessors/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-708-rating-voting-professor-review-ratemyprofessors/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-708-rating-voting-professor-review-ratemyprofessors/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-708-rating-voting-professor-review-ratemyprofessors/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..a18a5fed --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-708-rating-voting-professor-review-ratemyprofessors/steps/run/workdir/eval-schema.json @@ -0,0 +1,7 @@ +{ + "url_pattern": "ratemyprofessors\\.com/graphql", + "method": "POST", + "body": { + "operationName": "RateTeacherMutation" + } +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-708-rating-voting-professor-review-ratemyprofessors/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-708-rating-voting-professor-review-ratemyprofessors/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-708-rating-voting-professor-review-ratemyprofessors/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-708-rating-voting-professor-review-ratemyprofessors/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-708-rating-voting-professor-review-ratemyprofessors/steps/run/workdir/task.json new file mode 100644 index 00000000..6ef049a5 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-708-rating-voting-professor-review-ratemyprofessors/steps/run/workdir/task.json @@ -0,0 +1,28 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 708, + "metaclass": "rating-voting", + "class": "professor-review", + "description": "Submit a rating for a Computer Science professor on RateMyProfessors: quality 4, difficulty 3, would take again Yes, with exactly this comment: \"Clear lectures, fair exams.\"", + "sites_involved": [ + "ratemyprofessors.com" + ], + "platform": "ratemyprofessors", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Submit a rating for a Computer Science professor on RateMyProfessors: quality 4, difficulty 3, would take again Yes, with exactly this comment: \"Clear lectures, fair exams.\"", + "eval_schema": { + "url_pattern": "ratemyprofessors\\.com/graphql", + "method": "POST", + "body": { + "operationName": "RateTeacherMutation" + } + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-708-rating-voting-professor-review-ratemyprofessors/task.toml b/harbor/datasets/clawbench-v2/v2-708-rating-voting-professor-review-ratemyprofessors/task.toml new file mode 100644 index 00000000..6b09e85f --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-708-rating-voting-professor-review-ratemyprofessors/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-708-rating-voting-professor-review-ratemyprofessors" +description = "Submit a rating for a Computer Science professor on RateMyProfessors: quality 4, difficulty 3, would take again Yes, with exactly this comment: \"Clear lectures, fair exams.\"" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-708-rating-voting-professor-review-ratemyprofessors" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-711-creation-init-color-design-coolors/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-711-creation-init-color-design-coolors/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-711-creation-init-color-design-coolors/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-711-creation-init-color-design-coolors/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-711-creation-init-color-design-coolors/steps/run/instruction.md new file mode 100644 index 00000000..f74388f2 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-711-creation-init-color-design-coolors/steps/run/instruction.md @@ -0,0 +1,27 @@ +On Coolors, generate a 5-color palette, lock one color and adjust it to #FF6B6B, then export the palette to PDF. + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-711-creation-init-color-design-coolors/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-711-creation-init-color-design-coolors/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-711-creation-init-color-design-coolors/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-711-creation-init-color-design-coolors/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-711-creation-init-color-design-coolors/steps/run/tests/task.json new file mode 100644 index 00000000..e0e7bf7b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-711-creation-init-color-design-coolors/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 711, + "metaclass": "creation-init", + "class": "color-design", + "description": "On Coolors, generate a 5-color palette, lock one color and adjust it to #FF6B6B, then export the palette to PDF.", + "sites_involved": [ + "coolors.co" + ], + "platform": "coolors", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "On Coolors, generate a 5-color palette, lock one color and adjust it to #FF6B6B, then export the palette to PDF.", + "eval_schema": { + "url_pattern": "coolors\\.co/ajax/export-palette", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-711-creation-init-color-design-coolors/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-711-creation-init-color-design-coolors/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-711-creation-init-color-design-coolors/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-711-creation-init-color-design-coolors/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-711-creation-init-color-design-coolors/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..1e585b93 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-711-creation-init-color-design-coolors/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "coolors\\.co/ajax/export-palette", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-711-creation-init-color-design-coolors/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-711-creation-init-color-design-coolors/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-711-creation-init-color-design-coolors/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-711-creation-init-color-design-coolors/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-711-creation-init-color-design-coolors/steps/run/workdir/task.json new file mode 100644 index 00000000..e0e7bf7b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-711-creation-init-color-design-coolors/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 711, + "metaclass": "creation-init", + "class": "color-design", + "description": "On Coolors, generate a 5-color palette, lock one color and adjust it to #FF6B6B, then export the palette to PDF.", + "sites_involved": [ + "coolors.co" + ], + "platform": "coolors", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "On Coolors, generate a 5-color palette, lock one color and adjust it to #FF6B6B, then export the palette to PDF.", + "eval_schema": { + "url_pattern": "coolors\\.co/ajax/export-palette", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-711-creation-init-color-design-coolors/task.toml b/harbor/datasets/clawbench-v2/v2-711-creation-init-color-design-coolors/task.toml new file mode 100644 index 00000000..0561f25f --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-711-creation-init-color-design-coolors/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-711-creation-init-color-design-coolors" +description = "On Coolors, generate a 5-color palette, lock one color and adjust it to #FF6B6B, then export the palette to PDF." +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-711-creation-init-color-design-coolors" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-735-home-services-maintenance-house-cleaning-bark/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-735-home-services-maintenance-house-cleaning-bark/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-735-home-services-maintenance-house-cleaning-bark/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-735-home-services-maintenance-house-cleaning-bark/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-735-home-services-maintenance-house-cleaning-bark/steps/run/instruction.md new file mode 100644 index 00000000..973beb88 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-735-home-services-maintenance-house-cleaning-bark/steps/run/instruction.md @@ -0,0 +1,27 @@ +Post a home cleaning project on Bark, fill in the service description, address, and required date, and submit the project + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-735-home-services-maintenance-house-cleaning-bark/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-735-home-services-maintenance-house-cleaning-bark/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-735-home-services-maintenance-house-cleaning-bark/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-735-home-services-maintenance-house-cleaning-bark/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-735-home-services-maintenance-house-cleaning-bark/steps/run/tests/task.json new file mode 100644 index 00000000..c26a79fd --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-735-home-services-maintenance-house-cleaning-bark/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 735, + "metaclass": "home-services-maintenance", + "class": "house-cleaning", + "description": "Post a home cleaning project on Bark, fill in the service description, address, and required date, and submit the project", + "sites_involved": [ + "bark.com" + ], + "platform": "bark", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Post a home cleaning project on Bark, fill in the service description, address, and required date, and submit the project", + "eval_schema": { + "url_pattern": "api\\.bark\\.com/bark/pre", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-735-home-services-maintenance-house-cleaning-bark/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-735-home-services-maintenance-house-cleaning-bark/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-735-home-services-maintenance-house-cleaning-bark/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-735-home-services-maintenance-house-cleaning-bark/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-735-home-services-maintenance-house-cleaning-bark/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..b3268573 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-735-home-services-maintenance-house-cleaning-bark/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "api\\.bark\\.com/bark/pre", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-735-home-services-maintenance-house-cleaning-bark/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-735-home-services-maintenance-house-cleaning-bark/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-735-home-services-maintenance-house-cleaning-bark/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-735-home-services-maintenance-house-cleaning-bark/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-735-home-services-maintenance-house-cleaning-bark/steps/run/workdir/task.json new file mode 100644 index 00000000..c26a79fd --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-735-home-services-maintenance-house-cleaning-bark/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 735, + "metaclass": "home-services-maintenance", + "class": "house-cleaning", + "description": "Post a home cleaning project on Bark, fill in the service description, address, and required date, and submit the project", + "sites_involved": [ + "bark.com" + ], + "platform": "bark", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Post a home cleaning project on Bark, fill in the service description, address, and required date, and submit the project", + "eval_schema": { + "url_pattern": "api\\.bark\\.com/bark/pre", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-735-home-services-maintenance-house-cleaning-bark/task.toml b/harbor/datasets/clawbench-v2/v2-735-home-services-maintenance-house-cleaning-bark/task.toml new file mode 100644 index 00000000..df6e1cb3 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-735-home-services-maintenance-house-cleaning-bark/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-735-home-services-maintenance-house-cleaning-bark" +description = "Post a home cleaning project on Bark, fill in the service description, address, and required date, and submit the project" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-735-home-services-maintenance-house-cleaning-bark" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-737-home-services-maintenance-kitchen-remodel-lowes/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-737-home-services-maintenance-kitchen-remodel-lowes/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-737-home-services-maintenance-kitchen-remodel-lowes/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-737-home-services-maintenance-kitchen-remodel-lowes/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-737-home-services-maintenance-kitchen-remodel-lowes/steps/run/instruction.md new file mode 100644 index 00000000..50f1502e --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-737-home-services-maintenance-kitchen-remodel-lowes/steps/run/instruction.md @@ -0,0 +1,27 @@ +Schedule a free kitchen remodel consultation on Lowe's website + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-737-home-services-maintenance-kitchen-remodel-lowes/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-737-home-services-maintenance-kitchen-remodel-lowes/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-737-home-services-maintenance-kitchen-remodel-lowes/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-737-home-services-maintenance-kitchen-remodel-lowes/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-737-home-services-maintenance-kitchen-remodel-lowes/steps/run/tests/task.json new file mode 100644 index 00000000..802a4907 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-737-home-services-maintenance-kitchen-remodel-lowes/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 737, + "metaclass": "home-services-maintenance", + "class": "kitchen-remodel", + "description": "Schedule a free kitchen remodel consultation on Lowe's website", + "sites_involved": [ + "lowes.com" + ], + "platform": "lowes", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Schedule a free kitchen remodel consultation on Lowe's website", + "eval_schema": { + "url_pattern": "lowes\\.myhomeprojectcenter\\.com.*/lead|lowes\\.com.*/schedule|lowes\\.com.*/consultation|lowes\\.com.*/install-services|lowes\\.com.*/measurerequest", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-737-home-services-maintenance-kitchen-remodel-lowes/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-737-home-services-maintenance-kitchen-remodel-lowes/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-737-home-services-maintenance-kitchen-remodel-lowes/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-737-home-services-maintenance-kitchen-remodel-lowes/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-737-home-services-maintenance-kitchen-remodel-lowes/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..91194b63 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-737-home-services-maintenance-kitchen-remodel-lowes/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "lowes\\.myhomeprojectcenter\\.com.*/lead|lowes\\.com.*/schedule|lowes\\.com.*/consultation|lowes\\.com.*/install-services|lowes\\.com.*/measurerequest", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-737-home-services-maintenance-kitchen-remodel-lowes/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-737-home-services-maintenance-kitchen-remodel-lowes/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-737-home-services-maintenance-kitchen-remodel-lowes/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-737-home-services-maintenance-kitchen-remodel-lowes/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-737-home-services-maintenance-kitchen-remodel-lowes/steps/run/workdir/task.json new file mode 100644 index 00000000..802a4907 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-737-home-services-maintenance-kitchen-remodel-lowes/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 737, + "metaclass": "home-services-maintenance", + "class": "kitchen-remodel", + "description": "Schedule a free kitchen remodel consultation on Lowe's website", + "sites_involved": [ + "lowes.com" + ], + "platform": "lowes", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Schedule a free kitchen remodel consultation on Lowe's website", + "eval_schema": { + "url_pattern": "lowes\\.myhomeprojectcenter\\.com.*/lead|lowes\\.com.*/schedule|lowes\\.com.*/consultation|lowes\\.com.*/install-services|lowes\\.com.*/measurerequest", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-737-home-services-maintenance-kitchen-remodel-lowes/task.toml b/harbor/datasets/clawbench-v2/v2-737-home-services-maintenance-kitchen-remodel-lowes/task.toml new file mode 100644 index 00000000..e4c56018 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-737-home-services-maintenance-kitchen-remodel-lowes/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-737-home-services-maintenance-kitchen-remodel-lowes" +description = "Schedule a free kitchen remodel consultation on Lowe's website" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-737-home-services-maintenance-kitchen-remodel-lowes" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-763-automotive-vehicle-services-car-lease-autoslash/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-763-automotive-vehicle-services-car-lease-autoslash/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-763-automotive-vehicle-services-car-lease-autoslash/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-763-automotive-vehicle-services-car-lease-autoslash/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-763-automotive-vehicle-services-car-lease-autoslash/steps/run/instruction.md new file mode 100644 index 00000000..197271e7 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-763-automotive-vehicle-services-car-lease-autoslash/steps/run/instruction.md @@ -0,0 +1,27 @@ +Search AutoSlash for the cheapest economy car rental at Miami airport for a 3-day period, and complete the price-tracking registration so it auto-rebooks if prices drop + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-763-automotive-vehicle-services-car-lease-autoslash/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-763-automotive-vehicle-services-car-lease-autoslash/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-763-automotive-vehicle-services-car-lease-autoslash/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-763-automotive-vehicle-services-car-lease-autoslash/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-763-automotive-vehicle-services-car-lease-autoslash/steps/run/tests/task.json new file mode 100644 index 00000000..c6f41f93 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-763-automotive-vehicle-services-car-lease-autoslash/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 763, + "metaclass": "automotive-vehicle-services", + "class": "car-lease", + "description": "Search AutoSlash for the cheapest economy car rental at Miami airport for a 3-day period, and complete the price-tracking registration so it auto-rebooks if prices drop", + "sites_involved": [ + "autoslash.com" + ], + "platform": "autoslash", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Search AutoSlash for the cheapest economy car rental at Miami airport for a 3-day period, and complete the price-tracking registration so it auto-rebooks if prices drop", + "eval_schema": { + "url_pattern": "www\\.autoslash\\.com/quote/contact-info/[0-9a-f-]+", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-763-automotive-vehicle-services-car-lease-autoslash/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-763-automotive-vehicle-services-car-lease-autoslash/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-763-automotive-vehicle-services-car-lease-autoslash/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-763-automotive-vehicle-services-car-lease-autoslash/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-763-automotive-vehicle-services-car-lease-autoslash/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..d7deff35 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-763-automotive-vehicle-services-car-lease-autoslash/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "www\\.autoslash\\.com/quote/contact-info/[0-9a-f-]+", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-763-automotive-vehicle-services-car-lease-autoslash/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-763-automotive-vehicle-services-car-lease-autoslash/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-763-automotive-vehicle-services-car-lease-autoslash/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-763-automotive-vehicle-services-car-lease-autoslash/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-763-automotive-vehicle-services-car-lease-autoslash/steps/run/workdir/task.json new file mode 100644 index 00000000..c6f41f93 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-763-automotive-vehicle-services-car-lease-autoslash/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 763, + "metaclass": "automotive-vehicle-services", + "class": "car-lease", + "description": "Search AutoSlash for the cheapest economy car rental at Miami airport for a 3-day period, and complete the price-tracking registration so it auto-rebooks if prices drop", + "sites_involved": [ + "autoslash.com" + ], + "platform": "autoslash", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "Search AutoSlash for the cheapest economy car rental at Miami airport for a 3-day period, and complete the price-tracking registration so it auto-rebooks if prices drop", + "eval_schema": { + "url_pattern": "www\\.autoslash\\.com/quote/contact-info/[0-9a-f-]+", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-763-automotive-vehicle-services-car-lease-autoslash/task.toml b/harbor/datasets/clawbench-v2/v2-763-automotive-vehicle-services-car-lease-autoslash/task.toml new file mode 100644 index 00000000..4338e40f --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-763-automotive-vehicle-services-car-lease-autoslash/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-763-automotive-vehicle-services-car-lease-autoslash" +description = "Search AutoSlash for the cheapest economy car rental at Miami airport for a 3-day period, and complete the price-tracking registration so it auto-rebooks if prices drop" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-763-automotive-vehicle-services-car-lease-autoslash" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-774-nonprofit-charity-nonprofit-job-apply-charity-village/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-774-nonprofit-charity-nonprofit-job-apply-charity-village/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-774-nonprofit-charity-nonprofit-job-apply-charity-village/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-774-nonprofit-charity-nonprofit-job-apply-charity-village/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-774-nonprofit-charity-nonprofit-job-apply-charity-village/steps/run/instruction.md new file mode 100644 index 00000000..4603084a --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-774-nonprofit-charity-nonprofit-job-apply-charity-village/steps/run/instruction.md @@ -0,0 +1,27 @@ +On Charity Village, create an account and apply for a full-time volunteer coordinator position in the Youth sector in Calgary, upload a resume and fill in a cover letter + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-774-nonprofit-charity-nonprofit-job-apply-charity-village/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-774-nonprofit-charity-nonprofit-job-apply-charity-village/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-774-nonprofit-charity-nonprofit-job-apply-charity-village/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-774-nonprofit-charity-nonprofit-job-apply-charity-village/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-774-nonprofit-charity-nonprofit-job-apply-charity-village/steps/run/tests/task.json new file mode 100644 index 00000000..f5d8db58 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-774-nonprofit-charity-nonprofit-job-apply-charity-village/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 774, + "metaclass": "nonprofit-charity", + "class": "nonprofit-job-apply", + "description": "On Charity Village, create an account and apply for a full-time volunteer coordinator position in the Youth sector in Calgary, upload a resume and fill in a cover letter", + "sites_involved": [ + "charityvillage.com" + ], + "platform": "charity-village", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "On Charity Village, create an account and apply for a full-time volunteer coordinator position in the Youth sector in Calgary, upload a resume and fill in a cover letter", + "eval_schema": { + "url_pattern": "www\\.charityvillage\\.com/api/auth/callback/sign-in", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-774-nonprofit-charity-nonprofit-job-apply-charity-village/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-774-nonprofit-charity-nonprofit-job-apply-charity-village/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-774-nonprofit-charity-nonprofit-job-apply-charity-village/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-774-nonprofit-charity-nonprofit-job-apply-charity-village/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-774-nonprofit-charity-nonprofit-job-apply-charity-village/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..5676180a --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-774-nonprofit-charity-nonprofit-job-apply-charity-village/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "www\\.charityvillage\\.com/api/auth/callback/sign-in", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-774-nonprofit-charity-nonprofit-job-apply-charity-village/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-774-nonprofit-charity-nonprofit-job-apply-charity-village/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-774-nonprofit-charity-nonprofit-job-apply-charity-village/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-774-nonprofit-charity-nonprofit-job-apply-charity-village/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-774-nonprofit-charity-nonprofit-job-apply-charity-village/steps/run/workdir/task.json new file mode 100644 index 00000000..f5d8db58 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-774-nonprofit-charity-nonprofit-job-apply-charity-village/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 774, + "metaclass": "nonprofit-charity", + "class": "nonprofit-job-apply", + "description": "On Charity Village, create an account and apply for a full-time volunteer coordinator position in the Youth sector in Calgary, upload a resume and fill in a cover letter", + "sites_involved": [ + "charityvillage.com" + ], + "platform": "charity-village", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "On Charity Village, create an account and apply for a full-time volunteer coordinator position in the Youth sector in Calgary, upload a resume and fill in a cover letter", + "eval_schema": { + "url_pattern": "www\\.charityvillage\\.com/api/auth/callback/sign-in", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-774-nonprofit-charity-nonprofit-job-apply-charity-village/task.toml b/harbor/datasets/clawbench-v2/v2-774-nonprofit-charity-nonprofit-job-apply-charity-village/task.toml new file mode 100644 index 00000000..89d1addf --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-774-nonprofit-charity-nonprofit-job-apply-charity-village/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-774-nonprofit-charity-nonprofit-job-apply-charity-village" +description = "On Charity Village, create an account and apply for a full-time volunteer coordinator position in the Youth sector in Calgary, upload a resume and fill in a cover letter" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-774-nonprofit-charity-nonprofit-job-apply-charity-village" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-776-nonprofit-charity-volunteer-signup-idealist/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-776-nonprofit-charity-volunteer-signup-idealist/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-776-nonprofit-charity-volunteer-signup-idealist/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-776-nonprofit-charity-volunteer-signup-idealist/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-776-nonprofit-charity-volunteer-signup-idealist/steps/run/instruction.md new file mode 100644 index 00000000..60bcb869 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-776-nonprofit-charity-volunteer-signup-idealist/steps/run/instruction.md @@ -0,0 +1,27 @@ +On Idealist, search for Program Manager positions at nonprofits in Washington DC, select a qualifying position and submit a job application + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-776-nonprofit-charity-volunteer-signup-idealist/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-776-nonprofit-charity-volunteer-signup-idealist/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-776-nonprofit-charity-volunteer-signup-idealist/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-776-nonprofit-charity-volunteer-signup-idealist/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-776-nonprofit-charity-volunteer-signup-idealist/steps/run/tests/task.json new file mode 100644 index 00000000..751cf287 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-776-nonprofit-charity-volunteer-signup-idealist/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 776, + "metaclass": "nonprofit-charity", + "class": "volunteer-signup", + "description": "On Idealist, search for Program Manager positions at nonprofits in Washington DC, select a qualifying position and submit a job application", + "sites_involved": [ + "idealist.org" + ], + "platform": "idealist", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "On Idealist, search for Program Manager positions at nonprofits in Washington DC, select a qualifying position and submit a job application", + "eval_schema": { + "url_pattern": "www\\.idealist\\.org/data/userdashboard/missing-info", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-776-nonprofit-charity-volunteer-signup-idealist/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-776-nonprofit-charity-volunteer-signup-idealist/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-776-nonprofit-charity-volunteer-signup-idealist/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-776-nonprofit-charity-volunteer-signup-idealist/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-776-nonprofit-charity-volunteer-signup-idealist/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..e192e15b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-776-nonprofit-charity-volunteer-signup-idealist/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "www\\.idealist\\.org/data/userdashboard/missing-info", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-776-nonprofit-charity-volunteer-signup-idealist/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-776-nonprofit-charity-volunteer-signup-idealist/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-776-nonprofit-charity-volunteer-signup-idealist/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-776-nonprofit-charity-volunteer-signup-idealist/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-776-nonprofit-charity-volunteer-signup-idealist/steps/run/workdir/task.json new file mode 100644 index 00000000..751cf287 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-776-nonprofit-charity-volunteer-signup-idealist/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 776, + "metaclass": "nonprofit-charity", + "class": "volunteer-signup", + "description": "On Idealist, search for Program Manager positions at nonprofits in Washington DC, select a qualifying position and submit a job application", + "sites_involved": [ + "idealist.org" + ], + "platform": "idealist", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "On Idealist, search for Program Manager positions at nonprofits in Washington DC, select a qualifying position and submit a job application", + "eval_schema": { + "url_pattern": "www\\.idealist\\.org/data/userdashboard/missing-info", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-776-nonprofit-charity-volunteer-signup-idealist/task.toml b/harbor/datasets/clawbench-v2/v2-776-nonprofit-charity-volunteer-signup-idealist/task.toml new file mode 100644 index 00000000..5802b2d6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-776-nonprofit-charity-volunteer-signup-idealist/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-776-nonprofit-charity-volunteer-signup-idealist" +description = "On Idealist, search for Program Manager positions at nonprofits in Washington DC, select a qualifying position and submit a job application" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-776-nonprofit-charity-volunteer-signup-idealist" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-794-beauty-personal-care-salon-booking-styleseat/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-794-beauty-personal-care-salon-booking-styleseat/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-794-beauty-personal-care-salon-booking-styleseat/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-794-beauty-personal-care-salon-booking-styleseat/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-794-beauty-personal-care-salon-booking-styleseat/steps/run/instruction.md new file mode 100644 index 00000000..6dd7c606 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-794-beauty-personal-care-salon-booking-styleseat/steps/run/instruction.md @@ -0,0 +1,27 @@ +On StyleSeat, search for hairstylists in your city, select one rated 4.8 stars or above, and book a haircut appointment at the earliest available time slot + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-794-beauty-personal-care-salon-booking-styleseat/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-794-beauty-personal-care-salon-booking-styleseat/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-794-beauty-personal-care-salon-booking-styleseat/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-794-beauty-personal-care-salon-booking-styleseat/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-794-beauty-personal-care-salon-booking-styleseat/steps/run/tests/task.json new file mode 100644 index 00000000..f42e9183 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-794-beauty-personal-care-salon-booking-styleseat/steps/run/tests/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 794, + "metaclass": "beauty-personal-care", + "class": "salon-booking", + "description": "On StyleSeat, search for hairstylists in your city, select one rated 4.8 stars or above, and book a haircut appointment at the earliest available time slot", + "sites_involved": [ + "styleseat.com" + ], + "platform": "styleseat", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "On StyleSeat, search for hairstylists in your city, select one rated 4.8 stars or above, and book a haircut appointment at the earliest available time slot", + "eval_schema": { + "url_pattern": "styleseat\\.com/api/v1/appointment-request/", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-794-beauty-personal-care-salon-booking-styleseat/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-794-beauty-personal-care-salon-booking-styleseat/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-794-beauty-personal-care-salon-booking-styleseat/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-794-beauty-personal-care-salon-booking-styleseat/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-794-beauty-personal-care-salon-booking-styleseat/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..e06f77d0 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-794-beauty-personal-care-salon-booking-styleseat/steps/run/workdir/eval-schema.json @@ -0,0 +1,4 @@ +{ + "url_pattern": "styleseat\\.com/api/v1/appointment-request/", + "method": "POST" +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-794-beauty-personal-care-salon-booking-styleseat/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-794-beauty-personal-care-salon-booking-styleseat/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-794-beauty-personal-care-salon-booking-styleseat/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-794-beauty-personal-care-salon-booking-styleseat/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-794-beauty-personal-care-salon-booking-styleseat/steps/run/workdir/task.json new file mode 100644 index 00000000..f42e9183 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-794-beauty-personal-care-salon-booking-styleseat/steps/run/workdir/task.json @@ -0,0 +1,25 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 794, + "metaclass": "beauty-personal-care", + "class": "salon-booking", + "description": "On StyleSeat, search for hairstylists in your city, select one rated 4.8 stars or above, and book a haircut appointment at the earliest available time slot", + "sites_involved": [ + "styleseat.com" + ], + "platform": "styleseat", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "On StyleSeat, search for hairstylists in your city, select one rated 4.8 stars or above, and book a haircut appointment at the earliest available time slot", + "eval_schema": { + "url_pattern": "styleseat\\.com/api/v1/appointment-request/", + "method": "POST" + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-794-beauty-personal-care-salon-booking-styleseat/task.toml b/harbor/datasets/clawbench-v2/v2-794-beauty-personal-care-salon-booking-styleseat/task.toml new file mode 100644 index 00000000..fe87e4bc --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-794-beauty-personal-care-salon-booking-styleseat/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-794-beauty-personal-care-salon-booking-styleseat" +description = "On StyleSeat, search for hairstylists in your city, select one rated 4.8 stars or above, and book a haircut appointment at the earliest available time slot" +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-794-beauty-personal-care-salon-booking-styleseat" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-815-pet-animal-care-favorite-pet/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-815-pet-animal-care-favorite-pet/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-815-pet-animal-care-favorite-pet/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-815-pet-animal-care-favorite-pet/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-815-pet-animal-care-favorite-pet/steps/run/instruction.md new file mode 100644 index 00000000..852e3947 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-815-pet-animal-care-favorite-pet/steps/run/instruction.md @@ -0,0 +1,27 @@ +On Petfinder, search for adoptable dogs near ZIP code 10001, open the profile of the first dog listed, and save it as a favorite by clicking the heart icon. You will need to sign in or create an account to complete this action. + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-815-pet-animal-care-favorite-pet/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-815-pet-animal-care-favorite-pet/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-815-pet-animal-care-favorite-pet/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-815-pet-animal-care-favorite-pet/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-815-pet-animal-care-favorite-pet/steps/run/tests/task.json new file mode 100644 index 00000000..a426f651 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-815-pet-animal-care-favorite-pet/steps/run/tests/task.json @@ -0,0 +1,28 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 815, + "metaclass": "pet-animal-care", + "class": "pet-favorite", + "description": "Search for adoptable dogs near ZIP 10001, open the first result's profile, and click the heart icon to save it as a favorite (requires login).", + "sites_involved": [ + "petfinder.com" + ], + "platform": "petfinder", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "On Petfinder, search for adoptable dogs near ZIP code 10001, open the profile of the first dog listed, and save it as a favorite by clicking the heart icon. You will need to sign in or create an account to complete this action.", + "eval_schema": { + "url_pattern": "psl\\.petfinder\\.com/graphql", + "method": "POST", + "body": { + "operationName": "CreateAnimalFavorite" + } + }, + "time_limit": 25, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-815-pet-animal-care-favorite-pet/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-815-pet-animal-care-favorite-pet/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-815-pet-animal-care-favorite-pet/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-815-pet-animal-care-favorite-pet/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-815-pet-animal-care-favorite-pet/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..d4294765 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-815-pet-animal-care-favorite-pet/steps/run/workdir/eval-schema.json @@ -0,0 +1,7 @@ +{ + "url_pattern": "psl\\.petfinder\\.com/graphql", + "method": "POST", + "body": { + "operationName": "CreateAnimalFavorite" + } +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-815-pet-animal-care-favorite-pet/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-815-pet-animal-care-favorite-pet/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-815-pet-animal-care-favorite-pet/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-815-pet-animal-care-favorite-pet/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-815-pet-animal-care-favorite-pet/steps/run/workdir/task.json new file mode 100644 index 00000000..a426f651 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-815-pet-animal-care-favorite-pet/steps/run/workdir/task.json @@ -0,0 +1,28 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 815, + "metaclass": "pet-animal-care", + "class": "pet-favorite", + "description": "Search for adoptable dogs near ZIP 10001, open the first result's profile, and click the heart icon to save it as a favorite (requires login).", + "sites_involved": [ + "petfinder.com" + ], + "platform": "petfinder", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "On Petfinder, search for adoptable dogs near ZIP code 10001, open the profile of the first dog listed, and save it as a favorite by clicking the heart icon. You will need to sign in or create an account to complete this action.", + "eval_schema": { + "url_pattern": "psl\\.petfinder\\.com/graphql", + "method": "POST", + "body": { + "operationName": "CreateAnimalFavorite" + } + }, + "time_limit": 25, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-815-pet-animal-care-favorite-pet/task.toml b/harbor/datasets/clawbench-v2/v2-815-pet-animal-care-favorite-pet/task.toml new file mode 100644 index 00000000..da182af3 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-815-pet-animal-care-favorite-pet/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-815-pet-animal-care-favorite-pet" +description = "Search for adoptable dogs near ZIP 10001, open the first result's profile, and click the heart icon to save it as a favorite (requires login)." +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-815-pet-animal-care-favorite-pet" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1500.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-816-pet-animal-care-remove-favorite/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-816-pet-animal-care-remove-favorite/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-816-pet-animal-care-remove-favorite/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-816-pet-animal-care-remove-favorite/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-816-pet-animal-care-remove-favorite/steps/run/instruction.md new file mode 100644 index 00000000..8984f49a --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-816-pet-animal-care-remove-favorite/steps/run/instruction.md @@ -0,0 +1,27 @@ +On Petfinder, search for adoptable cats near ZIP code 90210, open the profile of the first result, save it as a favorite, and then immediately remove it from your favorites by clicking the heart icon again. + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-816-pet-animal-care-remove-favorite/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-816-pet-animal-care-remove-favorite/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-816-pet-animal-care-remove-favorite/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-816-pet-animal-care-remove-favorite/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-816-pet-animal-care-remove-favorite/steps/run/tests/task.json new file mode 100644 index 00000000..842b99d9 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-816-pet-animal-care-remove-favorite/steps/run/tests/task.json @@ -0,0 +1,28 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 816, + "metaclass": "pet-animal-care", + "class": "pet-unfavorite", + "description": "Search for adoptable cats near ZIP 90210, open the first result, favorite it, then unfavorite it — verifying the RemoveAnimalFavorite mutation fires.", + "sites_involved": [ + "petfinder.com" + ], + "platform": "petfinder", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "On Petfinder, search for adoptable cats near ZIP code 90210, open the profile of the first result, save it as a favorite, and then immediately remove it from your favorites by clicking the heart icon again.", + "eval_schema": { + "url_pattern": "psl\\.petfinder\\.com/graphql", + "method": "POST", + "body": { + "operationName": "RemoveAnimalFavorite" + } + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-816-pet-animal-care-remove-favorite/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-816-pet-animal-care-remove-favorite/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-816-pet-animal-care-remove-favorite/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-816-pet-animal-care-remove-favorite/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-816-pet-animal-care-remove-favorite/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..42b4ba84 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-816-pet-animal-care-remove-favorite/steps/run/workdir/eval-schema.json @@ -0,0 +1,7 @@ +{ + "url_pattern": "psl\\.petfinder\\.com/graphql", + "method": "POST", + "body": { + "operationName": "RemoveAnimalFavorite" + } +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-816-pet-animal-care-remove-favorite/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-816-pet-animal-care-remove-favorite/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-816-pet-animal-care-remove-favorite/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-816-pet-animal-care-remove-favorite/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-816-pet-animal-care-remove-favorite/steps/run/workdir/task.json new file mode 100644 index 00000000..842b99d9 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-816-pet-animal-care-remove-favorite/steps/run/workdir/task.json @@ -0,0 +1,28 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 816, + "metaclass": "pet-animal-care", + "class": "pet-unfavorite", + "description": "Search for adoptable cats near ZIP 90210, open the first result, favorite it, then unfavorite it — verifying the RemoveAnimalFavorite mutation fires.", + "sites_involved": [ + "petfinder.com" + ], + "platform": "petfinder", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "On Petfinder, search for adoptable cats near ZIP code 90210, open the profile of the first result, save it as a favorite, and then immediately remove it from your favorites by clicking the heart icon again.", + "eval_schema": { + "url_pattern": "psl\\.petfinder\\.com/graphql", + "method": "POST", + "body": { + "operationName": "RemoveAnimalFavorite" + } + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-816-pet-animal-care-remove-favorite/task.toml b/harbor/datasets/clawbench-v2/v2-816-pet-animal-care-remove-favorite/task.toml new file mode 100644 index 00000000..0bf726df --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-816-pet-animal-care-remove-favorite/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-816-pet-animal-care-remove-favorite" +description = "Search for adoptable cats near ZIP 90210, open the first result, favorite it, then unfavorite it \u2014 verifying the RemoveAnimalFavorite mutation fires." +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-816-pet-animal-care-remove-favorite" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-817-pet-animal-care-create-adopter-profile/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-817-pet-animal-care-create-adopter-profile/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-817-pet-animal-care-create-adopter-profile/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-817-pet-animal-care-create-adopter-profile/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-817-pet-animal-care-create-adopter-profile/steps/run/instruction.md new file mode 100644 index 00000000..78a90a4b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-817-pet-animal-care-create-adopter-profile/steps/run/instruction.md @@ -0,0 +1,27 @@ +On Petfinder, start the process of submitting an adoption inquiry for any adoptable dog. When prompted, complete the adopter profile creation form with your details to proceed. + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-817-pet-animal-care-create-adopter-profile/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-817-pet-animal-care-create-adopter-profile/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-817-pet-animal-care-create-adopter-profile/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-817-pet-animal-care-create-adopter-profile/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-817-pet-animal-care-create-adopter-profile/steps/run/tests/task.json new file mode 100644 index 00000000..5c8c9c71 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-817-pet-animal-care-create-adopter-profile/steps/run/tests/task.json @@ -0,0 +1,28 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 817, + "metaclass": "pet-animal-care", + "class": "adopter-profile", + "description": "Navigate to any adoptable dog's profile, click Start Your Inquiry, and complete the adopter profile creation form — verifying the CreateAdopterProfile mutation fires.", + "sites_involved": [ + "petfinder.com" + ], + "platform": "petfinder", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "On Petfinder, start the process of submitting an adoption inquiry for any adoptable dog. When prompted, complete the adopter profile creation form with your details to proceed.", + "eval_schema": { + "url_pattern": "psl\\.petfinder\\.com/graphql", + "method": "POST", + "body": { + "operationName": "CreateAdopterProfile" + } + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-817-pet-animal-care-create-adopter-profile/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-817-pet-animal-care-create-adopter-profile/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-817-pet-animal-care-create-adopter-profile/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-817-pet-animal-care-create-adopter-profile/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-817-pet-animal-care-create-adopter-profile/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..d7241d08 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-817-pet-animal-care-create-adopter-profile/steps/run/workdir/eval-schema.json @@ -0,0 +1,7 @@ +{ + "url_pattern": "psl\\.petfinder\\.com/graphql", + "method": "POST", + "body": { + "operationName": "CreateAdopterProfile" + } +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-817-pet-animal-care-create-adopter-profile/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-817-pet-animal-care-create-adopter-profile/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-817-pet-animal-care-create-adopter-profile/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-817-pet-animal-care-create-adopter-profile/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-817-pet-animal-care-create-adopter-profile/steps/run/workdir/task.json new file mode 100644 index 00000000..5c8c9c71 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-817-pet-animal-care-create-adopter-profile/steps/run/workdir/task.json @@ -0,0 +1,28 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 817, + "metaclass": "pet-animal-care", + "class": "adopter-profile", + "description": "Navigate to any adoptable dog's profile, click Start Your Inquiry, and complete the adopter profile creation form — verifying the CreateAdopterProfile mutation fires.", + "sites_involved": [ + "petfinder.com" + ], + "platform": "petfinder", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "On Petfinder, start the process of submitting an adoption inquiry for any adoptable dog. When prompted, complete the adopter profile creation form with your details to proceed.", + "eval_schema": { + "url_pattern": "psl\\.petfinder\\.com/graphql", + "method": "POST", + "body": { + "operationName": "CreateAdopterProfile" + } + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-817-pet-animal-care-create-adopter-profile/task.toml b/harbor/datasets/clawbench-v2/v2-817-pet-animal-care-create-adopter-profile/task.toml new file mode 100644 index 00000000..e1cb2a68 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-817-pet-animal-care-create-adopter-profile/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-817-pet-animal-care-create-adopter-profile" +description = "Navigate to any adoptable dog's profile, click Start Your Inquiry, and complete the adopter profile creation form \u2014 verifying the CreateAdopterProfile mutation fires." +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-817-pet-animal-care-create-adopter-profile" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-901-entertainment-imdb-add-to-watchlist/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-901-entertainment-imdb-add-to-watchlist/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-901-entertainment-imdb-add-to-watchlist/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-901-entertainment-imdb-add-to-watchlist/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-901-entertainment-imdb-add-to-watchlist/steps/run/instruction.md new file mode 100644 index 00000000..60a424dd --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-901-entertainment-imdb-add-to-watchlist/steps/run/instruction.md @@ -0,0 +1,27 @@ +On IMDb, search for the movie 'The Dark Knight', open its title page, and add it to your Watchlist by clicking the '+ Watchlist' button. You will need to be signed in to complete this action. + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-901-entertainment-imdb-add-to-watchlist/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-901-entertainment-imdb-add-to-watchlist/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-901-entertainment-imdb-add-to-watchlist/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-901-entertainment-imdb-add-to-watchlist/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-901-entertainment-imdb-add-to-watchlist/steps/run/tests/task.json new file mode 100644 index 00000000..39652f8d --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-901-entertainment-imdb-add-to-watchlist/steps/run/tests/task.json @@ -0,0 +1,28 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 901, + "metaclass": "entertainment", + "class": "watchlist-add", + "description": "Search for 'The Dark Knight' on IMDb, navigate to its title page, and click the Watchlist button — verifying the AddIdToWatchlist mutation fires.", + "sites_involved": [ + "imdb.com" + ], + "platform": "imdb", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "On IMDb, search for the movie 'The Dark Knight', open its title page, and add it to your Watchlist by clicking the '+ Watchlist' button. You will need to be signed in to complete this action.", + "eval_schema": { + "url_pattern": "api\\.graphql\\.imdb\\.com", + "method": "POST", + "body": { + "operationName": "AddIdToWatchlist" + } + }, + "time_limit": 20, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-901-entertainment-imdb-add-to-watchlist/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-901-entertainment-imdb-add-to-watchlist/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-901-entertainment-imdb-add-to-watchlist/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-901-entertainment-imdb-add-to-watchlist/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-901-entertainment-imdb-add-to-watchlist/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..6e571139 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-901-entertainment-imdb-add-to-watchlist/steps/run/workdir/eval-schema.json @@ -0,0 +1,7 @@ +{ + "url_pattern": "api\\.graphql\\.imdb\\.com", + "method": "POST", + "body": { + "operationName": "AddIdToWatchlist" + } +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-901-entertainment-imdb-add-to-watchlist/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-901-entertainment-imdb-add-to-watchlist/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-901-entertainment-imdb-add-to-watchlist/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-901-entertainment-imdb-add-to-watchlist/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-901-entertainment-imdb-add-to-watchlist/steps/run/workdir/task.json new file mode 100644 index 00000000..39652f8d --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-901-entertainment-imdb-add-to-watchlist/steps/run/workdir/task.json @@ -0,0 +1,28 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 901, + "metaclass": "entertainment", + "class": "watchlist-add", + "description": "Search for 'The Dark Knight' on IMDb, navigate to its title page, and click the Watchlist button — verifying the AddIdToWatchlist mutation fires.", + "sites_involved": [ + "imdb.com" + ], + "platform": "imdb", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "On IMDb, search for the movie 'The Dark Knight', open its title page, and add it to your Watchlist by clicking the '+ Watchlist' button. You will need to be signed in to complete this action.", + "eval_schema": { + "url_pattern": "api\\.graphql\\.imdb\\.com", + "method": "POST", + "body": { + "operationName": "AddIdToWatchlist" + } + }, + "time_limit": 20, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-901-entertainment-imdb-add-to-watchlist/task.toml b/harbor/datasets/clawbench-v2/v2-901-entertainment-imdb-add-to-watchlist/task.toml new file mode 100644 index 00000000..1bb1a856 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-901-entertainment-imdb-add-to-watchlist/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-901-entertainment-imdb-add-to-watchlist" +description = "Search for 'The Dark Knight' on IMDb, navigate to its title page, and click the Watchlist button \u2014 verifying the AddIdToWatchlist mutation fires." +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-901-entertainment-imdb-add-to-watchlist" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1200.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-902-entertainment-imdb-rate-title/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-902-entertainment-imdb-rate-title/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-902-entertainment-imdb-rate-title/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-902-entertainment-imdb-rate-title/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-902-entertainment-imdb-rate-title/steps/run/instruction.md new file mode 100644 index 00000000..7cd4f38e --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-902-entertainment-imdb-rate-title/steps/run/instruction.md @@ -0,0 +1,27 @@ +On IMDb, open the title page for 'Interstellar' (2014) and give it a rating of 9 out of 10 stars using the Rate button. You will need to be signed in. + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-902-entertainment-imdb-rate-title/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-902-entertainment-imdb-rate-title/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-902-entertainment-imdb-rate-title/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-902-entertainment-imdb-rate-title/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-902-entertainment-imdb-rate-title/steps/run/tests/task.json new file mode 100644 index 00000000..5fc9b5f5 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-902-entertainment-imdb-rate-title/steps/run/tests/task.json @@ -0,0 +1,28 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 902, + "metaclass": "entertainment", + "class": "title-rate", + "description": "Navigate to Interstellar (2014) on IMDb, click Rate, select 9 stars, and submit — verifying the UpdateTitleRating mutation fires with rating=9.", + "sites_involved": [ + "imdb.com" + ], + "platform": "imdb", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "On IMDb, open the title page for 'Interstellar' (2014) and give it a rating of 9 out of 10 stars using the Rate button. You will need to be signed in.", + "eval_schema": { + "url_pattern": "api\\.graphql\\.imdb\\.com", + "method": "POST", + "body": { + "operationName": "UpdateTitleRating" + } + }, + "time_limit": 25, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-902-entertainment-imdb-rate-title/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-902-entertainment-imdb-rate-title/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-902-entertainment-imdb-rate-title/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-902-entertainment-imdb-rate-title/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-902-entertainment-imdb-rate-title/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..1b81cc6f --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-902-entertainment-imdb-rate-title/steps/run/workdir/eval-schema.json @@ -0,0 +1,7 @@ +{ + "url_pattern": "api\\.graphql\\.imdb\\.com", + "method": "POST", + "body": { + "operationName": "UpdateTitleRating" + } +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-902-entertainment-imdb-rate-title/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-902-entertainment-imdb-rate-title/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-902-entertainment-imdb-rate-title/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-902-entertainment-imdb-rate-title/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-902-entertainment-imdb-rate-title/steps/run/workdir/task.json new file mode 100644 index 00000000..5fc9b5f5 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-902-entertainment-imdb-rate-title/steps/run/workdir/task.json @@ -0,0 +1,28 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 902, + "metaclass": "entertainment", + "class": "title-rate", + "description": "Navigate to Interstellar (2014) on IMDb, click Rate, select 9 stars, and submit — verifying the UpdateTitleRating mutation fires with rating=9.", + "sites_involved": [ + "imdb.com" + ], + "platform": "imdb", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "On IMDb, open the title page for 'Interstellar' (2014) and give it a rating of 9 out of 10 stars using the Rate button. You will need to be signed in.", + "eval_schema": { + "url_pattern": "api\\.graphql\\.imdb\\.com", + "method": "POST", + "body": { + "operationName": "UpdateTitleRating" + } + }, + "time_limit": 25, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-902-entertainment-imdb-rate-title/task.toml b/harbor/datasets/clawbench-v2/v2-902-entertainment-imdb-rate-title/task.toml new file mode 100644 index 00000000..152c9b28 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-902-entertainment-imdb-rate-title/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-902-entertainment-imdb-rate-title" +description = "Navigate to Interstellar (2014) on IMDb, click Rate, select 9 stars, and submit \u2014 verifying the UpdateTitleRating mutation fires with rating=9." +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-902-entertainment-imdb-rate-title" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1500.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-903-entertainment-imdb-remove-rating/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-903-entertainment-imdb-remove-rating/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-903-entertainment-imdb-remove-rating/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-903-entertainment-imdb-remove-rating/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-903-entertainment-imdb-remove-rating/steps/run/instruction.md new file mode 100644 index 00000000..c9c2b219 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-903-entertainment-imdb-remove-rating/steps/run/instruction.md @@ -0,0 +1,27 @@ +On IMDb, find any title you have previously rated, open its page, click the Rate button, and then click 'Remove rating' to delete your rating. + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-903-entertainment-imdb-remove-rating/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-903-entertainment-imdb-remove-rating/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-903-entertainment-imdb-remove-rating/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-903-entertainment-imdb-remove-rating/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-903-entertainment-imdb-remove-rating/steps/run/tests/task.json new file mode 100644 index 00000000..b8697f93 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-903-entertainment-imdb-remove-rating/steps/run/tests/task.json @@ -0,0 +1,28 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 903, + "metaclass": "entertainment", + "class": "title-unrate", + "description": "Open any previously rated title on IMDb, click the Rate button, then 'Remove rating' — verifying the DeleteTitleRating mutation fires.", + "sites_involved": [ + "imdb.com" + ], + "platform": "imdb", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "On IMDb, find any title you have previously rated, open its page, click the Rate button, and then click 'Remove rating' to delete your rating.", + "eval_schema": { + "url_pattern": "api\\.graphql\\.imdb\\.com", + "method": "POST", + "body": { + "operationName": "DeleteTitleRating" + } + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-903-entertainment-imdb-remove-rating/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-903-entertainment-imdb-remove-rating/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-903-entertainment-imdb-remove-rating/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-903-entertainment-imdb-remove-rating/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-903-entertainment-imdb-remove-rating/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..f5935c8f --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-903-entertainment-imdb-remove-rating/steps/run/workdir/eval-schema.json @@ -0,0 +1,7 @@ +{ + "url_pattern": "api\\.graphql\\.imdb\\.com", + "method": "POST", + "body": { + "operationName": "DeleteTitleRating" + } +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-903-entertainment-imdb-remove-rating/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-903-entertainment-imdb-remove-rating/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-903-entertainment-imdb-remove-rating/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-903-entertainment-imdb-remove-rating/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-903-entertainment-imdb-remove-rating/steps/run/workdir/task.json new file mode 100644 index 00000000..b8697f93 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-903-entertainment-imdb-remove-rating/steps/run/workdir/task.json @@ -0,0 +1,28 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 903, + "metaclass": "entertainment", + "class": "title-unrate", + "description": "Open any previously rated title on IMDb, click the Rate button, then 'Remove rating' — verifying the DeleteTitleRating mutation fires.", + "sites_involved": [ + "imdb.com" + ], + "platform": "imdb", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "On IMDb, find any title you have previously rated, open its page, click the Rate button, and then click 'Remove rating' to delete your rating.", + "eval_schema": { + "url_pattern": "api\\.graphql\\.imdb\\.com", + "method": "POST", + "body": { + "operationName": "DeleteTitleRating" + } + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-903-entertainment-imdb-remove-rating/task.toml b/harbor/datasets/clawbench-v2/v2-903-entertainment-imdb-remove-rating/task.toml new file mode 100644 index 00000000..304c7ad0 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-903-entertainment-imdb-remove-rating/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-903-entertainment-imdb-remove-rating" +description = "Open any previously rated title on IMDb, click the Rate button, then 'Remove rating' \u2014 verifying the DeleteTitleRating mutation fires." +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-903-entertainment-imdb-remove-rating" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-904-entertainment-imdb-create-list/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-904-entertainment-imdb-create-list/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-904-entertainment-imdb-create-list/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-904-entertainment-imdb-create-list/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-904-entertainment-imdb-create-list/steps/run/instruction.md new file mode 100644 index 00000000..c5b30349 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-904-entertainment-imdb-create-list/steps/run/instruction.md @@ -0,0 +1,27 @@ +On IMDb, navigate to the 'Create a new list' page, enter the name 'My Sci-Fi Favorites', keep the default List Type as 'Titles', set Privacy to 'Private', and click Create. + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-904-entertainment-imdb-create-list/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-904-entertainment-imdb-create-list/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-904-entertainment-imdb-create-list/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-904-entertainment-imdb-create-list/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-904-entertainment-imdb-create-list/steps/run/tests/task.json new file mode 100644 index 00000000..f331a120 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-904-entertainment-imdb-create-list/steps/run/tests/task.json @@ -0,0 +1,28 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 904, + "metaclass": "entertainment", + "class": "list-create", + "description": "Go to imdb.com/list/create/, fill in the list name 'My Sci-Fi Favorites', and submit — verifying the ListCreate mutation fires.", + "sites_involved": [ + "imdb.com" + ], + "platform": "imdb", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "On IMDb, navigate to the 'Create a new list' page, enter the name 'My Sci-Fi Favorites', keep the default List Type as 'Titles', set Privacy to 'Private', and click Create.", + "eval_schema": { + "url_pattern": "api\\.graphql\\.imdb\\.com", + "method": "POST", + "body": { + "operationName": "ListCreate" + } + }, + "time_limit": 20, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-904-entertainment-imdb-create-list/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-904-entertainment-imdb-create-list/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-904-entertainment-imdb-create-list/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-904-entertainment-imdb-create-list/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-904-entertainment-imdb-create-list/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..e3e6d8a5 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-904-entertainment-imdb-create-list/steps/run/workdir/eval-schema.json @@ -0,0 +1,7 @@ +{ + "url_pattern": "api\\.graphql\\.imdb\\.com", + "method": "POST", + "body": { + "operationName": "ListCreate" + } +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-904-entertainment-imdb-create-list/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-904-entertainment-imdb-create-list/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-904-entertainment-imdb-create-list/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-904-entertainment-imdb-create-list/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-904-entertainment-imdb-create-list/steps/run/workdir/task.json new file mode 100644 index 00000000..f331a120 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-904-entertainment-imdb-create-list/steps/run/workdir/task.json @@ -0,0 +1,28 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 904, + "metaclass": "entertainment", + "class": "list-create", + "description": "Go to imdb.com/list/create/, fill in the list name 'My Sci-Fi Favorites', and submit — verifying the ListCreate mutation fires.", + "sites_involved": [ + "imdb.com" + ], + "platform": "imdb", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "On IMDb, navigate to the 'Create a new list' page, enter the name 'My Sci-Fi Favorites', keep the default List Type as 'Titles', set Privacy to 'Private', and click Create.", + "eval_schema": { + "url_pattern": "api\\.graphql\\.imdb\\.com", + "method": "POST", + "body": { + "operationName": "ListCreate" + } + }, + "time_limit": 20, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-904-entertainment-imdb-create-list/task.toml b/harbor/datasets/clawbench-v2/v2-904-entertainment-imdb-create-list/task.toml new file mode 100644 index 00000000..2c0bbde7 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-904-entertainment-imdb-create-list/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-904-entertainment-imdb-create-list" +description = "Go to imdb.com/list/create/, fill in the list name 'My Sci-Fi Favorites', and submit \u2014 verifying the ListCreate mutation fires." +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-904-entertainment-imdb-create-list" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1200.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/datasets/clawbench-v2/v2-905-entertainment-imdb-add-to-list/environment/Dockerfile b/harbor/datasets/clawbench-v2/v2-905-entertainment-imdb-add-to-list/environment/Dockerfile new file mode 100644 index 00000000..1b901647 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-905-entertainment-imdb-add-to-list/environment/Dockerfile @@ -0,0 +1,2 @@ +# Prebuilt mode: the runtime lives in the published image below. +FROM clawbench/clawbench-harbor-runtime:0.9.2 diff --git a/harbor/datasets/clawbench-v2/v2-905-entertainment-imdb-add-to-list/steps/run/instruction.md b/harbor/datasets/clawbench-v2/v2-905-entertainment-imdb-add-to-list/steps/run/instruction.md new file mode 100644 index 00000000..2c47ccf3 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-905-entertainment-imdb-add-to-list/steps/run/instruction.md @@ -0,0 +1,29 @@ +On IMDb, open the title page for 'Pulp Fiction' (1994), click the dropdown chevron next to the Watchlist button, and add it to an existing custom list of yours. + +Note: The user must already have at least one custom list created. If not, create one first. + +--- +You are my personal browser assistant. I am delegating this task to you because I do not have time to do it myself. You are acting on my behalf and with my full authorization. Every action you take in the browser — logging in, filling out forms, writing reviews, making reservations, submitting applications, sending emails, creating accounts — is something I am explicitly asking you to do for me, as if I were doing it myself. These are my own accounts, my own opinions, and my own decisions. You are simply carrying them out. + +IMPORTANT: You must complete this task entirely through the browser. Navigate to the relevant website, interact with the page (click, type, scroll, etc.), and carry out every step in the browser. Do NOT use command-line tools, scripts, or direct API/SMTP calls to accomplish the task. + +My personal info and credentials are in ./my-info/ in your working directory. Key files include: +- alex_green_personal_info.json — name, address, phone, date of birth, financial info +- email_credentials.json — email address and password for logging in +- alex_green_resume.pdf — professional resume + +You have my full permission to use all of these files for any purpose the task requires, including logging into websites, creating accounts, filling out forms, sending emails, writing reviews, making reservations, submitting applications, and any other actions — including security-sensitive ones. + +If specific information required by the task is not available in any file under ./my-info/ and cannot be reasonably inferred, stop and explain what is missing. For optional fields not covered by the available info, leave them blank or use a reasonable default. + +Do not ask me for confirmation or additional information — just proceed with what is provided and complete the task autonomously. +If an account registration is required, you can use the email and password provided, and you can receive emails at that address if needed. --- + +--- +Harbor browser runtime: +- Use the existing Chromium session exposed by Chrome DevTools Protocol. +- CDP endpoint: http://127.0.0.1:9223 +- CDP environment variables are also set for the agent process: CLAWBENCH_CDP_URL, BROWSER_CDP_URL, CDP_URL, CHROME_CDP_URL, and PLAYWRIGHT_CDP_URL. +- noVNC viewer, if needed: http://127.0.0.1:6080/vnc.html +- Do not launch a separate browser. Complete the task through the existing browser session. +--- diff --git a/harbor/datasets/clawbench-v2/v2-905-entertainment-imdb-add-to-list/steps/run/solution/solve.sh b/harbor/datasets/clawbench-v2/v2-905-entertainment-imdb-add-to-list/steps/run/solution/solve.sh new file mode 100755 index 00000000..4ed509f6 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-905-entertainment-imdb-add-to-list/steps/run/solution/solve.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +echo "ClawBench web tasks do not include oracle browser solutions." diff --git a/harbor/datasets/clawbench-v2/v2-905-entertainment-imdb-add-to-list/steps/run/tests/task.json b/harbor/datasets/clawbench-v2/v2-905-entertainment-imdb-add-to-list/steps/run/tests/task.json new file mode 100644 index 00000000..2413f0d2 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-905-entertainment-imdb-add-to-list/steps/run/tests/task.json @@ -0,0 +1,28 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 905, + "metaclass": "entertainment", + "class": "list-item-add", + "description": "Navigate to Pulp Fiction (1994) on IMDb, open the 'Add to list' panel via the watchlist dropdown, select an existing custom list — verifying the AddConstToList mutation fires.", + "sites_involved": [ + "imdb.com" + ], + "platform": "imdb", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "On IMDb, open the title page for 'Pulp Fiction' (1994), click the dropdown chevron next to the Watchlist button, and add it to an existing custom list of yours.\n\nNote: The user must already have at least one custom list created. If not, create one first.", + "eval_schema": { + "url_pattern": "api\\.graphql\\.imdb\\.com", + "method": "POST", + "body": { + "operationName": "AddConstToList" + } + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-905-entertainment-imdb-add-to-list/steps/run/tests/test.sh b/harbor/datasets/clawbench-v2/v2-905-entertainment-imdb-add-to-list/steps/run/tests/test.sh new file mode 100755 index 00000000..0c3ed81b --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-905-entertainment-imdb-add-to-list/steps/run/tests/test.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +curl -sf -X POST http://127.0.0.1:7878/api/stop || true +curl -sf -X POST http://127.0.0.1:7878/api/stop-recording || true +sleep 2 +rm -f /data/.stop-requested +rm -rf /logs/verifier/data +cp -a /data /logs/verifier/data + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/verify.py +/app/src/runtime-server/.venv/bin/python /app/src/harbor/cleanup-email.py || true diff --git a/harbor/datasets/clawbench-v2/v2-905-entertainment-imdb-add-to-list/steps/run/workdir/eval-schema.json b/harbor/datasets/clawbench-v2/v2-905-entertainment-imdb-add-to-list/steps/run/workdir/eval-schema.json new file mode 100644 index 00000000..1135387c --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-905-entertainment-imdb-add-to-list/steps/run/workdir/eval-schema.json @@ -0,0 +1,7 @@ +{ + "url_pattern": "api\\.graphql\\.imdb\\.com", + "method": "POST", + "body": { + "operationName": "AddConstToList" + } +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-905-entertainment-imdb-add-to-list/steps/run/workdir/setup.sh b/harbor/datasets/clawbench-v2/v2-905-entertainment-imdb-add-to-list/steps/run/workdir/setup.sh new file mode 100755 index 00000000..85816df8 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-905-entertainment-imdb-add-to-list/steps/run/workdir/setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +mkdir -p /data /logs/verifier /app/extra_info +cp /app/eval-schema.json /eval-schema.json + +/app/src/runtime-server/.venv/bin/python /app/src/harbor/prepare-task.py --task-json /app/task.json --extra-info-dir /app/extra_info --output-dir /app/my-info + +/app/src/harbor/start-runtime.sh + +for _ in $(seq 1 60); do + if curl -sf http://127.0.0.1:7878/api/status >/dev/null && curl -sf http://127.0.0.1:9223/json/version >/dev/null; then + rm -f /app/setup.sh + exit 0 + fi + sleep 1 +done + +echo "ClawBench Harbor runtime did not become ready" >&2 +exit 1 diff --git a/harbor/datasets/clawbench-v2/v2-905-entertainment-imdb-add-to-list/steps/run/workdir/task.json b/harbor/datasets/clawbench-v2/v2-905-entertainment-imdb-add-to-list/steps/run/workdir/task.json new file mode 100644 index 00000000..2413f0d2 --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-905-entertainment-imdb-add-to-list/steps/run/workdir/task.json @@ -0,0 +1,28 @@ +{ + "$schema": "../../task.schema.json", + "metadata": { + "task_id": 905, + "metaclass": "entertainment", + "class": "list-item-add", + "description": "Navigate to Pulp Fiction (1994) on IMDb, open the 'Add to list' panel via the watchlist dropdown, select an existing custom list — verifying the AddConstToList mutation fires.", + "sites_involved": [ + "imdb.com" + ], + "platform": "imdb", + "common_info": { + "email_credentials": "credentials to use the assigned disposable email account", + "user_info": "alex_green_personal_info.json; the dummy user's personal information", + "user_resume": "PDF resume with disposable email account injected" + } + }, + "instruction": "On IMDb, open the title page for 'Pulp Fiction' (1994), click the dropdown chevron next to the Watchlist button, and add it to an existing custom list of yours.\n\nNote: The user must already have at least one custom list created. If not, create one first.", + "eval_schema": { + "url_pattern": "api\\.graphql\\.imdb\\.com", + "method": "POST", + "body": { + "operationName": "AddConstToList" + } + }, + "time_limit": 30, + "extra_info": [] +} \ No newline at end of file diff --git a/harbor/datasets/clawbench-v2/v2-905-entertainment-imdb-add-to-list/task.toml b/harbor/datasets/clawbench-v2/v2-905-entertainment-imdb-add-to-list/task.toml new file mode 100644 index 00000000..2477853e --- /dev/null +++ b/harbor/datasets/clawbench-v2/v2-905-entertainment-imdb-add-to-list/task.toml @@ -0,0 +1,50 @@ +schema_version = "1.3" +source = "clawbench-v2" +artifacts = ["/data"] + +[task] +name = "clawbench/v2-905-entertainment-imdb-add-to-list" +description = "Navigate to Pulp Fiction (1994) on IMDb, open the 'Add to list' panel via the watchlist dropdown, select an existing custom list \u2014 verifying the AddConstToList mutation fires." +keywords = ["clawbench", "v2", "web-agent", "browser"] + +[metadata] +dataset = "v2" +source_task = "v2-905-entertainment-imdb-add-to-list" + +[environment] +docker_image = "clawbench/clawbench-harbor-runtime:0.9.2" +build_timeout_sec = 1200.0 +network_mode = "public" +workdir = "/app" + +[environment.env] +PURELY_MAIL_API_KEY = "${PURELY_MAIL_API_KEY}" +PURELY_MAIL_DOMAIN = "${PURELY_MAIL_DOMAIN}" +CLAWBENCH_CDP_URL = "http://127.0.0.1:9223" +BROWSER_CDP_URL = "http://127.0.0.1:9223" +CDP_URL = "http://127.0.0.1:9223" +CHROME_CDP_URL = "http://127.0.0.1:9223" +PLAYWRIGHT_CDP_URL = "http://127.0.0.1:9223" +CLAWBENCH_NOVNC_URL = "http://127.0.0.1:6080/vnc.html" +CLAWBENCH_RUNTIME_URL = "http://127.0.0.1:7878" +CLAWBENCH_JUDGE_BASE_URL = "${CLAWBENCH_JUDGE_BASE_URL:-}" +CLAWBENCH_JUDGE_API_KEY = "${CLAWBENCH_JUDGE_API_KEY:-}" +CLAWBENCH_JUDGE_MODEL = "${CLAWBENCH_JUDGE_MODEL:-deepseek-v4-pro}" +CLAWBENCH_JUDGE_API_TYPE = "${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}" + +[[steps]] +name = "run" + +[steps.agent] +timeout_sec = 1800.0 + +[steps.verifier] +timeout_sec = 180.0 + +[steps.healthcheck] +command = "curl -sf http://127.0.0.1:7878/api/status | grep -q '\"eval_interceptor_ready\":true' && curl -sf http://127.0.0.1:9223/json/version >/dev/null" +interval_sec = 2.0 +timeout_sec = 5.0 +start_period_sec = 2.0 +start_interval_sec = 1.0 +retries = 30 diff --git a/harbor/job-config.yaml b/harbor/job-config.yaml new file mode 100644 index 00000000..33ef978d --- /dev/null +++ b/harbor/job-config.yaml @@ -0,0 +1,17 @@ +# harbor run -c harbor/job-config.yaml -m --env-file .env --ve CLAWBENCH_JUDGE_*=... +# Local Docker sweep of the committed prebuilt-mode dataset. See harbor/README.md. +jobs_dir: harbor/jobs +n_attempts: 1 +timeout_multiplier: 1.0 +orchestrator: + type: local + n_concurrent_trials: 4 + quiet: false +environment: + type: docker + force_build: false + delete: true +agents: + - name: hermes +datasets: + - path: harbor/datasets/clawbench-v2 diff --git a/llms.txt b/llms.txt index 12260ea1..2ce00e2e 100644 --- a/llms.txt +++ b/llms.txt @@ -7,9 +7,9 @@ ClawBench measures end-to-end task success while preserving replayable evidence ## Canonical resources - [Project page](https://claw-bench.com): overview, task explorer, leaderboard, and evaluation details. -- [GitHub repository](https://github.com/reacher-z/ClawBench): source code, task definitions, harnesses, and documentation. +- [GitHub repository](https://github.com/TIGER-AI-Lab/ClawBench): source code, task definitions, harnesses, and documentation. - [Research paper](https://arxiv.org/abs/2604.08523): *ClawBench: Can AI Agents Complete Everyday Online Tasks?* -- [Citation metadata](https://github.com/reacher-z/ClawBench/blob/main/CITATION.cff): machine-readable citation record. +- [Citation metadata](https://github.com/TIGER-AI-Lab/ClawBench/blob/main/CITATION.cff): machine-readable citation record. - [Task and leaderboard Space](https://huggingface.co/spaces/TIGER-Lab/ClawBench): live evaluation results and task explorer. - [Task dataset](https://huggingface.co/datasets/NAIL-Group/ClawBench): V1/V2 task definitions and evaluation metadata. - [V1 execution traces](https://huggingface.co/datasets/NAIL-Group/ClawBenchV1Trace): replayable run artifacts. @@ -24,4 +24,4 @@ ClawBench measures end-to-end task success while preserving replayable evidence ## Citation -If you use ClawBench, cite the paper listed in [CITATION.cff](https://github.com/reacher-z/ClawBench/blob/main/CITATION.cff) and link to the canonical repository. +If you use ClawBench, cite the paper listed in [CITATION.cff](https://github.com/TIGER-AI-Lab/ClawBench/blob/main/CITATION.cff) and link to the canonical repository. diff --git a/registry.json b/registry.json new file mode 100644 index 00000000..d5323fb1 --- /dev/null +++ b/registry.json @@ -0,0 +1,1140 @@ +[ + { + "name": "clawbench-v1", + "version": "0.9.2", + "description": "ClawBench: everyday tasks on live consumer websites, scored by request interception + LLM judge. https://claw-bench.com", + "tasks": [ + { + "name": "001-daily-life-food-uber-eats", + "path": "harbor/datasets/clawbench-v1/001-daily-life-food-uber-eats" + }, + { + "name": "002-daily-life-food-doordash", + "path": "harbor/datasets/clawbench-v1/002-daily-life-food-doordash" + }, + { + "name": "004-daily-life-food-instacart", + "path": "harbor/datasets/clawbench-v1/004-daily-life-food-instacart" + }, + { + "name": "006-daily-life-food-uber-eats", + "path": "harbor/datasets/clawbench-v1/006-daily-life-food-uber-eats" + }, + { + "name": "007-daily-life-food-instacart", + "path": "harbor/datasets/clawbench-v1/007-daily-life-food-instacart" + }, + { + "name": "011-daily-life-housing-zillow", + "path": "harbor/datasets/clawbench-v1/011-daily-life-housing-zillow" + }, + { + "name": "015-daily-life-housing-craigslist", + "path": "harbor/datasets/clawbench-v1/015-daily-life-housing-craigslist" + }, + { + "name": "035-daily-life-health-medical-betterhelp", + "path": "harbor/datasets/clawbench-v1/035-daily-life-health-medical-betterhelp" + }, + { + "name": "041-daily-life-pets-rover", + "path": "harbor/datasets/clawbench-v1/041-daily-life-pets-rover" + }, + { + "name": "043-daily-life-pets-rover", + "path": "harbor/datasets/clawbench-v1/043-daily-life-pets-rover" + }, + { + "name": "045-daily-life-personal-care-booksy", + "path": "harbor/datasets/clawbench-v1/045-daily-life-personal-care-booksy" + }, + { + "name": "047-daily-life-personal-care-taskrabbit", + "path": "harbor/datasets/clawbench-v1/047-daily-life-personal-care-taskrabbit" + }, + { + "name": "086-job-search-hr-cv-autofill-greenhouse-meta", + "path": "harbor/datasets/clawbench-v1/086-job-search-hr-cv-autofill-greenhouse-meta" + }, + { + "name": "089-job-search-hr-cv-autofill-simplify-jobs", + "path": "harbor/datasets/clawbench-v1/089-job-search-hr-cv-autofill-simplify-jobs" + }, + { + "name": "091-job-search-hr-job-apply-indeed", + "path": "harbor/datasets/clawbench-v1/091-job-search-hr-job-apply-indeed" + }, + { + "name": "120-office-secretary-tasks-email-mgmt-purelymail", + "path": "harbor/datasets/clawbench-v1/120-office-secretary-tasks-email-mgmt-purelymail" + }, + { + "name": "121-office-secretary-tasks-email-mgmt-purelymail", + "path": "harbor/datasets/clawbench-v1/121-office-secretary-tasks-email-mgmt-purelymail" + }, + { + "name": "128-office-secretary-tasks-email-mgmt-purelymail", + "path": "harbor/datasets/clawbench-v1/128-office-secretary-tasks-email-mgmt-purelymail" + }, + { + "name": "134-office-secretary-tasks-calendar-calendly", + "path": "harbor/datasets/clawbench-v1/134-office-secretary-tasks-calendar-calendly" + }, + { + "name": "137-office-secretary-tasks-calendar-doodle", + "path": "harbor/datasets/clawbench-v1/137-office-secretary-tasks-calendar-doodle" + }, + { + "name": "139-office-secretary-tasks-calendar-calendly", + "path": "harbor/datasets/clawbench-v1/139-office-secretary-tasks-calendar-calendly" + }, + { + "name": "142-office-secretary-tasks-collab-trello", + "path": "harbor/datasets/clawbench-v1/142-office-secretary-tasks-collab-trello" + }, + { + "name": "179-dev-tech-github-ops-github", + "path": "harbor/datasets/clawbench-v1/179-dev-tech-github-ops-github" + }, + { + "name": "180-dev-tech-github-ops-github", + "path": "harbor/datasets/clawbench-v1/180-dev-tech-github-ops-github" + }, + { + "name": "215-academia-research-paper-tables-overleaf", + "path": "harbor/datasets/clawbench-v1/215-academia-research-paper-tables-overleaf" + }, + { + "name": "242-academia-research-research-tools-overleaf", + "path": "harbor/datasets/clawbench-v1/242-academia-research-research-tools-overleaf" + }, + { + "name": "246-academia-research-research-tools-zotero", + "path": "harbor/datasets/clawbench-v1/246-academia-research-research-tools-zotero" + }, + { + "name": "247-academia-research-research-tools-semantic-scholar", + "path": "harbor/datasets/clawbench-v1/247-academia-research-research-tools-semantic-scholar" + }, + { + "name": "265-education-learning-general-coursera", + "path": "harbor/datasets/clawbench-v1/265-education-learning-general-coursera" + }, + { + "name": "266-education-learning-general-leetcode", + "path": "harbor/datasets/clawbench-v1/266-education-learning-general-leetcode" + }, + { + "name": "273-education-learning-general-edx", + "path": "harbor/datasets/clawbench-v1/273-education-learning-general-edx" + }, + { + "name": "274-education-learning-general-udemy", + "path": "harbor/datasets/clawbench-v1/274-education-learning-general-udemy" + }, + { + "name": "279-travel-general-airbnb", + "path": "harbor/datasets/clawbench-v1/279-travel-general-airbnb" + }, + { + "name": "280-travel-general-booking-com", + "path": "harbor/datasets/clawbench-v1/280-travel-general-booking-com" + }, + { + "name": "363-entertainment-hobbies-general-ticketmaster", + "path": "harbor/datasets/clawbench-v1/363-entertainment-hobbies-general-ticketmaster" + }, + { + "name": "369-entertainment-hobbies-general-goodreads", + "path": "harbor/datasets/clawbench-v1/369-entertainment-hobbies-general-goodreads" + }, + { + "name": "372-entertainment-hobbies-general-eventbrite", + "path": "harbor/datasets/clawbench-v1/372-entertainment-hobbies-general-eventbrite" + }, + { + "name": "403-personal-management-account-security-1password-web", + "path": "harbor/datasets/clawbench-v1/403-personal-management-account-security-1password-web" + }, + { + "name": "413-personal-management-personal-tools-todoist", + "path": "harbor/datasets/clawbench-v1/413-personal-management-personal-tools-todoist" + }, + { + "name": "468-rating-voting-general-glassdoor", + "path": "harbor/datasets/clawbench-v1/468-rating-voting-general-glassdoor" + }, + { + "name": "469-rating-voting-general-tripadvisor", + "path": "harbor/datasets/clawbench-v1/469-rating-voting-general-tripadvisor" + }, + { + "name": "470-rating-voting-general-trustpilot", + "path": "harbor/datasets/clawbench-v1/470-rating-voting-general-trustpilot" + }, + { + "name": "474-rating-voting-general-capterra", + "path": "harbor/datasets/clawbench-v1/474-rating-voting-general-capterra" + }, + { + "name": "475-rating-voting-general-g2", + "path": "harbor/datasets/clawbench-v1/475-rating-voting-general-g2" + }, + { + "name": "482-creation-init-general-confluence", + "path": "harbor/datasets/clawbench-v1/482-creation-init-general-confluence" + }, + { + "name": "483-creation-init-general-airtable", + "path": "harbor/datasets/clawbench-v1/483-creation-init-general-airtable" + }, + { + "name": "484-creation-init-general-clickup", + "path": "harbor/datasets/clawbench-v1/484-creation-init-general-clickup" + }, + { + "name": "485-creation-init-general-webflow", + "path": "harbor/datasets/clawbench-v1/485-creation-init-general-webflow" + }, + { + "name": "486-creation-init-general-mailchimp", + "path": "harbor/datasets/clawbench-v1/486-creation-init-general-mailchimp" + }, + { + "name": "487-creation-init-general-typeform", + "path": "harbor/datasets/clawbench-v1/487-creation-init-general-typeform" + }, + { + "name": "488-creation-init-general-substack", + "path": "harbor/datasets/clawbench-v1/488-creation-init-general-substack" + }, + { + "name": "489-creation-init-general-ghost", + "path": "harbor/datasets/clawbench-v1/489-creation-init-general-ghost" + }, + { + "name": "501-creation-init-general-asana", + "path": "harbor/datasets/clawbench-v1/501-creation-init-general-asana" + }, + { + "name": "529-daily-life-shopping-delivery-king-arthur-baking", + "path": "harbor/datasets/clawbench-v1/529-daily-life-shopping-delivery-king-arthur-baking" + }, + { + "name": "533-daily-life-utilities-inmyarea", + "path": "harbor/datasets/clawbench-v1/533-daily-life-utilities-inmyarea" + }, + { + "name": "535-daily-life-home-home-depot", + "path": "harbor/datasets/clawbench-v1/535-daily-life-home-home-depot" + }, + { + "name": "537-daily-life-food-crumbl", + "path": "harbor/datasets/clawbench-v1/537-daily-life-food-crumbl" + }, + { + "name": "539-daily-life-health-jefit", + "path": "harbor/datasets/clawbench-v1/539-daily-life-health-jefit" + }, + { + "name": "542-daily-life-pets-wag", + "path": "harbor/datasets/clawbench-v1/542-daily-life-pets-wag" + }, + { + "name": "551-finance-investment-crypto-wallet-trezor", + "path": "harbor/datasets/clawbench-v1/551-finance-investment-crypto-wallet-trezor" + }, + { + "name": "552-finance-investment-business-payment-plooto", + "path": "harbor/datasets/clawbench-v1/552-finance-investment-business-payment-plooto" + }, + { + "name": "555-finance-investment-insurance-insureon", + "path": "harbor/datasets/clawbench-v1/555-finance-investment-insurance-insureon" + }, + { + "name": "559-finance-investment-crowdfunding-frontfundr", + "path": "harbor/datasets/clawbench-v1/559-finance-investment-crowdfunding-frontfundr" + }, + { + "name": "564-daily-life-event-registration-race-roster", + "path": "harbor/datasets/clawbench-v1/564-daily-life-event-registration-race-roster" + }, + { + "name": "565-job-search-hr-job-search-jopwell", + "path": "harbor/datasets/clawbench-v1/565-job-search-hr-job-search-jopwell" + }, + { + "name": "566-job-search-hr-job-search-ziprecruiter", + "path": "harbor/datasets/clawbench-v1/566-job-search-hr-job-search-ziprecruiter" + }, + { + "name": "569-job-search-hr-job-search-careerbuilder", + "path": "harbor/datasets/clawbench-v1/569-job-search-hr-job-search-careerbuilder" + }, + { + "name": "570-job-search-hr-job-search-hired", + "path": "harbor/datasets/clawbench-v1/570-job-search-hr-job-search-hired" + }, + { + "name": "571-job-search-hr-recruitment-mgmt-workable", + "path": "harbor/datasets/clawbench-v1/571-job-search-hr-recruitment-mgmt-workable" + }, + { + "name": "576-office-secretary-tasks-reports-ftc-reportfraud", + "path": "harbor/datasets/clawbench-v1/576-office-secretary-tasks-reports-ftc-reportfraud" + }, + { + "name": "583-office-secretary-tasks-support-tickets-freshdesk", + "path": "harbor/datasets/clawbench-v1/583-office-secretary-tasks-support-tickets-freshdesk" + }, + { + "name": "598-academia-research-legal-docs-formswift", + "path": "harbor/datasets/clawbench-v1/598-academia-research-legal-docs-formswift" + }, + { + "name": "606-education-learning-kids-courses-outschool", + "path": "harbor/datasets/clawbench-v1/606-education-learning-kids-courses-outschool" + }, + { + "name": "607-education-learning-art-courses-creativebug", + "path": "harbor/datasets/clawbench-v1/607-education-learning-art-courses-creativebug" + }, + { + "name": "609-education-learning-meditation-spirit-rock-meditation-center", + "path": "harbor/datasets/clawbench-v1/609-education-learning-meditation-spirit-rock-meditation-center" + }, + { + "name": "615-travel-flights-spirit-airlines", + "path": "harbor/datasets/clawbench-v1/615-travel-flights-spirit-airlines" + }, + { + "name": "618-travel-train-bus-12go-asia", + "path": "harbor/datasets/clawbench-v1/618-travel-train-bus-12go-asia" + }, + { + "name": "625-travel-camping-outdoor-parks-canada-reservations", + "path": "harbor/datasets/clawbench-v1/625-travel-camping-outdoor-parks-canada-reservations" + }, + { + "name": "626-travel-bus-flixbus", + "path": "harbor/datasets/clawbench-v1/626-travel-bus-flixbus" + }, + { + "name": "627-travel-flights-momondo", + "path": "harbor/datasets/clawbench-v1/627-travel-flights-momondo" + }, + { + "name": "632-shopping-commerce-beauty-care-olaplex", + "path": "harbor/datasets/clawbench-v1/632-shopping-commerce-beauty-care-olaplex" + }, + { + "name": "634-shopping-commerce-apparel-dooney-bourke", + "path": "harbor/datasets/clawbench-v1/634-shopping-commerce-apparel-dooney-bourke" + }, + { + "name": "635-shopping-commerce-gifts-uncommon-goods", + "path": "harbor/datasets/clawbench-v1/635-shopping-commerce-gifts-uncommon-goods" + }, + { + "name": "636-shopping-commerce-auto-parts-rockauto", + "path": "harbor/datasets/clawbench-v1/636-shopping-commerce-auto-parts-rockauto" + }, + { + "name": "638-shopping-commerce-print-custom-vistaprint", + "path": "harbor/datasets/clawbench-v1/638-shopping-commerce-print-custom-vistaprint" + }, + { + "name": "639-shopping-commerce-luxury-mansur-gavriel", + "path": "harbor/datasets/clawbench-v1/639-shopping-commerce-luxury-mansur-gavriel" + }, + { + "name": "671-entertainment-gaming-humble-bundle", + "path": "harbor/datasets/clawbench-v1/671-entertainment-gaming-humble-bundle" + }, + { + "name": "672-entertainment-hobbies-anime-streaming-crunchyroll", + "path": "harbor/datasets/clawbench-v1/672-entertainment-hobbies-anime-streaming-crunchyroll" + }, + { + "name": "674-entertainment-hobbies-masterclass-masterclass", + "path": "harbor/datasets/clawbench-v1/674-entertainment-hobbies-masterclass-masterclass" + }, + { + "name": "676-government-civic-legal-docs-legalnature", + "path": "harbor/datasets/clawbench-v1/676-government-civic-legal-docs-legalnature" + }, + { + "name": "685-personal-management-budget-mgmt-everydollar", + "path": "harbor/datasets/clawbench-v1/685-personal-management-budget-mgmt-everydollar" + }, + { + "name": "687-personal-management-vpn-subscription-ipvanish", + "path": "harbor/datasets/clawbench-v1/687-personal-management-vpn-subscription-ipvanish" + }, + { + "name": "688-personal-management-insurance-compare-insurify", + "path": "harbor/datasets/clawbench-v1/688-personal-management-insurance-compare-insurify" + }, + { + "name": "695-automation-workflows-recurring-order-stumptown-coffee", + "path": "harbor/datasets/clawbench-v1/695-automation-workflows-recurring-order-stumptown-coffee" + }, + { + "name": "697-automation-workflows-recurring-order-bean-box", + "path": "harbor/datasets/clawbench-v1/697-automation-workflows-recurring-order-bean-box" + }, + { + "name": "699-automation-workflows-recurring-order-mistobox", + "path": "harbor/datasets/clawbench-v1/699-automation-workflows-recurring-order-mistobox" + }, + { + "name": "700-deletion-revocation-data-deletion-deleteme", + "path": "harbor/datasets/clawbench-v1/700-deletion-revocation-data-deletion-deleteme" + }, + { + "name": "705-rating-voting-wine-review-vivino", + "path": "harbor/datasets/clawbench-v1/705-rating-voting-wine-review-vivino" + }, + { + "name": "706-rating-voting-beer-review-beeradvocate", + "path": "harbor/datasets/clawbench-v1/706-rating-voting-beer-review-beeradvocate" + }, + { + "name": "707-rating-voting-social-wine-untappd", + "path": "harbor/datasets/clawbench-v1/707-rating-voting-social-wine-untappd" + }, + { + "name": "708-rating-voting-professor-review-ratemyprofessors", + "path": "harbor/datasets/clawbench-v1/708-rating-voting-professor-review-ratemyprofessors" + }, + { + "name": "709-rating-voting-service-review-angi", + "path": "harbor/datasets/clawbench-v1/709-rating-voting-service-review-angi" + }, + { + "name": "710-creation-init-interior-design-roomsketcher", + "path": "harbor/datasets/clawbench-v1/710-creation-init-interior-design-roomsketcher" + }, + { + "name": "711-creation-init-color-design-coolors", + "path": "harbor/datasets/clawbench-v1/711-creation-init-color-design-coolors" + }, + { + "name": "712-creation-init-website-create-squarespace", + "path": "harbor/datasets/clawbench-v1/712-creation-init-website-create-squarespace" + }, + { + "name": "713-creation-init-website-build-wix", + "path": "harbor/datasets/clawbench-v1/713-creation-init-website-build-wix" + }, + { + "name": "735-home-services-maintenance-house-cleaning-bark", + "path": "harbor/datasets/clawbench-v1/735-home-services-maintenance-house-cleaning-bark" + }, + { + "name": "736-home-services-maintenance-plumbing-ace-hardware", + "path": "harbor/datasets/clawbench-v1/736-home-services-maintenance-plumbing-ace-hardware" + }, + { + "name": "737-home-services-maintenance-kitchen-remodel-lowes", + "path": "harbor/datasets/clawbench-v1/737-home-services-maintenance-kitchen-remodel-lowes" + }, + { + "name": "738-home-services-maintenance-equipment-install-amazon-home-services", + "path": "harbor/datasets/clawbench-v1/738-home-services-maintenance-equipment-install-amazon-home-services" + }, + { + "name": "750-automotive-vehicle-services-car-insurance-compare-kanetix", + "path": "harbor/datasets/clawbench-v1/750-automotive-vehicle-services-car-insurance-compare-kanetix" + }, + { + "name": "751-automotive-vehicle-services-car-lease-sixt", + "path": "harbor/datasets/clawbench-v1/751-automotive-vehicle-services-car-lease-sixt" + }, + { + "name": "754-automotive-vehicle-services-used-car-listing-autotrader", + "path": "harbor/datasets/clawbench-v1/754-automotive-vehicle-services-used-car-listing-autotrader" + }, + { + "name": "763-automotive-vehicle-services-car-lease-autoslash", + "path": "harbor/datasets/clawbench-v1/763-automotive-vehicle-services-car-lease-autoslash" + }, + { + "name": "766-nonprofit-charity-donation-doctors-without-borders-msf", + "path": "harbor/datasets/clawbench-v1/766-nonprofit-charity-donation-doctors-without-borders-msf" + }, + { + "name": "768-nonprofit-charity-community-crowdfund-ioby", + "path": "harbor/datasets/clawbench-v1/768-nonprofit-charity-community-crowdfund-ioby" + }, + { + "name": "770-nonprofit-charity-volunteer-apply-on-make-a-wish-foundation-website-complete-and-submit-a-volunteer-application-form-selecting-the-wish-granter-role-and-entering-city-phoenix-az", + "path": "harbor/datasets/clawbench-v1/770-nonprofit-charity-volunteer-apply-on-make-a-wish-foundation-website-complete-and-submit-a-volunteer-application-form-selecting-the-wish-granter-role-and-entering-city-phoenix-az" + }, + { + "name": "774-nonprofit-charity-nonprofit-job-apply-charity-village", + "path": "harbor/datasets/clawbench-v1/774-nonprofit-charity-nonprofit-job-apply-charity-village" + }, + { + "name": "776-nonprofit-charity-volunteer-signup-idealist", + "path": "harbor/datasets/clawbench-v1/776-nonprofit-charity-volunteer-signup-idealist" + }, + { + "name": "778-nonprofit-charity-donation-globalgiving", + "path": "harbor/datasets/clawbench-v1/778-nonprofit-charity-donation-globalgiving" + }, + { + "name": "780-beauty-personal-care-skincare-purchase-soko-glam", + "path": "harbor/datasets/clawbench-v1/780-beauty-personal-care-skincare-purchase-soko-glam" + }, + { + "name": "781-beauty-personal-care-beauty-booking-bluemercury", + "path": "harbor/datasets/clawbench-v1/781-beauty-personal-care-beauty-booking-bluemercury" + }, + { + "name": "782-beauty-personal-care-skincare-purchase-paulas-choice", + "path": "harbor/datasets/clawbench-v1/782-beauty-personal-care-skincare-purchase-paulas-choice" + }, + { + "name": "783-beauty-personal-care-beauty-booking-ulta-beauty", + "path": "harbor/datasets/clawbench-v1/783-beauty-personal-care-beauty-booking-ulta-beauty" + }, + { + "name": "785-beauty-personal-care-skincare-curology", + "path": "harbor/datasets/clawbench-v1/785-beauty-personal-care-skincare-curology" + }, + { + "name": "788-beauty-personal-care-makeup-the-ordinary", + "path": "harbor/datasets/clawbench-v1/788-beauty-personal-care-makeup-the-ordinary" + }, + { + "name": "789-beauty-personal-care-makeup-fenty-beauty", + "path": "harbor/datasets/clawbench-v1/789-beauty-personal-care-makeup-fenty-beauty" + }, + { + "name": "793-beauty-personal-care-beauty-retail-mac-cosmetics", + "path": "harbor/datasets/clawbench-v1/793-beauty-personal-care-beauty-retail-mac-cosmetics" + }, + { + "name": "794-beauty-personal-care-salon-booking-styleseat", + "path": "harbor/datasets/clawbench-v1/794-beauty-personal-care-salon-booking-styleseat" + }, + { + "name": "795-pet-animal-care-pet-adoption-aspca", + "path": "harbor/datasets/clawbench-v1/795-pet-animal-care-pet-adoption-aspca" + }, + { + "name": "796-pet-animal-care-pet-supplies-grooming-petsmart", + "path": "harbor/datasets/clawbench-v1/796-pet-animal-care-pet-supplies-grooming-petsmart" + }, + { + "name": "801-pet-animal-care-pet-friendly-travel-bringfido", + "path": "harbor/datasets/clawbench-v1/801-pet-animal-care-pet-friendly-travel-bringfido" + }, + { + "name": "803-pet-animal-care-pet-medical-pawp", + "path": "harbor/datasets/clawbench-v1/803-pet-animal-care-pet-medical-pawp" + }, + { + "name": "807-pet-animal-care-pet-dna-embark", + "path": "harbor/datasets/clawbench-v1/807-pet-animal-care-pet-dna-embark" + }, + { + "name": "809-pet-animal-care-pet-adopt-petfinder", + "path": "harbor/datasets/clawbench-v1/809-pet-animal-care-pet-adopt-petfinder" + }, + { + "name": "812-pet-animal-care-pet-subscription-ollie", + "path": "harbor/datasets/clawbench-v1/812-pet-animal-care-pet-subscription-ollie" + }, + { + "name": "815-personal-management-records-mgmt-myheritage", + "path": "harbor/datasets/clawbench-v1/815-personal-management-records-mgmt-myheritage" + }, + { + "name": "821-education-learning-reading-self-study-blinkist", + "path": "harbor/datasets/clawbench-v1/821-education-learning-reading-self-study-blinkist" + }, + { + "name": "861-entertainment-hobbies-movies-cineplex", + "path": "harbor/datasets/clawbench-v1/861-entertainment-hobbies-movies-cineplex" + }, + { + "name": "862-entertainment-hobbies-movies-amc-theatres", + "path": "harbor/datasets/clawbench-v1/862-entertainment-hobbies-movies-amc-theatres" + }, + { + "name": "864-entertainment-hobbies-show-tickets-ticketmaster", + "path": "harbor/datasets/clawbench-v1/864-entertainment-hobbies-show-tickets-ticketmaster" + }, + { + "name": "865-travel-outdoor-hipcamp", + "path": "harbor/datasets/clawbench-v1/865-travel-outdoor-hipcamp" + }, + { + "name": "867-entertainment-hobbies-movies-fandango", + "path": "harbor/datasets/clawbench-v1/867-entertainment-hobbies-movies-fandango" + }, + { + "name": "872-daily-life-food-opentable", + "path": "harbor/datasets/clawbench-v1/872-daily-life-food-opentable" + }, + { + "name": "873-daily-life-food-resy", + "path": "harbor/datasets/clawbench-v1/873-daily-life-food-resy" + }, + { + "name": "876-entertainment-hobbies-show-tickets-vivid-seats", + "path": "harbor/datasets/clawbench-v1/876-entertainment-hobbies-show-tickets-vivid-seats" + }, + { + "name": "877-entertainment-hobbies-show-tickets-stubhub", + "path": "harbor/datasets/clawbench-v1/877-entertainment-hobbies-show-tickets-stubhub" + }, + { + "name": "878-travel-outdoor-ontario-parks", + "path": "harbor/datasets/clawbench-v1/878-travel-outdoor-ontario-parks" + }, + { + "name": "883-education-learning-hobby-class-sur-la-table", + "path": "harbor/datasets/clawbench-v1/883-education-learning-hobby-class-sur-la-table" + }, + { + "name": "884-entertainment-hobbies-experience-breakout-games", + "path": "harbor/datasets/clawbench-v1/884-entertainment-hobbies-experience-breakout-games" + }, + { + "name": "885-entertainment-hobbies-experience-bowlero", + "path": "harbor/datasets/clawbench-v1/885-entertainment-hobbies-experience-bowlero" + }, + { + "name": "886-entertainment-hobbies-experience-topgolf", + "path": "harbor/datasets/clawbench-v1/886-entertainment-hobbies-experience-topgolf" + } + ] + }, + { + "name": "clawbench-v2", + "version": "0.9.2", + "description": "ClawBench: everyday tasks on live consumer websites, scored by request interception + LLM judge. https://claw-bench.com", + "tasks": [ + { + "name": "v2-047-daily-life-personal-care-taskrabbit", + "path": "harbor/datasets/clawbench-v2/v2-047-daily-life-personal-care-taskrabbit" + }, + { + "name": "v2-086-job-search-hr-cv-autofill-greenhouse-meta", + "path": "harbor/datasets/clawbench-v2/v2-086-job-search-hr-cv-autofill-greenhouse-meta" + }, + { + "name": "v2-089-job-search-hr-cv-autofill-simplify-jobs", + "path": "harbor/datasets/clawbench-v2/v2-089-job-search-hr-cv-autofill-simplify-jobs" + }, + { + "name": "v2-091-job-search-hr-job-apply-indeed", + "path": "harbor/datasets/clawbench-v2/v2-091-job-search-hr-job-apply-indeed" + }, + { + "name": "v2-1010-rating-voting-review-myrecipes", + "path": "harbor/datasets/clawbench-v2/v2-1010-rating-voting-review-myrecipes" + }, + { + "name": "v2-1033-education-learning-registration-khanacademy", + "path": "harbor/datasets/clawbench-v2/v2-1033-education-learning-registration-khanacademy" + }, + { + "name": "v2-1035-education-learning-enrollment-edx", + "path": "harbor/datasets/clawbench-v2/v2-1035-education-learning-enrollment-edx" + }, + { + "name": "v2-1045-weworkremotely", + "path": "harbor/datasets/clawbench-v2/v2-1045-weworkremotely" + }, + { + "name": "v2-1065b-daily-life-home-services-handy", + "path": "harbor/datasets/clawbench-v2/v2-1065b-daily-life-home-services-handy" + }, + { + "name": "v2-1088-nonprofit-charity-petition-change", + "path": "harbor/datasets/clawbench-v2/v2-1088-nonprofit-charity-petition-change" + }, + { + "name": "v2-1093-personal-management-travel-tripit", + "path": "harbor/datasets/clawbench-v2/v2-1093-personal-management-travel-tripit" + }, + { + "name": "v2-1095-health-mealplan-eatthismuch", + "path": "harbor/datasets/clawbench-v2/v2-1095-health-mealplan-eatthismuch" + }, + { + "name": "v2-1097-beauty-personal-care-shopping-theordinary", + "path": "harbor/datasets/clawbench-v2/v2-1097-beauty-personal-care-shopping-theordinary" + }, + { + "name": "v2-1100-food-cooking-collection-myrecipes", + "path": "harbor/datasets/clawbench-v2/v2-1100-food-cooking-collection-myrecipes" + }, + { + "name": "v2-1101-hobbies-gaming-discussion-ravelry", + "path": "harbor/datasets/clawbench-v2/v2-1101-hobbies-gaming-discussion-ravelry" + }, + { + "name": "v2-1102-fitness-social-post-strava", + "path": "harbor/datasets/clawbench-v2/v2-1102-fitness-social-post-strava" + }, + { + "name": "v2-1103-fitness-club-strava", + "path": "harbor/datasets/clawbench-v2/v2-1103-fitness-club-strava" + }, + { + "name": "v2-1104-fitness-edit-activity-strava", + "path": "harbor/datasets/clawbench-v2/v2-1104-fitness-edit-activity-strava" + }, + { + "name": "v2-1107-hobbies-gaming-forum-boardgamegeek", + "path": "harbor/datasets/clawbench-v2/v2-1107-hobbies-gaming-forum-boardgamegeek" + }, + { + "name": "v2-1108-civic-petition-change", + "path": "harbor/datasets/clawbench-v2/v2-1108-civic-petition-change" + }, + { + "name": "v2-1111-civic-petition-change", + "path": "harbor/datasets/clawbench-v2/v2-1111-civic-petition-change" + }, + { + "name": "v2-1112-nonprofit-petition-update-change", + "path": "harbor/datasets/clawbench-v2/v2-1112-nonprofit-petition-update-change" + }, + { + "name": "v2-1113-health-mealplan-eatthismuch", + "path": "harbor/datasets/clawbench-v2/v2-1113-health-mealplan-eatthismuch" + }, + { + "name": "v2-1114-education-enrollment-edx", + "path": "harbor/datasets/clawbench-v2/v2-1114-education-enrollment-edx" + }, + { + "name": "v2-1115-education-enrollment-edx", + "path": "harbor/datasets/clawbench-v2/v2-1115-education-enrollment-edx" + }, + { + "name": "v2-1116-education-enrollment-edx", + "path": "harbor/datasets/clawbench-v2/v2-1116-education-enrollment-edx" + }, + { + "name": "v2-1117", + "path": "harbor/datasets/clawbench-v2/v2-1117" + }, + { + "name": "v2-1118", + "path": "harbor/datasets/clawbench-v2/v2-1118" + }, + { + "name": "v2-1120b-services-booking-handy", + "path": "harbor/datasets/clawbench-v2/v2-1120b-services-booking-handy" + }, + { + "name": "v2-1121b-services-booking-handy", + "path": "harbor/datasets/clawbench-v2/v2-1121b-services-booking-handy" + }, + { + "name": "v2-1122-education-course-khanacademy", + "path": "harbor/datasets/clawbench-v2/v2-1122-education-course-khanacademy" + }, + { + "name": "v2-1123-education-course-khanacademy", + "path": "harbor/datasets/clawbench-v2/v2-1123-education-course-khanacademy" + }, + { + "name": "v2-1124-education-course-khanacademy", + "path": "harbor/datasets/clawbench-v2/v2-1124-education-course-khanacademy" + }, + { + "name": "v2-1125-education-course-khanacademy", + "path": "harbor/datasets/clawbench-v2/v2-1125-education-course-khanacademy" + }, + { + "name": "v2-1126-food-cooking-collection-myrecipes", + "path": "harbor/datasets/clawbench-v2/v2-1126-food-cooking-collection-myrecipes" + }, + { + "name": "v2-1130-training-class-search-redcross", + "path": "harbor/datasets/clawbench-v2/v2-1130-training-class-search-redcross" + }, + { + "name": "v2-1131-training-add-to-cart-redcross", + "path": "harbor/datasets/clawbench-v2/v2-1131-training-add-to-cart-redcross" + }, + { + "name": "v2-1132-training-enroll-course-redcross", + "path": "harbor/datasets/clawbench-v2/v2-1132-training-enroll-course-redcross" + }, + { + "name": "v2-1133-newsletter-signup-redcross", + "path": "harbor/datasets/clawbench-v2/v2-1133-newsletter-signup-redcross" + }, + { + "name": "v2-1134-chapter-finder-redcross", + "path": "harbor/datasets/clawbench-v2/v2-1134-chapter-finder-redcross" + }, + { + "name": "v2-1135-events-tickets-ticketmaster", + "path": "harbor/datasets/clawbench-v2/v2-1135-events-tickets-ticketmaster" + }, + { + "name": "v2-1136-events-tickets-ticketmaster", + "path": "harbor/datasets/clawbench-v2/v2-1136-events-tickets-ticketmaster" + }, + { + "name": "v2-1137-entertainment-watchlist-trakt", + "path": "harbor/datasets/clawbench-v2/v2-1137-entertainment-watchlist-trakt" + }, + { + "name": "v2-1138-entertainment-watchlist-trakt", + "path": "harbor/datasets/clawbench-v2/v2-1138-entertainment-watchlist-trakt" + }, + { + "name": "v2-1139-travel-itinerary-tripit", + "path": "harbor/datasets/clawbench-v2/v2-1139-travel-itinerary-tripit" + }, + { + "name": "v2-1140-travel-itinerary-tripit", + "path": "harbor/datasets/clawbench-v2/v2-1140-travel-itinerary-tripit" + }, + { + "name": "v2-1141-travel-itinerary-tripit", + "path": "harbor/datasets/clawbench-v2/v2-1141-travel-itinerary-tripit" + }, + { + "name": "v2-1144-weworkremotely", + "path": "harbor/datasets/clawbench-v2/v2-1144-weworkremotely" + }, + { + "name": "v2-1145-weworkremotely", + "path": "harbor/datasets/clawbench-v2/v2-1145-weworkremotely" + }, + { + "name": "v2-1146-weworkremotely", + "path": "harbor/datasets/clawbench-v2/v2-1146-weworkremotely" + }, + { + "name": "v2-1201-beauty-cart-theordinary", + "path": "harbor/datasets/clawbench-v2/v2-1201-beauty-cart-theordinary" + }, + { + "name": "v2-1202-beauty-cart-quantity-theordinary", + "path": "harbor/datasets/clawbench-v2/v2-1202-beauty-cart-quantity-theordinary" + }, + { + "name": "v2-179-dev-tech-github-ops-github", + "path": "harbor/datasets/clawbench-v2/v2-179-dev-tech-github-ops-github" + }, + { + "name": "v2-180-dev-tech-github-ops-github", + "path": "harbor/datasets/clawbench-v2/v2-180-dev-tech-github-ops-github" + }, + { + "name": "v2-215-academia-research-paper-tables-overleaf", + "path": "harbor/datasets/clawbench-v2/v2-215-academia-research-paper-tables-overleaf" + }, + { + "name": "v2-234-daily-life-events-ticketmaster", + "path": "harbor/datasets/clawbench-v2/v2-234-daily-life-events-ticketmaster" + }, + { + "name": "v2-242-academia-research-research-tools-overleaf", + "path": "harbor/datasets/clawbench-v2/v2-242-academia-research-research-tools-overleaf" + }, + { + "name": "v2-247-academia-research-research-tools-semantic-scholar", + "path": "harbor/datasets/clawbench-v2/v2-247-academia-research-research-tools-semantic-scholar" + }, + { + "name": "v2-265-education-learning-general-coursera", + "path": "harbor/datasets/clawbench-v2/v2-265-education-learning-general-coursera" + }, + { + "name": "v2-266-education-learning-general-leetcode", + "path": "harbor/datasets/clawbench-v2/v2-266-education-learning-general-leetcode" + }, + { + "name": "v2-273-education-learning-general-edx", + "path": "harbor/datasets/clawbench-v2/v2-273-education-learning-general-edx" + }, + { + "name": "v2-274-office-secretary-tasks-calendar-when2meet", + "path": "harbor/datasets/clawbench-v2/v2-274-office-secretary-tasks-calendar-when2meet" + }, + { + "name": "v2-284-office-secretary-tasks-calendar-when2meet", + "path": "harbor/datasets/clawbench-v2/v2-284-office-secretary-tasks-calendar-when2meet" + }, + { + "name": "v2-369-entertainment-hobbies-general-goodreads", + "path": "harbor/datasets/clawbench-v2/v2-369-entertainment-hobbies-general-goodreads" + }, + { + "name": "v2-372-entertainment-hobbies-general-eventbrite", + "path": "harbor/datasets/clawbench-v2/v2-372-entertainment-hobbies-general-eventbrite" + }, + { + "name": "v2-413-personal-management-personal-tools-todoist", + "path": "harbor/datasets/clawbench-v2/v2-413-personal-management-personal-tools-todoist" + }, + { + "name": "v2-468-rating-voting-general-glassdoor", + "path": "harbor/datasets/clawbench-v2/v2-468-rating-voting-general-glassdoor" + }, + { + "name": "v2-469-rating-voting-general-tripadvisor", + "path": "harbor/datasets/clawbench-v2/v2-469-rating-voting-general-tripadvisor" + }, + { + "name": "v2-470-rating-voting-general-trustpilot", + "path": "harbor/datasets/clawbench-v2/v2-470-rating-voting-general-trustpilot" + }, + { + "name": "v2-474-rating-voting-general-capterra", + "path": "harbor/datasets/clawbench-v2/v2-474-rating-voting-general-capterra" + }, + { + "name": "v2-475-rating-voting-general-g2", + "path": "harbor/datasets/clawbench-v2/v2-475-rating-voting-general-g2" + }, + { + "name": "v2-483-creation-init-general-airtable", + "path": "harbor/datasets/clawbench-v2/v2-483-creation-init-general-airtable" + }, + { + "name": "v2-485-creation-init-general-webflow", + "path": "harbor/datasets/clawbench-v2/v2-485-creation-init-general-webflow" + }, + { + "name": "v2-486-creation-init-general-mailchimp", + "path": "harbor/datasets/clawbench-v2/v2-486-creation-init-general-mailchimp" + }, + { + "name": "v2-487-creation-init-general-typeform", + "path": "harbor/datasets/clawbench-v2/v2-487-creation-init-general-typeform" + }, + { + "name": "v2-488-creation-init-general-substack", + "path": "harbor/datasets/clawbench-v2/v2-488-creation-init-general-substack" + }, + { + "name": "v2-500-office-secretary-tasks-signup-doodle", + "path": "harbor/datasets/clawbench-v2/v2-500-office-secretary-tasks-signup-doodle" + }, + { + "name": "v2-501-office-secretary-tasks-signup-doodle", + "path": "harbor/datasets/clawbench-v2/v2-501-office-secretary-tasks-signup-doodle" + }, + { + "name": "v2-502-office-secretary-tasks-calendar-doodle", + "path": "harbor/datasets/clawbench-v2/v2-502-office-secretary-tasks-calendar-doodle" + }, + { + "name": "v2-503-office-secretary-tasks-calendar-doodle", + "path": "harbor/datasets/clawbench-v2/v2-503-office-secretary-tasks-calendar-doodle" + }, + { + "name": "v2-520-shopping-retail-target", + "path": "harbor/datasets/clawbench-v2/v2-520-shopping-retail-target" + }, + { + "name": "v2-521-shopping-retail-target", + "path": "harbor/datasets/clawbench-v2/v2-521-shopping-retail-target" + }, + { + "name": "v2-522-shopping-retail-target", + "path": "harbor/datasets/clawbench-v2/v2-522-shopping-retail-target" + }, + { + "name": "v2-523-shopping-retail-target", + "path": "harbor/datasets/clawbench-v2/v2-523-shopping-retail-target" + }, + { + "name": "v2-530-shopping-marketplace-etsy", + "path": "harbor/datasets/clawbench-v2/v2-530-shopping-marketplace-etsy" + }, + { + "name": "v2-531-shopping-marketplace-etsy", + "path": "harbor/datasets/clawbench-v2/v2-531-shopping-marketplace-etsy" + }, + { + "name": "v2-533-daily-life-utilities-inmyarea", + "path": "harbor/datasets/clawbench-v2/v2-533-daily-life-utilities-inmyarea" + }, + { + "name": "v2-535-daily-life-shopping-etsy", + "path": "harbor/datasets/clawbench-v2/v2-535-daily-life-shopping-etsy" + }, + { + "name": "v2-536-daily-life-shopping-etsy", + "path": "harbor/datasets/clawbench-v2/v2-536-daily-life-shopping-etsy" + }, + { + "name": "v2-560", + "path": "harbor/datasets/clawbench-v2/v2-560" + }, + { + "name": "v2-564", + "path": "harbor/datasets/clawbench-v2/v2-564" + }, + { + "name": "v2-571-job-search-hr-recruitment-mgmt-workable", + "path": "harbor/datasets/clawbench-v2/v2-571-job-search-hr-recruitment-mgmt-workable" + }, + { + "name": "v2-583-office-secretary-tasks-support-tickets-freshdesk", + "path": "harbor/datasets/clawbench-v2/v2-583-office-secretary-tasks-support-tickets-freshdesk" + }, + { + "name": "v2-596-daily-life-fitness-strava", + "path": "harbor/datasets/clawbench-v2/v2-596-daily-life-fitness-strava" + }, + { + "name": "v2-597-daily-life-fitness-strava", + "path": "harbor/datasets/clawbench-v2/v2-597-daily-life-fitness-strava" + }, + { + "name": "v2-598-academia-research-legal-docs-formswift", + "path": "harbor/datasets/clawbench-v2/v2-598-academia-research-legal-docs-formswift" + }, + { + "name": "v2-600-hobbies-crafts-community-ravelry", + "path": "harbor/datasets/clawbench-v2/v2-600-hobbies-crafts-community-ravelry" + }, + { + "name": "v2-601-hobbies-gaming-reviews-ravelry", + "path": "harbor/datasets/clawbench-v2/v2-601-hobbies-gaming-reviews-ravelry" + }, + { + "name": "v2-602-hobbies-crafts-community-ravelry", + "path": "harbor/datasets/clawbench-v2/v2-602-hobbies-crafts-community-ravelry" + }, + { + "name": "v2-603-hobbies-crafts-community-ravelry", + "path": "harbor/datasets/clawbench-v2/v2-603-hobbies-crafts-community-ravelry" + }, + { + "name": "v2-607-hobbies-gaming-reviews-boardgamegeek", + "path": "harbor/datasets/clawbench-v2/v2-607-hobbies-gaming-reviews-boardgamegeek" + }, + { + "name": "v2-608-education-learning-kids-courses-outschool", + "path": "harbor/datasets/clawbench-v2/v2-608-education-learning-kids-courses-outschool" + }, + { + "name": "v2-609-education-learning-meditation-spirit-rock-meditation-center", + "path": "harbor/datasets/clawbench-v2/v2-609-education-learning-meditation-spirit-rock-meditation-center" + }, + { + "name": "v2-610-hobbies-gaming-reviews-boardgamegeek", + "path": "harbor/datasets/clawbench-v2/v2-610-hobbies-gaming-reviews-boardgamegeek" + }, + { + "name": "v2-630-daily-life-productivity-habitica", + "path": "harbor/datasets/clawbench-v2/v2-630-daily-life-productivity-habitica" + }, + { + "name": "v2-631-daily-life-productivity-habitica", + "path": "harbor/datasets/clawbench-v2/v2-631-daily-life-productivity-habitica" + }, + { + "name": "v2-633-daily-life-productivity-habitica", + "path": "harbor/datasets/clawbench-v2/v2-633-daily-life-productivity-habitica" + }, + { + "name": "v2-634-daily-life-productivity-habitica", + "path": "harbor/datasets/clawbench-v2/v2-634-daily-life-productivity-habitica" + }, + { + "name": "v2-635-entertainment-tracking-trakt", + "path": "harbor/datasets/clawbench-v2/v2-635-entertainment-tracking-trakt" + }, + { + "name": "v2-638-entertainment-tracking-trakt", + "path": "harbor/datasets/clawbench-v2/v2-638-entertainment-tracking-trakt" + }, + { + "name": "v2-705-rating-voting-wine-review-vivino", + "path": "harbor/datasets/clawbench-v2/v2-705-rating-voting-wine-review-vivino" + }, + { + "name": "v2-706-rating-voting-beer-review-beeradvocate", + "path": "harbor/datasets/clawbench-v2/v2-706-rating-voting-beer-review-beeradvocate" + }, + { + "name": "v2-707-rating-voting-social-wine-untappd", + "path": "harbor/datasets/clawbench-v2/v2-707-rating-voting-social-wine-untappd" + }, + { + "name": "v2-708-rating-voting-professor-review-ratemyprofessors", + "path": "harbor/datasets/clawbench-v2/v2-708-rating-voting-professor-review-ratemyprofessors" + }, + { + "name": "v2-711-creation-init-color-design-coolors", + "path": "harbor/datasets/clawbench-v2/v2-711-creation-init-color-design-coolors" + }, + { + "name": "v2-735-home-services-maintenance-house-cleaning-bark", + "path": "harbor/datasets/clawbench-v2/v2-735-home-services-maintenance-house-cleaning-bark" + }, + { + "name": "v2-737-home-services-maintenance-kitchen-remodel-lowes", + "path": "harbor/datasets/clawbench-v2/v2-737-home-services-maintenance-kitchen-remodel-lowes" + }, + { + "name": "v2-763-automotive-vehicle-services-car-lease-autoslash", + "path": "harbor/datasets/clawbench-v2/v2-763-automotive-vehicle-services-car-lease-autoslash" + }, + { + "name": "v2-774-nonprofit-charity-nonprofit-job-apply-charity-village", + "path": "harbor/datasets/clawbench-v2/v2-774-nonprofit-charity-nonprofit-job-apply-charity-village" + }, + { + "name": "v2-776-nonprofit-charity-volunteer-signup-idealist", + "path": "harbor/datasets/clawbench-v2/v2-776-nonprofit-charity-volunteer-signup-idealist" + }, + { + "name": "v2-794-beauty-personal-care-salon-booking-styleseat", + "path": "harbor/datasets/clawbench-v2/v2-794-beauty-personal-care-salon-booking-styleseat" + }, + { + "name": "v2-815-pet-animal-care-favorite-pet", + "path": "harbor/datasets/clawbench-v2/v2-815-pet-animal-care-favorite-pet" + }, + { + "name": "v2-816-pet-animal-care-remove-favorite", + "path": "harbor/datasets/clawbench-v2/v2-816-pet-animal-care-remove-favorite" + }, + { + "name": "v2-817-pet-animal-care-create-adopter-profile", + "path": "harbor/datasets/clawbench-v2/v2-817-pet-animal-care-create-adopter-profile" + }, + { + "name": "v2-901-entertainment-imdb-add-to-watchlist", + "path": "harbor/datasets/clawbench-v2/v2-901-entertainment-imdb-add-to-watchlist" + }, + { + "name": "v2-902-entertainment-imdb-rate-title", + "path": "harbor/datasets/clawbench-v2/v2-902-entertainment-imdb-rate-title" + }, + { + "name": "v2-903-entertainment-imdb-remove-rating", + "path": "harbor/datasets/clawbench-v2/v2-903-entertainment-imdb-remove-rating" + }, + { + "name": "v2-904-entertainment-imdb-create-list", + "path": "harbor/datasets/clawbench-v2/v2-904-entertainment-imdb-create-list" + }, + { + "name": "v2-905-entertainment-imdb-add-to-list", + "path": "harbor/datasets/clawbench-v2/v2-905-entertainment-imdb-add-to-list" + } + ] + } +] diff --git a/scripts/harbor/build-runtime-image.sh b/scripts/harbor/build-runtime-image.sh new file mode 100755 index 00000000..812c29a0 --- /dev/null +++ b/scripts/harbor/build-runtime-image.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# Build (and optionally push) the ClawBench Harbor runtime image. +# +# The image is the environment every prebuilt-mode Harbor task points at via +# [environment].docker_image. It is built from the same Dockerfile the default +# (per-task environment/) mode copies into each task, so both modes are +# byte-identical at runtime. +# +# Usage: +# scripts/harbor/build-runtime-image.sh # local tag only +# scripts/harbor/build-runtime-image.sh --push # also push to Docker Hub +# IMAGE=ghcr.io/my-org/clawbench-harbor-runtime scripts/harbor/build-runtime-image.sh +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +RUNTIME="$ROOT/src/clawbench/runtime" +VERSION="$(grep -m1 '^version' "$ROOT/pyproject.toml" | sed -E 's/.*"([^"]+)".*/\1/')" +IMAGE="${IMAGE:-docker.io/clawbench/clawbench-harbor-runtime}" +ENGINE="${CONTAINER_ENGINE:-docker}" +PUSH=0 +[ "${1:-}" = "--push" ] && PUSH=1 + +# Stage a build context: the runtime tree plus resume_template.json, which the +# adapter normally copies into each task's environment/harbor/ at export time +# (setup.sh -> prepare-task.py reads /app/src/harbor/resume_template.json). +CTX="$(mktemp -d)" +trap 'rm -rf "$CTX"' EXIT +cp -r "$RUNTIME/." "$CTX/" +cp "$ROOT/src/clawbench/runner/run_support/resume_template.json" "$CTX/harbor/resume_template.json" + +echo "Building $IMAGE:$VERSION (engine=$ENGINE, context=$CTX)" +"$ENGINE" build \ + -f "$CTX/harbor/Dockerfile" \ + -t "$IMAGE:$VERSION" \ + -t "$IMAGE:latest" \ + "$CTX" + +if [ "$PUSH" = 1 ]; then + "$ENGINE" push "$IMAGE:$VERSION" + "$ENGINE" push "$IMAGE:latest" +fi +echo "Done: $IMAGE:$VERSION" diff --git a/scripts/harbor/build_registry.py b/scripts/harbor/build_registry.py new file mode 100644 index 00000000..a7564d89 --- /dev/null +++ b/scripts/harbor/build_registry.py @@ -0,0 +1,123 @@ +"""Build Harbor registry / manifest files from a generated ClawBench dataset. + +Two outputs, both derived from the task directories on disk so they never +drift from what is committed: + +* ``registry.json`` — the Harbor *git registry* consumed by + ``harbor run --repo TIGER-AI-Lab/ClawBench -d clawbench-v2``. Each task entry + is a repo-relative ``path``; ``git_url`` is omitted so Harbor resolves it + against the repo the registry was fetched from. +* ``harbor/dataset.toml`` — the *Hub* manifest consumed by ``harbor publish``. + Task digests are recomputed with Harbor's own Packager hash when the + ``harbor`` package is importable (``uv run --with harbor==0.22.0``), + otherwise the existing digests are kept. +""" + +from __future__ import annotations + +import argparse +import json +import re +import tomllib +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] + + +def task_dirs(dataset_dir: Path) -> list[Path]: + return sorted(p.parent for p in dataset_dir.glob("*/task.toml")) + + +def task_name(task_dir: Path) -> str: + config = tomllib.loads((task_dir / "task.toml").read_text()) + return config["task"]["name"] + + +def build_registry( + dataset_dir: Path, name: str, version: str, description: str +) -> dict: + tasks = [ + { + "name": task_name(d).split("/", 1)[1], + "path": d.relative_to(ROOT).as_posix(), + } + for d in task_dirs(dataset_dir) + ] + return [ + { + "name": name, + "version": version, + "description": description, + "tasks": tasks, + } + ] + + +def task_digest(task_dir: Path) -> str | None: + """Same digest ``harbor add`` records (Packager.compute_content_hash).""" + try: + from harbor.publisher.packager import Packager # type: ignore + except ImportError: + return None + content_hash, _files = Packager.compute_content_hash(task_dir) + return f"sha256:{content_hash}" + + +def update_manifest(dataset_dir: Path, manifest: Path) -> int: + text = manifest.read_text() + head, _, _ = text.partition("[[tasks]]") + existing = { + m.group(1): m.group(2) + for m in re.finditer( + r'\[\[tasks\]\]\s*\nname = "([^"]+)"\s*\ndigest = "([^"]+)"', text + ) + } + entries = [] + for d in task_dirs(dataset_dir): + name = task_name(d) + digest = task_digest(d) or existing.get(name, "sha256:UNSYNCED") + entries.append(f'[[tasks]]\nname = "{name}"\ndigest = "{digest}"\n') + manifest.write_text(head.rstrip("\n") + "\n\n" + "\n".join(entries)) + return len(entries) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--dataset-dir", type=Path, required=True) + parser.add_argument("--name", default="clawbench-v2") + parser.add_argument("--version", default="1.0") + parser.add_argument( + "--description", + default=( + "ClawBench: everyday tasks on live consumer websites, scored by " + "request interception + LLM judge. https://claw-bench.com" + ), + ) + parser.add_argument("--output", type=Path, help="Write registry.json here") + parser.add_argument("--manifest", type=Path, help="Update this dataset.toml") + args = parser.parse_args() + + dataset_dir = args.dataset_dir.resolve() + if args.output: + entry = build_registry(dataset_dir, args.name, args.version, args.description)[ + 0 + ] + registry = [] + if args.output.exists(): + registry = [ + e for e in json.loads(args.output.read_text()) if e["name"] != args.name + ] + registry.append(entry) + registry.sort(key=lambda e: e["name"]) + args.output.write_text(json.dumps(registry, indent=2) + "\n") + print(f"registry: {len(entry['tasks'])} tasks ({args.name}) -> {args.output}") + if args.manifest: + n = update_manifest(dataset_dir, args.manifest) + print(f"manifest: {n} tasks -> {args.manifest}") + if not (args.output or args.manifest): + parser.error("pass --output and/or --manifest") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/harbor/regenerate.sh b/scripts/harbor/regenerate.sh new file mode 100755 index 00000000..8c8a2b9d --- /dev/null +++ b/scripts/harbor/regenerate.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# Regenerate the committed prebuilt-mode Harbor dataset and registry.json. +# +# Run this whenever test-cases/v2/ or the Harbor adapter changes; CI +# (.github/workflows/validate-harbor.yml) fails if the committed copy is stale. +# +# Usage: scripts/harbor/regenerate.sh [--image ] +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +VERSION="$(grep -m1 '^version' "$ROOT/pyproject.toml" | sed -E 's/.*"([^"]+)".*/\1/')" +IMAGE="clawbench/clawbench-harbor-runtime:$VERSION" +if [ "${1:-}" = "--image" ]; then IMAGE="$2"; fi + +cd "$ROOT" +for SUITE in v2 v1; do + uv run clawbench-harbor-adapt \ + --suite "$SUITE" \ + --output-dir "harbor/datasets/clawbench-$SUITE" \ + --docker-image "$IMAGE" \ + --overwrite + + uv run --with harbor==0.22.0 python scripts/harbor/build_registry.py \ + --dataset-dir "harbor/datasets/clawbench-$SUITE" \ + --name "clawbench-$SUITE" \ + --version "$VERSION" \ + --output registry.json + + uv run --with harbor==0.22.0 python scripts/harbor/build_registry.py \ + --dataset-dir "harbor/datasets/clawbench-$SUITE" \ + --manifest "harbor/dataset-$SUITE.toml" +done + +echo "Regenerated harbor/datasets/clawbench-{v1,v2} (image=$IMAGE), registry.json, harbor/dataset-{v1,v2}.toml" diff --git a/src/clawbench/eval/harbor_adapter.py b/src/clawbench/eval/harbor_adapter.py index 4639bdce..9c84d3af 100644 --- a/src/clawbench/eval/harbor_adapter.py +++ b/src/clawbench/eval/harbor_adapter.py @@ -14,7 +14,7 @@ from clawbench.runner.run_support.task import build_instruction, validate_task_data from clawbench.utils.paths import RUNTIME_ROOT, asset_path -DEFAULT_CASES_DIR = asset_path("test-cases", "v2") +SUPPORTED_SUITES = ("v1", "v2") STEP_NAME = "run" @@ -130,13 +130,20 @@ def task_toml( dataset_name: str, timeout_sec: int, task_dir_name: str, + docker_image: str | None = None, ) -> str: escaped_description = json.dumps(description) + # Prebuilt mode: point Harbor at a published runtime image instead of the + # per-task environment/ build context. The image must be built from + # src/clawbench/runtime/harbor/Dockerfile (see scripts/harbor/). + docker_image_line = ( + f"docker_image = {json.dumps(docker_image)}\n" if docker_image else "" + ) escaped_dataset = json.dumps(dataset_name) escaped_source = json.dumps(task_dir_name) escaped_package = json.dumps(package_name) return f"""schema_version = "1.3" -source = "clawbench-v2" +source = "clawbench-{dataset_name}" artifacts = ["/data"] [task] @@ -149,7 +156,7 @@ def task_toml( source_task = {escaped_source} [environment] -build_timeout_sec = 1200.0 +{docker_image_line}build_timeout_sec = 1200.0 network_mode = "public" workdir = "/app" @@ -265,6 +272,7 @@ def write_harbor_task( output_name: str, org: str, dataset_name: str, + docker_image: str | None = None, ) -> Path: dest = output_root / output_name if dest.exists(): @@ -292,6 +300,7 @@ def write_harbor_task( dataset_name=dataset_name, timeout_sec=timeout_sec, task_dir_name=task_dir.name, + docker_image=docker_image, ) ) (step_dir / "instruction.md").write_text(harbor_instruction(task)) @@ -302,7 +311,16 @@ def write_harbor_task( (tests_dir / "task.json").write_text(json.dumps(task, indent=2, ensure_ascii=False)) write_text_executable(tests_dir / "test.sh", test_script()) write_text_executable(solution_dir / "solve.sh", solve_script()) - copy_environment(env_dir) + if docker_image is None: + copy_environment(env_dir) + else: + # Harbor's TaskModel.is_valid_dir requires environment/ to exist even + # when [environment].docker_image is set; a one-line Dockerfile keeps + # the task valid for local -p runs while still using the prebuilt image. + (env_dir / "Dockerfile").write_text( + "# Prebuilt mode: the runtime lives in the published image below.\n" + f"FROM {docker_image}\n" + ) return dest @@ -316,15 +334,23 @@ def build_parser() -> argparse.ArgumentParser: required=True, help="Directory to write the Harbor dataset", ) + parser.add_argument( + "--suite", + choices=SUPPORTED_SUITES, + default="v2", + help="Bundled corpus to convert when --cases-dir is not given", + ) parser.add_argument( "--cases-dir", type=Path, default=None, - help="ClawBench cases directory (defaults to bundled/source test-cases/v2)", + help="ClawBench cases directory (defaults to the bundled test-cases/)", ) parser.add_argument("--org", default="clawbench", help="Harbor package org prefix") parser.add_argument( - "--dataset-name", default="v2", help="Dataset name stored in metadata" + "--dataset-name", + default=None, + help="Dataset name stored in metadata (defaults to the suite name)", ) parser.add_argument( "--limit", type=int, default=None, help="Convert at most this many tasks" @@ -339,6 +365,15 @@ def build_parser() -> argparse.ArgumentParser: action="store_true", help="Overwrite an existing output directory", ) + parser.add_argument( + "--docker-image", + default=None, + help=( + "Prebuilt runtime image (e.g. ghcr.io/tiger-ai-lab/clawbench-harbor-runtime:0.9.2). " + "When set, tasks reference the image instead of shipping an environment/ build " + "context, which keeps a committed dataset small." + ), + ) return parser @@ -346,15 +381,13 @@ def main(argv: list[str] | None = None) -> int: parser = build_parser() args = parser.parse_args(argv) - cases_dir = (args.cases_dir or DEFAULT_CASES_DIR).resolve() - default_cases = args.cases_dir is None + cases_dir = (args.cases_dir or asset_path("test-cases", args.suite)).resolve() if not cases_dir.exists(): parser.error(f"cases directory not found: {cases_dir}") - if default_cases and cases_dir.name != "v2": - parser.error("default Harbor adapter supports only ClawBench V2") - if args.cases_dir is not None and cases_dir.name != "v2": + if args.cases_dir is not None and cases_dir.name not in SUPPORTED_SUITES: print( - f"WARNING: Harbor support is validated for V2 only; converting explicit cases dir {cases_dir}", + f"WARNING: Harbor support is validated for {'/'.join(SUPPORTED_SUITES)} only; " + f"converting explicit cases dir {cases_dir}", file=sys.stderr, ) @@ -383,7 +416,10 @@ def main(argv: list[str] | None = None) -> int: output_root=output_dir, output_name=out_name, org=args.org, - dataset_name=args.dataset_name, + dataset_name=args.dataset_name + or (args.cases_dir and cases_dir.name) + or args.suite, + docker_image=args.docker_image, ) ) diff --git a/tests/test_harbor_adapter.py b/tests/test_harbor_adapter.py index af67bee5..b1c8b350 100644 --- a/tests/test_harbor_adapter.py +++ b/tests/test_harbor_adapter.py @@ -120,6 +120,43 @@ def test_write_harbor_task_emits_expected_tree_and_extra_info(tmp_path: Path) -> assert "BROWSER_CDP_URL" in (out / "steps" / "run" / "instruction.md").read_text() +def test_write_harbor_task_prebuilt_image_mode(tmp_path: Path) -> None: + case = _write_case( + tmp_path / "v2", "v2-047-daily-life-personal-care-taskrabbit", _task() + ) + + out = write_harbor_task( + task_dir=case, + task=_task(), + output_root=tmp_path / "out", + output_name="v2-047-daily-life-personal-care-taskrabbit", + org="clawbench", + dataset_name="v2", + docker_image="ghcr.io/tiger-ai-lab/clawbench-harbor-runtime:0.9.2", + ) + + # No build context is shipped; a one-line Dockerfile references the + # prebuilt image (Harbor's is_valid_dir requires environment/ to exist). + env_files = list((out / "environment").rglob("*")) + assert [p.name for p in env_files] == ["Dockerfile"] + dockerfile = (out / "environment" / "Dockerfile").read_text() + assert ( + "FROM ghcr.io" in dockerfile + or "FROM clawbench/" in dockerfile + or "FROM " in dockerfile + ) + assert (out / "steps" / "run" / "workdir" / "setup.sh").is_file() + assert (out / "steps" / "run" / "tests" / "test.sh").is_file() + + config = tomllib.loads((out / "task.toml").read_text()) + assert ( + config["environment"]["docker_image"] + == "ghcr.io/tiger-ai-lab/clawbench-harbor-runtime:0.9.2" + ) + assert config["environment"]["workdir"] == "/app" + assert config["environment"]["network_mode"] == "public" + + def test_harbor_adapter_cli_smoke(tmp_path: Path) -> None: out = tmp_path / "harbor-v2" env = os.environ.copy()