feat(harbor): strict Harbor-format packaging — registry, Hub manifests, adapter package (V1+V2)

.github/workflows/publish-harbor-image.yml

@@ -0,0 +1,49 @@
+name: publish-harbor-image
+
+# Builds the prebuilt runtime image every committed Harbor task points at
+# ([environment].docker_image in harbor/datasets/clawbench-*/*/task.toml).
+# Runs on release tags and on demand; pushes to Docker Hub (clawbench/ namespace).
+# Requires repo secrets DOCKERHUB_USERNAME and DOCKERHUB_TOKEN.
+on:
+  push:
+    tags: ["v*"]
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: "Image tag (defaults to pyproject version)"
+        required: false
+
+permissions:
+  contents: read
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Resolve tag
+        id: meta
+        run: |
+          VERSION="$(grep -m1 '^version' pyproject.toml | sed -E 's/.*"([^"]+)".*/\1/')"
+          TAG="${{ github.event.inputs.tag }}"
+          echo "tag=${TAG:-$VERSION}" >> "$GITHUB_OUTPUT"
+          echo "image=docker.io/clawbench/clawbench-harbor-runtime" >> "$GITHUB_OUTPUT"
+
+      - uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - uses: docker/setup-buildx-action@v3
+
+      - uses: docker/build-push-action@v6
+        with:
+          context: src/clawbench/runtime
+          file: src/clawbench/runtime/harbor/Dockerfile
+          push: true
+          tags: |
+            ${{ steps.meta.outputs.image }}:${{ steps.meta.outputs.tag }}
+            ${{ steps.meta.outputs.image }}:latest
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

.github/workflows/validate-harbor.yml

@@ -0,0 +1,91 @@
+name: validate-harbor
+
+on:
+  pull_request:
+    branches: [main]
+    paths:
+      - "harbor/**"
+      - "registry.json"
+      - "scripts/harbor/**"
+      - "src/clawbench/eval/harbor_adapter.py"
+      - "src/clawbench/runtime/harbor/**"
+      - "test-cases/v2/**"
+      - ".github/workflows/validate-harbor.yml"
+  push:
+    branches: [main]
+    paths:
+      - "harbor/**"
+      - "registry.json"
+      - "scripts/harbor/**"
+      - "src/clawbench/eval/harbor_adapter.py"
+      - "test-cases/v2/**"
+  workflow_dispatch:
+
+jobs:
+  validate-harbor:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+
+      - uses: astral-sh/setup-uv@v6
+        with:
+          python-version: "3.11"
+
+      - name: Install ClawBench + Harbor
+        run: |
+          uv sync --frozen
+          uv pip install "harbor==0.22.0"
+
+      - name: Committed Harbor datasets are up to date
+        run: |
+          set -euo pipefail
+          VERSION="$(grep -m1 '^version' pyproject.toml | sed -E 's/.*"([^"]+)".*/\1/')"
+          for SUITE in v2 v1; do
+            uv run clawbench-harbor-adapt \
+              --suite "$SUITE" \
+              --output-dir "/tmp/clawbench-$SUITE" \
+              --docker-image "clawbench/clawbench-harbor-runtime:$VERSION" \
+              --overwrite
+            diff -r "/tmp/clawbench-$SUITE" "harbor/datasets/clawbench-$SUITE" \
+              || { echo "::error::harbor/datasets/clawbench-$SUITE is stale — run scripts/harbor/regenerate.sh"; exit 1; }
+            uv run python scripts/harbor/build_registry.py \
+              --dataset-dir "harbor/datasets/clawbench-$SUITE" --name "clawbench-$SUITE" \
+              --version "$VERSION" --output /tmp/registry.json
+          done
+          diff /tmp/registry.json registry.json \
+            || { echo "::error::registry.json is stale — run scripts/harbor/regenerate.sh"; exit 1; }
+
+      - name: Every task loads with Harbor's TaskConfig
+        run: |
+          uv run python - <<'PY'
+          import sys
+          from pathlib import Path
+          from harbor.models.task.task import Task
+          bad = []
+          total = 0
+          for root in (Path("harbor/datasets/clawbench-v2"), Path("harbor/datasets/clawbench-v1")):
+              for d in sorted(p.parent for p in root.glob("*/task.toml")):
+                  total += 1
+                  try:
+                      Task(task_dir=d).config
+                  except Exception as exc:  # noqa: BLE001
+                      bad.append(f"{d.name}: {exc}")
+          print(f"loaded {total - len(bad)} / {total} tasks")
+          if bad:
+              print("\n".join(bad)); sys.exit(1)
+          PY
+
+      - name: Manifests and registry agree on task names
+        run: |
+          uv run python - <<'PY'
+          import json, tomllib, sys
+          registry = {e["name"]: e for e in json.load(open("registry.json"))}
+          pairs = [("harbor/dataset.toml", "clawbench-v2"), ("harbor/dataset-v1.toml", "clawbench-v1")]
+          for path, name in pairs:
+              manifest = tomllib.loads(open(path).read())
+              m = {t["name"].split("/", 1)[1] for t in manifest["tasks"]}
+              r = {t["name"] for t in registry[name]["tasks"]}
+              if m != r:
+                  print(path, "manifest-only:", sorted(m - r)); print("registry-only:", sorted(r - m)); sys.exit(1)
+              print(f"{len(m)} tasks agree for {name}")
+          PY

.gitignore

@@ -19,3 +19,13 @@ build/
 run-logs/
 harbor-datasets/
 harbor-jobs/
+
+# macOS AppleDouble sidecars (created when files are copied from a Mac)
+._*
+
+# Harbor: full generated datasets are build output (see harbor/README.md);
+# only the committed prebuilt-mode dataset under harbor/datasets/ is tracked.
+harbor/datasets/*/
+!harbor/datasets/clawbench-v2/
+harbor/jobs/
+harbor/trials/

AGENTS.md

@@ -32,6 +32,12 @@ ClawBench/
     models.yaml                   # Model API configs (gitignored -- copy from example)
     models.example.yaml           # Template with placeholder keys
     model.schema.json             # JSON schema for model entries
+  harbor/                         # Harbor-format packaging: registry, dataset manifest, adapter package
+    README.md                     # How to run / publish ClawBench through Harbor
+    dataset.toml                  # Hub manifest (tiger-ai-lab/clawbench-v2)
+    adapter/                      # harbor-framework/harbor adapters/clawbench package
+    datasets/clawbench-{v1,v2}/   # Committed prebuilt-mode Harbor tasks (generated, CI-checked)
+  registry.json                   # Harbor git registry (harbor run --repo TIGER-AI-Lab/ClawBench)
   test-cases/
     task.schema.json              # JSON schema for task.json
     v1/                           # V1: 152 task directories

CHANGELOG.md

@@ -9,6 +9,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/).
 ## [Unreleased]
 ### Added
 - Added Kernel as a managed remote browser runtime with live view and downloaded replay recordings. Thanks to @[rgarcia](https://github.com/rgarcia).
+- Added a `harbor/` package: an in-repo Harbor registry (`registry.json` + committed `harbor/datasets/clawbench-{v1,v2}/`) so `harbor run --repo TIGER-AI-Lab/ClawBench -d clawbench-v2` works without conversion, a `dataset.toml` manifest for publishing to the Harbor Hub, and a Harbor-official adapter package (`harbor/adapter/`) for upstreaming to `harbor-framework/harbor`.
+- Added `clawbench-harbor-adapt --docker-image` (prebuilt-image mode): generated tasks reference the published `clawbench/clawbench-harbor-runtime` Docker Hub image instead of shipping a 280 KB `environment/` build context each (a one-line `FROM` Dockerfile remains, as Harbor requires `environment/` to exist).
+- Added `clawbench-harbor-adapt --suite v1|v2`: the V1 corpus (152 tasks) is now exported to Harbor alongside V2.
+
+### Fixed
+- README: removed the duplicated `## Star History` heading (EN + zh-CN).
 
 ## [0.9.2] - 2026-08-18
 ### Added

README.md

@@ -157,9 +157,9 @@ order food, book travel, apply for jobs, write reviews, manage projects.<br/>
 </td>
 <td align="center" valign="top">
 
-❓ **Have a question**<br/>
-[FAQ](#faq) · [Open an issue](https://github.com/TIGER-AI-Lab/ClawBench/issues/new/choose)<br/>
-<sub>Or ask on the HF dataset page</sub>
+🧭 **Run it via Harbor**<br/>
+[`harbor run --repo TIGER-AI-Lab/ClawBench`](harbor/README.md)<br/>
+<sub>[FAQ](#faq) · [Open an issue](https://github.com/TIGER-AI-Lab/ClawBench/issues/new/choose)</sub>
 
 </td>
 </tr>
@@ -333,6 +333,7 @@ Full registry: [`src/clawbench/runtime/harnesses/harnesses.yaml`](src/clawbench/
 | --- | --- |
 | Use a managed remote browser instead of a local container | [`docs/browser-runtimes.md`](docs/browser-runtimes.md) — Kernel and Browserbase setup, options, and recordings |
 | Run V2 through the Harbor framework (and run it fast) | [`docs/harbor.md`](docs/harbor.md) — conversion, judge wiring, concurrency, troubleshooting |
+| Pull ClawBench as a Harbor dataset (`harbor run --repo TIGER-AI-Lab/ClawBench -d clawbench-v2`) | [`harbor/README.md`](harbor/README.md) — in-repo registry, Hub publishing, adapter package |
 | See every CLI command and flag | [`docs/cli.md`](docs/cli.md) |
 
 <details>
@@ -1021,15 +1022,17 @@ Open to contributions — new test cases, bug fixes, or evaluation submissions f
 
 ## Star History
 
-## Star History
-
+<p align="center">
 <a href="https://www.star-history.com/?type=date&repos=TIGER-AI-Lab%2FClawBench">
  <picture>
    <source media="(prefers-color-scheme: dark)" srcset="https://api.star-history.com/chart?repos=TIGER-AI-Lab/ClawBench&type=date&theme=dark&legend=top-left&sealed_token=TZeXwgvoa-F4OXufUJG_yeQiaNJ3qK5xGDCyv4iJiF1UvtEWAfS4a3mN5_3tsdq8V-YHSGS6kjAsDJoAScK6tTCEqNl5hYsfec3LzAqQWDG2lNkdmTkIlg" />
    <source media="(prefers-color-scheme: light)" srcset="https://api.star-history.com/chart?repos=TIGER-AI-Lab/ClawBench&type=date&legend=top-left&sealed_token=TZeXwgvoa-F4OXufUJG_yeQiaNJ3qK5xGDCyv4iJiF1UvtEWAfS4a3mN5_3tsdq8V-YHSGS6kjAsDJoAScK6tTCEqNl5hYsfec3LzAqQWDG2lNkdmTkIlg" />
-   <img alt="Star History Chart" src="https://api.star-history.com/chart?repos=TIGER-AI-Lab/ClawBench&type=date&legend=top-left&sealed_token=TZeXwgvoa-F4OXufUJG_yeQiaNJ3qK5xGDCyv4iJiF1UvtEWAfS4a3mN5_3tsdq8V-YHSGS6kjAsDJoAScK6tTCEqNl5hYsfec3LzAqQWDG2lNkdmTkIlg" />
+   <img alt="Star History Chart" src="https://api.star-history.com/chart?repos=TIGER-AI-Lab/ClawBench&type=date&legend=top-left&sealed_token=TZeXwgvoa-F4OXufUJG_yeQiaNJ3qK5xGDCyv4iJiF1UvtEWAfS4a3mN5_3tsdq8V-YHSGS6kjAsDJoAScK6tTCEqNl5hYsfec3LzAqQWDG2lNkdmTkIlg" width="720" />
  </picture>
 </a>
+</p>
+
+<p align="center"><sub>Live chart — click to open on star-history.com. Sister projects: <a href="https://github.com/reacher-z/HarnessBench">HarnessBench</a> · <a href="https://github.com/TIGER-AI-Lab/RewardHarness">RewardHarness</a></sub></p>
 
 ## License & Acknowledgments
 

docs/README.zh-CN.md

@@ -105,9 +105,9 @@
 </td>
 <td align="center" valign="top">
 
-❓ **有问题**<br/>
-[FAQ](#faq) · [提 issue](https://github.com/TIGER-AI-Lab/ClawBench/issues/new/choose)<br/>
-<sub>也可在 HF 数据集页提问</sub>
+🧭 **用 Harbor 跑**<br/>
+[`harbor run --repo TIGER-AI-Lab/ClawBench`](../harbor/README.md)<br/>
+<sub>[FAQ](#faq) · [提 issue](https://github.com/TIGER-AI-Lab/ClawBench/issues/new/choose)</sub>
 
 </td>
 </tr>
@@ -279,6 +279,7 @@ harness 是驱动浏览器的智能体框架，和模型是两个独立维度。
 | --- | --- |
 | 用托管的远程浏览器代替本地容器 | [`docs/browser-runtimes.md`](browser-runtimes.md) —— Kernel 和 Browserbase 配置、参数与录制 |
 | 用 Harbor 框架跑 V2（并且跑得快） | [`docs/harbor.md`](harbor.md) —— 转换、judge 配置、并发、排错 |
+| 把 ClawBench 当作 Harbor 数据集直接拉取（`harbor run --repo TIGER-AI-Lab/ClawBench -d clawbench-v2`） | [`harbor/README.md`](../harbor/README.md) —— 仓库内 registry、发布到 Hub、adapter 包 |
 | 查所有 CLI 命令和参数 | [`docs/cli.md`](cli.md) |
 
 <details>
@@ -960,15 +961,17 @@ Pi 的 `agent.log`、`proxy.log` 等 harness 诊断日志不会复制到最终
 
 ## Star 历史
 
-## Star History
-
+<p align="center">
 <a href="https://www.star-history.com/?type=date&repos=TIGER-AI-Lab%2FClawBench">
  <picture>
    <source media="(prefers-color-scheme: dark)" srcset="https://api.star-history.com/chart?repos=TIGER-AI-Lab/ClawBench&type=date&theme=dark&legend=top-left&sealed_token=TZeXwgvoa-F4OXufUJG_yeQiaNJ3qK5xGDCyv4iJiF1UvtEWAfS4a3mN5_3tsdq8V-YHSGS6kjAsDJoAScK6tTCEqNl5hYsfec3LzAqQWDG2lNkdmTkIlg" />
    <source media="(prefers-color-scheme: light)" srcset="https://api.star-history.com/chart?repos=TIGER-AI-Lab/ClawBench&type=date&legend=top-left&sealed_token=TZeXwgvoa-F4OXufUJG_yeQiaNJ3qK5xGDCyv4iJiF1UvtEWAfS4a3mN5_3tsdq8V-YHSGS6kjAsDJoAScK6tTCEqNl5hYsfec3LzAqQWDG2lNkdmTkIlg" />
-   <img alt="Star History Chart" src="https://api.star-history.com/chart?repos=TIGER-AI-Lab/ClawBench&type=date&legend=top-left&sealed_token=TZeXwgvoa-F4OXufUJG_yeQiaNJ3qK5xGDCyv4iJiF1UvtEWAfS4a3mN5_3tsdq8V-YHSGS6kjAsDJoAScK6tTCEqNl5hYsfec3LzAqQWDG2lNkdmTkIlg" />
+   <img alt="Star History Chart" src="https://api.star-history.com/chart?repos=TIGER-AI-Lab/ClawBench&type=date&legend=top-left&sealed_token=TZeXwgvoa-F4OXufUJG_yeQiaNJ3qK5xGDCyv4iJiF1UvtEWAfS4a3mN5_3tsdq8V-YHSGS6kjAsDJoAScK6tTCEqNl5hYsfec3LzAqQWDG2lNkdmTkIlg" width="720" />
  </picture>
 </a>
+</p>
+
+<p align="center"><sub>实时图表 —— 点击在 star-history.com 打开。姊妹项目：<a href="https://github.com/reacher-z/HarnessBench">HarnessBench</a> · <a href="https://github.com/TIGER-AI-Lab/RewardHarness">RewardHarness</a></sub></p>
 
 ## 许可证与致谢
 

docs/harbor.md

@@ -4,6 +4,7 @@
 
 **Use Harbor when** you already run other benchmarks through it, you want Harbor's agent registry (`-a openclaw`, `-a hermes`, …) instead of ClawBench's own harnesses, or you need Harbor's retry/attempt semantics. **Use `clawbench-batch` instead** when you just want to score a model on ClawBench — it is the shorter path and it is what the leaderboard uses.
 
+- [Zero-conversion path: the git registry](#zero-conversion-path-the-git-registry)
 - [Prerequisites](#prerequisites)
 - [1. Convert V2 into a Harbor dataset](#1-convert-v2-into-a-harbor-dataset)
 - [2. Wire up the judge](#2-wire-up-the-judge)
@@ -12,6 +13,18 @@
 - [What the generated environment contains](#what-the-generated-environment-contains)
 - [Troubleshooting](#troubleshooting)
 
+## Zero-conversion path: the git registry
+
+The repo ships pre-generated Harbor datasets ([`harbor/datasets/clawbench-v2/`](../harbor/datasets/clawbench-v2/) — leaderboard corpus — and [`harbor/datasets/clawbench-v1/`](../harbor/datasets/clawbench-v1/), select with `-d clawbench-v1`) and a [`registry.json`](../registry.json), so Harbor can pull ClawBench straight from GitHub:
+
+```bash
+harbor run --repo TIGER-AI-Lab/ClawBench -d clawbench-v2 -a hermes -m deepseek/deepseek-v4-flash \
+  --env-file .env --ve CLAWBENCH_JUDGE_BASE_URL=... --ve CLAWBENCH_JUDGE_API_KEY=... \
+  --ve CLAWBENCH_JUDGE_MODEL=deepseek-v4-pro --ve CLAWBENCH_JUDGE_API_TYPE=openai-completions
+```
+
+Those tasks reference the published runtime image `clawbench/clawbench-harbor-runtime:<version>` (Docker Hub) instead of building one per task. Publishing to the Harbor Hub and upstreaming the adapter are covered in [`harbor/README.md`](../harbor/README.md). The rest of this page is the local-conversion path, which builds the image from source and is what you want when modifying the runtime.
+
 ## Prerequisites
 
 - **Docker.** Harbor runs use Harbor's Docker provider, so Docker must be available even if you normally use Podman for native ClawBench runs.
@@ -42,6 +55,7 @@ uv run clawbench-harbor-adapt \
 | `--task-ids <id> …` | Convert specific tasks by directory name or numeric `task_id` |
 | `--cases-dir <path>` | Convert a corpus other than V2 (defaults to `test-cases/v2/`) |
 | `--dataset-name`, `--org` | Metadata written into the generated `task.toml` |
+| `--docker-image <ref>` | Prebuilt mode: reference a published runtime image instead of copying `environment/` into every task (used for the committed dataset) |
 
 ## 2. Wire up the judge
 
@@ -59,7 +73,7 @@ Use `deepseek-v4-pro` if you want numbers comparable to the published leaderboar
 ## 3. Run it
 
 ```bash
-uvx --from harbor==0.15.0 harbor run \
+uvx --from harbor==0.22.0 harbor run \
   -p ./harbor-datasets/clawbench-v2 \
   -a "<agent>" \
   -m "<model>" \
@@ -70,15 +84,15 @@ uvx --from harbor==0.15.0 harbor run \
   --ve CLAWBENCH_JUDGE_API_TYPE="${CLAWBENCH_JUDGE_API_TYPE:-openai-completions}"
 ```
 
-Drop `uvx --from harbor==0.15.0` if Harbor is already installed.
+Drop `uvx --from harbor==0.22.0` if Harbor is already installed. Generated tasks use `schema_version = "1.3"` and load unchanged on Harbor 0.15 – 0.22.
 
 ### OpenClaw through an OpenAI-compatible endpoint
 
 ```bash
 export OPENAI_BASE_URL="https://openrouter.ai/api/v1"
 export OPENAI_API_KEY="$OPENROUTER_API_KEY"
 
-uvx --from harbor==0.15.0 harbor run \
+uvx --from harbor==0.22.0 harbor run \
   -p ./harbor-datasets/clawbench-v2 \
   -a openclaw \
   -m openai/deepseek/deepseek-v4-flash \
@@ -96,7 +110,7 @@ uvx --from harbor==0.15.0 harbor run \
 ```bash
 export OPENROUTER_API_KEY="your-openrouter-key"
 
-uvx --from harbor==0.15.0 harbor run \
+uvx --from harbor==0.22.0 harbor run \
   -p ./harbor-datasets/clawbench-v2 \
   -a hermes \
   -m deepseek/deepseek-v4-flash \
@@ -115,7 +129,7 @@ A full V2 sweep is 129 containerized browser sessions, each capped by the task's
 **1. Raise concurrency.** `-n / --n-concurrent` is the single biggest lever:
 
 ```bash
-uvx --from harbor==0.15.0 harbor run -p ./harbor-datasets/clawbench-v2 \
+uvx --from harbor==0.22.0 harbor run -p ./harbor-datasets/clawbench-v2 \
   -a hermes -m deepseek/deepseek-v4-flash -n 8 --env-file .env --ve ...
 ```
 
@@ -124,7 +138,7 @@ Each trial is a full Chromium container, so budget roughly **1 CPU core and ~2 G
 **2. Build the image once.** The first trial builds the ClawBench environment image; parallel cold starts all build at once. Warm the cache with the smoke dataset before the real sweep:
 
 ```bash
-uvx --from harbor==0.15.0 harbor run -p ./harbor-datasets/clawbench-v2-smoke \
+uvx --from harbor==0.22.0 harbor run -p ./harbor-datasets/clawbench-v2-smoke \
   -a hermes -m deepseek/deepseek-v4-flash --env-file .env --ve ...
 ```