fix(rescore): resolve paths from the workspace instead of a maintainer's home

src/clawbench/eval/rescore.py

@@ -36,7 +36,12 @@
 from pathlib import Path
 from typing import Any
 
-import yaml
+from clawbench.utils.model_config import (
+    MODELS_YAML,
+    ModelConfigError,
+    load_model_config,
+)
+from clawbench.utils.paths import WORKSPACE_ROOT
 
 JUDGE_FILE = {"strict": "judge.json", "lenient": "judge_llm.json"}
 
@@ -196,14 +201,16 @@ def write_eval_results(
     print(f"  eval_results written to {out_dir}/")
 
 
-def main() -> int:
+def build_parser() -> argparse.ArgumentParser:
     p = argparse.ArgumentParser(
         description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
     )
     p.add_argument(
         "--sweep-root",
         type=Path,
-        default=Path.home() / "work/ClawBench/claw-output/sweep",
+        default=WORKSPACE_ROOT / "test-output",
+        help="Directory tree to scan for completed runs "
+        "(default ./test-output, where the runner writes them).",
     )
     p.add_argument(
         "--judge-model",
@@ -214,7 +221,8 @@ def main() -> int:
     p.add_argument(
         "--models-yaml",
         type=Path,
-        default=Path.home() / "work/ClawBench/models/models.yaml",
+        default=MODELS_YAML,
+        help=f"Model definitions to resolve --judge-model against (default {MODELS_YAML}).",
     )
     p.add_argument(
         "--rubric",
@@ -242,19 +250,23 @@ def main() -> int:
     )
     p.add_argument("--limit", type=int, default=0)
     p.add_argument("--only-batch", type=Path, default=None)
-    args = p.parse_args()
+    return p
 
-    cfg_all = yaml.safe_load(args.models_yaml.read_text())
-    if args.judge_model not in cfg_all:
-        print(
-            f"ERROR: judge model {args.judge_model!r} not in {args.models_yaml}",
-            file=sys.stderr,
-        )
-        return 2
-    judge_cfg = dict(cfg_all[args.judge_model])
-    if not judge_cfg.get("api_key"):
-        print(f"ERROR: judge {args.judge_model!r} has no api_key", file=sys.stderr)
-        return 2
+
+def main() -> int:
+    args = build_parser().parse_args()
+
+    # Shared loader: validates the model exists, checks required fields, and
+    # normalizes the api_key/api_keys pair. Re-parsing the YAML here used to
+    # reject the api_keys list form that every other entry point accepts.
+    try:
+        judge_cfg = load_model_config(args.judge_model, args.models_yaml)
+    except ModelConfigError as e:
+        # A bad --judge-model is a typo on the command line, caught before any
+        # scoring work starts. Report it like run.py does rather than letting
+        # the traceback out.
+        print(f"ERROR: {e}")
+        return 1
 
     rubrics = [args.rubric] if args.rubric != "both" else ["lenient", "strict"]
     judge_funcs = {}
@@ -267,11 +279,25 @@ def main() -> int:
 
         judge_funcs["lenient"] = judge_lenient
 
-    run_dirs = (
-        find_run_dirs(args.only_batch)
-        if args.only_batch
-        else find_run_dirs(args.sweep_root)
-    )
+    scan_root = args.only_batch or args.sweep_root
+    if not scan_root.is_dir():
+        flag = "--only-batch" if args.only_batch else "--sweep-root"
+        print(f"ERROR: {flag} {scan_root} is not a directory.", file=sys.stderr)
+        print(
+            "       Point it at a tree containing completed runs "
+            "(the runner writes them under ./test-output by default).",
+            file=sys.stderr,
+        )
+        return 2
+
+    run_dirs = find_run_dirs(scan_root)
+    if not run_dirs:
+        print(
+            f"ERROR: no runs found under {scan_root} "
+            "(looked for run-meta.json at any depth).",
+            file=sys.stderr,
+        )
+        return 2
 
     pending = []
     for rd in run_dirs:

src/clawbench/runner/run_support/config.py

@@ -5,14 +5,18 @@
 import sys
 from pathlib import Path
 
-import yaml
-
 from clawbench.runner.run_support.harness_registry import (
     HARNESS_REGISTRY,
     HARNESS_REGISTRY_YAML,
     HarnessRegistry,
     load_harness_registry,
 )
+from clawbench.utils.model_config import (
+    MODELS_YAML,
+    ModelConfigError,
+    load_model_config,
+    load_models_yaml,
+)
 from clawbench.utils.paths import (
     ASSET_ROOT,
     WORKSPACE_ROOT,
@@ -44,17 +48,6 @@
     "resolve_test_case_path",
 ]
 
-
-class ModelConfigError(Exception):
-    """Raised when a model config in models/models.yaml is missing or invalid.
-
-    A plain Exception (not SystemExit) so callers that load a model mid-run
-    (e.g. the judge stage, after the agent has already produced results) can
-    catch it and continue instead of the process dying before run-meta.json
-    is written.
-    """
-
-
 HARNESSES = HARNESS_REGISTRY.harnesses
 DEFAULT_HARNESS = HARNESS_REGISTRY.default
 BASE_IMAGE = HARNESS_REGISTRY.base_image
@@ -99,7 +92,6 @@ def _detect_engine() -> str:
 
 
 ENGINE = _detect_engine()
-MODELS_YAML = WORKSPACE_ROOT / "models" / "models.yaml"
 
 
 def load_dotenv(path: Path) -> dict[str, str]:
@@ -117,20 +109,6 @@ def load_dotenv(path: Path) -> dict[str, str]:
     return env
 
 
-def load_models_yaml() -> dict:
-    """Load all model definitions from models/models.yaml.
-
-    Raises ModelConfigError rather than exiting, for the same reason
-    load_model_config does: this runs inside the judge stage too, where a
-    SystemExit would escape the handlers and lose the run's metadata.
-    """
-    if not MODELS_YAML.exists():
-        raise ModelConfigError(
-            f"{MODELS_YAML} not found (copy models.example.yaml and fill in your keys)"
-        )
-    return yaml.safe_load(MODELS_YAML.read_text()) or {}
-
-
 def load_runtime_env() -> dict[str, str]:
     """Load runtime credentials in increasing precedence order."""
     env = load_dotenv(bundled_path(".env"))
@@ -167,50 +145,3 @@ def resolve_task_file(path: Path) -> tuple[Path, Path, str]:
     if resolved.is_file():
         return resolved.parent, resolved, resolved.stem
     return resolved, resolved / "task.json", resolved.name
-
-
-def load_model_config(model: str) -> dict:
-    """Load a model config by name from models/models.yaml.
-
-    The YAML key is the model name (passed as MODEL_NAME to the container).
-    """
-    all_models = load_models_yaml()
-    if model not in all_models:
-        raise ModelConfigError(
-            f"model '{model}' not found in {MODELS_YAML}. "
-            f"Available models: {', '.join(sorted(all_models))}"
-        )
-
-    # Validate model name characters. Note: '/' and ':' are valid in
-    # vendor-prefixed ids like 'anthropic/claude-sonnet-4-6' or
-    # 'arcee-ai/trinity-large-preview:free' — they get sanitized to
-    # '--' before being used as path components. We only reject characters
-    # that could cause real trouble in shell/filesystem paths even after
-    # that sanitization.
-    bad = [c for c in ' \\*?"<>|' if c in model]
-    if bad:
-        raise ModelConfigError(
-            f"model name '{model}' contains illegal character(s): "
-            f"{' '.join(repr(c) for c in bad)}"
-        )
-
-    config = dict(all_models[model])
-    config["model"] = model  # the YAML key IS the model name
-
-    required = ["base_url", "api_type"]
-    missing = [k for k in required if not config.get(k)]
-    if missing:
-        raise ModelConfigError(
-            f"required field(s) missing for model '{model}': {', '.join(missing)}"
-        )
-
-    # Normalize API keys: api_keys list wins, else wrap api_key into list.
-    if config.get("api_keys"):
-        config["api_key"] = config["api_keys"][0]
-    elif config.get("api_key"):
-        config["api_keys"] = [config["api_key"]]
-
-    if not config.get("api_keys"):
-        raise ModelConfigError(f"no api_key or api_keys for model '{model}'")
-
-    return config

src/clawbench/utils/model_config.py

@@ -0,0 +1,92 @@
+"""Model definitions from models/models.yaml.
+
+Kept in `utils` rather than `runner.run_support.config` so that tools which
+only score existing runs — `clawbench-rescore`, for one — can resolve a judge
+model without importing the runner, which probes for a container engine at
+import time and exits when Docker and Podman are both absent.
+"""
+
+from pathlib import Path
+
+import yaml
+
+from clawbench.utils.paths import WORKSPACE_ROOT
+
+MODELS_YAML = WORKSPACE_ROOT / "models" / "models.yaml"
+
+
+class ModelConfigError(Exception):
+    """Raised when a model config in models/models.yaml is missing or invalid.
+
+    A plain Exception (not SystemExit) so callers that load a model mid-run
+    (e.g. the judge stage, after the agent has already produced results) can
+    catch it and continue instead of the process dying before run-meta.json
+    is written.
+    """
+
+
+def load_models_yaml(models_yaml: Path | None = None) -> dict:
+    """Load all model definitions from models/models.yaml.
+
+    `models_yaml` overrides the workspace-resolved default, for callers that
+    expose an explicit --models-yaml flag.
+
+    Raises ModelConfigError rather than exiting, for the same reason
+    load_model_config does: this runs inside the judge stage too, where a
+    SystemExit would escape the handlers and lose the run's metadata.
+    """
+    path = models_yaml or MODELS_YAML
+    if not path.exists():
+        raise ModelConfigError(
+            f"{path} not found (copy models.example.yaml and fill in your keys)"
+        )
+    return yaml.safe_load(path.read_text()) or {}
+
+
+def load_model_config(model: str, models_yaml: Path | None = None) -> dict:
+    """Load a model config by name from models/models.yaml.
+
+    The YAML key is the model name (passed as MODEL_NAME to the container).
+    `models_yaml` overrides the workspace-resolved default.
+    """
+    path = models_yaml or MODELS_YAML
+    all_models = load_models_yaml(models_yaml)
+    if model not in all_models:
+        raise ModelConfigError(
+            f"model '{model}' not found in {path}. "
+            f"Available models: {', '.join(sorted(all_models))}"
+        )
+
+    # Validate model name characters. Note: '/' and ':' are valid in
+    # vendor-prefixed ids like 'anthropic/claude-sonnet-4-6' or
+    # 'arcee-ai/trinity-large-preview:free' — they get sanitized to
+    # '--' before being used as path components. We only reject characters
+    # that could cause real trouble in shell/filesystem paths even after
+    # that sanitization.
+    bad = [c for c in ' \\*?"<>|' if c in model]
+    if bad:
+        raise ModelConfigError(
+            f"model name '{model}' contains illegal character(s): "
+            f"{' '.join(repr(c) for c in bad)}"
+        )
+
+    config = dict(all_models[model])
+    config["model"] = model  # the YAML key IS the model name
+
+    required = ["base_url", "api_type"]
+    missing = [k for k in required if not config.get(k)]
+    if missing:
+        raise ModelConfigError(
+            f"required field(s) missing for model '{model}': {', '.join(missing)}"
+        )
+
+    # Normalize API keys: api_keys list wins, else wrap api_key into list.
+    if config.get("api_keys"):
+        config["api_key"] = config["api_keys"][0]
+    elif config.get("api_key"):
+        config["api_keys"] = [config["api_key"]]
+
+    if not config.get("api_keys"):
+        raise ModelConfigError(f"no api_key or api_keys for model '{model}'")
+
+    return config