From 4fcef71d840c986ca29527164cfbc0879ad3df9c Mon Sep 17 00:00:00 2001 From: Benjamin Feuer Date: Tue, 1 Sep 2026 13:09:13 -0400 Subject: [PATCH] [eval] Cap TaskTrove verifiers at ten minutes Cap every TaskTrove Harbor evaluation verifier at 600 seconds and remove longer verifier overrides. Active SWE-ReBench runs held verifier phases for 69 and 81 minutes even though task.toml specified 600 seconds; the campaign YAMLs had replaced that value with 7,200- or 14,400-second limits. The config inventory test keeps all TaskTrove variants within the cap. --- .../snowball_tasktrove_v342_terminus2.yaml | 4 +-- hpc/harbor_yaml/eval/tasktrove_pi_ctx32k.yaml | 4 +-- .../eval/tasktrove_qwen3_coder_ctx32k.yaml | 2 +- .../eval/tasktrove_terminus2_ctx32k.yaml | 2 +- .../tasktrove_v44_glm52_terminus2_ctx32k.yaml | 4 +-- ...tasktrove_v44_qwen35_opencode_ctx131k.yaml | 4 +-- ...rove_v46_qwen3_coder_terminus2_ctx32k.yaml | 4 +-- tests/hpc/test_tasktrove_eval_configs.py | 29 +++++++++++++++++++ 8 files changed, 41 insertions(+), 12 deletions(-) create mode 100644 tests/hpc/test_tasktrove_eval_configs.py diff --git a/hpc/harbor_yaml/eval/snowball_tasktrove_v342_terminus2.yaml b/hpc/harbor_yaml/eval/snowball_tasktrove_v342_terminus2.yaml index 9b2d8a1b..ddd6ca30 100644 --- a/hpc/harbor_yaml/eval/snowball_tasktrove_v342_terminus2.yaml +++ b/hpc/harbor_yaml/eval/snowball_tasktrove_v342_terminus2.yaml @@ -28,8 +28,8 @@ environment: kwargs: auto_snapshot: true verifier: - override_timeout_sec: 14400 - max_timeout_sec: 14400 + override_timeout_sec: null + max_timeout_sec: 600 agents: - name: terminus-2 model_name: placeholder/override-at-runtime diff --git a/hpc/harbor_yaml/eval/tasktrove_pi_ctx32k.yaml b/hpc/harbor_yaml/eval/tasktrove_pi_ctx32k.yaml index 10546329..0f01d10a 100644 --- a/hpc/harbor_yaml/eval/tasktrove_pi_ctx32k.yaml +++ b/hpc/harbor_yaml/eval/tasktrove_pi_ctx32k.yaml @@ -23,8 +23,8 @@ environment: kwargs: auto_snapshot: true verifier: - override_timeout_sec: 7200 - max_timeout_sec: 7200 + override_timeout_sec: null + max_timeout_sec: 600 agents: - name: pi model_name: placeholder/override-at-runtime diff --git a/hpc/harbor_yaml/eval/tasktrove_qwen3_coder_ctx32k.yaml b/hpc/harbor_yaml/eval/tasktrove_qwen3_coder_ctx32k.yaml index fe256e71..29071786 100644 --- a/hpc/harbor_yaml/eval/tasktrove_qwen3_coder_ctx32k.yaml +++ b/hpc/harbor_yaml/eval/tasktrove_qwen3_coder_ctx32k.yaml @@ -23,7 +23,7 @@ environment: kwargs: auto_snapshot: true verifier: - max_timeout_sec: 14400 + max_timeout_sec: 600 agents: - name: terminus-2 model_name: placeholder/override-at-runtime diff --git a/hpc/harbor_yaml/eval/tasktrove_terminus2_ctx32k.yaml b/hpc/harbor_yaml/eval/tasktrove_terminus2_ctx32k.yaml index fe256e71..29071786 100644 --- a/hpc/harbor_yaml/eval/tasktrove_terminus2_ctx32k.yaml +++ b/hpc/harbor_yaml/eval/tasktrove_terminus2_ctx32k.yaml @@ -23,7 +23,7 @@ environment: kwargs: auto_snapshot: true verifier: - max_timeout_sec: 14400 + max_timeout_sec: 600 agents: - name: terminus-2 model_name: placeholder/override-at-runtime diff --git a/hpc/harbor_yaml/eval/tasktrove_v44_glm52_terminus2_ctx32k.yaml b/hpc/harbor_yaml/eval/tasktrove_v44_glm52_terminus2_ctx32k.yaml index c19f86c5..d0f70a20 100644 --- a/hpc/harbor_yaml/eval/tasktrove_v44_glm52_terminus2_ctx32k.yaml +++ b/hpc/harbor_yaml/eval/tasktrove_v44_glm52_terminus2_ctx32k.yaml @@ -26,8 +26,8 @@ environment: kwargs: auto_snapshot: true verifier: - override_timeout_sec: 14400 - max_timeout_sec: 14400 + override_timeout_sec: null + max_timeout_sec: 600 agents: - name: terminus-2 model_name: placeholder/override-at-runtime diff --git a/hpc/harbor_yaml/eval/tasktrove_v44_qwen35_opencode_ctx131k.yaml b/hpc/harbor_yaml/eval/tasktrove_v44_qwen35_opencode_ctx131k.yaml index 203d9311..411fa7c4 100644 --- a/hpc/harbor_yaml/eval/tasktrove_v44_qwen35_opencode_ctx131k.yaml +++ b/hpc/harbor_yaml/eval/tasktrove_v44_qwen35_opencode_ctx131k.yaml @@ -26,8 +26,8 @@ environment: kwargs: auto_snapshot: true verifier: - override_timeout_sec: 14400 - max_timeout_sec: 14400 + override_timeout_sec: null + max_timeout_sec: 600 agents: - name: opencode model_name: openai/override-at-runtime diff --git a/hpc/harbor_yaml/eval/tasktrove_v46_qwen3_coder_terminus2_ctx32k.yaml b/hpc/harbor_yaml/eval/tasktrove_v46_qwen3_coder_terminus2_ctx32k.yaml index e227a7b9..97e44642 100644 --- a/hpc/harbor_yaml/eval/tasktrove_v46_qwen3_coder_terminus2_ctx32k.yaml +++ b/hpc/harbor_yaml/eval/tasktrove_v46_qwen3_coder_terminus2_ctx32k.yaml @@ -27,8 +27,8 @@ environment: kwargs: auto_snapshot: true verifier: - override_timeout_sec: 14400 - max_timeout_sec: 14400 + override_timeout_sec: null + max_timeout_sec: 600 agents: - name: terminus-2 model_name: placeholder/override-at-runtime diff --git a/tests/hpc/test_tasktrove_eval_configs.py b/tests/hpc/test_tasktrove_eval_configs.py new file mode 100644 index 00000000..c4a06eb8 --- /dev/null +++ b/tests/hpc/test_tasktrove_eval_configs.py @@ -0,0 +1,29 @@ +from pathlib import Path + +import yaml + + +REPOSITORY_ROOT = Path(__file__).resolve().parents[2] +EVAL_CONFIG_DIR = REPOSITORY_ROOT / "hpc/harbor_yaml/eval" +MAX_VERIFIER_TIMEOUT = 600 + + +def tasktrove_eval_configs() -> list[Path]: + return sorted( + { + *EVAL_CONFIG_DIR.glob("tasktrove*.yaml"), + *EVAL_CONFIG_DIR.glob("snowball_tasktrove*.yaml"), + } + ) + + +def test_tasktrove_verifiers_are_capped_at_ten_minutes() -> None: + paths = tasktrove_eval_configs() + assert paths + + for path in paths: + config = yaml.safe_load(path.read_text()) + verifier = config["verifier"] + assert verifier["max_timeout_sec"] <= MAX_VERIFIER_TIMEOUT, path + override = verifier.get("override_timeout_sec") + assert override is None or override <= MAX_VERIFIER_TIMEOUT, path