diff --git a/data/nemotron_gym/converters/agent_calendar.py b/data/nemotron_gym/converters/agent_calendar.py index 036f6e4b..0d883ac6 100644 --- a/data/nemotron_gym/converters/agent_calendar.py +++ b/data/nemotron_gym/converters/agent_calendar.py @@ -1,8 +1,7 @@ """Convert nvidia/Nemotron-RL-agent-calendar_scheduling. -The expected output of the agent is a JSON list of events. The verifier -(calendar_constraints) checks duration / time-window / natural-language -constraint per event from `exp_cal_state`. +The expected output of the agent is a JSON list of events. The verifier checks +the exact event set, each event's local constraints, and pairwise non-overlap. """ from __future__ import annotations @@ -27,8 +26,9 @@ "You are scheduling events on a calendar. Read the conversation below and " "write your final calendar as a JSON list to `/app/answer.txt`. Each event " "must include `event_id` (int), `event_name` (str), `start_time` " - '("HH:MM"), and `duration` (minutes). The verifier checks duration, ' - "time-window, and any natural-language constraint per event.\n\n" + '("HH:MM"), and `duration` (minutes). Events must not overlap. The verifier ' + "checks the exact event set, duration, time window, declared constraints, " + "and pairwise overlap.\n\n" "---\n\n" ) _MAX_EVENTS = 32 diff --git a/data/nemotron_gym/upload_to_hf.py b/data/nemotron_gym/upload_to_hf.py index 86b5b0c2..1150facb 100644 --- a/data/nemotron_gym/upload_to_hf.py +++ b/data/nemotron_gym/upload_to_hf.py @@ -207,9 +207,9 @@ "knowledge-openqa-v2": "llm_judge (LiteLLM, default `openai/gpt-4o-mini`, scored against reference answers as rubric — semantic-equivalence judging for paraphrasable long-form answers)", "instruction-following": "ifeval_constraints (IFEval-style constraint checker — paragraphs / words / forbidden / formatting)", "instruction-following-structured": "json_schema (parse agent JSON, validate against Draft 2020-12 schema)", - "instruction-following-calendar": "calendar_constraints (parse agent JSON list, check duration/window/constraint per event)", + "instruction-following-calendar": "calendar_constraints (parse agent JSON list; check exact event set, duration, window, declared constraints, and pairwise non-overlap)", "reasoning-gym": "reasoning_gym (delegate to upstream reasoning_gym scorer; normalized-match fallback)", - "agent-calendar": "calendar_constraints (same as instruction-following-calendar)", + "agent-calendar": "calendar_constraints (same complete contract as instruction-following-calendar)", "agent-workplace": "tool_call_match (JSON `{name, arguments}` compared to ground_truth tool calls; lossy single-step substitute for upstream stateful env)", "safety": "llm_judge (LiteLLM, default `openai/gpt-4o-mini`, scored against `principle` rubric)", "safety-v2": "safety_judge (LiteLLM, default `openai/gpt-4o-mini`, scored against `principle` rubric, with a heuristic refusal-detection fallback for sandboxes without `OPENAI_API_KEY` and a `/logs/agent` scan for the agent's response when `/app/response.txt` is missing — v2 fixes the 0% solve rate of v1)", diff --git a/data/nemotron_gym/verifiers/calendar_constraints.py b/data/nemotron_gym/verifiers/calendar_constraints.py index 23c77267..8867f31b 100644 --- a/data/nemotron_gym/verifiers/calendar_constraints.py +++ b/data/nemotron_gym/verifiers/calendar_constraints.py @@ -155,9 +155,20 @@ def evaluate_calendar(expected: object, events: object) -> tuple[bool, list[str] intervals.append((start, end, event_id)) intervals.sort() - for previous, current in zip(intervals, intervals[1:]): - if current[0] < previous[1]: - errors.append(f"events {previous[2]} and {current[2]} overlap") + for index, previous in enumerate(intervals): + for current in intervals[index + 1 :]: + if current[0] >= previous[1]: + break + previous_start, previous_end, previous_id = previous + current_start, current_end, current_id = current + errors.append( + f"events {previous_id} " + f"[{previous_start // 60:02d}:{previous_start % 60:02d}, " + f"{previous_end // 60:02d}:{previous_end % 60:02d}) and " + f"{current_id} " + f"[{current_start // 60:02d}:{current_start % 60:02d}, " + f"{current_end // 60:02d}:{current_end % 60:02d}) overlap" + ) return not errors, errors diff --git a/data/openswe/generate.py b/data/openswe/generate.py index abb9040e..ed1edd1b 100644 --- a/data/openswe/generate.py +++ b/data/openswe/generate.py @@ -34,6 +34,11 @@ from tqdm import tqdm +from data.patchers.trusted_test_patch import ( + trusted_test_patch_command, + write_trusted_test_patch_installer, +) + # --------------------------------------------------------------------------- # Task configuration @@ -218,7 +223,7 @@ def _extract_test_section(eval_script: str) -> str: ) -def create_test_sh(eval_script: str, test_patch: str) -> str: +def create_test_sh(eval_script: str, test_patch: str, base_commit: str) -> str: """ Build Harbor test.sh. @@ -234,12 +239,13 @@ def create_test_sh(eval_script: str, test_patch: str) -> str: patch_block = "" if has_patch: - patch_block = """\ + install_command = trusted_test_patch_command(base_commit) + patch_block = f"""\ -# Apply test patch (adds new test files; does NOT contain the golden fix) -if [ -f /tests/test_patch.diff ]; then - git apply -v --allow-empty /tests/test_patch.diff || \\ - git apply -v --allow-empty --reject /tests/test_patch.diff || true +# Restore every hidden-test path from the immutable base commit before applying +# the trusted patch. Agent edits to product paths remain untouched. +if ! {install_command}; then + exit 1 fi """ @@ -332,7 +338,9 @@ def create_task_dir(datum: dict, out_root: Path, idx: int, prefix: str) -> None: eval_script = datum.get("eval_script") or "" test_patch = datum.get("test_patch") or "" test_sh_path = d / "tests" / "test.sh" - test_sh_path.write_text(create_test_sh(eval_script, test_patch), encoding="utf-8") + test_sh_path.write_text( + create_test_sh(eval_script, test_patch, base_commit), encoding="utf-8" + ) test_sh_path.chmod( test_sh_path.stat().st_mode | stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH ) @@ -340,6 +348,9 @@ def create_task_dir(datum: dict, out_root: Path, idx: int, prefix: str) -> None: # tests/test_patch.diff if test_patch: (d / "tests" / "test_patch.diff").write_text(test_patch, encoding="utf-8") + write_trusted_test_patch_installer( + d / "tests" / "install_trusted_test_patch.sh" + ) # tests/config.json — metadata without large blobs (stored elsewhere) config = { diff --git a/data/patchers/patch_openswe_tasks.py b/data/patchers/patch_openswe_tasks.py index d096009a..c863a338 100644 --- a/data/patchers/patch_openswe_tasks.py +++ b/data/patchers/patch_openswe_tasks.py @@ -49,6 +49,11 @@ from pathlib import Path from typing import Optional +from data.patchers.trusted_test_patch import ( + trusted_test_patch_command, + write_trusted_test_patch_installer, +) + # --------------------------------------------------------------------------- # Python version normalisation # --------------------------------------------------------------------------- @@ -437,6 +442,31 @@ def wrap_solve_sh(original_solve_sh: str) -> str: return header + body +def install_trusted_test_patch_in_verifier(test_sh: str, base_commit: str) -> str: + """Replace the legacy best-effort hidden-patch block with fail-closed setup.""" + if "/tests/install_trusted_test_patch.sh" in test_sh: + return test_sh + + legacy_block = """\ +# Apply test patch (adds new test files; does NOT contain the golden fix) +if [ -f /tests/test_patch.diff ]; then + git apply -v --allow-empty /tests/test_patch.diff || \\ + git apply -v --allow-empty --reject /tests/test_patch.diff || true +fi +""" + if legacy_block not in test_sh: + raise ValueError("OpenSWE verifier has an unrecognized hidden-test patch block") + + command = trusted_test_patch_command(base_commit) + trusted_block = f"""\ +# Restore hidden-test paths from the immutable base before applying the trusted patch. +if ! {command}; then + exit 1 +fi +""" + return test_sh.replace(legacy_block, trusted_block, 1) + + # --------------------------------------------------------------------------- # Per-task patching # --------------------------------------------------------------------------- @@ -510,6 +540,11 @@ def get_post_workdir(text: str) -> str: solve_path = src_dir / "solution" / "solve.sh" original_solve = solve_path.read_text() if solve_path.exists() else "" new_solve = wrap_solve_sh(original_solve) if original_solve else "" + test_sh_path = src_dir / "tests" / "test.sh" + test_patch_path = src_dir / "tests" / "test_patch.diff" + new_test_sh = test_sh_path.read_text() if test_sh_path.exists() else "" + if test_patch_path.exists(): + new_test_sh = install_trusted_test_patch_in_verifier(new_test_sh, base_commit) if dry_run: return result @@ -544,6 +579,13 @@ def get_post_workdir(text: str) -> str: new_solve_path.stat().st_mode | stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH ) + if test_patch_path.exists(): + output_test_sh = dst_dir / "tests" / "test.sh" + output_test_sh.write_text(new_test_sh) + write_trusted_test_patch_installer( + dst_dir / "tests" / "install_trusted_test_patch.sh" + ) + return result diff --git a/data/patchers/patch_swe_rebench_v2_tasks.py b/data/patchers/patch_swe_rebench_v2_tasks.py index 10f12890..a61669fd 100644 --- a/data/patchers/patch_swe_rebench_v2_tasks.py +++ b/data/patchers/patch_swe_rebench_v2_tasks.py @@ -68,6 +68,11 @@ from textwrap import dedent from typing import Iterable, Iterator +from data.patchers.trusted_test_patch import ( + trusted_test_patch_command, + write_trusted_test_patch_installer, +) + # --------------------------------------------------------------------------- # Base image map @@ -332,11 +337,10 @@ def _make_dockerfile(base_image: str, family: str) -> str: mkdir -p /logs/verifier - # Apply the hidden test patch (the tests the fix PR introduced) - cd /testbed - if [ -f /tests/test_patch.diff ]; then - git apply --verbose /tests/test_patch.diff || \\ - git apply --verbose --reject /tests/test_patch.diff || true + # Restore hidden-test paths from the immutable base before applying the + # trusted patch. Product-code edits made by the agent remain untouched. + if ! {trusted_test_patch_command}; then + exit 1 fi # Run the task's test command (per-row from install_config.test_cmd). @@ -1099,7 +1103,10 @@ def patch_row(row: dict, output_root: Path, dry_run: bool = False) -> dict: # 3. tests/test.sh (task_dir / "tests" / "test.sh").write_text( - TEST_SH_TEMPLATE.format(test_cmd=test_cmd) + TEST_SH_TEMPLATE.format( + test_cmd=test_cmd, + trusted_test_patch_command=trusted_test_patch_command(base_commit), + ) ) # 4. tests/test_state.py @@ -1126,6 +1133,9 @@ def patch_row(row: dict, output_root: Path, dry_run: bool = False) -> dict: # 6. tests/test_patch.diff if test_patch and test_patch.strip(): (task_dir / "tests" / "test_patch.diff").write_text(test_patch) + write_trusted_test_patch_installer( + task_dir / "tests" / "install_trusted_test_patch.sh" + ) # 7. solution/solve.sh (task_dir / "solution" / "solve.sh").write_text( diff --git a/data/patchers/trusted_test_patch.py b/data/patchers/trusted_test_patch.py new file mode 100644 index 00000000..dccdba25 --- /dev/null +++ b/data/patchers/trusted_test_patch.py @@ -0,0 +1,72 @@ +"""Install hidden test patches independently of agent workspace edits.""" + +from __future__ import annotations + +import stat +import shlex +from pathlib import Path + + +TRUSTED_TEST_PATCH_INSTALLER = r"""#!/bin/bash +set -Eeuo pipefail + +repository=${1:?repository path is required} +patch_path=${2:?test patch path is required} +base_commit=${3:?base commit is required} + +cd "$repository" +git -c safe.directory="$repository" cat-file -e "${base_commit}^{commit}" + +if [ ! -s "$patch_path" ]; then + exit 0 +fi + +# Validate the patch before changing the workspace. --numstat -z makes path +# records unambiguous even when a filename contains spaces. +git -c safe.directory="$repository" -c core.quotePath=false \ + apply --numstat -z "$patch_path" >/dev/null + +while IFS= read -r -d '' record; do + path=${record#*$'\t'} + path=${path#*$'\t'} + if [ "$path" = "$record" ]; then + echo "Malformed git-apply path record" >&2 + exit 1 + fi + case "$path" in + ""|/*|.|..|../*|*/..|*/../*) + echo "Unsafe hidden-test path: $path" >&2 + exit 1 + ;; + esac + + # The hidden patch owns these exact paths. Remove untracked/ignored agent + # replacements and restore tracked files from the immutable task base. + git -c safe.directory="$repository" clean -ffdx -- "$path" + if git -c safe.directory="$repository" \ + cat-file -e "${base_commit}:${path}" 2>/dev/null; then + git -c safe.directory="$repository" \ + restore --source="$base_commit" --staged --worktree -- "$path" + fi +done < <(git -c safe.directory="$repository" -c core.quotePath=false \ + apply --numstat -z "$patch_path") + +git -c safe.directory="$repository" apply --check "$patch_path" +git -c safe.directory="$repository" apply --verbose "$patch_path" +""" + + +def write_trusted_test_patch_installer(path: Path) -> None: + """Write the executable hidden-test installer used by packaged verifiers.""" + path.write_text(TRUSTED_TEST_PATCH_INSTALLER, encoding="utf-8") + path.chmod(path.stat().st_mode | stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH) + + +def trusted_test_patch_command(base_commit: str) -> str: + """Return the verifier command anchored to the task's immutable base commit.""" + if not base_commit: + raise ValueError("base commit is required for trusted hidden-test installation") + return ( + "bash /tests/install_trusted_test_patch.sh " + f"/testbed /tests/test_patch.diff {shlex.quote(base_commit)}" + ) diff --git a/data/tasktrove/assemble_v49_release.py b/data/tasktrove/assemble_v49_release.py new file mode 100644 index 00000000..ef3c5f3d --- /dev/null +++ b/data/tasktrove/assemble_v49_release.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +"""Validate and assemble the TaskTrove v4.9 remediation release.""" + +from __future__ import annotations + +import argparse +import json +import os +import shutil +from pathlib import Path + +import pyarrow.parquet as pq +from huggingface_hub import hf_hub_download + +from data.tasktrove.build_swe_verifier_isolation_v49 import ( + OPENSWE_STORAGE_MB, + REVISIONS, + SOURCE_REVISION, + SWE_REBENCH_MEMORY_MB, + SWE_REBENCH_STORAGE_MB, + file_sha256, + validate_output, +) + +MAX_IMAGES = 20 + + +def _hardlink_or_copy(source: Path, destination: Path) -> None: + destination.parent.mkdir(parents=True, exist_ok=True) + try: + os.link(source, destination) + except OSError: + shutil.copyfile(source, destination) + + +def assemble(repair_stage: Path, stage: Path) -> None: + if stage.exists() and any(stage.iterdir()): + raise ValueError(f"stage must be absent or empty: {stage}") + stage.mkdir(parents=True, exist_ok=True) + entries: list[dict[str, object]] = [] + images: set[str] = set() + for revision in REVISIONS: + source = repair_stage / "datasets" / revision.output / "tasks.parquet" + rows = pq.ParquetFile(source).metadata.num_rows + source_parquet = Path( + hf_hub_download( + "open-thoughts/TaskTrove", + f"{revision.source}/tasks.parquet", + repo_type="dataset", + revision=SOURCE_REVISION, + ) + ) + source_rows = pq.ParquetFile(source_parquet).metadata.num_rows + images.update(validate_output(source, revision, rows)) + relative = Path("datasets") / revision.output / "tasks.parquet" + _hardlink_or_copy(source, stage / relative) + entries.append( + { + "source": revision.source, + "source_sha256": revision.source_sha256, + "source_rows": source_rows, + "output": revision.output, + "output_rows": rows, + "rejected_rows": source_rows - rows, + "output_sha256": file_sha256(source), + "parquet": str(relative), + "verifier": revision.verifier, + "storage_mb": { + "swe_rebench": SWE_REBENCH_STORAGE_MB, + "openswe": OPENSWE_STORAGE_MB, + }.get(revision.verifier), + "memory_mb": { + "swe_rebench": SWE_REBENCH_MEMORY_MB, + }.get(revision.verifier), + } + ) + if len(images) > MAX_IMAGES: + raise ValueError(f"release uses {len(images)} images, above limit {MAX_IMAGES}") + manifest = { + "source_repo": "open-thoughts/TaskTrove", + "source_revision": SOURCE_REVISION, + "source_version": "4.8", + "target_version": "4.9", + "datasets": entries, + "release_unique_images": len(images), + } + (stage / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n") + print(f"assembled {len(entries)} replacements using {len(images)} unique images") + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--repair-stage", type=Path, required=True) + parser.add_argument("--stage", type=Path, required=True) + args = parser.parse_args() + assemble(args.repair_stage, args.stage) + + +if __name__ == "__main__": + main() diff --git a/data/tasktrove/build_data_quality_batch_a1.py b/data/tasktrove/build_data_quality_batch_a1.py index a1d38a09..9c083b43 100644 --- a/data/tasktrove/build_data_quality_batch_a1.py +++ b/data/tasktrove/build_data_quality_batch_a1.py @@ -32,6 +32,7 @@ TASKTROVE_V342_REVISION = "3e96fe6464ce5ab6209e98801caab29b4a1fe87a" MAX_BATCH_ROWS = 32 MIN_RETAINED_TASKS = 300 +CALENDAR_GRANULARITY_MINUTES = 5 TASK_SCHEMA = pa.schema([("path", pa.string()), ("task_binary", pa.binary())]) METHODS2TEST_BLOCK_REASON = ( "no certifiable >=300-task repair: 0/32 evenly spaced normalized oracles " @@ -237,7 +238,11 @@ def _feasible_calendar(expected: dict[str, dict]) -> list[dict] | None: return None starts = [ start - for start in range(minimum, maximum - duration + 1, 15) + for start in range( + minimum, + maximum - duration + 1, + CALENDAR_GRANULARITY_MINUTES, + ) if _CHECK_CALENDAR_CONSTRAINT( spec.get("constraint"), start, start + duration ) @@ -246,32 +251,38 @@ def _feasible_calendar(expected: dict[str, dict]) -> list[dict] | None: return None candidates[event_id] = starts - order = sorted( - candidates, key=lambda event_id: (len(candidates[event_id]), event_id) - ) - placed: dict[int, int] = {} - - def search(index: int) -> bool: - if index == len(order): - return True - event_id = order[index] - duration = expected[str(event_id)]["duration"] - for start in candidates[event_id]: - end = start + duration - if any( - start < other_start + expected[str(other_id)]["duration"] - and other_start < end - for other_id, other_start in placed.items() - ): + event_ids = sorted(candidates) + states: dict[int, tuple[int, dict[int, int]]] = {0: (0, {})} + for mask in range(1 << len(event_ids)): + state = states.get(mask) + if state is None: + continue + previous_end, placements = state + for index, event_id in enumerate(event_ids): + bit = 1 << index + if mask & bit: + continue + start = next( + ( + candidate + for candidate in candidates[event_id] + if candidate >= previous_end + ), + None, + ) + if start is None: + continue + end = start + expected[str(event_id)]["duration"] + next_mask = mask | bit + existing = states.get(next_mask) + if existing is not None and existing[0] <= end: continue - placed[event_id] = start - if search(index + 1): - return True - del placed[event_id] - return False + states[next_mask] = (end, {**placements, event_id: start}) - if not search(0): + complete = states.get((1 << len(event_ids)) - 1) + if complete is None: return None + placed = complete[1] return [ { "event_id": event_id, diff --git a/data/tasktrove/build_swe_verifier_isolation_v49.py b/data/tasktrove/build_swe_verifier_isolation_v49.py new file mode 100644 index 00000000..78e66527 --- /dev/null +++ b/data/tasktrove/build_swe_verifier_isolation_v49.py @@ -0,0 +1,514 @@ +#!/usr/bin/env python3 +"""Build TaskTrove v4.9 SWE sources with isolated hidden-test installation.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import shlex +import subprocess +from dataclasses import dataclass +from pathlib import Path + +import pyarrow as pa +import pyarrow.parquet as pq +from harbor_config.models.task.config import TaskConfig +from huggingface_hub import hf_hub_download + +from data.nemotron_gym.converters.agent_calendar import _calendar_names_from_prompt +from data.nemotron_gym.verifiers import CALENDAR_VERIFIER_PY +from data.patchers.trusted_test_patch import ( + TRUSTED_TEST_PATCH_INSTALLER, + trusted_test_patch_command, +) +from data.tasktrove.build_data_quality_batch_a1 import _feasible_calendar +from data.tasktrove.build_storage_repair import ( + MAX_BATCH_ROWS, + MIN_TASKS, + REQUIRED_MEMBERS, + TASK_SCHEMA, + file_sha256, + read_task, + task_toml_with_memory, + task_toml_with_storage, + write_task, +) + +TASKTROVE_REPO = "open-thoughts/TaskTrove" +SOURCE_REVISION = "35c1139e8932344e9b52b231bca806d95f5d14cb" +INSTALLER_PATH = "tests/install_trusted_test_patch.sh" +SWE_REBENCH_STORAGE_MB = 8192 +SWE_REBENCH_MEMORY_MB = 4096 +OPENSWE_STORAGE_MB = 4096 +EXCLUDED_TASKS = { + "DCAgent__swe_rebench_v2_patched_oracle": { + "aallam__openai-kotlin-127", + "actix__actix-web-2624", + "act-rules__act-rules.github.io-1277", + "aio-libs__aiohttp-10551", + "aio-libs__aiohttp-8636", + "algebraicjulia__catlab.jl-227", + "denvercoder1__readme-typing-svg-213", + }, + "laion__openswe-tasks-patched-v6-oracle-success": { + "openswe_oss-00715", + }, +} + + +class UnrecoverableCalendarTask(ValueError): + """Raised when a calendar task cannot support exact event-name grading.""" + + +@dataclass(frozen=True) +class Revision: + source: str + source_sha256: str + output: str + verifier: str + + +REVISIONS = ( + Revision( + source="DCAgent__swe_rebench_v2_patched_oracle", + source_sha256="afd827fc1fc5c930736fef88c2b115631bf4a237128188a2cb2f9c3009ac3774", + output="DCAgent__swe_rebench_v2_patched_oracle-v2", + verifier="swe_rebench", + ), + Revision( + source="laion__openswe-tasks-patched-v6-oracle-success", + source_sha256="f9904208d0736ea3e8079dee2dd5633c2711b8d2a8ca3060c82794be2ae9f46f", + output="laion__openswe-tasks-patched-v7-oracle-success", + verifier="openswe", + ), + Revision( + source="laion__nemotron-gym-instruction-following-calendar-v2", + source_sha256="3d89709363b11a28387ccb367163644a1fa6b5aa2ff919aedaa7b63369256864", + output="laion__nemotron-gym-instruction-following-calendar-v3", + verifier="calendar", + ), +) + +SWE_REBENCH_PATCH_BLOCK = b"""\ +# Apply the hidden test patch (the tests the fix PR introduced) +cd /testbed +if [ -f /tests/test_patch.diff ]; then + git apply --verbose /tests/test_patch.diff || \\ + git apply --verbose --reject /tests/test_patch.diff || true +fi +""" + +OPENSWE_PATCH_BLOCK = b"""\ +cd /testbed || exit 0 +if [ -s /tests/test_patch.diff ]; then + git apply --check --allow-empty /tests/test_patch.diff || exit 0 + git apply --verbose --allow-empty /tests/test_patch.diff || exit 0 +fi +""" + +OPENSWE_SUCCESS_BLOCK = b"""\ +if [ "$runner_rc" -eq 0 ] && [ "$junit_rc" -eq 0 ] && [ "$pytest_guard_rc" -eq 0 ]; then + echo 1 > "$logs_dir/reward.txt" +fi +exit 0 +""" + + +def _replace_once(source: bytes, old: bytes, new: bytes, label: str) -> bytes: + if source.count(old) != 1: + raise ValueError(f"{label} does not match the expected source verifier") + return source.replace(old, new, 1) + + +def patch_swe_rebench_verifier(test_sh: bytes, base_commit: str) -> bytes: + """Replace partial hidden-patch application with isolated installation.""" + command = trusted_test_patch_command(base_commit).encode() + replacement = ( + b"""\ +# Restore hidden-test paths from the immutable base before applying the trusted +# patch. Product-code edits made by the agent remain untouched. +if ! """ + + command + + b"""; then + exit 1 +fi +""" + ) + return _replace_once( + test_sh, + SWE_REBENCH_PATCH_BLOCK, + replacement, + "SWE-ReBench hidden-test block", + ) + + +def patch_openswe_verifier(test_sh: bytes, base_commit: str) -> bytes: + """Isolate trusted tests and reserve reward zero for executed test failures.""" + transformed = _replace_once( + test_sh, + b"# OpenSWE v6 verifier: provision dependencies and score only executed tests.\n", + b"# OpenSWE v7 verifier: isolate trusted tests and score only executed tests.\n", + "OpenSWE verifier version", + ) + transformed = _replace_once( + transformed, + b'echo 0 > "$logs_dir/reward.txt"\n', + b'rm -f "$logs_dir/reward.txt"\n', + "OpenSWE initial reward", + ) + transformed = _replace_once( + transformed, + b" exit 0\nfi\nsource /tmp/openswe-setup-environment.sh || exit 0\n", + b' exit "$setup_rc"\nfi\nsource /tmp/openswe-setup-environment.sh || exit 1\n', + "OpenSWE setup failure handling", + ) + command = trusted_test_patch_command(base_commit).encode() + patch_block = ( + b"""\ +cd /testbed || exit 1 +# Restore hidden-test paths from the immutable base before applying the trusted +# patch. Product-code edits made by the agent remain untouched. +if ! """ + + command + + b"""; then + exit 1 +fi +""" + ) + transformed = _replace_once( + transformed, + OPENSWE_PATCH_BLOCK, + patch_block, + "OpenSWE hidden-test block", + ) + transformed = _replace_once( + transformed, + OPENSWE_SUCCESS_BLOCK, + b"""\ +if [ "$runner_rc" -eq 0 ] && [ "$junit_rc" -eq 0 ] && [ "$pytest_guard_rc" -eq 0 ]; then + echo 1 > "$logs_dir/reward.txt" +else + echo 0 > "$logs_dir/reward.txt" +fi +exit 0 +""", + "OpenSWE final reward", + ) + return transformed + + +def patch_calendar_verifier(files: dict[str, bytes]) -> dict[str, bytes]: + """Replace the event-local verifier with the complete calendar verifier.""" + instruction = files["instruction.md"].decode("utf-8", errors="replace") + if "overlap" not in instruction.lower(): + raise ValueError("calendar instruction does not declare the overlap constraint") + verifier_data = json.loads(files["tests/verifier_data.json"]) + expected = verifier_data.get("expected_events") + if not isinstance(expected, dict) or not expected: + raise ValueError("calendar verifier data has no expected events") + names = _calendar_names_from_prompt(instruction) + repaired: dict[str, dict[str, object]] = {} + for key, spec in expected.items(): + if not isinstance(spec, dict): + raise ValueError(f"calendar event {key!r} is not an object") + event_id = spec.get("event_id") + if not isinstance(event_id, int) or isinstance(event_id, bool): + raise ValueError(f"calendar event {key!r} has an invalid ID") + event_name = spec.get("event_name") or names.get(event_id) + if not isinstance(event_name, str) or not event_name.strip(): + raise UnrecoverableCalendarTask( + f"calendar event {event_id} has no recoverable name" + ) + repaired[str(event_id)] = {**spec, "event_name": event_name.strip()} + oracle = _feasible_calendar(repaired) + if oracle is None: + raise UnrecoverableCalendarTask( + "calendar task has no feasible conflict-free schedule" + ) + transformed = dict(files) + transformed["tests/verifier.py"] = CALENDAR_VERIFIER_PY.encode() + transformed["tests/verifier_data.json"] = json.dumps( + {**verifier_data, "expected_events": repaired}, + ensure_ascii=False, + sort_keys=True, + indent=2, + ).encode() + transformed["solution/answer.json"] = json.dumps( + oracle, ensure_ascii=False, sort_keys=True, indent=2 + ).encode() + transformed["solution/solve.sh"] = ( + b"#!/bin/bash\nset -eu\ncp /solution/answer.json /app/answer.txt\n" + ) + return transformed + + +def patch_paths(patch: bytes) -> set[str]: + """Return paths named by standard ``diff --git`` headers.""" + paths: set[str] = set() + fallback_paths: set[str] = set() + for raw_line in patch.decode("utf-8", errors="replace").splitlines(): + if raw_line.startswith(("--- ", "+++ ")): + value = raw_line[4:].split("\t", 1)[0] + if value.startswith('"'): + fields = shlex.split(value) + if len(fields) != 1: + raise ValueError(f"malformed patch path: {raw_line!r}") + value = fields[0] + if value.startswith(("a/", "b/")): + value = value[2:] + if value != "/dev/null": + paths.add(value) + continue + if raw_line.startswith(("rename from ", "rename to ")): + paths.add(raw_line.split(" ", 2)[2]) + continue + if not raw_line.startswith("diff --git "): + continue + payload = raw_line.removeprefix("diff --git ") + if payload.startswith('"'): + fields = shlex.split(payload) + if len(fields) != 2: + raise ValueError(f"malformed diff header: {raw_line!r}") + else: + old, separator, new = payload.partition(" b/") + if not separator or not old.startswith("a/"): + raise ValueError(f"malformed diff header: {raw_line!r}") + fields = [old, "b/" + new] + fallback_paths.update(path[2:] for path in fields) + return paths | fallback_paths + + +def transform_files(files: dict[str, bytes], revision: Revision) -> dict[str, bytes]: + if revision.verifier == "calendar": + return patch_calendar_verifier(files) + + config = json.loads(files["tests/config.json"]) + base_commit = str(config.get("base_commit") or "") + if not base_commit: + raise ValueError("task has no immutable base commit") + test_patch = files.get("tests/test_patch.diff", b"") + overlap = patch_paths(test_patch) & patch_paths(files.get("solution/solve.sh", b"")) + if overlap: + raise ValueError(f"golden and hidden patches overlap: {sorted(overlap)}") + + transformed = dict(files) + if revision.verifier == "swe_rebench": + transformed["tests/test.sh"] = patch_swe_rebench_verifier( + files["tests/test.sh"], base_commit + ) + transformed["task.toml"] = task_toml_with_storage( + task_toml_with_memory(files["task.toml"], None, SWE_REBENCH_MEMORY_MB), + None, + SWE_REBENCH_STORAGE_MB, + ) + elif revision.verifier == "openswe": + transformed["tests/test.sh"] = patch_openswe_verifier( + files["tests/test.sh"], base_commit + ) + transformed["task.toml"] = task_toml_with_storage( + files["task.toml"], None, OPENSWE_STORAGE_MB + ) + else: + raise ValueError(f"unknown verifier family: {revision.verifier}") + transformed[INSTALLER_PATH] = TRUSTED_TEST_PATCH_INSTALLER.encode() + return transformed + + +def source_path(args: argparse.Namespace, revision: Revision) -> Path: + if args.source_root is not None: + return args.source_root / revision.source / "tasks.parquet" + return Path( + hf_hub_download( + TASKTROVE_REPO, + f"{revision.source}/tasks.parquet", + repo_type="dataset", + revision=SOURCE_REVISION, + ) + ) + + +def validate_output(path: Path, revision: Revision, expected_rows: int) -> set[str]: + parquet = pq.ParquetFile(path) + if parquet.schema_arrow != TASK_SCHEMA: + raise ValueError(f"unexpected output schema: {parquet.schema_arrow}") + if parquet.metadata.num_rows != expected_rows or expected_rows < MIN_TASKS: + raise ValueError("output row count is invalid") + seen: set[str] = set() + images: set[str] = set() + shell_hashes: set[str] = set() + for batch in parquet.iter_batches(batch_size=MAX_BATCH_ROWS): + for row in batch.to_pylist(): + task_path = row["path"] + if task_path in seen: + raise ValueError(f"duplicate task path: {task_path}") + seen.add(task_path) + files = read_task(row["task_binary"]) + images.add(hashlib.sha256(files["environment/Dockerfile"]).hexdigest()) + if not REQUIRED_MEMBERS <= files.keys(): + raise ValueError(f"incomplete repaired task: {task_path}") + if revision.verifier == "calendar": + if files["tests/verifier.py"] != CALENDAR_VERIFIER_PY.encode(): + raise ValueError(f"stale calendar verifier remains: {task_path}") + if not {"solution/answer.json", "solution/solve.sh"} <= files.keys(): + raise ValueError(f"calendar oracle missing: {task_path}") + namespace = {"__name__": "tasktrove_calendar_validation"} + exec(files["tests/verifier.py"], namespace) + data = json.loads(files["tests/verifier_data.json"]) + oracle = json.loads(files["solution/answer.json"]) + valid, errors = namespace["evaluate_calendar"]( + data["expected_events"], oracle + ) + if not valid: + raise ValueError(f"calendar oracle fails in {task_path}: {errors}") + continue + if INSTALLER_PATH not in files: + raise ValueError(f"trusted installer missing: {task_path}") + test_sh = files["tests/test.sh"] + if ( + b"--reject" in test_sh + or b"|| true" + in test_sh.split(b"install_trusted_test_patch.sh", 1)[0][-160:] + ): + raise ValueError(f"best-effort hidden patch remains: {task_path}") + if b"/tests/install_trusted_test_patch.sh" not in test_sh: + raise ValueError(f"trusted installer is not invoked: {task_path}") + if revision.verifier == "openswe": + if b'echo 0 > "$logs_dir/reward.txt"' not in test_sh: + raise ValueError(f"OpenSWE has no scoreable-zero path: {task_path}") + prefix = test_sh.split(b'echo ">>>>> Start Test Output"', 1)[0] + if b'echo 0 > "$logs_dir/reward.txt"' in prefix: + raise ValueError(f"OpenSWE prewrites reward zero: {task_path}") + config = TaskConfig.model_validate_toml(files["task.toml"].decode()) + if config.environment.storage_mb != OPENSWE_STORAGE_MB: + raise ValueError(f"wrong OpenSWE storage: {task_path}") + elif revision.verifier == "swe_rebench": + config = TaskConfig.model_validate_toml(files["task.toml"].decode()) + if config.environment.memory_mb != SWE_REBENCH_MEMORY_MB: + raise ValueError(f"wrong SWE-ReBench memory: {task_path}") + if config.environment.storage_mb != SWE_REBENCH_STORAGE_MB: + raise ValueError(f"wrong SWE-ReBench storage: {task_path}") + for content in (test_sh, files[INSTALLER_PATH]): + digest = str(hash(content)) + if digest in shell_hashes: + continue + checked = subprocess.run( + ["bash", "-n"], input=content, capture_output=True, check=False + ) + if checked.returncode: + raise ValueError( + f"invalid shell in {task_path}: " + + checked.stderr.decode(errors="replace") + ) + shell_hashes.add(digest) + return images + + +def build_revision(args: argparse.Namespace, revision: Revision) -> dict[str, object]: + source = source_path(args, revision) + if file_sha256(source) != revision.source_sha256: + raise ValueError(f"source hash mismatch: {revision.source}") + parquet = pq.ParquetFile(source) + if parquet.schema_arrow != TASK_SCHEMA: + raise ValueError(f"unexpected source schema: {parquet.schema_arrow}") + output = args.stage / "datasets" / revision.output / "tasks.parquet" + output.parent.mkdir(parents=True, exist_ok=True) + if output.exists(): + raise FileExistsError(output) + writer = pq.ParquetWriter( + output, + TASK_SCHEMA, + compression="zstd", + use_dictionary=False, + write_statistics=True, + ) + rows = 0 + rejected_rows = 0 + test_patch_rows = 0 + try: + for batch in parquet.iter_batches(batch_size=MAX_BATCH_ROWS): + transformed_rows = [] + for row in batch.to_pylist(): + if row["path"] in EXCLUDED_TASKS.get(revision.source, set()): + rejected_rows += 1 + continue + files = read_task(row["task_binary"]) + if not REQUIRED_MEMBERS <= files.keys(): + raise ValueError(f"incomplete task: {row['path']}") + if files.get("tests/test_patch.diff", b"").strip(): + test_patch_rows += 1 + try: + transformed = transform_files(files, revision) + except UnrecoverableCalendarTask: + rejected_rows += 1 + continue + changed = { + name + for name in files.keys() | transformed.keys() + if files.get(name) != transformed.get(name) + } + expected_changed = {"tests/test.sh", INSTALLER_PATH} + if revision.verifier == "swe_rebench": + expected_changed.add("task.toml") + elif revision.verifier == "openswe": + expected_changed.add("task.toml") + elif revision.verifier == "calendar": + expected_changed = { + "solution/answer.json", + "solution/solve.sh", + "tests/verifier.py", + "tests/verifier_data.json", + } + if changed != expected_changed: + raise ValueError(f"unexpected changed members: {sorted(changed)}") + transformed_rows.append( + {"path": row["path"], "task_binary": write_task(transformed)} + ) + writer.write_table( + pa.Table.from_pylist(transformed_rows, schema=TASK_SCHEMA) + ) + rows += len(transformed_rows) + finally: + writer.close() + validate_output(output, revision, rows) + return { + "source": revision.source, + "source_sha256": revision.source_sha256, + "source_rows": parquet.metadata.num_rows, + "output": revision.output, + "output_rows": rows, + "rejected_rows": rejected_rows, + "output_sha256": file_sha256(output), + "parquet": str(output.relative_to(args.stage)), + "verifier": revision.verifier, + "rows_with_test_patch": test_patch_rows, + "rows_without_test_patch": rows - test_patch_rows, + "storage_mb": { + "swe_rebench": SWE_REBENCH_STORAGE_MB, + "openswe": OPENSWE_STORAGE_MB, + }.get(revision.verifier), + "memory_mb": { + "swe_rebench": SWE_REBENCH_MEMORY_MB, + }.get(revision.verifier), + } + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--stage", type=Path, required=True) + parser.add_argument("--source-root", type=Path) + args = parser.parse_args() + reports = [build_revision(args, revision) for revision in REVISIONS] + manifest = { + "source_repo": TASKTROVE_REPO, + "source_revision": SOURCE_REVISION, + "source_version": "4.8", + "target_version": "4.9", + "datasets": reports, + } + (args.stage / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n") + print(json.dumps(manifest, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/data/tasktrove/publish_v49_release.py b/data/tasktrove/publish_v49_release.py new file mode 100644 index 00000000..ffa6c495 --- /dev/null +++ b/data/tasktrove/publish_v49_release.py @@ -0,0 +1,198 @@ +#!/usr/bin/env python3 +"""Publish, verify, tag, and retire standalones for TaskTrove v4.9.""" + +from __future__ import annotations + +import argparse +import json +import os +import re +from pathlib import Path + +from huggingface_hub import ( + CommitOperationAdd, + CommitOperationDelete, + HfApi, + hf_hub_download, +) + +REPO_ID = "open-thoughts/TaskTrove" +VERSION = "4.9" + + +def _repo_files(api: HfApi, revision: str) -> dict[str, object]: + return { + item.path: item + for item in api.list_repo_tree( + REPO_ID, + repo_type="dataset", + revision=revision, + recursive=True, + expand=True, + ) + if hasattr(item, "blob_id") + } + + +def _identity(item: object) -> tuple[str, int]: + lfs = getattr(item, "lfs", None) + digest = lfs.sha256 if lfs is not None else str(getattr(item, "blob_id")) + return digest, int(getattr(item, "size")) + + +def _release_note(manifest: dict[str, object]) -> str: + return ( + "> **v4.9 (current)** — trusted-test and calendar-verifier remediation — " + "replaces three sources. The SWE-ReBench and OpenSWE verifiers restore " + "hidden-test paths from the immutable base commit before applying the " + "trusted patch, so agent edits cannot suppress or replace target tests; " + "patch and setup failures now remain infrastructure failures rather than " + "scoreable zeros. SWE-ReBench explicitly requests 4 GiB memory and " + "8 GiB storage; " + "OpenSWE requests 4 GiB. The " + "instruction-following calendar verifier now rejects pairwise overlaps " + "using half-open intervals and reports both event IDs and intervals. The " + "agent-calendar sibling was audited and already contained this check. " + f"The release uses {int(manifest['release_unique_images'])} unique images, " + "and versioned sources are hosted only inside TaskTrove. Superseded source " + "versions remain available through earlier TaskTrove tags.\n>\n" + "> - `DCAgent/swe_rebench_v2_patched_oracle` → " + "`DCAgent/swe_rebench_v2_patched_oracle-v2`\n" + "> - `laion/openswe-tasks-patched-v6-oracle-success` → " + "`laion/openswe-tasks-patched-v7-oracle-success`\n" + "> - `laion/nemotron-gym-instruction-following-calendar-v2` → " + "`laion/nemotron-gym-instruction-following-calendar-v3` (5,673 retained; " + "2,714 without recoverable exact event names removed)\n>\n" + ) + + +def _updated_readme(source: str, manifest: dict[str, object]) -> str: + marker = "> **v4.8 (current)**" + if marker not in source: + raise ValueError("README does not identify v4.8 as current") + return source.replace(marker, _release_note(manifest) + "> **v4.8**", 1) + + +def publish(stage: Path) -> str: + manifest = json.loads((stage / "manifest.json").read_text()) + source_revision = str(manifest["source_revision"]) + api = HfApi(token=os.environ["HF_TOKEN"]) + if api.repo_info(REPO_ID, repo_type="dataset").sha != source_revision: + raise ValueError("TaskTrove changed after the v4.9 build") + if any( + ref.name == f"v{VERSION}" + for ref in api.list_repo_refs(REPO_ID, repo_type="dataset").tags + ): + raise ValueError(f"v{VERSION} already exists") + current = _repo_files(api, source_revision) + for item in manifest["datasets"]: + source = f"{item['source']}/tasks.parquet" + target = f"{item['output']}/tasks.parquet" + if ( + source not in current + or _identity(current[source])[0] != item["source_sha256"] + ): + raise ValueError(f"source mismatch: {source}") + if target in current: + raise ValueError(f"target already exists: {target}") + + readme_source = Path( + hf_hub_download( + REPO_ID, + "README.md", + repo_type="dataset", + revision=source_revision, + token=api.token, + ) + ).read_text() + readme = stage / "README-v4.9.md" + readme.write_text(_updated_readme(readme_source, manifest)) + operations: list[CommitOperationAdd | CommitOperationDelete] = [ + CommitOperationAdd("README.md", readme) + ] + for item in manifest["datasets"]: + operations.extend( + ( + CommitOperationDelete(str(item["source"])), + CommitOperationAdd( + f"{item['output']}/tasks.parquet", stage / str(item["parquet"]) + ), + ) + ) + commit = api.create_commit( + REPO_ID, + repo_type="dataset", + operations=operations, + commit_message="TaskTrove v4.9: repair trusted tests and calendar scoring", + parent_commit=source_revision, + num_threads=1, + ) + return commit.oid + + +def verify_and_retire(stage: Path, commit: str) -> None: + manifest = json.loads((stage / "manifest.json").read_text()) + source_revision = str(manifest["source_revision"]) + api = HfApi(token=os.environ["HF_TOKEN"]) + before = _repo_files(api, source_revision) + after = _repo_files(api, commit) + removed = tuple(f"{item['source']}/" for item in manifest["datasets"]) + expected = { + path for path in before if path != "README.md" and not path.startswith(removed) + } + expected.add("README.md") + expected.update(f"{item['output']}/tasks.parquet" for item in manifest["datasets"]) + if set(after) != expected: + raise ValueError( + f"unexpected tree: missing={sorted(expected - set(after))}, " + f"extra={sorted(set(after) - expected)}" + ) + for path, item in before.items(): + if path == "README.md" or path.startswith(removed): + continue + if _identity(after[path]) != _identity(item): + raise ValueError(f"untouched file changed: {path}") + for item in manifest["datasets"]: + target = f"{item['output']}/tasks.parquet" + if _identity(after[target])[0] != item["output_sha256"]: + raise ValueError(f"output hash mismatch: {target}") + readme = Path( + hf_hub_download( + REPO_ID, "README.md", repo_type="dataset", revision=commit, token=api.token + ) + ).read_text() + if not re.search(r"^> \*\*v4\.9 \(current\)\*\*", readme, re.MULTILINE): + raise ValueError("README does not identify v4.9 as current") + api.create_tag(REPO_ID, tag="v4.9", repo_type="dataset", revision=commit) + repositories = { + str(item[field]).replace("__", "/", 1) + for item in manifest["datasets"] + for field in ("source", "output") + } + for repository in sorted(repositories): + if api.repo_exists(repository, repo_type="dataset"): + api.delete_repo(repository, repo_type="dataset") + if api.repo_exists(repository, repo_type="dataset"): + raise ValueError(f"standalone remains: {repository}") + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--stage", type=Path, required=True) + parser.add_argument("--commit") + parser.add_argument("--publish", action="store_true") + parser.add_argument("--verify-and-retire", action="store_true") + args = parser.parse_args() + commit = args.commit + if args.publish: + commit = publish(args.stage) + print(commit) + if args.verify_and_retire: + if commit is None: + raise ValueError("--commit is required") + verify_and_retire(args.stage, commit) + print(f"verified {commit}, tagged v4.9, and retired standalones") + + +if __name__ == "__main__": + main() diff --git a/tests/data/nemotron_gym/test_adapter.py b/tests/data/nemotron_gym/test_adapter.py index 6b817871..54ff9104 100644 --- a/tests/data/nemotron_gym/test_adapter.py +++ b/tests/data/nemotron_gym/test_adapter.py @@ -376,6 +376,73 @@ def test_calendar_verifier_accepts_complete_non_overlapping_schedule(): assert valid, errors +def test_calendar_verifier_accepts_back_to_back_half_open_intervals(): + expected, events = _valid_calendar_case() + expected["1"].update(constraint=None, min_time="10:00") + events[1]["start_time"] = "10:30" + + valid, errors = _calendar_evaluator()(expected, events) + + assert valid, errors + + +def test_calendar_verifier_reports_overlapping_ids_and_intervals(): + expected, events = _valid_calendar_case() + expected["1"].update(constraint=None, min_time="10:00") + events[1]["start_time"] = "10:15" + + valid, errors = _calendar_evaluator()(expected, events) + + assert not valid + assert errors == ["events 0 [10:00, 10:30) and 1 [10:15, 11:00) overlap"] + + +def test_calendar_verifier_rejects_same_start_and_unsorted_outer_overlap(): + expected, events = _valid_calendar_case() + expected["1"].update(constraint=None, min_time="10:00") + events[0].update(start_time="10:00", duration=45) + expected["0"]["duration"] = 45 + events[1]["start_time"] = "10:00" + events.reverse() + + valid, errors = _calendar_evaluator()(expected, events) + + assert not valid + assert errors == ["events 0 [10:00, 10:45) and 1 [10:00, 10:45) overlap"] + + +def test_calendar_verifier_reports_every_conflicting_pair(): + expected, events = _valid_calendar_case() + expected["0"]["duration"] = 120 + expected["1"].update(duration=15, constraint=None, min_time="10:00") + expected["2"] = { + "event_id": 2, + "event_name": "C", + "duration": 15, + "min_time": "10:00", + "max_time": "16:00", + "constraint": None, + } + events = [ + {"event_id": 2, "event_name": "C", "start_time": "11:00", "duration": 15}, + { + "event_id": 0, + "event_name": "Design Review", + "start_time": "10:00", + "duration": 120, + }, + {"event_id": 1, "event_name": "Lunch", "start_time": "10:15", "duration": 15}, + ] + + valid, errors = _calendar_evaluator()(expected, events) + + assert not valid + assert errors == [ + "events 0 [10:00, 12:00) and 1 [10:15, 10:30) overlap", + "events 0 [10:00, 12:00) and 2 [11:00, 11:15) overlap", + ] + + def test_calendar_verifier_leaves_no_reward_for_corrupt_verifier_data(tmp_path): namespace = _calendar_namespace() namespace["DATA"] = tmp_path / "verifier_data.json" diff --git a/tests/data/patchers/test_trusted_test_patch.py b/tests/data/patchers/test_trusted_test_patch.py new file mode 100644 index 00000000..ca381dee --- /dev/null +++ b/tests/data/patchers/test_trusted_test_patch.py @@ -0,0 +1,97 @@ +from __future__ import annotations + +import subprocess +from pathlib import Path + +from data.patchers.trusted_test_patch import ( + TRUSTED_TEST_PATCH_INSTALLER, + write_trusted_test_patch_installer, +) + + +def _git(repo: Path, *args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + ["git", *args], + cwd=repo, + check=True, + capture_output=True, + text=True, + ) + + +def _base_repository(tmp_path: Path) -> tuple[Path, str]: + repo = tmp_path / "repo" + repo.mkdir() + _git(repo, "init", "--quiet") + _git(repo, "config", "user.email", "tests@example.com") + _git(repo, "config", "user.name", "Test Author") + (repo / "src").mkdir() + (repo / "tests").mkdir() + (repo / "src" / "feature.py").write_text("VALUE = 'base'\n") + (repo / "tests" / "test_feature.py").write_text("EXPECTED = 'old'\n") + _git(repo, "add", ".") + _git(repo, "commit", "--quiet", "-m", "base") + return repo, _git(repo, "rev-parse", "HEAD").stdout.strip() + + +def _trusted_patch(repo: Path, patch_path: Path) -> None: + (repo / "tests" / "test_feature.py").write_text("EXPECTED = 'trusted'\n") + (repo / "tests" / "test_hidden.py").write_text("HIDDEN = True\n") + _git(repo, "add", "--intent-to-add", "tests/test_hidden.py") + patch_path.write_text(_git(repo, "diff", "--binary").stdout) + _git(repo, "reset", "--hard", "--quiet", "HEAD") + _git(repo, "clean", "-fd", "--quiet") + + +def test_installer_replaces_agent_test_edits_and_preserves_source_edits( + tmp_path: Path, +) -> None: + repo, base_commit = _base_repository(tmp_path) + patch_path = tmp_path / "test_patch.diff" + _trusted_patch(repo, patch_path) + + (repo / "src" / "feature.py").write_text("VALUE = 'agent fix'\n") + (repo / "tests" / "test_feature.py").write_text("EXPECTED = 'agent bypass'\n") + (repo / "tests" / "test_hidden.py").write_text("HIDDEN = False\n") + + installer = tmp_path / "install_trusted_test_patch.sh" + write_trusted_test_patch_installer(installer) + subprocess.run( + ["bash", str(installer), str(repo), str(patch_path), base_commit], + check=True, + ) + + assert (repo / "src" / "feature.py").read_text() == "VALUE = 'agent fix'\n" + assert (repo / "tests" / "test_feature.py").read_text() == "EXPECTED = 'trusted'\n" + assert (repo / "tests" / "test_hidden.py").read_text() == "HIDDEN = True\n" + + +def test_installer_fails_when_trusted_patch_cannot_be_installed(tmp_path: Path) -> None: + repo, base_commit = _base_repository(tmp_path) + patch_path = tmp_path / "test_patch.diff" + patch_path.write_text( + "diff --git a/tests/missing.py b/tests/missing.py\n" + "--- a/tests/missing.py\n" + "+++ b/tests/missing.py\n" + "@@ -1 +1 @@\n" + "-missing\n" + "+trusted\n" + ) + installer = tmp_path / "install_trusted_test_patch.sh" + write_trusted_test_patch_installer(installer) + + result = subprocess.run( + ["bash", str(installer), str(repo), str(patch_path), base_commit], + check=False, + ) + + assert result.returncode != 0 + + +def test_installer_does_not_require_git_apply_allow_empty() -> None: + assert "--allow-empty" not in TRUSTED_TEST_PATCH_INSTALLER + + +def test_installer_marks_only_the_target_repository_as_safe() -> None: + assert 'git -c safe.directory="$repository"' in TRUSTED_TEST_PATCH_INSTALLER + assert "git config --global" not in TRUSTED_TEST_PATCH_INSTALLER diff --git a/tests/data/tasktrove/test_build_data_quality_batch_a1.py b/tests/data/tasktrove/test_build_data_quality_batch_a1.py index e54d4ee9..df54d551 100644 --- a/tests/data/tasktrove/test_build_data_quality_batch_a1.py +++ b/tests/data/tasktrove/test_build_data_quality_batch_a1.py @@ -6,7 +6,11 @@ import pytest -from data.tasktrove.build_data_quality_batch_a1 import build, read_task +from data.tasktrove.build_data_quality_batch_a1 import ( + _feasible_calendar, + build, + read_task, +) def _archive(members: list[tuple[tarfile.TarInfo, bytes]]) -> bytes: @@ -58,3 +62,25 @@ def test_rspec_build_is_blocked_without_output(tmp_path) -> None: assert report["status"] == "blocked" assert report["probe_false_positives"] == 3 assert not list(stage.rglob("*.parquet")) + + +def test_calendar_solver_supports_five_minute_constraints() -> None: + expected = { + "0": { + "event_id": 0, + "event_name": "Five-minute boundary", + "duration": 10, + "min_time": "10:05", + "max_time": "10:15", + "constraint": "at 10:05am", + } + } + + assert _feasible_calendar(expected) == [ + { + "event_id": 0, + "event_name": "Five-minute boundary", + "start_time": "10:05", + "duration": 10, + } + ] diff --git a/tests/data/tasktrove/test_build_swe_verifier_isolation_v49.py b/tests/data/tasktrove/test_build_swe_verifier_isolation_v49.py new file mode 100644 index 00000000..cbc50fe9 --- /dev/null +++ b/tests/data/tasktrove/test_build_swe_verifier_isolation_v49.py @@ -0,0 +1,187 @@ +import json + +import pytest +from harbor_config.models.task.config import TaskConfig + +from data.tasktrove.build_swe_verifier_isolation_v49 import ( + OPENSWE_PATCH_BLOCK, + OPENSWE_SUCCESS_BLOCK, + SWE_REBENCH_PATCH_BLOCK, + REVISIONS, + patch_paths, + patch_openswe_verifier, + patch_calendar_verifier, + patch_swe_rebench_verifier, + read_task, + transform_files, + write_task, +) +from data.nemotron_gym.verifiers import CALENDAR_VERIFIER_PY + + +BASE_COMMIT = "0123456789abcdef0123456789abcdef01234567" + + +def test_swe_rebench_verifier_rejects_partial_hidden_patch_application() -> None: + transformed = patch_swe_rebench_verifier( + b"#!/bin/bash\n" + SWE_REBENCH_PATCH_BLOCK + b"echo test-run\n", + BASE_COMMIT, + ) + + assert b"install_trusted_test_patch.sh" in transformed + assert b"--reject" not in transformed + assert b"exit 1" in transformed + + +def test_openswe_setup_failure_does_not_emit_reward() -> None: + source = ( + b"#!/bin/bash\n" + b"# OpenSWE v6 verifier: provision dependencies and score only executed tests.\n" + b"logs_dir=/logs/verifier\n" + b'mkdir -p "$logs_dir"\n' + b'echo 0 > "$logs_dir/reward.txt"\n' + b"bash /tests/setup_files/setup.sh\n" + b"setup_rc=$?\n" + b'if [ "$setup_rc" -ne 0 ]; then\n' + b' echo "OPENSWE_SETUP_EXIT_CODE=$setup_rc"\n' + b" exit 0\n" + b"fi\n" + b"source /tmp/openswe-setup-environment.sh || exit 0\n" + + OPENSWE_PATCH_BLOCK + + b'echo ">>>>> Start Test Output"\n' + + OPENSWE_SUCCESS_BLOCK + ) + + transformed = patch_openswe_verifier(source, BASE_COMMIT) + before_tests = transformed.split(b'echo ">>>>> Start Test Output"', 1)[0] + + assert b'rm -f "$logs_dir/reward.txt"' in before_tests + assert b'echo 0 > "$logs_dir/reward.txt"' not in before_tests + assert b'exit "$setup_rc"' in before_tests + assert b"install_trusted_test_patch.sh" in before_tests + assert b'echo 0 > "$logs_dir/reward.txt"' in transformed + + +def test_verifier_transform_rejects_unknown_source_contract() -> None: + with pytest.raises(ValueError, match="does not match"): + patch_swe_rebench_verifier(b"#!/bin/bash\necho changed\n", BASE_COMMIT) + + +def test_patch_paths_preserves_unquoted_spaces() -> None: + patch = ( + b"diff --git a/Sources/Parsable Types/Value.swift " + b"b/Sources/Parsable Types/Value.swift\n" + ) + + assert patch_paths(patch) == {"Sources/Parsable Types/Value.swift"} + + +def test_swe_rebench_successor_requests_explicit_memory_and_storage() -> None: + files = { + "task.toml": b'version = "1.0"\n', + "tests/config.json": json.dumps({"base_commit": BASE_COMMIT}).encode(), + "tests/test.sh": b"#!/bin/bash\n" + SWE_REBENCH_PATCH_BLOCK, + "tests/test_patch.diff": b"", + "solution/solve.sh": b"#!/bin/bash\n", + } + + transformed = transform_files(files, REVISIONS[0]) + config = TaskConfig.model_validate_toml(transformed["task.toml"].decode()) + + assert config.environment.memory_mb == 4096 + assert config.environment.storage_mb == 8192 + + +def test_openswe_successor_requests_four_gibibytes_storage() -> None: + files = { + "task.toml": b'version = "1.0"\n', + "tests/config.json": json.dumps({"base_commit": BASE_COMMIT}).encode(), + "tests/test.sh": ( + b"#!/bin/bash\n" + b"# OpenSWE v6 verifier: provision dependencies and score only executed tests.\n" + b'logs_dir=/logs/verifier\nmkdir -p "$logs_dir"\n' + b'echo 0 > "$logs_dir/reward.txt"\n' + b"bash /tests/setup_files/setup.sh\nsetup_rc=$?\n" + b'if [ "$setup_rc" -ne 0 ]; then\n' + b' echo "OPENSWE_SETUP_EXIT_CODE=$setup_rc"\n exit 0\nfi\n' + b"source /tmp/openswe-setup-environment.sh || exit 0\n" + + OPENSWE_PATCH_BLOCK + + b'echo ">>>>> Start Test Output"\n' + + OPENSWE_SUCCESS_BLOCK + ), + "tests/test_patch.diff": b"", + "solution/solve.sh": b"#!/bin/bash\n", + } + + transformed = transform_files(files, REVISIONS[1]) + config = TaskConfig.model_validate_toml(transformed["task.toml"].decode()) + + assert config.environment.storage_mb == 4096 + + +def test_calendar_successor_replaces_stale_packaged_verifier() -> None: + stale = b"def evaluate_calendar(expected, events):\n return True, []\n" + + transformed = patch_calendar_verifier( + { + "instruction.md": b"Ensure that there are no conflicts (overlapping events).", + "tests/verifier.py": stale, + "tests/verifier_data.json": json.dumps( + { + "expected_events": { + "0": { + "event_id": 0, + "event_name": "A", + "duration": 30, + "min_time": "10:00", + "max_time": "12:00", + "constraint": None, + }, + "1": { + "event_id": 1, + "event_name": "B", + "duration": 30, + "min_time": "10:00", + "max_time": "12:00", + "constraint": None, + }, + } + } + ).encode(), + } + ) + + packaged = read_task(write_task(transformed)) + namespace = {"__name__": "packaged_calendar_test"} + exec(packaged["tests/verifier.py"], namespace) + expected = { + "0": { + "event_id": 0, + "event_name": "A", + "duration": 30, + "min_time": "10:00", + "max_time": "12:00", + "constraint": None, + }, + "1": { + "event_id": 1, + "event_name": "B", + "duration": 30, + "min_time": "10:00", + "max_time": "12:00", + "constraint": None, + }, + } + overlap = [ + {"event_id": 0, "event_name": "A", "start_time": "10:00", "duration": 30}, + {"event_id": 1, "event_name": "B", "start_time": "10:15", "duration": 30}, + ] + back_to_back = [overlap[0], {**overlap[1], "start_time": "10:30"}] + + assert packaged["tests/verifier.py"] == CALENDAR_VERIFIER_PY.encode() + assert packaged["instruction.md"].count(b"overlap") == 1 + oracle = json.loads(packaged["solution/answer.json"]) + assert namespace["evaluate_calendar"](expected, oracle) == (True, []) + assert b"/solution/answer.json /app/answer.txt" in packaged["solution/solve.sh"] + assert namespace["evaluate_calendar"](expected, overlap)[0] is False + assert namespace["evaluate_calendar"](expected, back_to_back) == (True, [])