Repair SWE test isolation and calendar overlap grading

data/nemotron_gym/converters/agent_calendar.py

@@ -1,8 +1,7 @@
 """Convert nvidia/Nemotron-RL-agent-calendar_scheduling.
 
-The expected output of the agent is a JSON list of events. The verifier
-(calendar_constraints) checks duration / time-window / natural-language
-constraint per event from `exp_cal_state`.
+The expected output of the agent is a JSON list of events. The verifier checks
+the exact event set, each event's local constraints, and pairwise non-overlap.
 """
 
 from __future__ import annotations
@@ -27,8 +26,9 @@
     "You are scheduling events on a calendar. Read the conversation below and "
     "write your final calendar as a JSON list to `/app/answer.txt`. Each event "
     "must include `event_id` (int), `event_name` (str), `start_time` "
-    '("HH:MM"), and `duration` (minutes). The verifier checks duration, '
-    "time-window, and any natural-language constraint per event.\n\n"
+    '("HH:MM"), and `duration` (minutes). Events must not overlap. The verifier '
+    "checks the exact event set, duration, time window, declared constraints, "
+    "and pairwise overlap.\n\n"
     "---\n\n"
 )
 _MAX_EVENTS = 32

data/nemotron_gym/upload_to_hf.py

@@ -207,9 +207,9 @@
     "knowledge-openqa-v2": "llm_judge (LiteLLM, default `openai/gpt-4o-mini`, scored against reference answers as rubric — semantic-equivalence judging for paraphrasable long-form answers)",
     "instruction-following": "ifeval_constraints (IFEval-style constraint checker — paragraphs / words / forbidden / formatting)",
     "instruction-following-structured": "json_schema (parse agent JSON, validate against Draft 2020-12 schema)",
-    "instruction-following-calendar": "calendar_constraints (parse agent JSON list, check duration/window/constraint per event)",
+    "instruction-following-calendar": "calendar_constraints (parse agent JSON list; check exact event set, duration, window, declared constraints, and pairwise non-overlap)",
     "reasoning-gym": "reasoning_gym (delegate to upstream reasoning_gym scorer; normalized-match fallback)",
-    "agent-calendar": "calendar_constraints (same as instruction-following-calendar)",
+    "agent-calendar": "calendar_constraints (same complete contract as instruction-following-calendar)",
     "agent-workplace": "tool_call_match (JSON `{name, arguments}` compared to ground_truth tool calls; lossy single-step substitute for upstream stateful env)",
     "safety": "llm_judge (LiteLLM, default `openai/gpt-4o-mini`, scored against `principle` rubric)",
     "safety-v2": "safety_judge (LiteLLM, default `openai/gpt-4o-mini`, scored against `principle` rubric, with a heuristic refusal-detection fallback for sandboxes without `OPENAI_API_KEY` and a `/logs/agent` scan for the agent's response when `/app/response.txt` is missing — v2 fixes the 0% solve rate of v1)",

data/nemotron_gym/verifiers/calendar_constraints.py

@@ -155,9 +155,20 @@ def evaluate_calendar(expected: object, events: object) -> tuple[bool, list[str]
         intervals.append((start, end, event_id))
 
     intervals.sort()
-    for previous, current in zip(intervals, intervals[1:]):
-        if current[0] < previous[1]:
-            errors.append(f"events {previous[2]} and {current[2]} overlap")
+    for index, previous in enumerate(intervals):
+        for current in intervals[index + 1 :]:
+            if current[0] >= previous[1]:
+                break
+            previous_start, previous_end, previous_id = previous
+            current_start, current_end, current_id = current
+            errors.append(
+                f"events {previous_id} "
+                f"[{previous_start // 60:02d}:{previous_start % 60:02d}, "
+                f"{previous_end // 60:02d}:{previous_end % 60:02d}) and "
+                f"{current_id} "
+                f"[{current_start // 60:02d}:{current_start % 60:02d}, "
+                f"{current_end // 60:02d}:{current_end % 60:02d}) overlap"
+            )
     return not errors, errors
 
 

data/openswe/generate.py

@@ -34,6 +34,11 @@
 
 from tqdm import tqdm
 
+from data.patchers.trusted_test_patch import (
+    trusted_test_patch_command,
+    write_trusted_test_patch_installer,
+)
+
 
 # ---------------------------------------------------------------------------
 # Task configuration
@@ -218,7 +223,7 @@ def _extract_test_section(eval_script: str) -> str:
     )
 
 
-def create_test_sh(eval_script: str, test_patch: str) -> str:
+def create_test_sh(eval_script: str, test_patch: str, base_commit: str) -> str:
     """
     Build Harbor test.sh.
 
@@ -234,12 +239,13 @@ def create_test_sh(eval_script: str, test_patch: str) -> str:
 
     patch_block = ""
     if has_patch:
-        patch_block = """\
+        install_command = trusted_test_patch_command(base_commit)
+        patch_block = f"""\
 
-# Apply test patch (adds new test files; does NOT contain the golden fix)
-if [ -f /tests/test_patch.diff ]; then
-    git apply -v --allow-empty /tests/test_patch.diff || \\
-    git apply -v --allow-empty --reject /tests/test_patch.diff || true
+# Restore every hidden-test path from the immutable base commit before applying
+# the trusted patch. Agent edits to product paths remain untouched.
+if ! {install_command}; then
+    exit 1
 fi
 """
 
@@ -332,14 +338,19 @@ def create_task_dir(datum: dict, out_root: Path, idx: int, prefix: str) -> None:
     eval_script = datum.get("eval_script") or ""
     test_patch = datum.get("test_patch") or ""
     test_sh_path = d / "tests" / "test.sh"
-    test_sh_path.write_text(create_test_sh(eval_script, test_patch), encoding="utf-8")
+    test_sh_path.write_text(
+        create_test_sh(eval_script, test_patch, base_commit), encoding="utf-8"
+    )
     test_sh_path.chmod(
         test_sh_path.stat().st_mode | stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH
     )
 
     # tests/test_patch.diff
     if test_patch:
         (d / "tests" / "test_patch.diff").write_text(test_patch, encoding="utf-8")
+        write_trusted_test_patch_installer(
+            d / "tests" / "install_trusted_test_patch.sh"
+        )
 
     # tests/config.json — metadata without large blobs (stored elsewhere)
     config = {

data/patchers/patch_openswe_tasks.py

@@ -49,6 +49,11 @@
 from pathlib import Path
 from typing import Optional
 
+from data.patchers.trusted_test_patch import (
+    trusted_test_patch_command,
+    write_trusted_test_patch_installer,
+)
+
 # ---------------------------------------------------------------------------
 # Python version normalisation
 # ---------------------------------------------------------------------------
@@ -437,6 +442,31 @@ def wrap_solve_sh(original_solve_sh: str) -> str:
     return header + body
 
 
+def install_trusted_test_patch_in_verifier(test_sh: str, base_commit: str) -> str:
+    """Replace the legacy best-effort hidden-patch block with fail-closed setup."""
+    if "/tests/install_trusted_test_patch.sh" in test_sh:
+        return test_sh
+
+    legacy_block = """\
+# Apply test patch (adds new test files; does NOT contain the golden fix)
+if [ -f /tests/test_patch.diff ]; then
+    git apply -v --allow-empty /tests/test_patch.diff || \\
+    git apply -v --allow-empty --reject /tests/test_patch.diff || true
+fi
+"""
+    if legacy_block not in test_sh:
+        raise ValueError("OpenSWE verifier has an unrecognized hidden-test patch block")
+
+    command = trusted_test_patch_command(base_commit)
+    trusted_block = f"""\
+# Restore hidden-test paths from the immutable base before applying the trusted patch.
+if ! {command}; then
+    exit 1
+fi
+"""
+    return test_sh.replace(legacy_block, trusted_block, 1)
+
+
 # ---------------------------------------------------------------------------
 # Per-task patching
 # ---------------------------------------------------------------------------
@@ -510,6 +540,11 @@ def get_post_workdir(text: str) -> str:
     solve_path = src_dir / "solution" / "solve.sh"
     original_solve = solve_path.read_text() if solve_path.exists() else ""
     new_solve = wrap_solve_sh(original_solve) if original_solve else ""
+    test_sh_path = src_dir / "tests" / "test.sh"
+    test_patch_path = src_dir / "tests" / "test_patch.diff"
+    new_test_sh = test_sh_path.read_text() if test_sh_path.exists() else ""
+    if test_patch_path.exists():
+        new_test_sh = install_trusted_test_patch_in_verifier(new_test_sh, base_commit)
 
     if dry_run:
         return result
@@ -544,6 +579,13 @@ def get_post_workdir(text: str) -> str:
             new_solve_path.stat().st_mode | stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH
         )
 
+    if test_patch_path.exists():
+        output_test_sh = dst_dir / "tests" / "test.sh"
+        output_test_sh.write_text(new_test_sh)
+        write_trusted_test_patch_installer(
+            dst_dir / "tests" / "install_trusted_test_patch.sh"
+        )
+
     return result
 
 

data/patchers/patch_swe_rebench_v2_tasks.py

@@ -68,6 +68,11 @@
 from textwrap import dedent
 from typing import Iterable, Iterator
 
+from data.patchers.trusted_test_patch import (
+    trusted_test_patch_command,
+    write_trusted_test_patch_installer,
+)
+
 
 # ---------------------------------------------------------------------------
 # Base image map
@@ -332,11 +337,10 @@ def _make_dockerfile(base_image: str, family: str) -> str:
 
     mkdir -p /logs/verifier
 
-    # Apply the hidden test patch (the tests the fix PR introduced)
-    cd /testbed
-    if [ -f /tests/test_patch.diff ]; then
-        git apply --verbose /tests/test_patch.diff || \\
-            git apply --verbose --reject /tests/test_patch.diff || true
+    # Restore hidden-test paths from the immutable base before applying the
+    # trusted patch. Product-code edits made by the agent remain untouched.
+    if ! {trusted_test_patch_command}; then
+        exit 1
     fi
 
     # Run the task's test command (per-row from install_config.test_cmd).
@@ -1099,7 +1103,10 @@ def patch_row(row: dict, output_root: Path, dry_run: bool = False) -> dict:
 
     # 3. tests/test.sh
     (task_dir / "tests" / "test.sh").write_text(
-        TEST_SH_TEMPLATE.format(test_cmd=test_cmd)
+        TEST_SH_TEMPLATE.format(
+            test_cmd=test_cmd,
+            trusted_test_patch_command=trusted_test_patch_command(base_commit),
+        )
     )
 
     # 4. tests/test_state.py
@@ -1126,6 +1133,9 @@ def patch_row(row: dict, output_root: Path, dry_run: bool = False) -> dict:
     # 6. tests/test_patch.diff
     if test_patch and test_patch.strip():
         (task_dir / "tests" / "test_patch.diff").write_text(test_patch)
+        write_trusted_test_patch_installer(
+            task_dir / "tests" / "install_trusted_test_patch.sh"
+        )
 
     # 7. solution/solve.sh
     (task_dir / "solution" / "solve.sh").write_text(

data/patchers/trusted_test_patch.py

@@ -0,0 +1,72 @@
+"""Install hidden test patches independently of agent workspace edits."""
+
+from __future__ import annotations
+
+import stat
+import shlex
+from pathlib import Path
+
+
+TRUSTED_TEST_PATCH_INSTALLER = r"""#!/bin/bash
+set -Eeuo pipefail
+
+repository=${1:?repository path is required}
+patch_path=${2:?test patch path is required}
+base_commit=${3:?base commit is required}
+
+cd "$repository"
+git -c safe.directory="$repository" cat-file -e "${base_commit}^{commit}"
+
+if [ ! -s "$patch_path" ]; then
+    exit 0
+fi
+
+# Validate the patch before changing the workspace. --numstat -z makes path
+# records unambiguous even when a filename contains spaces.
+git -c safe.directory="$repository" -c core.quotePath=false \
+    apply --numstat -z "$patch_path" >/dev/null
+
+while IFS= read -r -d '' record; do
+    path=${record#*$'\t'}
+    path=${path#*$'\t'}
+    if [ "$path" = "$record" ]; then
+        echo "Malformed git-apply path record" >&2
+        exit 1
+    fi
+    case "$path" in
+        ""|/*|.|..|../*|*/..|*/../*)
+            echo "Unsafe hidden-test path: $path" >&2
+            exit 1
+            ;;
+    esac
+
+    # The hidden patch owns these exact paths. Remove untracked/ignored agent
+    # replacements and restore tracked files from the immutable task base.
+    git -c safe.directory="$repository" clean -ffdx -- "$path"
+    if git -c safe.directory="$repository" \
+        cat-file -e "${base_commit}:${path}" 2>/dev/null; then
+        git -c safe.directory="$repository" \
+            restore --source="$base_commit" --staged --worktree -- "$path"
+    fi
+done < <(git -c safe.directory="$repository" -c core.quotePath=false \
+    apply --numstat -z "$patch_path")
+
+git -c safe.directory="$repository" apply --check "$patch_path"
+git -c safe.directory="$repository" apply --verbose "$patch_path"
+"""
+
+
+def write_trusted_test_patch_installer(path: Path) -> None:
+    """Write the executable hidden-test installer used by packaged verifiers."""
+    path.write_text(TRUSTED_TEST_PATCH_INSTALLER, encoding="utf-8")
+    path.chmod(path.stat().st_mode | stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH)
+
+
+def trusted_test_patch_command(base_commit: str) -> str:
+    """Return the verifier command anchored to the task's immutable base commit."""
+    if not base_commit:
+        raise ValueError("base commit is required for trusted hidden-test installation")
+    return (
+        "bash /tests/install_trusted_test_patch.sh "
+        f"/testbed /tests/test_patch.diff {shlex.quote(base_commit)}"
+    )