fix: stop awm code verifiers from passing on an untouched database

envs/agent_world_model_env/server/_verifier_runner.py

@@ -27,8 +27,17 @@
         For mode == "code":
             {"result": "complete"|"others",
              "execution_status": "success"|"error",
+             "uses_final_answer"?: bool,
              "raw_result"?: <verifier return>,
-             "error_message"?: str}
+             "error_message"?: str,
+             "error_kind"?: "compile"|"not_found"|"execution"|"invalid_return"}
+
+    ``uses_final_answer`` reports the verifier contract: whether the
+    verifier function consumes the final answer (a ``final_answer``
+    parameter, or a positional parameter in that slot). ``error_kind``
+    separates failures raised by the verifier's own code ("execution")
+    from broken verifier code; subprocess/infrastructure failures are
+    produced by the parent process and carry neither field.
 """
 
 from __future__ import annotations
@@ -291,6 +300,20 @@ def _sanitize_for_json(obj: Any) -> Any:
         return str(obj)
 
 
+def _verifier_uses_final_answer(verify_func: Any) -> bool:
+    """Whether calling the verifier passes it the final answer.
+
+    Mirrors the dispatch in _call_verifier_func: the answer is passed
+    when the function declares a ``final_answer`` parameter, or when it
+    is called positionally with at least three parameters.
+    """
+    params = inspect.signature(verify_func).parameters
+    if "final_answer" in params:
+        return True
+    named = {"initial_db_path", "final_db_path"} & params.keys()
+    return not named and len(params) >= 3
+
+
 def _call_verifier_func(
     verify_func: Any,
     initial_db_path: str,
@@ -341,6 +364,7 @@ def _run(payload: dict[str, Any]) -> dict[str, Any]:
             return {
                 "result": "others",
                 "execution_status": "error",
+                "error_kind": "compile",
                 "error_message": f"Compile error: {e}",
             }
         return {"execution_status": "error", "error_message": f"Compile error: {e}"}
@@ -351,6 +375,7 @@ def _run(payload: dict[str, Any]) -> dict[str, Any]:
             return {
                 "result": "others",
                 "execution_status": "error",
+                "error_kind": "not_found",
                 "error_message": f"Function '{function_name}' not found",
             }
         return {
@@ -367,6 +392,7 @@ def _run(payload: dict[str, Any]) -> dict[str, Any]:
             return {
                 "result": "others",
                 "execution_status": "error",
+                "error_kind": "execution",
                 "error_message": f"Execution error: {e}",
             }
         return {"execution_status": "error", "error_message": f"Execution error: {e}"}
@@ -376,6 +402,7 @@ def _run(payload: dict[str, Any]) -> dict[str, Any]:
             return {
                 "result": "others",
                 "execution_status": "error",
+                "error_kind": "invalid_return",
                 "error_message": f"Invalid return format: {type(result).__name__}",
             }
         result_value = result.get("result", "others")
@@ -384,6 +411,7 @@ def _run(payload: dict[str, Any]) -> dict[str, Any]:
         return {
             "result": result_value,
             "execution_status": "success",
+            "uses_final_answer": _verifier_uses_final_answer(verify_func),
             "raw_result": _sanitize_for_json(result),
         }
 

envs/agent_world_model_env/server/verifier.py

@@ -10,9 +10,11 @@
 import logging
 import os
 import re
+import sqlite3
 import subprocess
 import sys
 import textwrap
+from pathlib import Path
 from typing import Any
 
 from openai import AsyncOpenAI
@@ -184,6 +186,18 @@ def execute_code_verifier(
     return result
 
 
+def _database_contents_equal(initial_db_path: str, final_db_path: str) -> bool:
+    """Compare SQLite contents, including committed WAL changes."""
+    dumps = []
+    for path in (initial_db_path, final_db_path):
+        conn = sqlite3.connect(Path(path).resolve().as_uri() + "?mode=ro", uri=True)
+        try:
+            dumps.append("\n".join(conn.iterdump()))
+        finally:
+            conn.close()
+    return dumps[0] == dumps[1]
+
+
 def run_verifier(
     verifier_entry: dict,
     verifier_mode: str,
@@ -219,7 +233,45 @@ def run_verifier(
         )
         if result.get("execution_status") == "error":
             return "judge_error", result
-        return result.get("result", "others"), result
+        reward_type = result.get("result", "others")
+        if reward_type == "complete":
+            try:
+                unchanged = _database_contents_equal(initial_db_path, final_db_path)
+            except (OSError, sqlite3.Error) as exc:
+                return "judge_error", {
+                    "execution_status": "error",
+                    "error_message": f"Database comparison failed: {exc}",
+                }
+            if unchanged:
+                if (
+                    final_answer
+                    and final_answer.strip()
+                    and result.get("uses_final_answer") is not False
+                ):
+                    # Answer-task baseline contract: rerun the same
+                    # verifier with an empty answer on the same databases.
+                    # The two runs differ only in the answer, so a
+                    # different outcome is the answer's doing:
+                    # - baseline "others": the answer earned the pass.
+                    # - any baseline error fails closed as judge_error.
+                    #   Verifiers for answer tasks must return "others"
+                    #   for an empty answer before parsing it (for
+                    #   example before float(final_answer) or
+                    #   json.loads(final_answer)). An execution error
+                    #   during the baseline, whether from parsing, a
+                    #   locked database, or an unrelated conversion,
+                    #   cannot be attributed to the answer, so it never
+                    #   preserves a pass.
+                    baseline = execute_code_verifier(
+                        code, func_name, initial_db_path, final_db_path, None
+                    )
+                    if baseline.get("execution_status") == "error":
+                        return "judge_error", baseline
+                    if baseline.get("result") == "others":
+                        return reward_type, result
+                result = {**result, "result": "others", "no_db_change": True}
+                return "others", result
+        return reward_type, result
 
     result = execute_sql_verifier(code, func_name, initial_db_path, final_db_path)
     if isinstance(result, dict) and result.get("execution_status") == "error":