Make TaskProtocol retry settings take effect

CHANGELOG.md

@@ -9,10 +9,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- Reports now include an `attempts` field listing the status, error, usage, and timeout of every attempt of a task repetition. `ResultLogger` writes it too. (PR: #71)
+
 ### Changed
 
+- tau2 and gaia2 `load_tasks()` now default to `max_retries=0`, matching the original benchmarks, which do not retry failed runs. (PR: #71)
+
 ### Fixed
 
+- `TaskProtocol.max_retries` and `TimeoutAction.RETRY`/`EXTEND` now take effect. Failures outside the agent's control (`setup_failed`, `environment_error`, `user_error`, `unknown_execution_error`) are retried up to `max_retries` times. Timeouts are retried once with the same (`RETRY`) or doubled (`EXTEND`) timeout. Previously both settings were silently ignored. `TaskProtocol` now rejects negative `max_retries`. (PR: #71)
+
 ### Removed
 
 ## [0.5.1] - 2026-07-05

docs/guides/exception-handling.md

@@ -190,7 +190,13 @@ The success rate (65.22%) reflects `60 / 92` rather than `60 / 100`.
 
 ## Rerunning Failed Tasks
 
-Infrastructure errors are often transient. Tasks with infrastructure failures can be rerun:
+Infrastructure errors are often transient. To retry them automatically, set `max_retries` on the task's protocol. This retries `setup_failed`, `environment_error`, `user_error`, and `unknown_execution_error` with the same seeds. Every attempt is listed in the report's `attempts` field.
+
+```python
+task = Task(query="...", protocol=TaskProtocol(max_retries=2))
+```
+
+Alternatively, tasks with infrastructure failures can be rerun manually:
 
 ```python
 results = benchmark.run(tasks)

maseval/benchmark/gaia2/data_loader.py

@@ -94,7 +94,7 @@ class Gaia2JudgeEngineConfig:
 
 # ARE scenarios/config.py:20: DEFAULT_SCENARIO_TIMEOUT = 1860
 DEFAULT_TIMEOUT_SECONDS = 1860.0  # 31 minutes per task (matching ARE)
-DEFAULT_MAX_RETRIES = 1
+DEFAULT_MAX_RETRIES = 0  # ARE does not retry failed scenarios
 
 # HuggingFace dataset info
 HF_DATASET_ID = "meta-agents-research-environments/gaia2"
@@ -129,7 +129,7 @@ def load_tasks(
         limit: Maximum number of tasks to load (across all capabilities)
         timeout_seconds: Maximum execution time per task. Default 1860 (31 minutes,
             matching ARE's DEFAULT_SCENARIO_TIMEOUT). Set to None to disable timeout.
-        max_retries: Maximum retry attempts. Default 1 (skip on failure).
+        max_retries: Maximum retries after an infrastructure failure (see `TaskProtocol`). Default 0 (no retries).
 
     Returns:
         TaskQueue with Task objects.
@@ -230,7 +230,7 @@ def _convert_gaia2_to_maseval(
         row: Raw row from HuggingFace dataset
         scenario: ARE BenchmarkScenario object
         timeout_seconds: Maximum execution time per task
-        max_retries: Maximum retry attempts
+        max_retries: Maximum retries after an infrastructure failure
         config_capability: The capability from the HuggingFace config name
 
     Returns:

maseval/benchmark/tau2/data_loader.py

@@ -39,7 +39,7 @@
 
 # Default execution protocol settings
 DEFAULT_TIMEOUT_SECONDS = 600.0  # 10 minutes per task
-DEFAULT_MAX_RETRIES = 1  # Skip on first failure
+DEFAULT_MAX_RETRIES = 0  # tau2-bench v0.2.0 does not retry failed simulations
 
 # GitHub raw content URLs for v0.2.0 tag
 GITHUB_BASE = "https://raw.githubusercontent.com/sierra-research/tau2-bench"
@@ -274,7 +274,7 @@ def load_tasks(
         limit: Maximum number of tasks to load
         timeout_seconds: Maximum execution time per task in seconds. Default 600 (10 minutes).
             Set to None to disable timeout.
-        max_retries: Maximum retry attempts for transient failures. Default 1 (skip on failure).
+        max_retries: Maximum retries after an infrastructure failure (see `TaskProtocol`). Default 0 (no retries).
 
     Returns:
         TaskQueue containing Task objects with:
@@ -349,7 +349,7 @@ def _convert_tau2_task_to_maseval(
         split: Split name
         domain_config: Domain configuration with policy and db_path
         timeout_seconds: Maximum execution time per task in seconds
-        max_retries: Maximum retry attempts for transient failures
+        max_retries: Maximum retries after an infrastructure failure
 
     Returns:
         MASEval Task object

maseval/core/benchmark.py

@@ -8,7 +8,7 @@
 import logging
 
 from .evaluator import Evaluator
-from .task import Task, BaseTaskQueue, SequentialTaskQueue
+from .task import Task, BaseTaskQueue, SequentialTaskQueue, TimeoutAction
 from .environment import Environment
 from .agent import AgentAdapter
 from .model import ModelAdapter
@@ -68,6 +68,17 @@ class TaskExecutionStatus(Enum):
     TASK_EXECUTION_FAILED = "task_execution_failed"
 
 
+# Failures outside the agent's control, retried up to `TaskProtocol.max_retries` times.
+_RETRYABLE_STATUSES = frozenset(
+    {
+        TaskExecutionStatus.SETUP_FAILED,
+        TaskExecutionStatus.ENVIRONMENT_ERROR,
+        TaskExecutionStatus.USER_ERROR,
+        TaskExecutionStatus.UNKNOWN_EXECUTION_ERROR,
+    }
+)
+
+
 class Benchmark(ABC):
     """Abstract base class for orchestrating multi-agent system execution and evaluation.
 
@@ -552,7 +563,7 @@ def _build_report(
 
         Returns:
             Report dictionary with keys: ``task_id``, ``repeat_idx``, ``status``,
-            ``error``, ``traces``, ``config``, ``usage``, ``eval``, ``task``.
+            ``error``, ``traces``, ``config``, ``usage``, ``eval``, ``attempts``, ``task``.
         """
         if status is not TaskExecutionStatus.SUCCESS and error is None:
             # Defensive: a non-success report must always carry error details.
@@ -570,6 +581,7 @@ def _build_report(
             "config": config if config is not None else {},
             "usage": usage,
             "eval": eval_results,
+            "attempts": [],  # Filled in by _execute_task_repetition
             "task": {
                 "query": task.query,
                 "metadata": dict(task.metadata),
@@ -1072,16 +1084,65 @@ def _execute_task_repetition(
         agent_data: Dict[str, Any],
         repeat_idx: int,
     ) -> Dict[str, Any]:
-        """Execute a single task repetition with timeout handling.
+        """Execute a single task repetition, retrying failed attempts as configured in ``task.protocol``.
+
+        Called from both sequential and parallel execution paths.
+
+        Args:
+            task: The task to execute.
+            agent_data: Agent configuration for this task.
+            repeat_idx: Repetition index (0 to n_task_repeats-1).
+
+        Returns:
+            Report of the last attempt, with ``attempts`` listing every attempt.
+        """
+        protocol = task.protocol
+        timeout_seconds = protocol.timeout_seconds
+        attempts: List[Dict[str, Any]] = []
+        failure_retries = 0
+        timeout_retried = False
+
+        while True:
+            report = self._execute_task_attempt(task, agent_data, repeat_idx, timeout_seconds)
+            attempts.append(
+                {
+                    "status": report["status"],
+                    "error": report["error"],
+                    "usage": report["usage"],
+                    "timeout_seconds": timeout_seconds,
+                }
+            )
+            status = TaskExecutionStatus(report["status"])
+
+            if status in _RETRYABLE_STATUSES and failure_retries < protocol.max_retries:
+                # Infrastructure failure: retry up to max_retries
+                failure_retries += 1
+            elif status is TaskExecutionStatus.TASK_TIMEOUT and protocol.timeout_action is not TimeoutAction.SKIP and not timeout_retried:
+                # Timeout: retry once, EXTEND doubles the timeout
+                timeout_retried = True
+                if protocol.timeout_action is TimeoutAction.EXTEND and timeout_seconds is not None:
+                    timeout_seconds *= 2
+            else:
+                report["attempts"] = attempts
+                return report
+
+    def _execute_task_attempt(
+        self,
+        task: Task,
+        agent_data: Dict[str, Any],
+        repeat_idx: int,
+        timeout_seconds: Optional[float],
+    ) -> Dict[str, Any]:
+        """Execute a single attempt of a task repetition with timeout handling.
 
-        This method encapsulates the complete execution of one task repetition,
-        including setup, execution, trace collection, and evaluation. It is
-        designed to be called from both sequential and parallel execution paths.
+        This method encapsulates the complete execution of one attempt,
+        including setup, execution, trace collection, and evaluation.
 
         Args:
             task: The task to execute.
             agent_data: Agent configuration for this task.
             repeat_idx: Repetition index (0 to n_task_repeats-1).
+            timeout_seconds: Timeout for this attempt, or None for no timeout.
 
         Returns:
             Report dictionary containing execution results.
@@ -1097,8 +1158,7 @@ def _execute_task_repetition(
         agents_dict: Dict[str, AgentAdapter] = {}
 
         # Create execution context with optional timeout
-        timeout = task.protocol.timeout_seconds
-        context = TaskContext(deadline=timeout)
+        context = TaskContext(deadline=timeout_seconds)
 
         # Create scoped seed generator for this task+repetition
         task_seed_gen = self._seed_generator.for_task(str(task.id)).for_repetition(repeat_idx)

maseval/core/callbacks/result_logger.py

@@ -170,6 +170,7 @@ def _filter_report(self, report: Dict) -> Dict:
             "repeat_idx": report.get("repeat_idx"),
             "status": report.get("status"),
             "error": report.get("error"),
+            "attempts": report.get("attempts"),
         }
 
         if self.include_traces and "traces" in report:

maseval/core/task.py

@@ -19,11 +19,14 @@
 
 
 class TimeoutAction(Enum):
-    """Action to take when a task timeout occurs."""
+    """Action to take when a task timeout occurs.
+
+    A timed-out task is retried at most once, independent of `TaskProtocol.max_retries`.
+    """
 
     SKIP = "skip"  # Mark as timed out, continue to next task
     RETRY = "retry"  # Retry once with same timeout
-    EXTEND = "extend"  # Double timeout and retry
+    EXTEND = "extend"  # Double timeout and retry once
 
 
 @dataclass
@@ -34,17 +37,26 @@ class TaskProtocol:
     task content (query, environment_data, etc.). It controls the
     interface between the task and MASEval's execution engine.
 
+    A retry reruns the full task repetition with the same seeds. Every attempt is
+    listed in the report's `attempts` field. With a benchmark `fail_on_*` flag set,
+    errors are raised without retrying.
+
     Note:
         Timeout checking is cooperative and currently only occurs at execution phase
         boundaries (after setup, before execution, before evaluation). Timeout detection
         during agent execution is not yet supported.
 
     Attributes:
         timeout_seconds: Maximum execution time for this task. None means no timeout.
-        timeout_action: Action to take when timeout occurs.
-        max_retries: Maximum retry attempts for transient failures (not timeouts).
+        timeout_action: Action to take when timeout occurs. See `TimeoutAction`.
+        max_retries: Maximum retries after a failure outside the agent's control (`setup_failed`,
+            `environment_error`, `user_error`, `unknown_execution_error`). Agent errors and
+            evaluation failures are never retried.
         priority: Execution priority (higher = sooner). Used by adaptive task queues.
         tags: Arbitrary tags for filtering or grouping tasks.
+
+    Raises:
+        ValueError: If `max_retries` is negative.
     """
 
     timeout_seconds: Optional[float] = None
@@ -53,6 +65,10 @@ class TaskProtocol:
     priority: int = 0
     tags: Dict[str, Any] = field(default_factory=dict)
 
+    def __post_init__(self) -> None:
+        if self.max_retries < 0:
+            raise ValueError(f"TaskProtocol.max_retries must be 0 or greater, got {self.max_retries}.")
+
     def to_dict(self) -> Dict[str, Any]:
         """Convert to a JSON-serializable dictionary.
 

tests/test_core/test_benchmark/test_report_schema.py

@@ -25,7 +25,7 @@
 
 
 # Canonical top-level keys every report must carry.
-REPORT_KEYS = {"task_id", "repeat_idx", "status", "error", "traces", "config", "usage", "eval", "task"}
+REPORT_KEYS = {"task_id", "repeat_idx", "status", "error", "traces", "config", "usage", "eval", "attempts", "task"}
 ERROR_KEYS = {"error_type", "error_message", "traceback"}