Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions scripts/agent_loop_benchmark.py
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@
"repair_turns.total",
)
MAX_DRIVER_RECEIPT_BYTES = 64 * 1024
MAX_DRIVER_RECEIPT_DEPTH = 128
MAX_FIXTURE_ORACLE_BYTES = 1024 * 1024
MAX_CASE_MANIFEST_BYTES = 1024 * 1024
MAX_GIT_PATH_BYTES = 64 * 1024
Expand Down Expand Up @@ -444,6 +445,15 @@ def _load_driver_receipt(path: Path) -> dict[str, Any]:
raise BenchmarkError(f"driver result is not valid JSON: {exc}") from exc
except (RecursionError, ValueError) as exc:
raise BenchmarkError("driver result is not valid bounded JSON") from exc
pending = [(value, 0)]
while pending:
item, depth = pending.pop()
if depth > MAX_DRIVER_RECEIPT_DEPTH:
raise BenchmarkError("driver result is not valid bounded JSON")
if isinstance(item, dict):
pending.extend((child, depth + 1) for child in item.values())
elif isinstance(item, list):
pending.extend((child, depth + 1) for child in item)
if not isinstance(value, dict):
raise BenchmarkError("driver result must be a JSON object")
status = value.get("status")
Expand Down
25 changes: 24 additions & 1 deletion scripts/tests/test_agent_loop_benchmark.py
Original file line number Diff line number Diff line change
Expand Up @@ -490,6 +490,27 @@ def test_malformed_receipt_status_is_a_contract_error(self) -> None:
with self.assertRaisesRegex(benchmark.BenchmarkError, "status must be"):
benchmark._load_driver_receipt(receipt)

def test_driver_receipt_depth_has_an_explicit_inclusive_limit(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
receipt = Path(tmp) / "receipt.json"
for kind in ("array", "object"):
for depth in (128, 129):
with self.subTest(kind=kind, depth=depth):
value: object = "private-marker"
for _ in range(depth - 1):
value = [value] if kind == "array" else {"value": value}
document = {"status": "unsupported", "extra": value}
receipt.write_text(json.dumps(document), encoding="utf-8")
self.assertLess(receipt.stat().st_size, benchmark.MAX_DRIVER_RECEIPT_BYTES)
if depth == 128:
self.assertEqual(benchmark._load_driver_receipt(receipt), document)
else:
with self.assertRaisesRegex(
benchmark.BenchmarkError,
"^driver result is not valid bounded JSON$",
):
benchmark._load_driver_receipt(receipt)

def test_driver_receipt_pathological_json_is_a_contract_error(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
Expand All @@ -499,7 +520,9 @@ def test_driver_receipt_pathological_json_is_a_contract_error(self) -> None:
):
receipt = root / name
receipt.write_text(raw, encoding="utf-8")
with self.assertRaisesRegex(benchmark.BenchmarkError, "bounded JSON"):
# Parser exceptions differ for excessive integers across Python versions.
pattern = "bounded JSON" if name == "deep.json" else r"not valid (?:bounded )?JSON"
with self.assertRaisesRegex(benchmark.BenchmarkError, pattern):
benchmark._load_driver_receipt(receipt)

@unittest.skipIf(os.name == "nt", "FIFO is POSIX-only")
Expand Down
Loading