From b36302774b7f5c686a3e7ce83b63e5cbcab98eba Mon Sep 17 00:00:00 2001 From: vbares Date: Fri, 4 Sep 2026 16:33:00 +0200 Subject: [PATCH 1/5] Add support for rendering during training in benchmark configurations --- pufferlib/config/evaluation/benchmark.yaml | 13 +++++ pufferlib/config_schema.py | 6 +++ .../evaluation_utils/evaluation_utils.py | 3 ++ pufferlib/pufferl.py | 3 +- tests/eval/test_eval.py | 51 +++++++++++++++++++ 5 files changed, 74 insertions(+), 2 deletions(-) diff --git a/pufferlib/config/evaluation/benchmark.yaml b/pufferlib/config/evaluation/benchmark.yaml index 077157026a..1f02172e95 100644 --- a/pufferlib/config/evaluation/benchmark.yaml +++ b/pufferlib/config/evaluation/benchmark.yaml @@ -58,6 +58,19 @@ benchmarks: control_mode: control_vehicles map_dir: pufferlib/resources/drive/binaries/carla + - name: carla_render + seed: 42 + num_scenarios: 8 + env: + simulation_mode: gigaflow + num_maps: 8 + max_agents_per_env: 50 + max_scenarios_per_batch: null + scenario_length: 500 + control_mode: control_vehicles + map_dir: pufferlib/resources/drive/binaries/carla + eval_training_render: true + - name: womd_multi seed: 42 num_scenarios: 1000 diff --git a/pufferlib/config_schema.py b/pufferlib/config_schema.py index 2fcb92cba6..3f643b495b 100644 --- a/pufferlib/config_schema.py +++ b/pufferlib/config_schema.py @@ -791,6 +791,12 @@ def normalize_puffer_drive_benchmarks(environment_config, benchmarks, context, v control_mode = benchmark_environment.get("control_mode") if not isinstance(control_mode, str) or not control_mode: _raise_config_error(context, f"{benchmark_path}.env.control_mode", "must be a non-empty string") + if benchmark_environment.get("eval_training_render") and simulation_mode != "gigaflow": + _raise_config_error( + context, + f"{benchmark_path}.env.eval_training_render", + "is only supported in gigaflow mode", + ) seed = benchmark.get("seed") if seed is None: diff --git a/pufferlib/ocean/evaluation_utils/evaluation_utils.py b/pufferlib/ocean/evaluation_utils/evaluation_utils.py index 88a0bb10de..be90bfcb46 100644 --- a/pufferlib/ocean/evaluation_utils/evaluation_utils.py +++ b/pufferlib/ocean/evaluation_utils/evaluation_utils.py @@ -239,6 +239,9 @@ def _finalize_benchmark_args(args, cli_overrides, eval_training_render, validati def build_benchmark_args(base_args, benchmark, environment_config, cli_overrides=()): """Compose and validate final arguments for one benchmark evaluation.""" + if benchmark["env"].get("eval_training_render"): + base_args = copy.deepcopy(base_args) + base_args["env"]["eval_training_render"] = True eval_training_render = base_args["env"]["eval_training_render"] validation_context = f"evaluation.{benchmark['name']}" if eval_training_render: diff --git a/pufferlib/pufferl.py b/pufferlib/pufferl.py index 03eed64a00..1a8f8ad8df 100644 --- a/pufferlib/pufferl.py +++ b/pufferlib/pufferl.py @@ -1681,12 +1681,10 @@ def eval( selected_benchmarks = benchmark_names if benchmark_names is not None else eval_config["benchmarks"] eval_config["benchmarks"] = selected_benchmarks output_name = eval_config["output_name"] - render_scenarios = eval_config["render_scenarios"] render_filter = eval_config["render_filter"] max_rendered_failures = eval_config["max_rendered_failures"] failure_replay_csv = eval_config["failure_replay_csv"] eval_training_render = args["env"]["eval_training_render"] - render_scenarios = render_scenarios or eval_training_render report_to_wandb = bool(args["wandb"]) and not use_training_config environment_config, benchmarks = drive_benchmark.load_benchmark_config(benchmark_config_path, selected_benchmarks) @@ -1722,6 +1720,7 @@ def eval( environment_config, cli_overrides, ) + render_scenarios = eval_config["render_scenarios"] or run_args["env"]["eval_training_render"] output_directory_name = benchmark["name"] if output_name is not None: output_directory_name = f"{output_directory_name}_{output_name}" diff --git a/tests/eval/test_eval.py b/tests/eval/test_eval.py index 337e25745d..1f4e14e1fd 100644 --- a/tests/eval/test_eval.py +++ b/tests/eval/test_eval.py @@ -1,3 +1,4 @@ +import copy import json import random import struct @@ -13,6 +14,7 @@ import yaml import pufferlib +import pufferlib.pytorch from pufferlib import pufferl from pufferlib.config_schema import validate_puffer_drive_config from pufferlib.ocean.drive.drive import Drive @@ -25,6 +27,7 @@ CARLA_WORKER_COUNT = 2 CARLA_MAP_COUNT = 4 CARLA_SCENARIO_LENGTH = 32 +BENCHMARK_SCENARIO_LENGTH = 48 TRAIN_EPOCH_COUNT = 7 TRAIN_EVAL_INTERVAL = 3 TRAIN_HORIZON = 16 @@ -453,6 +456,54 @@ def record_vector_make(*make_args, **make_kwargs): assert all('class="payload-chunk"' in page.read_text() for page in rendered_pages) +def _write_render_benchmark_config(tmp_path): + """One plain and one render benchmark, differing only by eval_training_render.""" + config_path = _write_benchmark_config( + tmp_path / "render_benchmark.yaml", + name="carla_test", + map_dir=CARLA_MAP_DIR, + simulation_mode="gigaflow", + num_maps=CARLA_MAP_COUNT, + num_scenarios=CARLA_SCENARIO_COUNT, + scenario_length=BENCHMARK_SCENARIO_LENGTH, + max_agents_per_env=8, + control_mode="control_vehicles", + use_neighbor_cache=True, + ) + config = yaml.safe_load(config_path.read_text()) + render_benchmark = copy.deepcopy(config["benchmarks"][0]) + render_benchmark["name"] = "carla_test_render" + render_benchmark["env"]["eval_training_render"] = True + config["benchmarks"].append(render_benchmark) + config_path.write_text(yaml.safe_dump(config, sort_keys=False)) + return config_path + + +def test_render_benchmark_scopes_training_render_to_its_own_entry(tmp_path): + benchmark_config_path = _write_render_benchmark_config(tmp_path) + selected_benchmarks = "carla_test,carla_test_render" + base_args = _standalone_eval_args(benchmark_config_path) + base_args["eval"]["benchmarks"] = selected_benchmarks + environment_config, benchmarks = drive_benchmark.load_benchmark_config( + benchmark_config_path, + selected_benchmarks, + ) + run_args = { + benchmark["name"]: drive_benchmark.build_benchmark_args(base_args, benchmark, environment_config) + for benchmark in benchmarks + } + + plain_args = run_args["carla_test"] + assert plain_args["env"]["eval_training_render"] is False + assert plain_args["env"]["scenario_length"] == BENCHMARK_SCENARIO_LENGTH + assert plain_args["eval"]["action_selection"] == "mode" + + render_args = run_args["carla_test_render"] + assert render_args["env"]["eval_training_render"] is True + assert render_args["env"]["scenario_length"] == CARLA_SCENARIO_LENGTH + assert render_args["eval"]["action_selection"] == pufferlib.pytorch.ACTION_SELECT_SAMPLE + + def _write_training_benchmark(tmp_path): return _write_benchmark_config( tmp_path / "training_benchmark.yaml", From 13622ec7a7426c2bff7228c48e1851fa483dd066 Mon Sep 17 00:00:00 2001 From: vbares Date: Fri, 4 Sep 2026 16:35:21 +0200 Subject: [PATCH 2/5] Remove unused render benchmark configuration and related tests --- tests/eval/test_eval.py | 50 ----------------------------------------- 1 file changed, 50 deletions(-) diff --git a/tests/eval/test_eval.py b/tests/eval/test_eval.py index 1f4e14e1fd..1e24659ba6 100644 --- a/tests/eval/test_eval.py +++ b/tests/eval/test_eval.py @@ -1,4 +1,3 @@ -import copy import json import random import struct @@ -14,7 +13,6 @@ import yaml import pufferlib -import pufferlib.pytorch from pufferlib import pufferl from pufferlib.config_schema import validate_puffer_drive_config from pufferlib.ocean.drive.drive import Drive @@ -456,54 +454,6 @@ def record_vector_make(*make_args, **make_kwargs): assert all('class="payload-chunk"' in page.read_text() for page in rendered_pages) -def _write_render_benchmark_config(tmp_path): - """One plain and one render benchmark, differing only by eval_training_render.""" - config_path = _write_benchmark_config( - tmp_path / "render_benchmark.yaml", - name="carla_test", - map_dir=CARLA_MAP_DIR, - simulation_mode="gigaflow", - num_maps=CARLA_MAP_COUNT, - num_scenarios=CARLA_SCENARIO_COUNT, - scenario_length=BENCHMARK_SCENARIO_LENGTH, - max_agents_per_env=8, - control_mode="control_vehicles", - use_neighbor_cache=True, - ) - config = yaml.safe_load(config_path.read_text()) - render_benchmark = copy.deepcopy(config["benchmarks"][0]) - render_benchmark["name"] = "carla_test_render" - render_benchmark["env"]["eval_training_render"] = True - config["benchmarks"].append(render_benchmark) - config_path.write_text(yaml.safe_dump(config, sort_keys=False)) - return config_path - - -def test_render_benchmark_scopes_training_render_to_its_own_entry(tmp_path): - benchmark_config_path = _write_render_benchmark_config(tmp_path) - selected_benchmarks = "carla_test,carla_test_render" - base_args = _standalone_eval_args(benchmark_config_path) - base_args["eval"]["benchmarks"] = selected_benchmarks - environment_config, benchmarks = drive_benchmark.load_benchmark_config( - benchmark_config_path, - selected_benchmarks, - ) - run_args = { - benchmark["name"]: drive_benchmark.build_benchmark_args(base_args, benchmark, environment_config) - for benchmark in benchmarks - } - - plain_args = run_args["carla_test"] - assert plain_args["env"]["eval_training_render"] is False - assert plain_args["env"]["scenario_length"] == BENCHMARK_SCENARIO_LENGTH - assert plain_args["eval"]["action_selection"] == "mode" - - render_args = run_args["carla_test_render"] - assert render_args["env"]["eval_training_render"] is True - assert render_args["env"]["scenario_length"] == CARLA_SCENARIO_LENGTH - assert render_args["eval"]["action_selection"] == pufferlib.pytorch.ACTION_SELECT_SAMPLE - - def _write_training_benchmark(tmp_path): return _write_benchmark_config( tmp_path / "training_benchmark.yaml", From 6e7a8e78c27a3a2d20117d019483b3d9ef64cb31 Mon Sep 17 00:00:00 2001 From: vbares Date: Mon, 7 Sep 2026 11:03:35 +0200 Subject: [PATCH 3/5] Enhance evaluation rendering: add option to keep intermediate .replay.zlib files --- README.md | 3 ++- docs/evaluation.md | 10 ++++++---- pufferlib/config/puffer_drive.yaml | 2 ++ pufferlib/config_schema.py | 1 + pufferlib/ocean/evaluation_utils/eval_replay.py | 16 +++++++++++----- pufferlib/pufferl.py | 10 ++++++---- tests/eval/test_eval.py | 10 ++++++++-- 7 files changed, 36 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 64920781b6..0904b571d3 100644 --- a/README.md +++ b/README.md @@ -145,7 +145,8 @@ puffer eval puffer_drive carla \ ``` Scenario renders are captured during the benchmark pass and written as -interactive HTML with retained `.replay.zlib` files. To render only episodes +self-contained interactive HTML; the intermediate `.replay.zlib` bundles are +deleted afterwards unless `eval.keep_zlib_replays=true`. To render only episodes where `offroad_rate > 0` instead: ```bash diff --git a/docs/evaluation.md b/docs/evaluation.md index 536c38464a..9c64f00281 100644 --- a/docs/evaluation.md +++ b/docs/evaluation.md @@ -105,8 +105,10 @@ puffer eval puffer_drive carla_fast \ `eval.render_scenarios=true` records each of the benchmark's configured `num_scenarios` during the metrics rollout. It writes the completed -`.replay.zlib` files incrementally, then renders one interactive HTML page per -scenario and builds a navigable `index.html`. +`.replay.zlib` files incrementally in `replays_zlib/`, then renders one +interactive HTML page per scenario and builds a navigable `index.html`. The +compressed files are removed after rendering by default; use +`eval.keep_zlib_replays=true` to keep them. `eval.capture_observations=true` also stores policy observations. @@ -168,7 +170,7 @@ eval/[_]// ├── resolved_benchmark.yaml ├── episode_metrics.csv ├── evaluation_summary.json -├── replays/ # only when render_scenarios=true +├── replays_zlib/ # only when render_scenarios=true and keep_zlib_replays=true │ └── *.replay.zlib ├── rendered_replays/ # only when render_scenarios=true │ ├── *.html @@ -177,7 +179,7 @@ eval/[_]// ├── selected_failures.csv ├── episode_metrics.csv ├── evaluation_summary.json - ├── replays/ + ├── replays_zlib/ # only when keep_zlib_replays=true │ └── *.replay.zlib └── rendered_replays/ ├── *.html diff --git a/pufferlib/config/puffer_drive.yaml b/pufferlib/config/puffer_drive.yaml index 469f53b5ee..922b0e80de 100644 --- a/pufferlib/config/puffer_drive.yaml +++ b/pufferlib/config/puffer_drive.yaml @@ -303,6 +303,8 @@ eval: output_dir_name: eval # Capture and render every scenario during the standard benchmark pass. render_scenarios: false + # Keep the intermediate .replay.zlib bundles after rendering HTML. + keep_zlib_replays: false # Render scenarios where any configured metric is positive. # Use all_infractions for every infraction type; null disables it. render_filter: null diff --git a/pufferlib/config_schema.py b/pufferlib/config_schema.py index 3f643b495b..37567bb0d1 100644 --- a/pufferlib/config_schema.py +++ b/pufferlib/config_schema.py @@ -418,6 +418,7 @@ class EvaluationConfig: output_name: str | None = MISSING output_dir_name: str = _constrained_field(NONEMPTY_STRING_CONSTRAINT) render_scenarios: bool = MISSING + keep_zlib_replays: bool = MISSING render_filter: Any = MISSING max_rendered_failures: int | None = _constrained_field(POSITIVE_INT_CONSTRAINT) failure_replay_csv: str | None = MISSING diff --git a/pufferlib/ocean/evaluation_utils/eval_replay.py b/pufferlib/ocean/evaluation_utils/eval_replay.py index f8bdaa3c01..7b63b378c6 100644 --- a/pufferlib/ocean/evaluation_utils/eval_replay.py +++ b/pufferlib/ocean/evaluation_utils/eval_replay.py @@ -1,6 +1,7 @@ import numbers import os import pickle +import shutil import zlib from concurrent.futures import ThreadPoolExecutor @@ -12,6 +13,10 @@ import pufferlib.viz +ZLIB_REPLAY_DIR_NAME = "replays_zlib" +ZLIB_REPLAY_SUFFIX = ".replay.zlib" + + def _eval_replay_stem(summary, episode_id): map_stem = os.path.splitext(os.path.basename(summary["map_name"]))[0] return f"{map_stem}__seed_{summary['seed']}__episode_{episode_id:06d}" @@ -140,7 +145,7 @@ def queue_replay(self, summary, episode_id): for replay_key, history_values in self.policy_history.items(): replay[replay_key] = history_values[:episode_length, global_agent_start:global_agent_end] replay_stem = _eval_replay_stem(summary, self.episode_id_offset + episode_id) - replay_path = os.path.abspath(os.path.join(self.replay_output_dir, f"{replay_stem}.replay.zlib")) + replay_path = os.path.abspath(os.path.join(self.replay_output_dir, f"{replay_stem}{ZLIB_REPLAY_SUFFIX}")) self.pending_replays.append((replay_environment["scenario"], replay, replay_path)) summary["has_replay"] = 1 summary["replay_path"] = replay_path @@ -159,7 +164,7 @@ def write_pending(self): self.pending_replays = [] -def _render_eval_replays(episode_summaries, out_dir): +def _render_eval_replays(episode_summaries, out_dir, keep_zlib_replays): """Render captured eval replays as navigable HTML pages plus an index.""" render_dir = os.path.join(out_dir, "rendered_replays") os.makedirs(render_dir, exist_ok=True) @@ -172,10 +177,9 @@ def _render_eval_replays(episode_summaries, out_dir): if not replay_path or not os.path.isfile(replay_path): raise RuntimeError(f"Cannot render episode {episode_id}: replay file is missing: {replay_path}") replay_filename = os.path.basename(replay_path) - replay_suffix = ".replay.zlib" - if not replay_filename.endswith(replay_suffix): + if not replay_filename.endswith(ZLIB_REPLAY_SUFFIX): raise RuntimeError(f"Cannot render episode {episode_id}: unexpected replay filename: {replay_filename}") - html_filename = f"{replay_filename[: -len(replay_suffix)]}.html" + html_filename = f"{replay_filename[: -len(ZLIB_REPLAY_SUFFIX)]}.html" output_path = os.path.join(render_dir, html_filename) replay_paths.append(replay_path) output_paths.append(output_path) @@ -197,4 +201,6 @@ def _render_eval_replays(episode_summaries, out_dir): pufferlib.viz.build_gallery_index(render_dir, file_metrics=file_metrics) print(f"Rendered {len(episode_summaries)} replay pages into {render_dir}") print(f"Wrote replay index to {os.path.join(render_dir, 'index.html')}") + if not keep_zlib_replays: + shutil.rmtree(os.path.join(out_dir, ZLIB_REPLAY_DIR_NAME)) return render_dir diff --git a/pufferlib/pufferl.py b/pufferlib/pufferl.py index 1a8f8ad8df..73c86f960c 100644 --- a/pufferlib/pufferl.py +++ b/pufferlib/pufferl.py @@ -1766,7 +1766,9 @@ def eval( capture_replay=render_scenarios, ) print(f"Evaluation {benchmark['name']}: {num_scenarios} scenarios across {num_workers} workers") - replay_output_dir = os.path.join(benchmark_output_dir, "replays") if render_scenarios else None + replay_output_dir = ( + os.path.join(benchmark_output_dir, drive_eval_replay.ZLIB_REPLAY_DIR_NAME) if render_scenarios else None + ) summaries = _run_eval_rollout( run_args, env_name, @@ -1786,7 +1788,7 @@ def eval( } if render_scenarios: - drive_eval_replay._render_eval_replays(summaries, benchmark_output_dir) + drive_eval_replay._render_eval_replays(summaries, benchmark_output_dir, eval_config["keep_zlib_replays"]) elif render_filter is not None: _render_eval_failures( env_name, @@ -2271,7 +2273,7 @@ def _render_eval_failures( ) replay_agent_capacity = failure_args["env"]["max_agents_per_env"] failure_args["env"]["num_agents"] = replay_agent_capacity - replay_output_dir = os.path.join(failures_dir, "replays") + replay_output_dir = os.path.join(failures_dir, drive_eval_replay.ZLIB_REPLAY_DIR_NAME) os.makedirs(replay_output_dir, exist_ok=True) agents_per_batch_values = selected_rows["agents_per_batch"].unique() if len(agents_per_batch_values) != 1: @@ -2308,7 +2310,7 @@ def _render_eval_failures( ) summaries.extend(wave_summaries) summary = drive_benchmark._write_eval_reports(summaries, failures_dir, len(pairs)) - drive_eval_replay._render_eval_replays(summaries, failures_dir) + drive_eval_replay._render_eval_replays(summaries, failures_dir, run_args["eval"]["keep_zlib_replays"]) return { "episodes": summaries, "summary": summary, diff --git a/tests/eval/test_eval.py b/tests/eval/test_eval.py index 1e24659ba6..ed945ce04f 100644 --- a/tests/eval/test_eval.py +++ b/tests/eval/test_eval.py @@ -411,7 +411,7 @@ def record_vector_make(*make_args, **make_kwargs): return original_vector_make(*make_args, **make_kwargs) monkeypatch.setattr(pufferlib.vector, "make", record_vector_make) - replay_output_dir = tmp_path / "replays" + replay_output_dir = tmp_path / drive_eval_replay.ZLIB_REPLAY_DIR_NAME summaries = pufferl._run_eval_rollout( args, "puffer_drive", @@ -447,11 +447,17 @@ def record_vector_make(*make_args, **make_kwargs): assert header["obs_dim"] > 0 assert required_chunks <= set(header["chunks"]) - render_dir = Path(drive_eval_replay._render_eval_replays(summaries, str(tmp_path))) + render_dir = Path(drive_eval_replay._render_eval_replays(summaries, str(tmp_path), keep_zlib_replays=True)) rendered_pages = sorted(path for path in render_dir.glob("*.html") if path.name != "index.html") assert len(rendered_pages) == 2 assert (render_dir / "index.html").is_file() assert all('class="payload-chunk"' in page.read_text() for page in rendered_pages) + assert all(replay_path.is_file() for replay_path in replay_paths) + + drive_eval_replay._render_eval_replays(summaries, str(tmp_path), keep_zlib_replays=False) + assert not replay_output_dir.exists() + assert (render_dir / "index.html").is_file() + assert len(sorted(path for path in render_dir.glob("*.html") if path.name != "index.html")) == 2 def _write_training_benchmark(tmp_path): From f14040cb7b9d1289420f301aca3b56334b111791 Mon Sep 17 00:00:00 2001 From: vbares Date: Wed, 9 Sep 2026 10:42:15 +0200 Subject: [PATCH 4/5] Update evaluation documentation and enhance benchmark argument handling for training rendering --- README.md | 2 ++ docs/evaluation.md | 3 +-- pufferlib/ocean/evaluation_utils/evaluation_utils.py | 2 ++ 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 0904b571d3..54bcd4805c 100644 --- a/README.md +++ b/README.md @@ -159,6 +159,8 @@ puffer eval puffer_drive carla \ Set `env.eval_training_render=true` to evaluate and render the Gigaflow environment distribution saved in the checkpoint's adjacent `config.yaml`. +The selected benchmark's `map_dir` and `num_maps` override the checkpoint values, +so checkpoints remain portable across machines with different dataset paths. Use `eval.num_agents`, not `env.num_agents`, to configure evaluation capacity. Evaluation outputs are written under diff --git a/docs/evaluation.md b/docs/evaluation.md index 9c64f00281..f46d0be0b5 100644 --- a/docs/evaluation.md +++ b/docs/evaluation.md @@ -117,8 +117,7 @@ To evaluate and render the environment distribution used during training, run: ```bash puffer eval puffer_drive carla_fast \ load_model_path=path/to/model.pt \ - env.eval_training_render=true \ - env.map_dir=pufferlib/resources/drive/binaries/carla + env.eval_training_render=true ``` ## Filtered replay and rendering diff --git a/pufferlib/ocean/evaluation_utils/evaluation_utils.py b/pufferlib/ocean/evaluation_utils/evaluation_utils.py index be90bfcb46..044fddabeb 100644 --- a/pufferlib/ocean/evaluation_utils/evaluation_utils.py +++ b/pufferlib/ocean/evaluation_utils/evaluation_utils.py @@ -201,6 +201,8 @@ def _build_benchmark_args(base_args, benchmark, environment_config): args["vec"]["seed"] = seed if eval_training_render: args["eval"]["action_selection"] = pufferlib.pytorch.ACTION_SELECT_SAMPLE + args["env"]["map_dir"] = benchmark_environment_config["map_dir"] + args["env"]["num_maps"] = benchmark_environment_config["num_maps"] else: args["env"].update(copy.deepcopy(environment_config)) args["env"].update(copy.deepcopy(benchmark_environment_config)) From 19f1ce7de20e43b4c5b1ef94cf890848b4d1bb89 Mon Sep 17 00:00:00 2001 From: vbares Date: Wed, 9 Sep 2026 11:18:46 +0200 Subject: [PATCH 5/5] Refactor benchmark configuration: remove unused parameters and validate eval_training_render --- pufferlib/config/evaluation/benchmark.yaml | 4 ---- pufferlib/config_schema.py | 15 ++++++++++++--- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/pufferlib/config/evaluation/benchmark.yaml b/pufferlib/config/evaluation/benchmark.yaml index 1f02172e95..6980b98c40 100644 --- a/pufferlib/config/evaluation/benchmark.yaml +++ b/pufferlib/config/evaluation/benchmark.yaml @@ -64,10 +64,6 @@ benchmarks: env: simulation_mode: gigaflow num_maps: 8 - max_agents_per_env: 50 - max_scenarios_per_batch: null - scenario_length: 500 - control_mode: control_vehicles map_dir: pufferlib/resources/drive/binaries/carla eval_training_render: true diff --git a/pufferlib/config_schema.py b/pufferlib/config_schema.py index 37567bb0d1..2d2650457d 100644 --- a/pufferlib/config_schema.py +++ b/pufferlib/config_schema.py @@ -789,10 +789,19 @@ def normalize_puffer_drive_benchmarks(environment_config, benchmarks, context, v simulation_mode = benchmark_environment.get("simulation_mode") if simulation_mode not in ("gigaflow", "replay"): _raise_config_error(context, f"{benchmark_path}.env.simulation_mode", "must be 'gigaflow' or 'replay'") + eval_training_render = benchmark_environment.get("eval_training_render", False) + if not isinstance(eval_training_render, bool): + _raise_config_error( + context, + f"{benchmark_path}.env.eval_training_render", + "must be a boolean", + ) control_mode = benchmark_environment.get("control_mode") - if not isinstance(control_mode, str) or not control_mode: + if control_mode is not None and (not isinstance(control_mode, str) or not control_mode): + _raise_config_error(context, f"{benchmark_path}.env.control_mode", "must be a non-empty string") + if control_mode is None and not eval_training_render: _raise_config_error(context, f"{benchmark_path}.env.control_mode", "must be a non-empty string") - if benchmark_environment.get("eval_training_render") and simulation_mode != "gigaflow": + if eval_training_render and simulation_mode != "gigaflow": _raise_config_error( context, f"{benchmark_path}.env.eval_training_render", @@ -821,7 +830,7 @@ def normalize_puffer_drive_benchmarks(environment_config, benchmarks, context, v max_agents_per_env = benchmark_environment.get("max_agents_per_env") single_agent_replay = simulation_mode == "replay" and control_mode == "control_sdc_only" - if max_agents_per_env is None and not single_agent_replay: + if max_agents_per_env is None and not single_agent_replay and not eval_training_render: _raise_config_error(context, f"{benchmark_path}.env.max_agents_per_env", "must be a positive integer") if max_agents_per_env is not None: _validate_value_constraint(