diff --git a/.dockerignore b/.dockerignore index e5abb4fde6..a32d70b9a8 100644 --- a/.dockerignore +++ b/.dockerignore @@ -7,7 +7,6 @@ build *.egg-info experiments wandb -.neptune .pytest_cache .ruff_cache diff --git a/.gitignore b/.gitignore index 52b9244e2f..f556be62b8 100644 --- a/.gitignore +++ b/.gitignore @@ -149,7 +149,6 @@ checkpoints/ experiments/ benchmark*/ wandb/ -.neptune/ raylib*/ box2d*/ diff --git a/pufferlib/config/puffer_drive.yaml b/pufferlib/config/puffer_drive.yaml index 1cf8a923f8..d6618192df 100644 --- a/pufferlib/config/puffer_drive.yaml +++ b/pufferlib/config/puffer_drive.yaml @@ -8,9 +8,6 @@ wandb_group: debug # Unique run identifier. Doubles as the logger run id, so relaunching with the # same run_name and train.data_dir resumes that run instead of starting a new one. run_name: default_run -neptune: false -neptune_name: pufferai -neptune_project: ablations tb: false local_rank: 0 tag: null diff --git a/pufferlib/config_schema.py b/pufferlib/config_schema.py index c7efc743ca..55811adbc6 100644 --- a/pufferlib/config_schema.py +++ b/pufferlib/config_schema.py @@ -449,9 +449,6 @@ class PufferDriveConfig: wandb_project: str = _constrained_field(NONEMPTY_STRING_CONSTRAINT) wandb_group: str = _constrained_field(NONEMPTY_STRING_CONSTRAINT) run_name: str = _constrained_field(NONEMPTY_STRING_CONSTRAINT) - neptune: bool = MISSING - neptune_name: str = MISSING - neptune_project: str = MISSING tb: bool = MISSING local_rank: int = MISSING tag: str | None = MISSING @@ -540,8 +537,8 @@ def _validate_string_selection(value, context, path, *, allow_none=True): def _validate_cross_field_constraints(config, context): """Validate relationships and context-dependent rules spanning config fields.""" - if config["load_id"] is not None and not (config["wandb"] or config["neptune"]): - _raise_config_error(context, "load_id", "requires wandb or neptune") + if config["load_id"] is not None and not config["wandb"]: + _raise_config_error(context, "load_id", "requires wandb") env = config["env"] if env["min_agents_per_env"] > env["max_agents_per_env"]: diff --git a/pufferlib/ocean/drive/drive.h b/pufferlib/ocean/drive/drive.h index 50eec3d0d5..9b825c49b5 100644 --- a/pufferlib/ocean/drive/drive.h +++ b/pufferlib/ocean/drive/drive.h @@ -187,6 +187,7 @@ struct Drive { int *tracks_to_predict; // Simulation int timestep; + int autoreset_pending; int init_step; float dt; float base_max_speed_mps; @@ -4436,6 +4437,7 @@ static void move_dynamics(Drive *env, int action_idx, int agent_idx) { #include "idm.h" void c_reset(Drive *env) { + env->autoreset_pending = 0; if (env->timestep == 0) { for (int i = 0; i < env->num_total_agents; i++) { copy_pose_to_prev(&env->agents[i]); @@ -4549,18 +4551,10 @@ void c_step(Drive *env) { memset(env->rewards, 0, env->active_agent_count * sizeof(float)); memset(env->terminals, 0, env->active_agent_count * sizeof(unsigned char)); memset(env->truncations, 0, env->active_agent_count * sizeof(unsigned char)); - - // Update masks for stopped/removed agents - for (int i = 0; i < env->active_agent_count; i++) { - int agent_idx = env->active_agent_indices[i]; - Agent *a = &env->agents[agent_idx]; - if (a->stopped || a->removed || a->is_blind_partner || a->is_phantom_braker) { - env->masks[i] = 0; - } else { - env->masks[i] = 1; - } + if (env->autoreset_pending) { + c_reset(env); + return; } - env->timestep++; // -> 1. Apply actions and move agents @@ -4615,8 +4609,10 @@ void c_step(Drive *env) { // Mark terminals for stopped or removed agents for (int i = 0; i < env->active_agent_count; i++) { - int agent_idx = env->active_agent_indices[i]; - if (env->agents[agent_idx].stopped || env->agents[agent_idx].removed) { + Agent *agent = &env->agents[env->active_agent_indices[i]]; + // Masks describe the next action's eligibility; terminal rewards belong to the previous action. + env->masks[i] = !agent->stopped && !agent->removed && !agent->is_blind_partner && !agent->is_phantom_braker; + if (agent->stopped || agent->removed) { env->terminals[i] = 1; } } @@ -4656,7 +4652,10 @@ void c_step(Drive *env) { env->eval_episode_done = 1; return; } - c_reset(env); + // Expose the terminal state for value bootstrap; the next call resets without applying its action. + compute_observations(env); + memset(env->masks, 0, env->active_agent_count * sizeof(unsigned char)); + env->autoreset_pending = 1; return; } @@ -4699,6 +4698,8 @@ void c_step(Drive *env) { if (!regen_ok) { invalidate_agent(agent); agent->removed = 1; + env->masks[i] = 0; + env->terminals[i] = 1; } } } diff --git a/pufferlib/ocean/drive/drive.py b/pufferlib/ocean/drive/drive.py index c34e1d9b15..c93c6a5798 100644 --- a/pufferlib/ocean/drive/drive.py +++ b/pufferlib/ocean/drive/drive.py @@ -546,6 +546,7 @@ def reset(self, seed=None): else: binding.vec_reset(self.c_envs) self.tick = 0 + self._resample_pending = False self.truncations[:] = 0 if self.capture_replay: self._initialize_replay_captures() @@ -560,17 +561,23 @@ def step(self, actions): if self.capture_replay: self._capture_replay_step() self.actions[:] = actions - binding.vec_step(self.c_envs) - self.tick += 1 + if not self._resample_pending: + binding.vec_step(self.c_envs) + self.tick += 1 info = [] # vec_log is the training aggregate; it resets env->log, which eval reads # per episode, so it must not run in eval mode. - if not self.eval_mode and self.tick % self.report_interval == 0: + if not self.eval_mode and not self._resample_pending and self.tick % self.report_interval == 0: log = binding.vec_log(self.c_envs, self.num_agents) if log: info.append(log) - # print(log) if self.tick > 0 and self.resample_frequency > 0 and self.tick % self.resample_frequency == 0: + if not self.eval_mode and not self._resample_pending: + self._resample_pending = True + self.truncations[:] = 1 + self.masks[:] = 0 + return (self.observations, self.rewards, self.terminals, self.truncations, info) + self._resample_pending = False self.tick = 0 will_resample = 1 if will_resample: @@ -643,8 +650,10 @@ def step(self, actions): binding.vec_reset(self.c_envs) if self.capture_replay: self._initialize_replay_captures() - # Map resampling is an external reset boundary (dataset/map switch). Treat as truncation. - self.truncations[:] = 1 + self.truncations[:] = self.eval_mode + if not self.eval_mode: + self.rewards[:] = 0 + self.terminals[:] = 0 return (self.observations, self.rewards, self.terminals, self.truncations, info) def get_global_agent_state(self): diff --git a/pufferlib/ocean/evaluation_utils/eval_replay.py b/pufferlib/ocean/evaluation_utils/eval_replay.py index 7b63b378c6..8dcd97fb6f 100644 --- a/pufferlib/ocean/evaluation_utils/eval_replay.py +++ b/pufferlib/ocean/evaluation_utils/eval_replay.py @@ -164,7 +164,7 @@ def write_pending(self): self.pending_replays = [] -def _render_eval_replays(episode_summaries, out_dir, keep_zlib_replays): +def render_eval_replays(episode_summaries, out_dir, keep_zlib_replays): """Render captured eval replays as navigable HTML pages plus an index.""" render_dir = os.path.join(out_dir, "rendered_replays") os.makedirs(render_dir, exist_ok=True) diff --git a/pufferlib/ocean/evaluation_utils/evaluation_utils.py b/pufferlib/ocean/evaluation_utils/evaluation_utils.py index 044fddabeb..51caccaace 100644 --- a/pufferlib/ocean/evaluation_utils/evaluation_utils.py +++ b/pufferlib/ocean/evaluation_utils/evaluation_utils.py @@ -56,7 +56,7 @@ def _load_yaml_mapping(path, label): return _require_mapping(value, label) -def _resolve_map_indices(map_dir, map_names): +def resolve_map_indices(map_dir, map_names): """Map each logged map name back to its index in the sorted .bin map set.""" if os.path.isfile(map_dir) and str(map_dir).endswith(".bin"): map_files = [map_dir] @@ -270,7 +270,7 @@ def build_benchmark_args(base_args, benchmark, environment_config, cli_overrides ) -def _plan_benchmark_eval_workers(args, num_scenarios, num_workers, scenario_length, capture_replay=False): +def plan_benchmark_eval_workers(args, num_scenarios, num_workers, scenario_length, capture_replay=False): """One disjoint contiguous map window per worker; together they cover the set once.""" scenarios_per_worker, remainder = divmod(num_scenarios, num_workers) worker_env_kwargs = [] @@ -290,7 +290,7 @@ def _plan_benchmark_eval_workers(args, num_scenarios, num_workers, scenario_leng return worker_env_kwargs, max_scenarios_per_worker * scenario_length -def _plan_failure_replay_workers(args, map_seed_pairs, num_workers, scenario_length): +def plan_failure_replay_workers(args, map_seed_pairs, num_workers, scenario_length): """Split the (map, seed) pairs across workers; each worker cycles through its pairs in fit-aware batches (num_agents from config bounds a batch).""" pairs_per_worker, remainder = divmod(len(map_seed_pairs), num_workers) @@ -347,7 +347,7 @@ def _build_eval_report(episode_summaries, num_scenarios): return df, summary -def _write_eval_reports(episode_summaries, out_dir, num_scenarios): +def write_eval_reports(episode_summaries, out_dir, num_scenarios): """Write a per-episode metrics CSV and a JSON of metric averages to out_dir.""" import json diff --git a/pufferlib/pufferl.py b/pufferlib/pufferl.py index a1ed9a6bc6..cc2cefac7f 100644 --- a/pufferlib/pufferl.py +++ b/pufferlib/pufferl.py @@ -11,43 +11,39 @@ warnings.filterwarnings("error", category=RuntimeWarning) -import os -import sys -import traceback import glob -import time -import random -import shutil -import subprocess import importlib +import os import platform +import random import shlex +import shutil +import subprocess +import sys +import time +import traceback +from collections import defaultdict, deque from datetime import datetime from threading import Thread -from collections import defaultdict, deque -import yaml -from hydra import compose, initialize_config_dir -from omegaconf import OmegaConf import numpy as np import psutil - +import rich +import rich.traceback import torch import torch.distributed +import yaml +from hydra import compose, initialize_config_dir +from omegaconf import OmegaConf +from rich.console import Console +from rich.table import Table from torch.distributed.elastic.multiprocessing.errors import record +from tqdm import tqdm import pufferlib -from pufferlib.ocean.evaluation_utils import evaluation_utils as drive_benchmark -from pufferlib.ocean.evaluation_utils import eval_replay as drive_eval_replay -import pufferlib.sweep -import pufferlib.utils -import pufferlib.vector -import pufferlib.pytorch -from pufferlib.config_schema import ( - normalize_puffer_drive_config, - validate_puffer_drive_config, - validate_puffer_drive_resources, -) +from pufferlib import config_schema, pytorch, utils, vector +from pufferlib import sweep as sweep_module +from pufferlib.ocean.evaluation_utils import eval_replay, evaluation_utils try: @@ -57,20 +53,17 @@ "Failed to import C/CUDA advantage kernel. If you have non-default PyTorch, try installing with --no-build-isolation" ) -import rich -import rich.traceback -from rich.table import Table -from rich.console import Console -from tqdm import tqdm rich.traceback.install(show_locals=False) import signal # Aggressively exit on ctrl+c + signal.signal(signal.SIGINT, lambda sig, frame: os._exit(0)) from torch.utils.cpp_extension import CUDA_HOME, ROCM_HOME # noqa: E402 + # Assume advantage kernel has been built if torch has been compiled with CUDA or HIP support # and can find CUDA or HIP in the system ADVANTAGE_CUDA = bool(CUDA_HOME or ROCM_HOME) @@ -100,40 +93,6 @@ PROFILE_SIM_STEPS_PER_CYCLE = 16_384 -def torch_device(device): - if isinstance(device, int): - return torch.device("cuda", device) if torch.cuda.is_available() else torch.device("cpu") - return device - - -def is_cuda_device(device): - if isinstance(device, int): - return torch.cuda.is_available() - device = torch.device(device) - return device.type == "cuda" - - -def base_policy(policy): - return policy.module if hasattr(policy, "module") else policy - - -def clean_state_key(key): - prefixes = ("module.", "_orig_mod.") - while key.startswith(prefixes): - key = key.split(".", 1)[1] - return key - - -def clean_policy_state_dict(state_dict): - return {clean_state_key(k): v for k, v in state_dict.items()} - - -def logits_to_float(logits): - if isinstance(logits, torch.distributions.Normal): - return torch.distributions.Normal(logits.loc.float(), logits.scale.float()) - return logits.float() - - class PuffeRL: def __init__(self, config, vecenv, policy, logger=None): # Backend perf optimization @@ -154,13 +113,13 @@ def __init__(self, config, vecenv, policy, logger=None): self.env_continuous = isinstance(vecenv.single_action_space, pufferlib.spaces.Box) obs_space = vecenv.single_observation_space # Custom policy attributes live on the base module, not the DDP/compile wrapper. - unwrapped_policy = base_policy(policy) + unwrapped_policy = utils.base_policy(policy) if self.env_continuous and not unwrapped_policy.is_continuous: action_shape = () action_dtype = torch.int32 else: action_shape = vecenv.single_action_space.shape - action_dtype = pufferlib.pytorch.numpy_to_torch_dtype_dict[vecenv.single_action_space.dtype] + action_dtype = pytorch.numpy_to_torch_dtype_dict[vecenv.single_action_space.dtype] total_agents = vecenv.num_agents self.total_agents = total_agents @@ -177,13 +136,13 @@ def __init__(self, config, vecenv, policy, logger=None): device = config["device"] precision = config["precision"] - use_cuda = is_cuda_device(device) + use_cuda = utils.is_cuda_device(device) if precision == "bfloat16" and use_cuda and not torch.cuda.is_bf16_supported(): raise pufferlib.APIUsageError("bfloat16 precision requires a CUDA device with bf16 support") rollout_dtype = config.get("rollout_dtype", "float32") if rollout_dtype == "float32": - obs_dtype = pufferlib.pytorch.numpy_to_torch_dtype_dict[obs_space.dtype] + obs_dtype = pytorch.numpy_to_torch_dtype_dict[obs_space.dtype] elif rollout_dtype == "float16" and np.issubdtype(obs_space.dtype, np.floating): obs_dtype = getattr(torch, rollout_dtype) else: @@ -233,7 +192,7 @@ def __init__(self, config, vecenv, policy, logger=None): self.minibatch_segments = self.minibatch_size // horizon # Torch compile - self.uncompiled_policy = base_policy(policy) + self.uncompiled_policy = utils.base_policy(policy) self.policy = policy if config["compile"]: compile_kwargs = { @@ -242,7 +201,7 @@ def __init__(self, config, vecenv, policy, logger=None): } self.policy = torch.compile(policy, **compile_kwargs) self.policy.forward_eval = torch.compile(self.uncompiled_policy.forward_eval, **compile_kwargs) - pufferlib.pytorch.sample_logits = torch.compile(pufferlib.pytorch.sample_logits, **compile_kwargs) + pytorch.sample_logits = torch.compile(pytorch.sample_logits, **compile_kwargs) # Optimizer if config["optimizer"] == "adam": @@ -323,7 +282,7 @@ def __init__(self, config, vecenv, policy, logger=None): self.print_dashboard(clear=True) def load_training_state(self, path): - device = torch_device(self.config["device"]) + device = utils.torch_device(self.config["device"]) state = torch.load(path, map_location=device, weights_only=False) policy_state = state.get("policy_state_dict") if policy_state is None: @@ -331,7 +290,7 @@ def load_training_state(self, path): model_path = os.path.join(os.path.dirname(path), "models", model_name) policy_state = torch.load(model_path, map_location=device) - policy_state = clean_policy_state_dict(policy_state) + policy_state = utils.clean_policy_state_dict(policy_state) self.uncompiled_policy.load_state_dict(policy_state) self.optimizer.load_state_dict(state["optimizer_state_dict"]) @@ -343,7 +302,7 @@ def load_training_state(self, path): self.last_log_step = self.global_step self.best_score = state.get("best_score", self.best_score) self.ema_max = state.get("ema_max", self.ema_max) - restore_rng_state(state) + utils.restore_rng_state(state) print(f"Resumed training from {path}: epoch={self.epoch}, global_step={self.global_step}") @property @@ -405,20 +364,20 @@ def evaluate(self): profile("eval_forward", epoch) with torch.no_grad(), self.amp_context: - state = dict( - reward=r, - done=done_mask, - env_id=env_id, - mask=mask, - ) + state = { + "reward": r, + "done": done_mask, + "env_id": env_id, + "mask": mask, + } if config["use_rnn"]: state["lstm_h"] = self.lstm_h[env_id.start] state["lstm_c"] = self.lstm_c[env_id.start] logits, value = self.policy.forward_eval(o_device, state) - logits = logits_to_float(logits) - action, logprob, _, cont_action = pufferlib.pytorch.sample_logits( + logits = utils.logits_to_float(logits) + action, logprob, _, cont_action = pytorch.sample_logits( logits, env_continuous=self.env_continuous, policy=self.uncompiled_policy ) if config["normalize_rewards"]: @@ -441,19 +400,14 @@ def evaluate(self): self.actions[batch_rows, l] = action self.logprobs[batch_rows, l] = logprob.float() - # Truncation bootstrap hack for auto-reset envs. - # Ideally we add `gamma * V(s_{t+1})` on truncation steps, but Drive resets in C so - # the value at index `l` is post-reset. We use `values[..., l-1]` as a heuristic - # proxy for the pre-reset terminal value (bootstrap term is not clipped). - if l > 0 and config["use_value_bootstrapping"]: + if config["use_value_bootstrapping"]: trunc_mask = (t > 0) & (d == 0) - r = r + trunc_mask.to(r.dtype) * config["gamma"] * self.values[batch_rows, l - 1] + r = r + trunc_mask.to(r.dtype) * config["gamma"] * value.flatten().float() self.rewards[batch_rows, l] = r self.terminals[batch_rows, l] = done_mask.bool() self.values[batch_rows, l] = value.flatten().float() self.masks[batch_rows, l] = m - # Note: We are not yet handling masks in this version self.ep_lengths[env_id] += 1 if l + 1 >= config["bptt_horizon"]: num_full = env_id.stop - env_id.start @@ -488,7 +442,7 @@ def evaluate(self): self.ep_indices = torch.arange(self.total_agents, dtype=torch.int32) self.ep_lengths.zero_() profile.end() - return pufferlib.utils.reduce_environment_metrics(self.stats) + return utils.reduce_environment_metrics(self.stats) @record def train(self): @@ -528,14 +482,14 @@ def train(self): def _ppo_loss(self, mb_obs, mb_actions, mb_logprobs, mb_values, mb_returns, mb_adv, adv_weights=None): config = self.config - state = dict(action=mb_actions, lstm_h=None, lstm_c=None) + state = {"action": mb_actions, "lstm_h": None, "lstm_c": None} if self.compress_observations: mb_obs = mb_obs.float() with self.amp_context: logits, newvalue = self.policy(mb_obs, state) - logits = logits_to_float(logits) + logits = utils.logits_to_float(logits) newvalue = newvalue.float() - _, newlogprob, entropy, _ = pufferlib.pytorch.sample_logits(logits, action=mb_actions) + _, newlogprob, entropy, _ = pytorch.sample_logits(logits, action=mb_actions) newlogprob = newlogprob.float().view_as(mb_logprobs) newvalue = newvalue.view_as(mb_returns) @@ -628,7 +582,7 @@ def _train_ppo_trajectory(self, losses, profile, epoch): for mb in range(self.total_minibatches): profile("train_misc", epoch) - advantages, returns, masks = self._compute_advantages( + advantages, returns, _ = self._compute_advantages( self.ratio, config["vtrace_rho_clip"], config["vtrace_c_clip"], @@ -813,23 +767,20 @@ def _train_ppo_transition(self, losses, profile, epoch): def mean_and_log(self): config = self.config - self.stats = pufferlib.utils.reduce_environment_metrics(self.stats) + self.stats = utils.reduce_environment_metrics(self.stats) device = config["device"] - agent_steps = int(dist_sum(self.global_step, device)) + agent_steps = int(utils.dist_sum(self.global_step, device)) self.agent_steps = agent_steps logs = { - "SPS": dist_sum(self.sps, device), + "SPS": utils.dist_sum(self.sps, device), "agent_steps": agent_steps, "uptime": time.time() - self.start_time, - "epoch": int(dist_sum(self.epoch, device)), # VB Why it is a sum ? + "epoch": self.epoch, "learning_rate": self.optimizer.param_groups[0]["lr"], **{f"environment/{k}": v for k, v in self.stats.items()}, **{f"losses/{k}": v for k, v in self.losses.items()}, **{f"performance/{k}": v["elapsed"] for k, v in self.profile}, - # **{f'environment/{k}': dist_mean(v, device) for k, v in self.stats.items()}, - # **{f'losses/{k}': dist_mean(v, device) for k, v in self.losses.items()}, - # **{f'performance/{k}': dist_sum(v['elapsed'], device) for k, v in self.profile}, } if torch.distributed.is_initialized() and torch.distributed.get_rank() != 0: @@ -888,7 +839,7 @@ def save_checkpoint(self): "env": self.config["env"], "best_score": self.best_score, "ema_max": self.ema_max, - "rng_state": capture_rng_state(), + "rng_state": utils.capture_rng_state(), } state_path = os.path.join(path, "trainer_state.pt") torch.save(state, state_path + ".tmp") @@ -906,11 +857,17 @@ def save_checkpoint(self): return model_path def print_dashboard( - self, clear=False, idx=[0], c1="[cyan]", c2="[dim default]", b1="[bright_cyan]", b2="[default]" + self, + clear=False, + idx=[0], + c1="[cyan]", + c2="[dim default]", + b1="[bright_cyan]", + b2="[default]", ): config = self.config - sps = dist_sum(self.sps, config["device"]) - agent_steps = dist_sum(self.global_step, config["device"]) + sps = utils.dist_sum(self.sps, config["device"]) + agent_steps = utils.dist_sum(self.global_step, config["device"]) if torch.distributed.is_initialized() and torch.distributed.get_rank() != 0: return @@ -927,7 +884,7 @@ def print_dashboard( table.add_column(justify="right", width=13) table.add_row( - f"{b1}PufferLib {b2}3.0 {idx[0] * ' '}:blowfish:", + f"{b1}PufferDrive {b2}3.0 {idx[0] * ' '}:car:", f"{c1}CPU: {b2}{np.mean(self.utilization.cpu_util):.1f}{c2}%", f"{c1}GPU: {b2}{np.mean(self.utilization.gpu_util):.1f}{c2}%", f"{c1}DRAM: {b2}{np.mean(self.utilization.cpu_mem):.1f}{c2}%", @@ -938,16 +895,16 @@ def print_dashboard( s = Table(box=None, expand=True) remaining = f"{b2}A hair past a freckle{c2}" if sps != 0: - remaining = duration((config["total_timesteps"] - agent_steps) / sps, b2, c2) + remaining = utils.duration((config["total_timesteps"] - agent_steps) / sps, b2, c2) s.add_column(f"{c1}Summary", justify="left", vertical="top", width=10) s.add_column(f"{c1}Value", justify="right", vertical="top", width=14) s.add_row(f"{b2}Env", f"{b2}{config['env']}") - s.add_row(f"{b2}Params", abbreviate(self.model_size, b2, c2)) - s.add_row(f"{b2}Steps", abbreviate(agent_steps, b2, c2)) - s.add_row(f"{b2}SPS", abbreviate(sps, b2, c2)) + s.add_row(f"{b2}Params", utils.abbreviate(self.model_size, b2, c2)) + s.add_row(f"{b2}Steps", utils.abbreviate(agent_steps, b2, c2)) + s.add_row(f"{b2}SPS", utils.abbreviate(sps, b2, c2)) s.add_row(f"{b2}Epoch", f"{b2}{self.epoch}") - s.add_row(f"{b2}Uptime", duration(self.uptime, b2, c2)) + s.add_row(f"{b2}Uptime", utils.duration(self.uptime, b2, c2)) s.add_row(f"{b2}Remaining", remaining) delta = profile.eval["buffer"] + profile.train["buffer"] @@ -955,16 +912,16 @@ def print_dashboard( p.add_column(f"{c1}Performance", justify="left", width=10) p.add_column(f"{c1}Time", justify="right", width=8) p.add_column(f"{c1}%", justify="right", width=4) - p.add_row(*fmt_perf("Evaluate", b1, delta, profile.eval, b2, c2)) - p.add_row(*fmt_perf(" Forward", b2, delta, profile.eval_forward, b2, c2)) - p.add_row(*fmt_perf(" Env", b2, delta, profile.env, b2, c2)) - p.add_row(*fmt_perf(" Copy", b2, delta, profile.eval_copy, b2, c2)) - p.add_row(*fmt_perf(" Misc", b2, delta, profile.eval_misc, b2, c2)) - p.add_row(*fmt_perf("Train", b1, delta, profile.train, b2, c2)) - p.add_row(*fmt_perf(" Forward", b2, delta, profile.train_forward, b2, c2)) - p.add_row(*fmt_perf(" Learn", b2, delta, profile.learn, b2, c2)) - p.add_row(*fmt_perf(" Copy", b2, delta, profile.train_copy, b2, c2)) - p.add_row(*fmt_perf(" Misc", b2, delta, profile.train_misc, b2, c2)) + p.add_row(*utils.fmt_perf("Evaluate", b1, delta, profile.eval, b2, c2)) + p.add_row(*utils.fmt_perf(" Forward", b2, delta, profile.eval_forward, b2, c2)) + p.add_row(*utils.fmt_perf(" Env", b2, delta, profile.env, b2, c2)) + p.add_row(*utils.fmt_perf(" Copy", b2, delta, profile.eval_copy, b2, c2)) + p.add_row(*utils.fmt_perf(" Misc", b2, delta, profile.eval_misc, b2, c2)) + p.add_row(*utils.fmt_perf("Train", b1, delta, profile.train, b2, c2)) + p.add_row(*utils.fmt_perf(" Forward", b2, delta, profile.train_forward, b2, c2)) + p.add_row(*utils.fmt_perf(" Learn", b2, delta, profile.learn, b2, c2)) + p.add_row(*utils.fmt_perf(" Copy", b2, delta, profile.train_copy, b2, c2)) + p.add_row(*utils.fmt_perf(" Misc", b2, delta, profile.train_misc, b2, c2)) l = Table( box=None, @@ -1018,7 +975,15 @@ def print_dashboard( def compute_puff_advantage( - values, rewards, terminals, ratio, advantages, gamma, gae_lambda, vtrace_rho_clip, vtrace_c_clip + values, + rewards, + terminals, + ratio, + advantages, + gamma, + gae_lambda, + vtrace_rho_clip, + vtrace_c_clip, ): """CUDA kernel for puffer advantage with automatic CPU fallback. You need nvcc (in cuda-dev-tools or in a cuda-dev docker base) for PufferLib to @@ -1033,7 +998,15 @@ def compute_puff_advantage( advantages = advantages.cpu() torch.ops.pufferlib.compute_puff_advantage( - values, rewards, terminals, ratio, advantages, gamma, gae_lambda, vtrace_rho_clip, vtrace_c_clip + values, + rewards, + terminals, + ratio, + advantages, + gamma, + gae_lambda, + vtrace_rho_clip, + vtrace_c_clip, ) if not ADVANTAGE_CUDA: @@ -1042,73 +1015,6 @@ def compute_puff_advantage( return advantages -def abbreviate(num, b2, c2): - if num < 1e3: - return f"{b2}{num}{c2}" - elif num < 1e6: - return f"{b2}{num / 1e3:.1f}{c2}K" - elif num < 1e9: - return f"{b2}{num / 1e6:.1f}{c2}M" - elif num < 1e12: - return f"{b2}{num / 1e9:.1f}{c2}B" - else: - return f"{b2}{num / 1e12:.2f}{c2}T" - - -def duration(seconds, b2, c2): - if seconds < 0: - return f"{b2}0{c2}s" - seconds = int(seconds) - h = seconds // 3600 - m = (seconds % 3600) // 60 - s = seconds % 60 - return f"{b2}{h}{c2}h {b2}{m}{c2}m {b2}{s}{c2}s" if h else f"{b2}{m}{c2}m {b2}{s}{c2}s" if m else f"{b2}{s}{c2}s" - - -def fmt_perf(name, color, delta_ref, prof, b2, c2): - percent = 0 if delta_ref == 0 else int(100 * prof["buffer"] / delta_ref - 1e-5) - return f"{color}{name}", duration(prof["elapsed"], b2, c2), f"{b2}{percent:2d}{c2}%" - - -def dist_sum(value, device): - if not torch.distributed.is_initialized(): - return value - - tensor = torch.tensor(value, device=device) - torch.distributed.all_reduce(tensor, op=torch.distributed.ReduceOp.SUM) - return tensor.item() - - -def dist_mean(value, device): - if not torch.distributed.is_initialized(): - return value - - return dist_sum(value, device) / torch.distributed.get_world_size() - - -def capture_rng_state(): - state = { - "python": random.getstate(), - "numpy": np.random.get_state(), - "torch": torch.get_rng_state(), - } - if torch.cuda.is_available(): - state["cuda"] = torch.cuda.get_rng_state_all() - return state - - -def restore_rng_state(state): - rng_state = state.get("rng_state") - if not rng_state: - return - - random.setstate(rng_state["python"]) - np.random.set_state(rng_state["numpy"]) - torch.set_rng_state(rng_state["torch"].cpu()) - if torch.cuda.is_available() and "cuda" in rng_state: - torch.cuda.set_rng_state_all([state.cpu() for state in rng_state["cuda"]]) - - class Profile: def __init__(self, frequency=5): self.profiles = defaultdict(lambda: defaultdict(float)) @@ -1197,27 +1103,6 @@ def stop(self): self.stopped = True -def downsample(data_list, num_points): - if not data_list or num_points <= 0: - return [] - if num_points == 1: - return [data_list[-1]] - if len(data_list) <= num_points: - return data_list - - last = data_list[-1] - data_list = data_list[:-1] - - data_np = np.array(data_list) - num_points -= 1 # one down for the last one - - n = (len(data_np) // num_points) * num_points - data_np = data_np[-n:] if n > 0 else data_np - downsampled = data_np.reshape(num_points, -1).mean(axis=1) - - return downsampled.tolist() + [last] - - class NoLogger: def __init__(self, args, run_id=None): self.run_id = run_id or args["run_name"] @@ -1229,48 +1114,6 @@ def close(self, model_path, early_stop): pass -class NeptuneLogger: - def __init__(self, args, load_id=None, mode="async"): - import neptune as nept - - neptune_name = args["neptune_name"] - neptune_project = args["neptune_project"] - neptune = nept.init_run( - project=f"{neptune_name}/{neptune_project}", - capture_hardware_metrics=False, - capture_stdout=False, - capture_stderr=False, - capture_traceback=False, - with_id=load_id, - # Neptune's resume-by-name key, mirroring the wandb id above. - custom_run_id=None if load_id else args["run_name"], - mode=mode, - tags=[args["tag"]] if args["tag"] is not None else [], - ) - self.run_id = neptune._sys_id - self.neptune = neptune - for k, v in pufferlib.unroll_nested_dict(args): - neptune[k].append(v) - self.should_upload_model = not args["no_model_upload"] - - def log(self, logs, step): - for k, v in logs.items(): - self.neptune[k].append(v, step=step) - - def upload_model(self, model_path): - self.neptune["model"].track_files(model_path) - - def close(self, model_path, early_stop): - self.neptune["early_stop"] = early_stop - if self.should_upload_model: - self.upload_model(model_path) - self.neptune.stop() - - def download(self): - self.neptune["model"].download(destination="artifacts") - return f"artifacts/{self.run_id}.pt" - - class WandbLogger: def __init__(self, args, load_id=None, resume="allow", upload_config=True, disable_meta=False): import wandb @@ -1304,7 +1147,7 @@ def upload_model(self, model_path): artifact = self.wandb.Artifact(self.run_id, type="model") artifact.add_file(model_path) # Ship the config with the weights; load_checkpoint_architecture reads it off disk. - config_path = os.path.join(drive_benchmark.resolve_run_dir(model_path), "config.yaml") + config_path = os.path.join(evaluation_utils.resolve_run_dir(model_path), "config.yaml") if os.path.isfile(config_path): artifact.add_file(config_path) self.wandb.run.log_artifact(artifact) @@ -1344,72 +1187,12 @@ def close(self, model_path, early_stop): self.local_writer.close() -def _get_git_metadata(): - git_metadata = { - "commit_hash": os.environ.get("GITHUB_SHA") or os.environ.get("COMMIT_SHA"), - } - - try: - repo_root = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) - if shutil.which("git") is None: - return git_metadata - - subprocess.check_output( - ["git", "rev-parse", "--is-inside-work-tree"], - cwd=repo_root, - stderr=subprocess.DEVNULL, - text=True, - ) - - if git_metadata["commit_hash"] is None: - git_metadata["commit_hash"] = subprocess.check_output( - ["git", "rev-parse", "HEAD"], - cwd=repo_root, - stderr=subprocess.DEVNULL, - text=True, - ).strip() - except (OSError, subprocess.SubprocessError): - pass - - return git_metadata - - -def _save_experiment_config(args, path): - import yaml - import json - - experiment_dir = path - os.makedirs(experiment_dir, exist_ok=True) - - # Save config as yaml - config_yaml_path = os.path.join(experiment_dir, "config.yaml") - with open(config_yaml_path, "w") as f: - # Convert defaultdict to dict for cleaner output - config = json.loads(json.dumps(args)) - yaml.dump(config, f) - - -def _global_agent_steps(pufferl): - world_size = torch.distributed.get_world_size() if torch.distributed.is_initialized() else 1 - return int(pufferl.global_step * world_size) - - -def derive_rank_seeds(vec_seed, train_seed, world_size, global_rank): - """Deterministic per-rank (torch_seed, env_seed): DDP ranks share weights, so identical seeds - would collect duplicate experience. global_rank is torchrun's global RANK, not LOCAL_RANK.""" - torch_seed = train_seed * world_size + global_rank - env_seed = vec_seed - if env_seed is not None: - env_seed = int(np.random.SeedSequence([env_seed, train_seed, global_rank]).generate_state(1)[0]) - return torch_seed, env_seed - - def train(env_name, args=None, vecenv=None, policy=None, logger=None, early_stop_fn=None): args = args or load_config(env_name) # Fine-tuning: reload network, observation configuration from config.yaml and override the args --> only change new reward / new maps / new simulation mode if args["load_model_path"]: - experiment_dir = drive_benchmark.resolve_run_dir(args["load_model_path"]) + experiment_dir = evaluation_utils.resolve_run_dir(args["load_model_path"]) config_yaml_path = os.path.join(experiment_dir, "config.yaml") KEYS_OF_INTEREST = { "action_type", @@ -1458,11 +1241,11 @@ def train(env_name, args=None, vecenv=None, policy=None, logger=None, early_stop "policy/observation architecture instead of the checkpoint's." ) - args = normalize_puffer_drive_config(args, "training") - validate_puffer_drive_config(args, "training") + args = config_schema.normalize_puffer_drive_config(args, "training") + config_schema.validate_puffer_drive_config(args, "training") if vecenv is None: - validate_puffer_drive_resources(args, "training") - training_evaluation_scheduled = drive_benchmark.validate_training_evaluation_config(args) + config_schema.validate_puffer_drive_resources(args, "training") + training_evaluation_scheduled = evaluation_utils.validate_training_evaluation_config(args) # Assume TorchRun DDP is used if LOCAL_RANK is set world_size = int(os.environ.get("WORLD_SIZE", 1)) @@ -1478,7 +1261,7 @@ def train(env_name, args=None, vecenv=None, policy=None, logger=None, early_stop if train_seed is None: train_seed = time.time_ns() & 0xFFFFFFFF - torch_seed, env_seed = derive_rank_seeds(args["vec"]["seed"], train_seed, world_size, global_rank) + torch_seed, env_seed = utils.derive_rank_seeds(args["vec"]["seed"], train_seed, world_size, global_rank) torch.manual_seed(torch_seed) vecenv = vecenv or load_env(env_name, args, seed=env_seed) policy = policy or load_policy(args, vecenv, env_name) @@ -1496,16 +1279,14 @@ def train(env_name, args=None, vecenv=None, policy=None, logger=None, early_stop policy = model.to(local_rank) # Set before the logger so the run config the logger uploads carries it too. - args["git"] = _get_git_metadata() + args["git"] = utils.get_git_metadata() # Under DDP only rank 0 owns the run logger; other ranks keep logger=None, # which PuffeRL wraps in a NoLogger. Without this gate every rank calls - # wandb.init()/NeptuneLogger and you get world_size duplicate runs. + # wandb.init() and you get world_size duplicate runs. is_rank0 = (not torch.distributed.is_initialized()) or torch.distributed.get_rank() == 0 if is_rank0: - if args["neptune"]: - logger = NeptuneLogger(args) - elif args["wandb"]: + if args["wandb"]: logger = WandbLogger(args) elif args["tb"]: logger = TensorBoardLogger( @@ -1540,7 +1321,7 @@ def train(env_name, args=None, vecenv=None, policy=None, logger=None, early_stop # save_checkpoint writes models/model__.pt and trainer_state.pt # (sibling of models/) — so trainer_state.pt is one dir above the .pt path. if args.get("load_model_path"): - trainer_state_path = os.path.join(drive_benchmark.resolve_run_dir(args["load_model_path"]), "trainer_state.pt") + trainer_state_path = os.path.join(evaluation_utils.resolve_run_dir(args["load_model_path"]), "trainer_state.pt") if os.path.exists(trainer_state_path): print(f"Resuming optimizer/step state from {trainer_state_path}") # weights_only=False as in load_training_state: the state carries the @@ -1558,7 +1339,7 @@ def train(env_name, args=None, vecenv=None, policy=None, logger=None, early_stop path = args["train"]["data_dir"] if is_rank0: - _save_experiment_config(args, path) + utils.save_experiment_config(args, path) # Sweep needs data for early stopped runs, so send data when steps > 100M logging_threshold = min(0.20 * train_config["total_timesteps"], 100_000_000) @@ -1566,7 +1347,7 @@ def train(env_name, args=None, vecenv=None, policy=None, logger=None, early_stop last_training_evaluation_epoch = None while pufferl.global_step < train_config["total_timesteps"]: - if is_cuda_device(train_config["device"]): + if utils.is_cuda_device(train_config["device"]): torch.compiler.cudagraph_mark_step_begin() try: pufferl.evaluate() @@ -1576,7 +1357,7 @@ def train(env_name, args=None, vecenv=None, policy=None, logger=None, early_stop if torch.distributed.is_initialized(): torch.distributed.destroy_process_group() raise - if is_cuda_device(train_config["device"]): + if utils.is_cuda_device(train_config["device"]): torch.compiler.cudagraph_mark_step_begin() try: logs = pufferl.train() @@ -1596,7 +1377,7 @@ def train(env_name, args=None, vecenv=None, policy=None, logger=None, early_stop policy=pufferl.uncompiled_policy, logger=pufferl.logger, epoch=pufferl.epoch, - global_step=_global_agent_steps(pufferl), + global_step=utils.global_agent_steps(pufferl), run_dir=path, ) @@ -1618,15 +1399,6 @@ def train(env_name, args=None, vecenv=None, policy=None, logger=None, early_stop pufferl.logger.close(model_path, early_stop=True) return all_logs - # Final eval. You can reset the env here, but depending on - # your env, this can skew data (i.e. you only collect the shortest - # rollouts within a fixed number of epochs) - # i = 0 - # stats = {} - # while i < 32 or not stats: - # stats = pufferl.evaluate() - # i += 1 - if training_evaluation_scheduled and last_training_evaluation_epoch != pufferl.epoch and is_rank0: run_training_evaluation( env_name=env_name, @@ -1634,7 +1406,7 @@ def train(env_name, args=None, vecenv=None, policy=None, logger=None, early_stop policy=pufferl.uncompiled_policy, logger=pufferl.logger, epoch=pufferl.epoch, - global_step=_global_agent_steps(pufferl), + global_step=utils.global_agent_steps(pufferl), run_dir=path, ) @@ -1673,7 +1445,7 @@ def eval( eval_training_render = args["env"]["eval_training_render"] report_to_wandb = bool(args["wandb"]) and not use_training_config - environment_config, benchmarks = drive_benchmark.load_benchmark_config(benchmark_config_path, selected_benchmarks) + environment_config, benchmarks = evaluation_utils.load_benchmark_config(benchmark_config_path, selected_benchmarks) if use_training_config: if policy is None: raise pufferlib.APIUsageError("Training evaluation requires the live policy") @@ -1682,14 +1454,14 @@ def eval( environment_config["obs_dropout_boundary"] = base_args["env"]["obs_dropout_boundary"] checkpoint_config_path = None else: - base_args, checkpoint_config_path = drive_benchmark.load_checkpoint_architecture(args) + base_args, checkpoint_config_path = evaluation_utils.load_checkpoint_architecture(args) base_args["env"]["eval_training_render"] = eval_training_render wandb_run_identity = ( - drive_benchmark.load_checkpoint_run_identity(checkpoint_config_path) if report_to_wandb else None + evaluation_utils.load_checkpoint_run_identity(checkpoint_config_path) if report_to_wandb else None ) if eval_output_dir is None: - run_dir = drive_benchmark.resolve_run_dir(base_args["load_model_path"]) + run_dir = evaluation_utils.resolve_run_dir(base_args["load_model_path"]) eval_output_dir = os.path.join(run_dir, eval_config["output_dir_name"]) if eval_output_subdir is None: eval_output_subdir = datetime.now().strftime("%Y%m%d-%H%M%S") @@ -1700,7 +1472,7 @@ def eval( benchmark_results = {} evaluation_policy_cache = {"policy": policy} for benchmark in benchmarks: - run_args = drive_benchmark.build_benchmark_args( + run_args = evaluation_utils.build_benchmark_args( base_args, benchmark, environment_config, @@ -1719,7 +1491,7 @@ def eval( benchmark_output_dir = os.path.join(benchmark_output_dir, eval_output_subdir) os.makedirs(benchmark_output_dir) resolved_benchmark_output_dir = benchmark_output_dir - drive_benchmark.write_resolved_benchmark_config( + evaluation_utils.write_resolved_benchmark_config( run_args, benchmark, benchmark_config_path, @@ -1744,7 +1516,7 @@ def eval( num_scenarios = run_args["num_scenarios"] num_workers = min(run_args["vec"]["num_envs"], num_scenarios) - worker_env_kwargs, total_steps = drive_benchmark._plan_benchmark_eval_workers( + worker_env_kwargs, total_steps = evaluation_utils.plan_benchmark_eval_workers( run_args, num_scenarios, num_workers, @@ -1753,7 +1525,7 @@ def eval( ) print(f"Evaluation {benchmark['name']}: {num_scenarios} scenarios across {num_workers} workers") replay_output_dir = ( - os.path.join(benchmark_output_dir, drive_eval_replay.ZLIB_REPLAY_DIR_NAME) if render_scenarios else None + os.path.join(benchmark_output_dir, eval_replay.ZLIB_REPLAY_DIR_NAME) if render_scenarios else None ) summaries = _run_eval_rollout( run_args, @@ -1767,14 +1539,14 @@ def eval( capture_observations=render_scenarios and eval_config["capture_observations"], evaluation_policy_cache=evaluation_policy_cache, ) - summary = drive_benchmark._write_eval_reports(summaries, benchmark_output_dir, num_scenarios) + summary = evaluation_utils.write_eval_reports(summaries, benchmark_output_dir, num_scenarios) benchmark_results[benchmark["name"]] = { "episodes": summaries, "summary": summary, } if render_scenarios: - drive_eval_replay._render_eval_replays(summaries, benchmark_output_dir, eval_config["keep_zlib_replays"]) + eval_replay.render_eval_replays(summaries, benchmark_output_dir, eval_config["keep_zlib_replays"]) elif render_filter is not None: _render_eval_failures( env_name, @@ -1795,7 +1567,7 @@ def eval( def report_eval_to_wandb(args, benchmark_results, wandb_run_identity, output_dir_name): """Attach standalone eval results to the wandb run that trained the checkpoint.""" - metrics = drive_benchmark.summarize_benchmark_metrics(benchmark_results, f"final_{output_dir_name}_") + metrics = evaluation_utils.summarize_benchmark_metrics(benchmark_results, f"final_{output_dir_name}_") if not metrics: print("No evaluation metrics to report to wandb.") return @@ -1813,16 +1585,16 @@ def report_eval_to_wandb(args, benchmark_results, wandb_run_identity, output_dir def sweep(args=None, env_name=None): args = args or load_config(env_name) - if not args["wandb"] and not args["neptune"] and not args["tb"]: - raise pufferlib.APIUsageError("Sweeps require either wandb, neptune, or tb") + if not args["wandb"] and not args["tb"]: + raise pufferlib.APIUsageError("Sweeps require either wandb or tb") method = args["sweep"].pop("method") try: - sweep_cls = getattr(pufferlib.sweep, method) + sweep_cls = getattr(sweep_module, method) except: raise pufferlib.APIUsageError(f"Invalid sweep method {method}. See pufferlib.sweep") - sweep = sweep_cls(args["sweep"]) + sweep_runner = sweep_cls(args["sweep"]) points_per_run = args["sweep"]["downsample"] target_key = f"environment/{args['sweep']['metric']}" @@ -1831,16 +1603,16 @@ def sweep(args=None, env_name=None): random.seed(seed) np.random.seed(seed) torch.manual_seed(seed) - sweep.suggest(args) + sweep_runner.suggest(args) total_timesteps = args["train"]["total_timesteps"] all_logs = train(env_name, args=args) all_logs = [e for e in all_logs if target_key in e] - scores = downsample([log[target_key] for log in all_logs], points_per_run) - costs = downsample([log["uptime"] for log in all_logs], points_per_run) - timesteps = downsample([log["agent_steps"] for log in all_logs], points_per_run) + scores = utils.downsample([log[target_key] for log in all_logs], points_per_run) + costs = utils.downsample([log["uptime"] for log in all_logs], points_per_run) + timesteps = utils.downsample([log["agent_steps"] for log in all_logs], points_per_run) for score, cost, timestep in zip(scores, costs, timesteps): args["train"]["total_timesteps"] = timestep - sweep.observe(args, score, cost) + sweep_runner.observe(args, score, cost) # Prevent logging final eval steps as training steps args["train"]["total_timesteps"] = total_timesteps @@ -1852,8 +1624,8 @@ def controlled_exp(env_name, args=None): from copy import deepcopy args = args or load_config(env_name) - if not args["wandb"] and not args["neptune"]: - raise pufferlib.APIUsageError("Targeted experiments require either wandb or neptune") + if not args["wandb"]: + raise pufferlib.APIUsageError("Targeted experiments require wandb") # Check if controlled_exp config exists if "controlled_exp" not in args: @@ -2004,7 +1776,6 @@ def profile(env_name): profile_config = args["profile"] profile_mode = profile_config["mode"] args["wandb"] = False - args["neptune"] = False args["tb"] = False args["load_id"] = None args["env"]["compute_eval_metrics"] = False @@ -2017,8 +1788,8 @@ def profile(env_name): args["train"]["minibatch_size"] = args["env"]["num_agents"] * args["train"]["bptt_horizon"] // 16 args["train"]["checkpoint_interval"] = profile_config["warmup_cycles"] + profile_config["trace_cycles"] + 1 validation_context = "simulation profiling" if profile_mode == "sim" else "profiling" - validate_puffer_drive_config(args, validation_context) - validate_puffer_drive_resources(args, "profiling") + config_schema.validate_puffer_drive_config(args, validation_context) + config_schema.validate_puffer_drive_resources(args, "profiling") # Acknowledged FIFO commands make the trace boundaries exact. perf_control = open(os.environ[PROFILE_CONTROL_ENV], "w") @@ -2027,7 +1798,7 @@ def profile(env_name): train_seed = args["train"]["seed"] if train_seed is None: train_seed = time.time_ns() & 0xFFFFFFFF - torch_seed, env_seed = derive_rank_seeds(args["vec"]["seed"], train_seed, 1, 0) + torch_seed, env_seed = utils.derive_rank_seeds(args["vec"]["seed"], train_seed, 1, 0) if profile_mode == "sim": perf_control.write("enable\n") @@ -2048,14 +1819,15 @@ def profile(env_name): vecenv.recv() profiler = contextlib.nullcontext() else: - from torch.profiler import ProfilerActivity, profile as torch_profile, record_function + from torch.profiler import ProfilerActivity, record_function + from torch.profiler import profile as torch_profile torch.manual_seed(torch_seed) vecenv = load_env(env_name, args, seed=env_seed) policy = load_policy(args, vecenv, env_name) train_config = dict(**args["train"], env=env_name, eval=args.get("eval", {}), run_name=args["run_name"]) pufferl = PuffeRL(train_config, vecenv, policy) - use_cuda = is_cuda_device(train_config["device"]) + use_cuda = utils.is_cuda_device(train_config["device"]) profile_rollouts = profile_mode != "training" activities = [ProfilerActivity.CPU] if use_cuda: @@ -2123,7 +1895,7 @@ def autotune(args=None, env_name=None, vecenv=None, policy=None): env_module = importlib.import_module(module_name) env_name = args["env_name"] make_env = env_module.env_creator(env_name) - pufferlib.vector.autotune(make_env, batch_size=args["train"]["env_batch_size"]) + vector.autotune(make_env, batch_size=args["train"]["env_batch_size"]) def _require_finite_eval_batch(batch_array, what, num_workers, worker_env_kwargs): @@ -2160,7 +1932,7 @@ def _run_eval_rollout( module_name = "pufferlib.ocean" if package == "ocean" else f"pufferlib.environments.{package}" env_module = importlib.import_module(module_name) make_env = env_module.env_creator(env_name) - vecenv = pufferlib.vector.make( + vecenv = vector.make( [make_env] * num_workers, env_args=[[]] * num_workers, env_kwargs=worker_env_kwargs, @@ -2196,7 +1968,7 @@ def _run_eval_rollout( policy.eval() if "policy_forward_eval" not in evaluation_policy_cache: policy_forward_eval = policy.forward_eval - eval_sample_logits = pufferlib.pytorch.sample_logits + eval_sample_logits = pytorch.sample_logits if args["train"]["compile"]: compile_kwargs = { "mode": args["train"]["compile_mode"], @@ -2211,11 +1983,11 @@ def _run_eval_rollout( # A discrete policy on a continuous env emits a discrete class that the # policy's own table maps back to the continuous action the env expects. action_selection = args["eval"]["action_selection"] - uncompiled_policy = base_policy(policy) + uncompiled_policy = utils.base_policy(policy) env_continuous = isinstance(vecenv.single_action_space, pufferlib.spaces.Box) discrete_policy_on_continuous_env = env_continuous and not uncompiled_policy.is_continuous - device = torch_device(args["train"]["device"]) - use_bfloat16 = args["train"]["precision"] == "bfloat16" and is_cuda_device(device) + device = utils.torch_device(args["train"]["device"]) + use_bfloat16 = args["train"]["precision"] == "bfloat16" and utils.is_cuda_device(device) if use_bfloat16 and not torch.cuda.is_bf16_supported(): raise pufferlib.APIUsageError("bfloat16 evaluation requires CUDA BF16 support") eval_amp_context = torch.amp.autocast(device_type="cuda", dtype=torch.bfloat16, enabled=use_bfloat16) @@ -2239,7 +2011,7 @@ def _run_eval_rollout( capture_batch_steps = worker_env_kwargs[0]["resample_frequency"] replay_capture = None if replay_output_dir is not None: - replay_capture = drive_eval_replay.EvalReplayCapture( + replay_capture = eval_replay.EvalReplayCapture( args, policy, replay_output_dir, @@ -2347,7 +2119,7 @@ def run_training_evaluation(env_name, args, policy, logger, epoch, global_step, eval_output_dir = os.path.join(run_dir, "eval", "training") eval_output_subdir = f"epoch_{epoch:06d}_step_{global_step}" - rng_state = capture_rng_state() + rng_state = utils.capture_rng_state() policy_was_training = bool(getattr(policy, "training", False)) try: benchmark_results = eval( @@ -2358,7 +2130,7 @@ def run_training_evaluation(env_name, args, policy, logger, epoch, global_step, eval_output_subdir=eval_output_subdir, use_training_config=True, ) - metrics = drive_benchmark.summarize_benchmark_metrics(benchmark_results, TRAINING_EVAL_KEY_PREFIX) + metrics = evaluation_utils.summarize_benchmark_metrics(benchmark_results, TRAINING_EVAL_KEY_PREFIX) if metrics: logger.log(metrics, global_step) return benchmark_results @@ -2369,7 +2141,7 @@ def run_training_evaluation(env_name, args, policy, logger, epoch, global_step, finally: if hasattr(policy, "train"): policy.train(policy_was_training) - restore_rng_state({"rng_state": rng_state}) + utils.restore_rng_state({"rng_state": rng_state}) def _render_eval_failures( @@ -2384,7 +2156,7 @@ def _render_eval_failures( evaluation_policy_cache=None, ): configured_render_filter = run_args["eval"]["render_filter"] - selected_rows = drive_benchmark.select_render_rows(metrics_path, configured_render_filter) + selected_rows = evaluation_utils.select_render_rows(metrics_path, configured_render_filter) if max_rendered_failures is not None: selected_rows = selected_rows.head(max_rendered_failures).copy() failures_dir = os.path.join(benchmark_output_dir, "failures") @@ -2395,7 +2167,7 @@ def _render_eval_failures( print(f"No failures matched for benchmark {benchmark['name']}; wrote {selected_path}") return {"episodes": [], "summary": None} - map_indices = drive_benchmark._resolve_map_indices( + map_indices = evaluation_utils.resolve_map_indices( run_args["env"]["map_dir"], selected_rows["map_name"].tolist(), ) @@ -2413,7 +2185,7 @@ def _render_eval_failures( ) replay_agent_capacity = failure_args["env"]["max_agents_per_env"] failure_args["env"]["num_agents"] = replay_agent_capacity - replay_output_dir = os.path.join(failures_dir, drive_eval_replay.ZLIB_REPLAY_DIR_NAME) + replay_output_dir = os.path.join(failures_dir, eval_replay.ZLIB_REPLAY_DIR_NAME) os.makedirs(replay_output_dir, exist_ok=True) agents_per_batch_values = selected_rows["agents_per_batch"].unique() if len(agents_per_batch_values) != 1: @@ -2425,7 +2197,7 @@ def _render_eval_failures( replay_pairs = pairs[replay_pair_start : replay_pair_start + replay_wave_size] num_workers = min(configured_worker_count, len(replay_pairs)) failure_args["vec"]["num_envs"] = num_workers - worker_env_kwargs, total_steps = drive_benchmark._plan_failure_replay_workers( + worker_env_kwargs, total_steps = evaluation_utils.plan_failure_replay_workers( failure_args, replay_pairs, num_workers, @@ -2449,8 +2221,8 @@ def _render_eval_failures( evaluation_policy_cache=evaluation_policy_cache, ) summaries.extend(wave_summaries) - summary = drive_benchmark._write_eval_reports(summaries, failures_dir, len(pairs)) - drive_eval_replay._render_eval_replays(summaries, failures_dir, run_args["eval"]["keep_zlib_replays"]) + summary = evaluation_utils.write_eval_reports(summaries, failures_dir, len(pairs)) + eval_replay.render_eval_replays(summaries, failures_dir, run_args["eval"]["keep_zlib_replays"]) return { "episodes": summaries, "summary": summary, @@ -2465,7 +2237,7 @@ def load_env(env_name, args, seed=None): vec_kwargs = dict(args["vec"]) if seed is not None: vec_kwargs["seed"] = seed - return pufferlib.vector.make(make_env, env_kwargs=args["env"], **vec_kwargs) + return vector.make(make_env, env_kwargs=args["env"], **vec_kwargs) def load_policy(args, vecenv, env_name=""): @@ -2473,7 +2245,7 @@ def load_policy(args, vecenv, env_name=""): module_name = "pufferlib.ocean" if package == "ocean" else f"pufferlib.environments.{package}" env_module = importlib.import_module(module_name) - device = torch_device(args["train"]["device"]) + device = utils.torch_device(args["train"]["device"]) policy_cls = getattr(env_module.torch, args["policy_name"]) policy = policy_cls(vecenv.driver_env, **args["policy"]) @@ -2486,13 +2258,10 @@ def load_policy(args, vecenv, env_name=""): load_id = args["load_id"] if load_id is not None: - if args["neptune"]: - path = NeptuneLogger(args, load_id, mode="read-only").download() - elif args["wandb"]: - path = WandbLogger(args, load_id).download() + path = WandbLogger(args, load_id).download() state_dict = torch.load(path, map_location=device) - policy.load_state_dict(clean_policy_state_dict(state_dict)) + policy.load_state_dict(utils.clean_policy_state_dict(state_dict)) load_path = args["load_model_path"] if load_path == "latest": @@ -2500,10 +2269,7 @@ def load_policy(args, vecenv, env_name=""): if load_path is not None: state_dict = torch.load(load_path, map_location=device) - policy.load_state_dict(clean_policy_state_dict(state_dict)) - # state_path = os.path.join(*load_path.split('/')[:-1], 'state.pt') - # optim_state = torch.load(state_path)['optimizer_state_dict'] - # pufferl.optimizer.load_state_dict(optim_state) + policy.load_state_dict(utils.clean_policy_state_dict(state_dict)) return policy @@ -2527,7 +2293,7 @@ def load_config(env_name, config_dir=None): args = defaultdict(dict, OmegaConf.to_container(cfg, resolve=True, throw_on_missing=True)) args["train"]["use_rnn"] = args["rnn_name"] is not None - return defaultdict(dict, normalize_puffer_drive_config(args, "load")) + return defaultdict(dict, config_schema.normalize_puffer_drive_config(args, "load")) def main(): diff --git a/pufferlib/utils.py b/pufferlib/utils.py index ce6a1677a8..2e22413fd0 100644 --- a/pufferlib/utils.py +++ b/pufferlib/utils.py @@ -1,6 +1,193 @@ import numbers +import os +import random +import shutil +import subprocess import numpy as np +import torch + + +def torch_device(device): + if isinstance(device, int): + return torch.device("cuda", device) if torch.cuda.is_available() else torch.device("cpu") + return device + + +def is_cuda_device(device): + if isinstance(device, int): + return torch.cuda.is_available() + device = torch.device(device) + return device.type == "cuda" + + +def base_policy(policy): + return policy.module if hasattr(policy, "module") else policy + + +def clean_state_key(key): + prefixes = ("module.", "_orig_mod.") + while key.startswith(prefixes): + key = key.split(".", 1)[1] + return key + + +def clean_policy_state_dict(state_dict): + return {clean_state_key(k): v for k, v in state_dict.items()} + + +def logits_to_float(logits): + if isinstance(logits, torch.distributions.Normal): + return torch.distributions.Normal(logits.loc.float(), logits.scale.float()) + return logits.float() + + +def abbreviate(num, b2, c2): + if num < 1e3: + return f"{b2}{num}{c2}" + elif num < 1e6: + return f"{b2}{num / 1e3:.1f}{c2}K" + elif num < 1e9: + return f"{b2}{num / 1e6:.1f}{c2}M" + elif num < 1e12: + return f"{b2}{num / 1e9:.1f}{c2}B" + else: + return f"{b2}{num / 1e12:.2f}{c2}T" + + +def duration(seconds, b2, c2): + if seconds < 0: + return f"{b2}0{c2}s" + seconds = int(seconds) + h = seconds // 3600 + m = (seconds % 3600) // 60 + s = seconds % 60 + return f"{b2}{h}{c2}h {b2}{m}{c2}m {b2}{s}{c2}s" if h else f"{b2}{m}{c2}m {b2}{s}{c2}s" if m else f"{b2}{s}{c2}s" + + +def fmt_perf(name, color, delta_ref, prof, b2, c2): + percent = 0 if delta_ref == 0 else int(100 * prof["buffer"] / delta_ref - 1e-5) + return f"{color}{name}", duration(prof["elapsed"], b2, c2), f"{b2}{percent:2d}{c2}%" + + +def dist_sum(value, device): + if not torch.distributed.is_initialized(): + return value + + tensor = torch.tensor(value, device=device) + torch.distributed.all_reduce(tensor, op=torch.distributed.ReduceOp.SUM) + return tensor.item() + + +def dist_mean(value, device): + if not torch.distributed.is_initialized(): + return value + + return dist_sum(value, device) / torch.distributed.get_world_size() + + +def capture_rng_state(): + state = { + "python": random.getstate(), + "numpy": np.random.get_state(), + "torch": torch.get_rng_state(), + } + if torch.cuda.is_available(): + state["cuda"] = torch.cuda.get_rng_state_all() + return state + + +def restore_rng_state(state): + rng_state = state.get("rng_state") + if not rng_state: + return + + random.setstate(rng_state["python"]) + np.random.set_state(rng_state["numpy"]) + torch.set_rng_state(rng_state["torch"].cpu()) + if torch.cuda.is_available() and "cuda" in rng_state: + torch.cuda.set_rng_state_all([state.cpu() for state in rng_state["cuda"]]) + + +def downsample(data_list, num_points): + if not data_list or num_points <= 0: + return [] + if num_points == 1: + return [data_list[-1]] + if len(data_list) <= num_points: + return data_list + + last = data_list[-1] + data_list = data_list[:-1] + + data_np = np.array(data_list) + num_points -= 1 # one down for the last one + + n = (len(data_np) // num_points) * num_points + data_np = data_np[-n:] if n > 0 else data_np + downsampled = data_np.reshape(num_points, -1).mean(axis=1) + + return downsampled.tolist() + [last] + + +def get_git_metadata(): + git_metadata = { + "commit_hash": os.environ.get("GITHUB_SHA") or os.environ.get("COMMIT_SHA"), + } + + try: + repo_root = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) + if shutil.which("git") is None: + return git_metadata + + subprocess.check_output( + ["git", "rev-parse", "--is-inside-work-tree"], + cwd=repo_root, + stderr=subprocess.DEVNULL, + text=True, + ) + + if git_metadata["commit_hash"] is None: + git_metadata["commit_hash"] = subprocess.check_output( + ["git", "rev-parse", "HEAD"], + cwd=repo_root, + stderr=subprocess.DEVNULL, + text=True, + ).strip() + except (OSError, subprocess.SubprocessError): + pass + + return git_metadata + + +def save_experiment_config(args, path): + import yaml + import json + + experiment_dir = path + os.makedirs(experiment_dir, exist_ok=True) + + # Save config as yaml + config_yaml_path = os.path.join(experiment_dir, "config.yaml") + with open(config_yaml_path, "w") as f: + # Convert defaultdict to dict for cleaner output + config = json.loads(json.dumps(args)) + yaml.dump(config, f) + + +def global_agent_steps(pufferl): + world_size = torch.distributed.get_world_size() if torch.distributed.is_initialized() else 1 + return int(pufferl.global_step * world_size) + + +def derive_rank_seeds(vec_seed, train_seed, world_size, global_rank): + """Deterministic per-rank (torch_seed, env_seed): DDP ranks share weights, so identical seeds + would collect duplicate experience. global_rank is torchrun's global RANK, not LOCAL_RANK.""" + torch_seed = train_seed * world_size + global_rank + env_seed = vec_seed + if env_seed is not None: + env_seed = int(np.random.SeedSequence([env_seed, train_seed, global_rank]).generate_state(1)[0]) + return torch_seed, env_seed def reduce_environment_metrics(metric_lists): diff --git a/scripts/baseline_10G/config-reward-dense.yaml b/scripts/baseline_10G/config-reward-dense.yaml deleted file mode 100644 index 0c1386b24e..0000000000 --- a/scripts/baseline_10G/config-reward-dense.yaml +++ /dev/null @@ -1,384 +0,0 @@ -controlled_exp: - train: - ent_coef: - values: - - 0.01 - - 0.005 - learning_rate: - values: - - 0.001 - - 0.003 - - 0.01 -env: - action_type: discrete - collision_behavior: 1 - compute_eval_metrics: false - control_mode: control_vehicles - dt: 0.1 - dynamics_model: jerk - goal_on_lane: true - goal_radius: 2.0 - goal_speed: 1000.0 - inactive_agent_threshold: 0.4 - init_mode: create_all_valid - init_step: 0 - map_dir: "pufferlib/resources/drive/binaries/carla" - max_agents_per_env: 80 - max_waypoint_spacing: 40.0 - min_agents_per_env: 1 - min_waypoint_spacing: 20.0 - non_sdc_controller: policy - non_vehicle_controller: auto - num_agents: 2048 - num_maps: 8 - num_target_waypoints: 3 - obs_dropout_boundary: 0.0 - obs_dropout_lane: 0.0 - obs_norm_goal_offset_m: 120.0 - obs_norm_road_seg_length_m: 10.0 - obs_norm_road_seg_width_m: 5.0 - obs_norm_veh_length_m: 10.0 - obs_norm_veh_width_m: 5.0 - obs_norm_xy_offset_m: 120.0 - obs_range_partner_m: 120.0 - obs_range_road_behind_m: 30.0 - obs_range_road_front_m: 120.0 - obs_range_road_side_m: 40.0 - obs_range_traffic_control_m: 120.0 - obs_slots_boundary_n: 30 - obs_slots_lane_n: 50 - obs_slots_partners_n: 12 - obs_slots_traffic_controls_n: 4 - offroad_behavior: 1 - partner_blindness_prob: 0.0 - partner_blindness_trigger_prob: 0.0 - phantom_braking_duration: 10 - phantom_braking_prob: 0.0 - phantom_braking_trigger_prob: 0.0 - resample_frequency: 102400 - reward_ade: 0.0 - reward_center_bias: 0.0 - reward_collision: 1.5 - reward_comfort: 0.5 - reward_conditioning: false - reward_goal: 0.2 - reward_lane_align: 0.5 - reward_lane_center: 0.05 - reward_offroad: 1.5 - reward_overspeed: 0.1 - reward_randomization: false - reward_reverse: 0.005 - reward_stop_line: 1.0 - reward_timestep: 0.0 - reward_vel_align: 1.0 - reward_velocity: 0.1 - scenario_length: 1024 - sdc_controller: policy - simulation_mode: gigaflow - spawn_initial_speed: 0.0 - goal_regen_mode: finite - goal_source: route - termination_mode: true - traffic_light_behavior: 1 - use_map_cache: true -env_name: puffer_drive -eval: - behaviors_defaults: - clean: 'true' - enabled: 'false' - env: - control_mode: control_sdc_only - init_mode: create_all_valid - obs_slots_partners_n: 32 - scenario_length: 201 - simulation_mode: replay - eval: - num_scenarios: 50 - render_max_steps: 200 - render_num_scenarios: 2 - interval: 250 - mode: inline - render: 'true' - render_backend: triage_html - behaviors_full_dir: - enabled: 'true' - env: - map_dir: /scratch/ev2237/data/nuplan/nuplan_mini_train_bins - inherits: behaviors_defaults - type: behavior_class - behaviors_hard_stop: - enabled: 'true' - env: - map_dir: /scratch/ev2237/data/nuplan/categories_v021/hard_stop - inherits: behaviors_defaults - type: behavior_class - behaviors_highway_straight: - enabled: 'true' - env: - map_dir: /scratch/ev2237/data/nuplan/categories_v021/highway_straight - inherits: behaviors_defaults - type: behavior_class - behaviors_lane_change: - enabled: 'true' - env: - map_dir: /scratch/ev2237/data/nuplan/categories_v021/lane_change - inherits: behaviors_defaults - type: behavior_class - behaviors_merge: - enabled: 'true' - env: - map_dir: /scratch/ev2237/data/nuplan/categories_v021/merge - inherits: behaviors_defaults - type: behavior_class - behaviors_parked_cars: - enabled: 'true' - env: - map_dir: /scratch/ev2237/data/nuplan/categories_v021/parked_cars - inherits: behaviors_defaults - type: behavior_class - behaviors_roundabout: - enabled: 'true' - env: - map_dir: /scratch/ev2237/data/nuplan/categories_v021/roundabout - inherits: behaviors_defaults - type: behavior_class - behaviors_stopped_traffic: - enabled: 'true' - env: - map_dir: /scratch/ev2237/data/nuplan/categories_v021/stopped_traffic - inherits: behaviors_defaults - type: behavior_class - behaviors_traffic_light_green: - enabled: 'true' - env: - map_dir: /scratch/ev2237/data/nuplan/categories_v021/traffic_light_green - inherits: behaviors_defaults - type: behavior_class - behaviors_traffic_light_stop: - enabled: 'true' - env: - map_dir: /scratch/ev2237/data/nuplan/categories_v021/traffic_light_stop - inherits: behaviors_defaults - type: behavior_class - behaviors_unprotected_left: - enabled: 'true' - env: - map_dir: /scratch/ev2237/data/nuplan/categories_v021/unprotected_left - inherits: behaviors_defaults - type: behavior_class - behaviors_unprotected_right: - enabled: 'true' - env: - map_dir: /scratch/ev2237/data/nuplan/categories_v021/unprotected_right - inherits: behaviors_defaults - type: behavior_class - dnf_triage: - enabled: 'false' - env: - map_dir: pufferlib/resources/drive/binaries/carla/opendrive__Town10HD.bin - max_agents_per_env: 1 - min_agents_per_env: 1 - num_maps: 1 - resample_frequency: 500 - scenario_length: 500 - simulation_mode: gigaflow - eval: - num_scenarios: 32 - render_max_steps: 300 - render_num_scenarios: 16 - inherits: validation_defaults - render: 'true' - render_backend: triage_html - type: multi_scenario - validation_defaults: - clean: 'true' - enabled: 'true' - env: - collision_behavior: 1 - eval_mode: 1 - goal_speed: 3.0 - num_agents: 1024 - obs_dropout_boundary: 0.0 - obs_dropout_lane: 0.0 - obs_slots_boundary_n: 80 - obs_slots_lane_n: 80 - offroad_behavior: 1 - reward_ade: 0.0 - reward_collision: 3.0 - reward_comfort: 0.05 - reward_goal: 1.0 - reward_lane_align: 0.025 - reward_lane_center: 0.0038 - reward_offroad: 3.0 - reward_overspeed: 0.05 - reward_randomization: false - reward_reverse: 0.005 - reward_stop_line: 1.0 - reward_timestep: 2.5e-05 - reward_velocity: 0.0025 - goal_source: route - goal_regen_mode: finite - termination_mode: false - traffic_light_behavior: 0 - eval: - export_episode_csv: 'true' - num_scenarios: 250 - verify_coverage: 'true' - interval: 250 - mode: inline - validation_gigaflow: - enabled: 'true' - env: - map_dir: pufferlib/resources/drive/binaries/carla - max_agents_per_env: 40 - min_agents_per_env: 40 - num_agents: 1024 - num_maps: 8 - resample_frequency: 500 - scenario_length: 500 - simulation_mode: gigaflow - eval: - render_max_steps: 300 - render_num_scenarios: 8 - inherits: validation_defaults - render: 'true' - render_backend: triage_html - type: multi_scenario - validation_replay: - enabled: 'true' - env: - control_mode: control_sdc_only - map_dir: /scratch/ev2237/data/nuplan/nuplan_mini_train_bins - max_agents_per_env: 64 - num_maps: 250 - resample_frequency: 200 - scenario_length: 200 - simulation_mode: replay - eval: - render_max_steps: 200 - render_num_scenarios: 5 - inherits: validation_defaults - render: 'true' - render_backend: triage_html - type: multi_scenario - wosac: - clean: 'true' - enabled: 'false' - env: - control_mode: control_wosac - goal_radius: 2.0 - init_mode: create_all_valid - init_step: 10 - eval: - wosac_aggregate_results: 'true' - wosac_num_agents: 256 - wosac_num_rollouts: 32 - wosac_sanity_check: 'false' - interval: 500 - mode: subprocess - render: 'false' - type: wosac -eval_simulation: null -git: - commit_hash: 9d3fcc5c09db50b9674a374b8c3404b6349c4cb5 -load_id: null -load_model_path: null -local_rank: 0 -max_runs: 200 -max_suggestion_cost: 3600 -mine: - num_episodes: 100 - output_dir: '' - render: 'true' - score_threshold: -inf -neptune: false -neptune_name: pufferai -neptune_project: ablations -no_model_upload: {} -num_scenarios: 3 -package: ocean -policy: - actor_hidden_size: 512 - actor_num_layers: 0 - backbone_activation: gelu - backbone_hidden_size: 512 - backbone_layer_norm: false - backbone_num_layers: 2 - boundary_input_size: 128 - context_input_size: 128 - critic_hidden_size: 512 - critic_num_layers: 0 - ego_input_size: 128 - encoder_activation: relu - encoder_layer_norm: true - lane_input_size: 128 - mask_padded_features: false - partner_input_size: 128 - shared_network: true - traffic_control_input_size: 128 - action_type: discrete -policy_name: Drive -rnn: - hidden_size: 512 - input_size: 512 -rnn_name: null -run_name: rew-dense-seed4 -sweep: - downsample: 10 - goal: maximize - method: Protein - metric: score -tag: null -tb: false -train: - adam_beta1: 0.9 - adam_beta2: 0.999 - adam_eps: 1.0e-08 - adv_filter_ewma_beta: 0.25 - adv_filter_threshold_scale: 0.01 - adv_sampling_prio_alpha: 0.8499999999999999 - adv_sampling_prio_beta0: 0.8499999999999999 - amp: true - anneal_lr: true - batch_size: auto - bptt_horizon: 128 - checkpoint_interval: 50 - clip_coef: 0.2 - compile: true - compile_fullgraph: false - compile_mode: default - cpu_offload: false - data_dir: /pufferdrive/training_output/ - device: cuda - ent_coef: 0.01 - gae_lambda: 0.95 - gamma: 0.999 - learning_rate: 0.0005 - max_grad_norm: 0.5 - max_minibatch_size: 98304 - minibatch_size: 98304 - name: pufferai - normalize_rewards: true - optimizer: adamw - precision: bfloat16 - project: ablations - resume_state_path: null - seed: 4 - torch_deterministic: false - total_timesteps: 10000000000 - update_epochs: 2 - use_rnn: false - vf_clip_coef: null - vf_coef: 0.5 - vtrace_c_clip: 1 - vtrace_rho_clip: 1 -vec: - backend: Multiprocessing - batch_size: auto - num_envs: 24 - num_workers: auto - seed: 42 - zero_copy: true -wandb: true -wandb_group: pr-smoke-runs -wandb_project: pr-smoke-runs diff --git a/scripts/baseline_10G/config-reward-sparse.yaml b/scripts/baseline_10G/config-reward-sparse.yaml deleted file mode 100644 index cda5e9d4bc..0000000000 --- a/scripts/baseline_10G/config-reward-sparse.yaml +++ /dev/null @@ -1,384 +0,0 @@ -controlled_exp: - train: - ent_coef: - values: - - 0.01 - - 0.005 - learning_rate: - values: - - 0.001 - - 0.003 - - 0.01 -env: - action_type: discrete - collision_behavior: 1 - compute_eval_metrics: false - control_mode: control_vehicles - dt: 0.1 - dynamics_model: jerk - goal_on_lane: true - goal_radius: 2.0 - goal_speed: 1000.0 - inactive_agent_threshold: 0.4 - init_mode: create_all_valid - init_step: 0 - map_dir: "pufferlib/resources/drive/binaries/carla" - max_agents_per_env: 80 - max_waypoint_spacing: 40.0 - min_agents_per_env: 1 - min_waypoint_spacing: 20.0 - non_sdc_controller: policy - non_vehicle_controller: auto - num_agents: 2048 - num_maps: 8 - num_target_waypoints: 3 - obs_dropout_boundary: 0.0 - obs_dropout_lane: 0.0 - obs_norm_goal_offset_m: 120.0 - obs_norm_road_seg_length_m: 10.0 - obs_norm_road_seg_width_m: 5.0 - obs_norm_veh_length_m: 10.0 - obs_norm_veh_width_m: 5.0 - obs_norm_xy_offset_m: 120.0 - obs_range_partner_m: 120.0 - obs_range_road_behind_m: 30.0 - obs_range_road_front_m: 120.0 - obs_range_road_side_m: 40.0 - obs_range_traffic_control_m: 120.0 - obs_slots_boundary_n: 30 - obs_slots_lane_n: 50 - obs_slots_partners_n: 12 - obs_slots_traffic_controls_n: 4 - offroad_behavior: 1 - partner_blindness_prob: 0.0 - partner_blindness_trigger_prob: 0.0 - phantom_braking_duration: 10 - phantom_braking_prob: 0.0 - phantom_braking_trigger_prob: 0.0 - resample_frequency: 102400 - reward_ade: 0.0 - reward_center_bias: 0.0 - reward_collision: 1.5 - reward_comfort: 0.05 - reward_conditioning: false - reward_goal: 0.5 - reward_lane_align: 0.025 - reward_lane_center: 0.0038 - reward_offroad: 1.5 - reward_overspeed: 0.05 - reward_randomization: false - reward_reverse: 0.005 - reward_stop_line: 1.0 - reward_timestep: 0.0 - reward_vel_align: 1.0 - reward_velocity: 0.0025 - scenario_length: 1024 - sdc_controller: policy - simulation_mode: gigaflow - spawn_initial_speed: 0.0 - goal_source: route - goal_regen_mode: finite - termination_mode: true - traffic_light_behavior: 1 - use_map_cache: true -env_name: puffer_drive -eval: - behaviors_defaults: - clean: 'true' - enabled: 'false' - env: - control_mode: control_sdc_only - init_mode: create_all_valid - obs_slots_partners_n: 32 - scenario_length: 201 - simulation_mode: replay - eval: - num_scenarios: 50 - render_max_steps: 200 - render_num_scenarios: 2 - interval: 250 - mode: inline - render: 'true' - render_backend: triage_html - behaviors_full_dir: - enabled: 'true' - env: - map_dir: /scratch/ev2237/data/nuplan/nuplan_mini_train_bins - inherits: behaviors_defaults - type: behavior_class - behaviors_hard_stop: - enabled: 'true' - env: - map_dir: /scratch/ev2237/data/nuplan/categories_v021/hard_stop - inherits: behaviors_defaults - type: behavior_class - behaviors_highway_straight: - enabled: 'true' - env: - map_dir: /scratch/ev2237/data/nuplan/categories_v021/highway_straight - inherits: behaviors_defaults - type: behavior_class - behaviors_lane_change: - enabled: 'true' - env: - map_dir: /scratch/ev2237/data/nuplan/categories_v021/lane_change - inherits: behaviors_defaults - type: behavior_class - behaviors_merge: - enabled: 'true' - env: - map_dir: /scratch/ev2237/data/nuplan/categories_v021/merge - inherits: behaviors_defaults - type: behavior_class - behaviors_parked_cars: - enabled: 'true' - env: - map_dir: /scratch/ev2237/data/nuplan/categories_v021/parked_cars - inherits: behaviors_defaults - type: behavior_class - behaviors_roundabout: - enabled: 'true' - env: - map_dir: /scratch/ev2237/data/nuplan/categories_v021/roundabout - inherits: behaviors_defaults - type: behavior_class - behaviors_stopped_traffic: - enabled: 'true' - env: - map_dir: /scratch/ev2237/data/nuplan/categories_v021/stopped_traffic - inherits: behaviors_defaults - type: behavior_class - behaviors_traffic_light_green: - enabled: 'true' - env: - map_dir: /scratch/ev2237/data/nuplan/categories_v021/traffic_light_green - inherits: behaviors_defaults - type: behavior_class - behaviors_traffic_light_stop: - enabled: 'true' - env: - map_dir: /scratch/ev2237/data/nuplan/categories_v021/traffic_light_stop - inherits: behaviors_defaults - type: behavior_class - behaviors_unprotected_left: - enabled: 'true' - env: - map_dir: /scratch/ev2237/data/nuplan/categories_v021/unprotected_left - inherits: behaviors_defaults - type: behavior_class - behaviors_unprotected_right: - enabled: 'true' - env: - map_dir: /scratch/ev2237/data/nuplan/categories_v021/unprotected_right - inherits: behaviors_defaults - type: behavior_class - dnf_triage: - enabled: 'false' - env: - map_dir: pufferlib/resources/drive/binaries/carla/opendrive__Town10HD.bin - max_agents_per_env: 1 - min_agents_per_env: 1 - num_maps: 1 - resample_frequency: 500 - scenario_length: 500 - simulation_mode: gigaflow - eval: - num_scenarios: 32 - render_max_steps: 300 - render_num_scenarios: 16 - inherits: validation_defaults - render: 'true' - render_backend: triage_html - type: multi_scenario - validation_defaults: - clean: 'true' - enabled: 'true' - env: - collision_behavior: 1 - eval_mode: 1 - goal_speed: 3.0 - num_agents: 1024 - obs_dropout_boundary: 0.0 - obs_dropout_lane: 0.0 - obs_slots_boundary_n: 80 - obs_slots_lane_n: 80 - offroad_behavior: 1 - reward_ade: 0.0 - reward_collision: 3.0 - reward_comfort: 0.05 - reward_goal: 1.0 - reward_lane_align: 0.025 - reward_lane_center: 0.0038 - reward_offroad: 3.0 - reward_overspeed: 0.05 - reward_randomization: false - reward_reverse: 0.005 - reward_stop_line: 1.0 - reward_timestep: 2.5e-05 - reward_velocity: 0.0025 - goal_source: route - goal_regen_mode: finite - termination_mode: false - traffic_light_behavior: 0 - eval: - export_episode_csv: 'true' - num_scenarios: 250 - verify_coverage: 'true' - interval: 250 - mode: inline - validation_gigaflow: - enabled: 'true' - env: - map_dir: pufferlib/resources/drive/binaries/carla - max_agents_per_env: 40 - min_agents_per_env: 40 - num_agents: 1024 - num_maps: 8 - resample_frequency: 500 - scenario_length: 500 - simulation_mode: gigaflow - eval: - render_max_steps: 300 - render_num_scenarios: 8 - inherits: validation_defaults - render: 'true' - render_backend: triage_html - type: multi_scenario - validation_replay: - enabled: 'true' - env: - control_mode: control_sdc_only - map_dir: /scratch/ev2237/data/nuplan/nuplan_mini_train_bins - max_agents_per_env: 64 - num_maps: 250 - resample_frequency: 200 - scenario_length: 200 - simulation_mode: replay - eval: - render_max_steps: 200 - render_num_scenarios: 5 - inherits: validation_defaults - render: 'true' - render_backend: triage_html - type: multi_scenario - wosac: - clean: 'true' - enabled: 'false' - env: - control_mode: control_wosac - goal_radius: 2.0 - init_mode: create_all_valid - init_step: 10 - eval: - wosac_aggregate_results: 'true' - wosac_num_agents: 256 - wosac_num_rollouts: 32 - wosac_sanity_check: 'false' - interval: 500 - mode: subprocess - render: 'false' - type: wosac -eval_simulation: null -git: - commit_hash: 9d3fcc5c09db50b9674a374b8c3404b6349c4cb5 -load_id: null -load_model_path: null -local_rank: 0 -max_runs: 200 -max_suggestion_cost: 3600 -mine: - num_episodes: 100 - output_dir: '' - render: 'true' - score_threshold: -inf -neptune: false -neptune_name: pufferai -neptune_project: ablations -no_model_upload: {} -num_scenarios: 3 -package: ocean -policy: - actor_hidden_size: 512 - actor_num_layers: 0 - backbone_activation: gelu - backbone_hidden_size: 512 - backbone_layer_norm: false - backbone_num_layers: 2 - boundary_input_size: 128 - context_input_size: 128 - critic_hidden_size: 512 - critic_num_layers: 0 - ego_input_size: 128 - encoder_activation: relu - encoder_layer_norm: true - lane_input_size: 128 - mask_padded_features: false - partner_input_size: 128 - shared_network: true - traffic_control_input_size: 128 - action_type: discrete -policy_name: Drive -rnn: - hidden_size: 512 - input_size: 512 -rnn_name: null -run_name: rew-sparse-seed4 -sweep: - downsample: 10 - goal: maximize - method: Protein - metric: score -tag: null -tb: false -train: - adam_beta1: 0.9 - adam_beta2: 0.999 - adam_eps: 1.0e-08 - adv_filter_ewma_beta: 0.25 - adv_filter_threshold_scale: 0.01 - adv_sampling_prio_alpha: 0.8499999999999999 - adv_sampling_prio_beta0: 0.8499999999999999 - amp: true - anneal_lr: true - batch_size: auto - bptt_horizon: 128 - checkpoint_interval: 50 - clip_coef: 0.2 - compile: true - compile_fullgraph: false - compile_mode: default - cpu_offload: false - data_dir: /pufferdrive/training_output/ - device: cuda - ent_coef: 0.01 - gae_lambda: 0.95 - gamma: 0.999 - learning_rate: 0.0005 - max_grad_norm: 0.5 - max_minibatch_size: 98304 - minibatch_size: 98304 - name: pufferai - normalize_rewards: true - optimizer: adamw - precision: bfloat16 - project: ablations - resume_state_path: null - seed: 4 - torch_deterministic: false - total_timesteps: 10000000000 - update_epochs: 2 - use_rnn: false - vf_clip_coef: null - vf_coef: 0.5 - vtrace_c_clip: 1 - vtrace_rho_clip: 1 -vec: - backend: Multiprocessing - batch_size: auto - num_envs: 24 - num_workers: auto - seed: 42 - zero_copy: true -wandb: true -wandb_group: pr-smoke-runs -wandb_project: pr-smoke-runs diff --git a/scripts/config-reward-dense.yaml b/scripts/config-reward-dense.yaml deleted file mode 100644 index 0c1386b24e..0000000000 --- a/scripts/config-reward-dense.yaml +++ /dev/null @@ -1,384 +0,0 @@ -controlled_exp: - train: - ent_coef: - values: - - 0.01 - - 0.005 - learning_rate: - values: - - 0.001 - - 0.003 - - 0.01 -env: - action_type: discrete - collision_behavior: 1 - compute_eval_metrics: false - control_mode: control_vehicles - dt: 0.1 - dynamics_model: jerk - goal_on_lane: true - goal_radius: 2.0 - goal_speed: 1000.0 - inactive_agent_threshold: 0.4 - init_mode: create_all_valid - init_step: 0 - map_dir: "pufferlib/resources/drive/binaries/carla" - max_agents_per_env: 80 - max_waypoint_spacing: 40.0 - min_agents_per_env: 1 - min_waypoint_spacing: 20.0 - non_sdc_controller: policy - non_vehicle_controller: auto - num_agents: 2048 - num_maps: 8 - num_target_waypoints: 3 - obs_dropout_boundary: 0.0 - obs_dropout_lane: 0.0 - obs_norm_goal_offset_m: 120.0 - obs_norm_road_seg_length_m: 10.0 - obs_norm_road_seg_width_m: 5.0 - obs_norm_veh_length_m: 10.0 - obs_norm_veh_width_m: 5.0 - obs_norm_xy_offset_m: 120.0 - obs_range_partner_m: 120.0 - obs_range_road_behind_m: 30.0 - obs_range_road_front_m: 120.0 - obs_range_road_side_m: 40.0 - obs_range_traffic_control_m: 120.0 - obs_slots_boundary_n: 30 - obs_slots_lane_n: 50 - obs_slots_partners_n: 12 - obs_slots_traffic_controls_n: 4 - offroad_behavior: 1 - partner_blindness_prob: 0.0 - partner_blindness_trigger_prob: 0.0 - phantom_braking_duration: 10 - phantom_braking_prob: 0.0 - phantom_braking_trigger_prob: 0.0 - resample_frequency: 102400 - reward_ade: 0.0 - reward_center_bias: 0.0 - reward_collision: 1.5 - reward_comfort: 0.5 - reward_conditioning: false - reward_goal: 0.2 - reward_lane_align: 0.5 - reward_lane_center: 0.05 - reward_offroad: 1.5 - reward_overspeed: 0.1 - reward_randomization: false - reward_reverse: 0.005 - reward_stop_line: 1.0 - reward_timestep: 0.0 - reward_vel_align: 1.0 - reward_velocity: 0.1 - scenario_length: 1024 - sdc_controller: policy - simulation_mode: gigaflow - spawn_initial_speed: 0.0 - goal_regen_mode: finite - goal_source: route - termination_mode: true - traffic_light_behavior: 1 - use_map_cache: true -env_name: puffer_drive -eval: - behaviors_defaults: - clean: 'true' - enabled: 'false' - env: - control_mode: control_sdc_only - init_mode: create_all_valid - obs_slots_partners_n: 32 - scenario_length: 201 - simulation_mode: replay - eval: - num_scenarios: 50 - render_max_steps: 200 - render_num_scenarios: 2 - interval: 250 - mode: inline - render: 'true' - render_backend: triage_html - behaviors_full_dir: - enabled: 'true' - env: - map_dir: /scratch/ev2237/data/nuplan/nuplan_mini_train_bins - inherits: behaviors_defaults - type: behavior_class - behaviors_hard_stop: - enabled: 'true' - env: - map_dir: /scratch/ev2237/data/nuplan/categories_v021/hard_stop - inherits: behaviors_defaults - type: behavior_class - behaviors_highway_straight: - enabled: 'true' - env: - map_dir: /scratch/ev2237/data/nuplan/categories_v021/highway_straight - inherits: behaviors_defaults - type: behavior_class - behaviors_lane_change: - enabled: 'true' - env: - map_dir: /scratch/ev2237/data/nuplan/categories_v021/lane_change - inherits: behaviors_defaults - type: behavior_class - behaviors_merge: - enabled: 'true' - env: - map_dir: /scratch/ev2237/data/nuplan/categories_v021/merge - inherits: behaviors_defaults - type: behavior_class - behaviors_parked_cars: - enabled: 'true' - env: - map_dir: /scratch/ev2237/data/nuplan/categories_v021/parked_cars - inherits: behaviors_defaults - type: behavior_class - behaviors_roundabout: - enabled: 'true' - env: - map_dir: /scratch/ev2237/data/nuplan/categories_v021/roundabout - inherits: behaviors_defaults - type: behavior_class - behaviors_stopped_traffic: - enabled: 'true' - env: - map_dir: /scratch/ev2237/data/nuplan/categories_v021/stopped_traffic - inherits: behaviors_defaults - type: behavior_class - behaviors_traffic_light_green: - enabled: 'true' - env: - map_dir: /scratch/ev2237/data/nuplan/categories_v021/traffic_light_green - inherits: behaviors_defaults - type: behavior_class - behaviors_traffic_light_stop: - enabled: 'true' - env: - map_dir: /scratch/ev2237/data/nuplan/categories_v021/traffic_light_stop - inherits: behaviors_defaults - type: behavior_class - behaviors_unprotected_left: - enabled: 'true' - env: - map_dir: /scratch/ev2237/data/nuplan/categories_v021/unprotected_left - inherits: behaviors_defaults - type: behavior_class - behaviors_unprotected_right: - enabled: 'true' - env: - map_dir: /scratch/ev2237/data/nuplan/categories_v021/unprotected_right - inherits: behaviors_defaults - type: behavior_class - dnf_triage: - enabled: 'false' - env: - map_dir: pufferlib/resources/drive/binaries/carla/opendrive__Town10HD.bin - max_agents_per_env: 1 - min_agents_per_env: 1 - num_maps: 1 - resample_frequency: 500 - scenario_length: 500 - simulation_mode: gigaflow - eval: - num_scenarios: 32 - render_max_steps: 300 - render_num_scenarios: 16 - inherits: validation_defaults - render: 'true' - render_backend: triage_html - type: multi_scenario - validation_defaults: - clean: 'true' - enabled: 'true' - env: - collision_behavior: 1 - eval_mode: 1 - goal_speed: 3.0 - num_agents: 1024 - obs_dropout_boundary: 0.0 - obs_dropout_lane: 0.0 - obs_slots_boundary_n: 80 - obs_slots_lane_n: 80 - offroad_behavior: 1 - reward_ade: 0.0 - reward_collision: 3.0 - reward_comfort: 0.05 - reward_goal: 1.0 - reward_lane_align: 0.025 - reward_lane_center: 0.0038 - reward_offroad: 3.0 - reward_overspeed: 0.05 - reward_randomization: false - reward_reverse: 0.005 - reward_stop_line: 1.0 - reward_timestep: 2.5e-05 - reward_velocity: 0.0025 - goal_source: route - goal_regen_mode: finite - termination_mode: false - traffic_light_behavior: 0 - eval: - export_episode_csv: 'true' - num_scenarios: 250 - verify_coverage: 'true' - interval: 250 - mode: inline - validation_gigaflow: - enabled: 'true' - env: - map_dir: pufferlib/resources/drive/binaries/carla - max_agents_per_env: 40 - min_agents_per_env: 40 - num_agents: 1024 - num_maps: 8 - resample_frequency: 500 - scenario_length: 500 - simulation_mode: gigaflow - eval: - render_max_steps: 300 - render_num_scenarios: 8 - inherits: validation_defaults - render: 'true' - render_backend: triage_html - type: multi_scenario - validation_replay: - enabled: 'true' - env: - control_mode: control_sdc_only - map_dir: /scratch/ev2237/data/nuplan/nuplan_mini_train_bins - max_agents_per_env: 64 - num_maps: 250 - resample_frequency: 200 - scenario_length: 200 - simulation_mode: replay - eval: - render_max_steps: 200 - render_num_scenarios: 5 - inherits: validation_defaults - render: 'true' - render_backend: triage_html - type: multi_scenario - wosac: - clean: 'true' - enabled: 'false' - env: - control_mode: control_wosac - goal_radius: 2.0 - init_mode: create_all_valid - init_step: 10 - eval: - wosac_aggregate_results: 'true' - wosac_num_agents: 256 - wosac_num_rollouts: 32 - wosac_sanity_check: 'false' - interval: 500 - mode: subprocess - render: 'false' - type: wosac -eval_simulation: null -git: - commit_hash: 9d3fcc5c09db50b9674a374b8c3404b6349c4cb5 -load_id: null -load_model_path: null -local_rank: 0 -max_runs: 200 -max_suggestion_cost: 3600 -mine: - num_episodes: 100 - output_dir: '' - render: 'true' - score_threshold: -inf -neptune: false -neptune_name: pufferai -neptune_project: ablations -no_model_upload: {} -num_scenarios: 3 -package: ocean -policy: - actor_hidden_size: 512 - actor_num_layers: 0 - backbone_activation: gelu - backbone_hidden_size: 512 - backbone_layer_norm: false - backbone_num_layers: 2 - boundary_input_size: 128 - context_input_size: 128 - critic_hidden_size: 512 - critic_num_layers: 0 - ego_input_size: 128 - encoder_activation: relu - encoder_layer_norm: true - lane_input_size: 128 - mask_padded_features: false - partner_input_size: 128 - shared_network: true - traffic_control_input_size: 128 - action_type: discrete -policy_name: Drive -rnn: - hidden_size: 512 - input_size: 512 -rnn_name: null -run_name: rew-dense-seed4 -sweep: - downsample: 10 - goal: maximize - method: Protein - metric: score -tag: null -tb: false -train: - adam_beta1: 0.9 - adam_beta2: 0.999 - adam_eps: 1.0e-08 - adv_filter_ewma_beta: 0.25 - adv_filter_threshold_scale: 0.01 - adv_sampling_prio_alpha: 0.8499999999999999 - adv_sampling_prio_beta0: 0.8499999999999999 - amp: true - anneal_lr: true - batch_size: auto - bptt_horizon: 128 - checkpoint_interval: 50 - clip_coef: 0.2 - compile: true - compile_fullgraph: false - compile_mode: default - cpu_offload: false - data_dir: /pufferdrive/training_output/ - device: cuda - ent_coef: 0.01 - gae_lambda: 0.95 - gamma: 0.999 - learning_rate: 0.0005 - max_grad_norm: 0.5 - max_minibatch_size: 98304 - minibatch_size: 98304 - name: pufferai - normalize_rewards: true - optimizer: adamw - precision: bfloat16 - project: ablations - resume_state_path: null - seed: 4 - torch_deterministic: false - total_timesteps: 10000000000 - update_epochs: 2 - use_rnn: false - vf_clip_coef: null - vf_coef: 0.5 - vtrace_c_clip: 1 - vtrace_rho_clip: 1 -vec: - backend: Multiprocessing - batch_size: auto - num_envs: 24 - num_workers: auto - seed: 42 - zero_copy: true -wandb: true -wandb_group: pr-smoke-runs -wandb_project: pr-smoke-runs diff --git a/scripts/config-reward-sparse.yaml b/scripts/config-reward-sparse.yaml deleted file mode 100644 index cda5e9d4bc..0000000000 --- a/scripts/config-reward-sparse.yaml +++ /dev/null @@ -1,384 +0,0 @@ -controlled_exp: - train: - ent_coef: - values: - - 0.01 - - 0.005 - learning_rate: - values: - - 0.001 - - 0.003 - - 0.01 -env: - action_type: discrete - collision_behavior: 1 - compute_eval_metrics: false - control_mode: control_vehicles - dt: 0.1 - dynamics_model: jerk - goal_on_lane: true - goal_radius: 2.0 - goal_speed: 1000.0 - inactive_agent_threshold: 0.4 - init_mode: create_all_valid - init_step: 0 - map_dir: "pufferlib/resources/drive/binaries/carla" - max_agents_per_env: 80 - max_waypoint_spacing: 40.0 - min_agents_per_env: 1 - min_waypoint_spacing: 20.0 - non_sdc_controller: policy - non_vehicle_controller: auto - num_agents: 2048 - num_maps: 8 - num_target_waypoints: 3 - obs_dropout_boundary: 0.0 - obs_dropout_lane: 0.0 - obs_norm_goal_offset_m: 120.0 - obs_norm_road_seg_length_m: 10.0 - obs_norm_road_seg_width_m: 5.0 - obs_norm_veh_length_m: 10.0 - obs_norm_veh_width_m: 5.0 - obs_norm_xy_offset_m: 120.0 - obs_range_partner_m: 120.0 - obs_range_road_behind_m: 30.0 - obs_range_road_front_m: 120.0 - obs_range_road_side_m: 40.0 - obs_range_traffic_control_m: 120.0 - obs_slots_boundary_n: 30 - obs_slots_lane_n: 50 - obs_slots_partners_n: 12 - obs_slots_traffic_controls_n: 4 - offroad_behavior: 1 - partner_blindness_prob: 0.0 - partner_blindness_trigger_prob: 0.0 - phantom_braking_duration: 10 - phantom_braking_prob: 0.0 - phantom_braking_trigger_prob: 0.0 - resample_frequency: 102400 - reward_ade: 0.0 - reward_center_bias: 0.0 - reward_collision: 1.5 - reward_comfort: 0.05 - reward_conditioning: false - reward_goal: 0.5 - reward_lane_align: 0.025 - reward_lane_center: 0.0038 - reward_offroad: 1.5 - reward_overspeed: 0.05 - reward_randomization: false - reward_reverse: 0.005 - reward_stop_line: 1.0 - reward_timestep: 0.0 - reward_vel_align: 1.0 - reward_velocity: 0.0025 - scenario_length: 1024 - sdc_controller: policy - simulation_mode: gigaflow - spawn_initial_speed: 0.0 - goal_source: route - goal_regen_mode: finite - termination_mode: true - traffic_light_behavior: 1 - use_map_cache: true -env_name: puffer_drive -eval: - behaviors_defaults: - clean: 'true' - enabled: 'false' - env: - control_mode: control_sdc_only - init_mode: create_all_valid - obs_slots_partners_n: 32 - scenario_length: 201 - simulation_mode: replay - eval: - num_scenarios: 50 - render_max_steps: 200 - render_num_scenarios: 2 - interval: 250 - mode: inline - render: 'true' - render_backend: triage_html - behaviors_full_dir: - enabled: 'true' - env: - map_dir: /scratch/ev2237/data/nuplan/nuplan_mini_train_bins - inherits: behaviors_defaults - type: behavior_class - behaviors_hard_stop: - enabled: 'true' - env: - map_dir: /scratch/ev2237/data/nuplan/categories_v021/hard_stop - inherits: behaviors_defaults - type: behavior_class - behaviors_highway_straight: - enabled: 'true' - env: - map_dir: /scratch/ev2237/data/nuplan/categories_v021/highway_straight - inherits: behaviors_defaults - type: behavior_class - behaviors_lane_change: - enabled: 'true' - env: - map_dir: /scratch/ev2237/data/nuplan/categories_v021/lane_change - inherits: behaviors_defaults - type: behavior_class - behaviors_merge: - enabled: 'true' - env: - map_dir: /scratch/ev2237/data/nuplan/categories_v021/merge - inherits: behaviors_defaults - type: behavior_class - behaviors_parked_cars: - enabled: 'true' - env: - map_dir: /scratch/ev2237/data/nuplan/categories_v021/parked_cars - inherits: behaviors_defaults - type: behavior_class - behaviors_roundabout: - enabled: 'true' - env: - map_dir: /scratch/ev2237/data/nuplan/categories_v021/roundabout - inherits: behaviors_defaults - type: behavior_class - behaviors_stopped_traffic: - enabled: 'true' - env: - map_dir: /scratch/ev2237/data/nuplan/categories_v021/stopped_traffic - inherits: behaviors_defaults - type: behavior_class - behaviors_traffic_light_green: - enabled: 'true' - env: - map_dir: /scratch/ev2237/data/nuplan/categories_v021/traffic_light_green - inherits: behaviors_defaults - type: behavior_class - behaviors_traffic_light_stop: - enabled: 'true' - env: - map_dir: /scratch/ev2237/data/nuplan/categories_v021/traffic_light_stop - inherits: behaviors_defaults - type: behavior_class - behaviors_unprotected_left: - enabled: 'true' - env: - map_dir: /scratch/ev2237/data/nuplan/categories_v021/unprotected_left - inherits: behaviors_defaults - type: behavior_class - behaviors_unprotected_right: - enabled: 'true' - env: - map_dir: /scratch/ev2237/data/nuplan/categories_v021/unprotected_right - inherits: behaviors_defaults - type: behavior_class - dnf_triage: - enabled: 'false' - env: - map_dir: pufferlib/resources/drive/binaries/carla/opendrive__Town10HD.bin - max_agents_per_env: 1 - min_agents_per_env: 1 - num_maps: 1 - resample_frequency: 500 - scenario_length: 500 - simulation_mode: gigaflow - eval: - num_scenarios: 32 - render_max_steps: 300 - render_num_scenarios: 16 - inherits: validation_defaults - render: 'true' - render_backend: triage_html - type: multi_scenario - validation_defaults: - clean: 'true' - enabled: 'true' - env: - collision_behavior: 1 - eval_mode: 1 - goal_speed: 3.0 - num_agents: 1024 - obs_dropout_boundary: 0.0 - obs_dropout_lane: 0.0 - obs_slots_boundary_n: 80 - obs_slots_lane_n: 80 - offroad_behavior: 1 - reward_ade: 0.0 - reward_collision: 3.0 - reward_comfort: 0.05 - reward_goal: 1.0 - reward_lane_align: 0.025 - reward_lane_center: 0.0038 - reward_offroad: 3.0 - reward_overspeed: 0.05 - reward_randomization: false - reward_reverse: 0.005 - reward_stop_line: 1.0 - reward_timestep: 2.5e-05 - reward_velocity: 0.0025 - goal_source: route - goal_regen_mode: finite - termination_mode: false - traffic_light_behavior: 0 - eval: - export_episode_csv: 'true' - num_scenarios: 250 - verify_coverage: 'true' - interval: 250 - mode: inline - validation_gigaflow: - enabled: 'true' - env: - map_dir: pufferlib/resources/drive/binaries/carla - max_agents_per_env: 40 - min_agents_per_env: 40 - num_agents: 1024 - num_maps: 8 - resample_frequency: 500 - scenario_length: 500 - simulation_mode: gigaflow - eval: - render_max_steps: 300 - render_num_scenarios: 8 - inherits: validation_defaults - render: 'true' - render_backend: triage_html - type: multi_scenario - validation_replay: - enabled: 'true' - env: - control_mode: control_sdc_only - map_dir: /scratch/ev2237/data/nuplan/nuplan_mini_train_bins - max_agents_per_env: 64 - num_maps: 250 - resample_frequency: 200 - scenario_length: 200 - simulation_mode: replay - eval: - render_max_steps: 200 - render_num_scenarios: 5 - inherits: validation_defaults - render: 'true' - render_backend: triage_html - type: multi_scenario - wosac: - clean: 'true' - enabled: 'false' - env: - control_mode: control_wosac - goal_radius: 2.0 - init_mode: create_all_valid - init_step: 10 - eval: - wosac_aggregate_results: 'true' - wosac_num_agents: 256 - wosac_num_rollouts: 32 - wosac_sanity_check: 'false' - interval: 500 - mode: subprocess - render: 'false' - type: wosac -eval_simulation: null -git: - commit_hash: 9d3fcc5c09db50b9674a374b8c3404b6349c4cb5 -load_id: null -load_model_path: null -local_rank: 0 -max_runs: 200 -max_suggestion_cost: 3600 -mine: - num_episodes: 100 - output_dir: '' - render: 'true' - score_threshold: -inf -neptune: false -neptune_name: pufferai -neptune_project: ablations -no_model_upload: {} -num_scenarios: 3 -package: ocean -policy: - actor_hidden_size: 512 - actor_num_layers: 0 - backbone_activation: gelu - backbone_hidden_size: 512 - backbone_layer_norm: false - backbone_num_layers: 2 - boundary_input_size: 128 - context_input_size: 128 - critic_hidden_size: 512 - critic_num_layers: 0 - ego_input_size: 128 - encoder_activation: relu - encoder_layer_norm: true - lane_input_size: 128 - mask_padded_features: false - partner_input_size: 128 - shared_network: true - traffic_control_input_size: 128 - action_type: discrete -policy_name: Drive -rnn: - hidden_size: 512 - input_size: 512 -rnn_name: null -run_name: rew-sparse-seed4 -sweep: - downsample: 10 - goal: maximize - method: Protein - metric: score -tag: null -tb: false -train: - adam_beta1: 0.9 - adam_beta2: 0.999 - adam_eps: 1.0e-08 - adv_filter_ewma_beta: 0.25 - adv_filter_threshold_scale: 0.01 - adv_sampling_prio_alpha: 0.8499999999999999 - adv_sampling_prio_beta0: 0.8499999999999999 - amp: true - anneal_lr: true - batch_size: auto - bptt_horizon: 128 - checkpoint_interval: 50 - clip_coef: 0.2 - compile: true - compile_fullgraph: false - compile_mode: default - cpu_offload: false - data_dir: /pufferdrive/training_output/ - device: cuda - ent_coef: 0.01 - gae_lambda: 0.95 - gamma: 0.999 - learning_rate: 0.0005 - max_grad_norm: 0.5 - max_minibatch_size: 98304 - minibatch_size: 98304 - name: pufferai - normalize_rewards: true - optimizer: adamw - precision: bfloat16 - project: ablations - resume_state_path: null - seed: 4 - torch_deterministic: false - total_timesteps: 10000000000 - update_epochs: 2 - use_rnn: false - vf_clip_coef: null - vf_coef: 0.5 - vtrace_c_clip: 1 - vtrace_rho_clip: 1 -vec: - backend: Multiprocessing - batch_size: auto - num_envs: 24 - num_workers: auto - seed: 42 - zero_copy: true -wandb: true -wandb_group: pr-smoke-runs -wandb_project: pr-smoke-runs diff --git a/setup.py b/setup.py index 134826fdfc..54928090df 100644 --- a/setup.py +++ b/setup.py @@ -223,7 +223,6 @@ def run(self): "imageio", "pyro-ppl", "heavyball", - "neptune", "wandb", "wandb-workspaces", "tensorboard", diff --git a/tests/eval/test_eval.py b/tests/eval/test_eval.py index 964b62953a..03bcc8dde9 100644 --- a/tests/eval/test_eval.py +++ b/tests/eval/test_eval.py @@ -172,7 +172,6 @@ def _standalone_eval_args(benchmark_config_path): } ) args["wandb"] = False - args["neptune"] = False args["tb"] = False return args @@ -308,12 +307,12 @@ def test_seed_replay_writes_exactly_identical_metrics(carla_evaluation): environment_config["obs_dropout_boundary"] = args["env"]["obs_dropout_boundary"] replay_args = drive_benchmark.build_benchmark_args(args, benchmarks[0], environment_config) - map_indices = drive_benchmark._resolve_map_indices( + map_indices = drive_benchmark.resolve_map_indices( replay_args["env"]["map_dir"], standard_metrics["map_name"].tolist(), ) map_seed_pairs = [(map_idx, int(seed)) for map_idx, seed in zip(map_indices, standard_metrics["seed"].tolist())] - worker_env_kwargs, total_steps = drive_benchmark._plan_failure_replay_workers( + worker_env_kwargs, total_steps = drive_benchmark.plan_failure_replay_workers( replay_args, map_seed_pairs, CARLA_WORKER_COUNT, @@ -332,7 +331,7 @@ def test_seed_replay_writes_exactly_identical_metrics(carla_evaluation): policy=ZeroPolicy(action_count=63), ) replay_output_dir = carla_evaluation["output_root"] / "seed_replay" - replay_summary = drive_benchmark._write_eval_reports( + replay_summary = drive_benchmark.write_eval_reports( replay_summaries, replay_output_dir, CARLA_SCENARIO_COUNT, @@ -415,7 +414,7 @@ def _read_replay_float32_chunk(replay_path, chunk_name): def test_multiprocess_replay_capture_renders_zlib_to_html(tmp_path, monkeypatch, capture_observations): args = _replay_render_args() map_seed_pairs = [(0, 1234), (0, 5678)] - worker_env_kwargs, total_steps = drive_benchmark._plan_failure_replay_workers( + worker_env_kwargs, total_steps = drive_benchmark.plan_failure_replay_workers( args, map_seed_pairs, num_workers=2, @@ -483,7 +482,7 @@ def record_vector_make(*make_args, **make_kwargs): agent_goal_radii = agent_frames[..., header["agent_goal_radius_field"]] np.testing.assert_allclose(goal_radius_coefs, agent_goal_radii, atol=1e-6) - render_dir = Path(drive_eval_replay._render_eval_replays(summaries, str(tmp_path), keep_zlib_replays=True)) + render_dir = Path(drive_eval_replay.render_eval_replays(summaries, str(tmp_path), keep_zlib_replays=True)) rendered_pages = sorted(path for path in render_dir.glob("*.html") if path.name != "index.html") assert len(rendered_pages) == 2 assert (render_dir / "index.html").is_file() @@ -493,7 +492,7 @@ def record_vector_make(*make_args, **make_kwargs): assert all("ctx.arc(g.x,g.y,g.radius" in html for html in rendered_html) assert all(replay_path.is_file() for replay_path in replay_paths) - drive_eval_replay._render_eval_replays(summaries, str(tmp_path), keep_zlib_replays=False) + drive_eval_replay.render_eval_replays(summaries, str(tmp_path), keep_zlib_replays=False) assert not replay_output_dir.exists() assert (render_dir / "index.html").is_file() assert len(sorted(path for path in render_dir.glob("*.html") if path.name != "index.html")) == 2 @@ -589,7 +588,6 @@ def _training_args(tmp_path, benchmark_config_path, evaluation_enabled): } ) args["wandb"] = False - args["neptune"] = False args["tb"] = False return args @@ -765,7 +763,6 @@ def _sdc_eval_args(benchmark_config_path, benchmark_name, map_dir): } ) args["wandb"] = False - args["neptune"] = False args["tb"] = False return args diff --git a/tests/smoke_tests/test_drive_rollout.py b/tests/smoke_tests/test_drive_rollout.py index 69f581858b..ab459d6486 100644 --- a/tests/smoke_tests/test_drive_rollout.py +++ b/tests/smoke_tests/test_drive_rollout.py @@ -83,7 +83,6 @@ def _build_config(): }, ) args["wandb"] = False - args["neptune"] = False args["eval"] = None return args diff --git a/tests/smoke_tests/test_drive_train.py b/tests/smoke_tests/test_drive_train.py index b06a50be53..c4bab35f4a 100644 --- a/tests/smoke_tests/test_drive_train.py +++ b/tests/smoke_tests/test_drive_train.py @@ -165,7 +165,6 @@ def _build_config(): _set_existing(args["rnn"], {"input_size": 256, "hidden_size": 256}) args["wandb"] = False - args["neptune"] = False args["eval"] = None # disable all evaluators during the smoke run return args diff --git a/tests/unit_tests/test_rank_seeding.py b/tests/unit_tests/test_rank_seeding.py index c86cf0af61..fae410cd6f 100644 --- a/tests/unit_tests/test_rank_seeding.py +++ b/tests/unit_tests/test_rank_seeding.py @@ -3,7 +3,7 @@ Under DDP every rank holds identical policy weights; if ranks also shared identical torch and env seeds they would collect (near-)duplicate experience and multi-node training would degenerate to single-node. train() derives -per-rank seeds via `pufferl.derive_rank_seeds(vec_seed, train_seed, +per-rank seeds via `utils.derive_rank_seeds(vec_seed, train_seed, world_size, global_rank)`; these tests pin its contract: - every GPU (rank) gets a distinct torch seed and env seed @@ -19,14 +19,14 @@ import pytest -from pufferlib.pufferl import derive_rank_seeds +from pufferlib import utils VEC_SEED = 42 TRAIN_SEED = 42 def _seeds_for_all_ranks(world_size, vec_seed=VEC_SEED, train_seed=TRAIN_SEED): - return [derive_rank_seeds(vec_seed, train_seed, world_size, rank) for rank in range(world_size)] + return [utils.derive_rank_seeds(vec_seed, train_seed, world_size, rank) for rank in range(world_size)] @pytest.mark.parametrize("world_size", [2, 4, 8]) @@ -55,7 +55,7 @@ def test_multi_node_ranks_with_same_local_rank_differ(): def test_single_process_torch_seed_unchanged(): # Backward compatibility: a non-distributed run must seed torch with the # plain train seed, exactly as before the per-rank derivation existed. - torch_seed, _ = derive_rank_seeds(VEC_SEED, 123, world_size=1, global_rank=0) + torch_seed, _ = utils.derive_rank_seeds(VEC_SEED, 123, world_size=1, global_rank=0) assert torch_seed == 123 @@ -65,21 +65,21 @@ def test_derivation_is_deterministic(): # runs would not be reproducible. The pinned value guards the mixing # scheme itself; update it only on a deliberate scheme change (which # breaks scenario-stream comparability with older runs). - assert derive_rank_seeds(42, 42, 8, 0) == (336, 1921063561) - assert derive_rank_seeds(42, 42, 8, 3) == derive_rank_seeds(42, 42, 8, 3) + assert utils.derive_rank_seeds(42, 42, 8, 0) == (336, 1921063561) + assert utils.derive_rank_seeds(42, 42, 8, 3) == utils.derive_rank_seeds(42, 42, 8, 3) def test_train_seed_sweep_varies_env_seed(): # Sweeping train.seed alone must vary the env scenario stream; before the # per-rank derivation, sweeps only changed network init and sampling. - env_seeds = {derive_rank_seeds(VEC_SEED, train_seed, 1, 0)[1] for train_seed in range(5)} + env_seeds = {utils.derive_rank_seeds(VEC_SEED, train_seed, 1, 0)[1] for train_seed in range(5)} assert len(env_seeds) == 5 def test_unseeded_envs_pass_through(): # vec.seed=None means "do not seed the envs"; the derivation must not # manufacture a seed for them. - _, env_seed = derive_rank_seeds(None, TRAIN_SEED, 8, 3) + _, env_seed = utils.derive_rank_seeds(None, TRAIN_SEED, 8, 3) assert env_seed is None @@ -87,5 +87,5 @@ def test_no_env_seed_collisions_across_sweep_by_rank_grid(): # A realistic experiment grid (5 sweep seeds x 64 ranks) must produce # all-distinct env seeds; any collision means two runs/ranks replay the # same scenario sequence. - grid = [derive_rank_seeds(VEC_SEED, train_seed, 64, rank)[1] for train_seed in range(5) for rank in range(64)] + grid = [utils.derive_rank_seeds(VEC_SEED, train_seed, 64, rank)[1] for train_seed in range(5) for rank in range(64)] assert len(set(grid)) == len(grid) diff --git a/weights/06_02_checkpoint/config.yaml b/weights/06_02_checkpoint/config.yaml deleted file mode 100644 index beb215a1e9..0000000000 --- a/weights/06_02_checkpoint/config.yaml +++ /dev/null @@ -1,390 +0,0 @@ -agent_index: null -controlled_exp: - train: - ent_coef: - values: - - 0.01 - - 0.005 - learning_rate: - values: - - 0.001 - - 0.003 - - 0.01 -env: - action_type: discrete - collision_behavior: 1 - compute_eval_metrics: false - control_mode: control_vehicles - dt: 0.3 - dynamics_model: jerk - goal_on_lane: true - goal_radius: 2.0 - goal_speed: 3.0 - inactive_agent_threshold: 0.4 - init_mode: create_all_valid - init_step: 0 - map_dir: pufferlib/resources/drive/binaries/carla - max_agents_per_env: 150 - max_waypoint_spacing: 60.0 - min_agents_per_env: 1 - min_waypoint_spacing: 20.0 - num_agents: 2048 - num_maps: 8 - num_target_waypoints: 3 - obs_dropout_boundary: 0.4 - obs_dropout_lane: 0.5 - obs_norm_goal_offset_m: 200.0 - obs_norm_road_seg_length_m: 10.0 - obs_norm_road_seg_width_m: 5.0 - obs_norm_veh_length_m: 15.0 - obs_norm_veh_width_m: 10.0 - obs_norm_xy_offset_m: 200.0 - obs_range_partner_m: 200.0 - obs_range_road_behind_m: 40.0 - obs_range_road_front_m: 200.0 - obs_range_road_side_m: 50.0 - obs_range_traffic_control_m: 100.0 - obs_slots_boundary_n: 80 - obs_slots_lane_n: 80 - obs_slots_partners_n: 16 - obs_slots_traffic_controls_n: 4 - offroad_behavior: 1 - partner_blindness_prob: 0.03 - partner_blindness_trigger_prob: 0.05 - phantom_braking_duration: 10 - phantom_braking_prob: 0.02 - phantom_braking_trigger_prob: 0.02 - resample_frequency: 0 - reward_ade: 0.0 - reward_center_bias: 0.0 - reward_collision: 1.5 - reward_comfort: 0.05 - reward_conditioning: true - reward_goal: 1.0 - reward_lane_align: 0.025 - reward_lane_center: 0.005 - reward_offroad: 1.5 - reward_overspeed: 0.05 - reward_randomization: true - reward_reverse: 0.005 - reward_stop_line: 1.0 - reward_timestep: 2.5e-05 - reward_vel_align: 1.0 - reward_velocity: 0.0025 - scenario_length: 1200 - simulation_mode: gigaflow - spawn_initial_speed: 0.0 - target_type: static - termination_mode: 1 - traffic_light_behavior: 1 - use_map_cache: 1 -env_name: puffer_drive -eval: - behaviors_defaults: - clean: 'true' - enabled: 'false' - env: - control_mode: control_sdc_only - init_mode: create_all_valid - obs_slots_partners_n: 32 - scenario_length: 201 - simulation_mode: replay - eval: - num_scenarios: 50 - render_max_steps: 200 - render_num_scenarios: 2 - interval: 250 - mode: inline - render: 'true' - render_views: - - sim_state - - bev - behaviors_full_dir: - enabled: 0 - env: - map_dir: /scratch/ev2237/data/nuplan/nuplan_mini_train_bins - inherits: behaviors_defaults - type: behavior_class - behaviors_hard_stop: - enabled: 0 - env: - map_dir: /scratch/ev2237/data/nuplan/categories_v021/hard_stop - inherits: behaviors_defaults - type: behavior_class - behaviors_highway_straight: - enabled: 0 - env: - map_dir: /scratch/ev2237/data/nuplan/categories_v021/highway_straight - inherits: behaviors_defaults - type: behavior_class - behaviors_lane_change: - enabled: 0 - env: - map_dir: /scratch/ev2237/data/nuplan/categories_v021/lane_change - inherits: behaviors_defaults - type: behavior_class - behaviors_merge: - enabled: 0 - env: - map_dir: /scratch/ev2237/data/nuplan/categories_v021/merge - inherits: behaviors_defaults - type: behavior_class - behaviors_parked_cars: - enabled: 0 - env: - map_dir: /scratch/ev2237/data/nuplan/categories_v021/parked_cars - inherits: behaviors_defaults - type: behavior_class - behaviors_roundabout: - enabled: 0 - env: - map_dir: /scratch/ev2237/data/nuplan/categories_v021/roundabout - inherits: behaviors_defaults - type: behavior_class - behaviors_stopped_traffic: - enabled: 0 - env: - map_dir: /scratch/ev2237/data/nuplan/categories_v021/stopped_traffic - inherits: behaviors_defaults - type: behavior_class - behaviors_traffic_light_green: - enabled: 0 - env: - map_dir: /scratch/ev2237/data/nuplan/categories_v021/traffic_light_green - inherits: behaviors_defaults - type: behavior_class - behaviors_traffic_light_stop: - enabled: 0 - env: - map_dir: /scratch/ev2237/data/nuplan/categories_v021/traffic_light_stop - inherits: behaviors_defaults - type: behavior_class - behaviors_unprotected_left: - enabled: 0 - env: - map_dir: /scratch/ev2237/data/nuplan/categories_v021/unprotected_left - inherits: behaviors_defaults - type: behavior_class - behaviors_unprotected_right: - enabled: 0 - env: - map_dir: /scratch/ev2237/data/nuplan/categories_v021/unprotected_right - inherits: behaviors_defaults - type: behavior_class - dnf_triage: - enabled: 'false' - env: - map_dir: pufferlib/resources/drive/binaries/carla/opendrive__Town10HD.bin - max_agents_per_env: 1 - min_agents_per_env: 1 - num_maps: 1 - resample_frequency: 500 - scenario_length: 500 - simulation_mode: gigaflow - eval: - num_scenarios: 32 - render_max_steps: 300 - render_num_scenarios: 16 - inherits: validation_defaults - render: 'true' - render_backend: triage_html - type: multi_scenario - validation_defaults: - clean: 'true' - enabled: 'true' - env: - collision_behavior: 1 - eval_mode: 1 - goal_speed: 3.0 - num_agents: 1024 - obs_dropout_boundary: 0.0 - obs_dropout_lane: 0.0 - obs_slots_boundary_n: 80 - obs_slots_lane_n: 80 - offroad_behavior: 1 - reward_ade: 0.0 - reward_collision: 3.0 - reward_comfort: 0.05 - reward_goal: 1.0 - reward_lane_align: 0.025 - reward_lane_center: 0.0038 - reward_offroad: 3.0 - reward_overspeed: 0.05 - reward_randomization: false - reward_reverse: 0.005 - reward_stop_line: 1.0 - reward_timestep: 2.5e-05 - reward_velocity: 0.0025 - target_type: static - termination_mode: 0 - traffic_light_behavior: 0 - eval: - export_episode_csv: 'true' - num_scenarios: 250 - verify_coverage: 'true' - interval: 250 - mode: inline - validation_gigaflow: - enabled: 'true' - env: - map_dir: pufferlib/resources/drive/binaries/carla - max_agents_per_env: 40 - min_agents_per_env: 40 - num_agents: 1024 - num_maps: 8 - resample_frequency: 500 - scenario_length: 500 - simulation_mode: gigaflow - eval: - render_max_steps: 300 - render_num_scenarios: 8 - inherits: validation_defaults - render: false - render_backend: egl - render_views: - - sim_state - - bev - type: multi_scenario - validation_replay: - enabled: 0 - env: - control_mode: control_sdc_only - map_dir: /scratch/ev2237/data/nuplan/nuplan_mini_train_bins - max_agents_per_env: 64 - num_maps: 250 - resample_frequency: 200 - scenario_length: 200 - simulation_mode: replay - eval: - render_max_steps: 200 - render_num_scenarios: 5 - inherits: validation_defaults - render: 'true' - render_backend: triage_html - type: multi_scenario - wosac: - clean: 'true' - enabled: 'false' - env: - control_mode: control_wosac - goal_radius: 2.0 - init_mode: create_all_valid - init_step: 10 - eval: - wosac_aggregate_results: 'true' - wosac_num_agents: 256 - wosac_num_rollouts: 32 - wosac_sanity_check: 'false' - interval: 500 - mode: subprocess - render: 'false' - type: wosac -eval_simulation: null -fps: 15 -gif_path: eval.gif -git: - commit_hash: aca6960c5c9368422a945fd62b81985378bc0031 -load_id: null -load_model_path: null -local_rank: 0 -max_runs: 200 -max_suggestion_cost: 3600 -mine: - num_episodes: 100 - output_dir: '' - render: 'true' - score_threshold: -inf -neptune: false -neptune_name: pufferai -neptune_project: ablations -no_model_upload: {} -num_scenarios: 3 -package: ocean -policy: - actor_hidden_size: 1024 - actor_num_layers: 0 - backbone_hidden_size: 1024 - backbone_num_layers: 3 - critic_hidden_size: 1024 - critic_num_layers: 0 - dropout: 0.0 - encoder_gigaflow: true - input_size: 256 - split_network: true -policy_name: Drive -render: 0 -render_mode: auto -rnn: - hidden_size: 512 - input_size: 512 -rnn_name: null -run_name: null -save_frames: 0 -sweep: - downsample: 10 - goal: maximize - method: Protein - metric: score -tag: 2026-06-02_local_4gpu -tb: false -train: - adam_beta1: 0.9 - adam_beta2: 0.999 - adam_eps: 1.0e-08 - adv_filter_ewma_beta: 0.25 - adv_filter_threshold_scale: 0.01 - adv_sampling_prio_alpha: 0.8499999999999999 - adv_sampling_prio_beta0: 0.8499999999999999 - amp: true - anneal_lr: true - batch_size: auto - bptt_horizon: 128 - checkpoint_interval: 500 - clip_coef: 0.2 - compile: true - compile_fullgraph: false - compile_mode: default - cpu_offload: false - data_dir: experiments - device: cuda - ent_coef: 0.01 - gae_lambda: 0.95 - gamma: 0.999 - learning_rate: 0.0005 - max_grad_norm: 0.5 - max_minibatch_size: 153600 - minibatch_size: 153600 - name: pufferai - normalize_rewards: false - obs_only: true - optimizer: adamw - ppo_granularity: auto - precision: bfloat16 - project: ablations - render: false - render_interval: 1000 - render_map: none - resume_state_path: null - seed: 0 - show_grid: false - show_human_logs: true - show_lasers: false - torch_deterministic: false - total_timesteps: 10000000000 - update_epochs: 3 - use_rnn: false - vf_clip_coef: null - vf_coef: 0.5 - vtrace_c_clip: 1 - vtrace_rho_clip: 1 -vec: - backend: Multiprocessing - batch_size: auto - num_envs: 20 - num_workers: auto - seed: 42 - zero_copy: true -video_path: videos -wandb: true -wandb_group: Nightly_MultiAgent -wandb_project: nightly-multi-agent diff --git a/weights/06_02_checkpoint/models/model_puffer_drive_001500.pt b/weights/06_02_checkpoint/models/model_puffer_drive_001500.pt deleted file mode 100644 index fbdc4da259..0000000000 Binary files a/weights/06_02_checkpoint/models/model_puffer_drive_001500.pt and /dev/null differ diff --git a/weights/06_02_checkpoint/trainer_state.pt b/weights/06_02_checkpoint/trainer_state.pt deleted file mode 100644 index 6f6af480a6..0000000000 Binary files a/weights/06_02_checkpoint/trainer_state.pt and /dev/null differ diff --git a/weights/mimolette/config.yaml b/weights/mimolette/config.yaml deleted file mode 100644 index 8189a681ab..0000000000 --- a/weights/mimolette/config.yaml +++ /dev/null @@ -1,222 +0,0 @@ -agent_index: null -controlled_exp: - train: - ent_coef: - values: - - 0.01 - - 0.005 - learning_rate: - values: - - 0.001 - - 0.003 - - 0.01 -env: - action_type: discrete - collision_behavior: 1 - compute_eval_metrics: false - control_mode: control_vehicles - dt: 0.3 - dynamics_model: jerk - goal_radius: 2.0 - goal_regen_mode: finite - goal_source: map - goal_speed: 3.0 - inactive_agent_threshold: 0.4 - init_mode: create_all_valid - init_step: 0 - map_dir: "" - max_agents_per_env: 120 - max_goal_spacing: 200.0 - min_agents_per_env: 1 - min_goal_spacing: 20.0 - num_agents: 3200 - num_goals: 3 - num_maps: 72 - obs_boundary_stride: 1 - obs_dropout_boundary: 0.4 - obs_dropout_lane: 0.3 - obs_goal_lane_distance: true - obs_lane_stride: 2 - obs_norm_goal_offset_m: 200.0 - obs_norm_road_seg_length_m: 10.0 - obs_norm_road_seg_width_m: 5.0 - obs_norm_veh_length_m: 10.0 - obs_norm_veh_width_m: 5.0 - obs_norm_xy_offset_m: 200.0 - obs_range_partner_m: 200.0 - obs_range_road_behind_m: 60.0 - obs_range_road_front_m: 200.0 - obs_range_road_side_m: 50.0 - obs_range_traffic_control_m: 200.0 - obs_slots_boundary_n: 50 - obs_slots_lane_n: 70 - obs_slots_partners_n: 16 - obs_slots_traffic_controls_n: 4 - offroad_behavior: 1 - partner_blindness_prob: 0.02 - partner_blindness_trigger_prob: 0.03 - phantom_braking_duration_seconds: 3.0 - phantom_braking_prob: 0.02 - phantom_braking_trigger_prob: 0.03 - resample_frequency: 256000 - reward_ade: 0.0 - reward_center_bias: 0.0 - reward_collision: 1.5 - reward_comfort: 0.05 - reward_conditioning: true - reward_goal: 1.0 - reward_lane_align: 0.025 - reward_lane_center: 0.001 - reward_offroad: 1.5 - reward_overspeed: 0.05 - reward_randomization: true - reward_reverse: 0.005 - reward_stop_line: 1.0 - reward_timestep: 2.5e-05 - reward_vel_align: 1.0 - reward_velocity: 0.0025 - scenario_length: 2560 - simulation_mode: gigaflow - spawn_initial_speed: 0.0 - termination_mode: 1 - traffic_lights_enabled: true - stop_signs_enabled: false - yield_signs_enabled: false - traffic_light_behavior: 1 - stop_sign_behavior: 1 - use_map_cache: 1 -env_name: puffer_drive -eval: - backend: PufferEnv - benchmark: false - benchmark_config: pufferlib/ocean/competition/benchmark_catalog.yaml - benchmark_datasets: nuplan_multi,nuplan_single - benchmark_sdc_num_envs: 8 - eval_interval: 20 - human_replay_control_mode: '"control_sdc_only" ; Control only the self-driving car' - human_replay_eval: false - human_replay_num_agents: 64 ; This equals the number of scenarios, since we control - one agent in each - num_agents: 512 - render: false - render_obs: false - render_only: false - wosac_aggregate_results: True ; Only return aggregate results across all scenes - wosac_control_mode: '"control_wosac" ; Control the tracks to predict' - wosac_goal_radius: 2.0 ; Can shrink goal radius for WOSAC evaluation - wosac_init_mode: '"create_all_valid" ; Initialize from the tracks to predict' - wosac_init_steps: 10 ; When to start the simulation - wosac_num_agents: 256 ; Total number of WOSAC agents to evaluate - wosac_num_rollouts: 32 ; Number of policy rollouts per scene - wosac_realism_eval: false - wosac_sanity_check: false -fps: 15 -gif_path: eval.gif -git: - commit_hash: bc5b8940370d20a5bfc714555c0be657cd6bcb6e -load_id: null -load_model_path: null -local_rank: 0 -max_runs: 200 -max_suggestion_cost: 3600 -neptune: false -neptune_name: pufferai -neptune_project: ablations -num_maps: 1 -num_scenarios: 3 -package: ocean -policy: - actor_hidden_size: 256 - actor_num_layers: 1 - backbone_activation: gelu - backbone_hidden_size: 1024 - backbone_layer_norm: false - backbone_num_layers: 3 - boundary_input_size: 256 - context_input_size: 128 - critic_hidden_size: 256 - critic_num_layers: 1 - ego_input_size: 128 - encoder_activation: relu - encoder_layer_norm: true - lane_input_size: 256 - mask_padded_features: false - partner_input_size: 256 - shared_network: false - traffic_control_input_size: 128 - action_type: discrete -policy_name: Drive -render: 0 -render_mode: matplotlib -render_obs: 0 -rnn: - hidden_size: 512 - input_size: 512 -rnn_name: null -save_frames: 0 -sweep: - downsample: 10 - goal: maximize - method: Protein - metric: score -tag: null -tb: true -train: - adam_beta1: 0.9 - adam_beta2: 0.999 - adam_eps: 1.0e-08 - adv_filter_ewma_beta: 0.25 - adv_filter_threshold_scale: 0.01 - adv_sampling_prio_alpha: 0.8499999999999999 - adv_sampling_prio_beta0: 0.8499999999999999 - anneal_lr: true - batch_size: auto - bptt_horizon: 128 - checkpoint_interval: 100 - clip_coef: 0.2 - compile: true - compile_fullgraph: false - compile_mode: default - cpu_offload: false - data_dir: /pufferdrive/training_output/ - device: cuda - ent_coef: 0.01 - gae_lambda: 0.95 - gamma: 0.999 - learning_rate: 0.0005 - max_grad_norm: 0.5 - max_minibatch_size: 64000 - minibatch_size: 256000 - name: pufferai - normalize_rewards: false - obs_only: true - optimizer: adamw - precision: bfloat16 - project: ablations - render: false - render_interval: 1000 - render_map: none - resume_state_path: null - seed: 42 - show_grid: false - show_human_logs: true - show_lasers: false - torch_deterministic: false - total_timesteps: 62500000000 - update_epochs: 3 - use_rnn: false - vf_clip_coef: null - vf_coef: 0.5 - vtrace_c_clip: 1 - vtrace_rho_clip: 1 -vec: - backend: Multiprocessing - batch_size: auto - num_envs: 40 - num_workers: auto - seed: 42 - zero_copy: true -video_path: videos -wandb: false -wandb_group: debug -wandb_project: pufferlib diff --git a/weights/mimolette/events.out.tfevents.1782743642.cmle-training-workerpool0-f23ab4b69a-0-28wbj.165.0 b/weights/mimolette/events.out.tfevents.1782743642.cmle-training-workerpool0-f23ab4b69a-0-28wbj.165.0 deleted file mode 100644 index 1a4c7d6145..0000000000 Binary files a/weights/mimolette/events.out.tfevents.1782743642.cmle-training-workerpool0-f23ab4b69a-0-28wbj.165.0 and /dev/null differ diff --git a/weights/mimolette/models/model_puffer_drive_003815.pt b/weights/mimolette/models/model_puffer_drive_003815.pt deleted file mode 100644 index fd6eae3d01..0000000000 Binary files a/weights/mimolette/models/model_puffer_drive_003815.pt and /dev/null differ diff --git a/weights/mimolette/puffer_drive_20260629-143401.pt b/weights/mimolette/puffer_drive_20260629-143401.pt deleted file mode 100644 index fd6eae3d01..0000000000 Binary files a/weights/mimolette/puffer_drive_20260629-143401.pt and /dev/null differ diff --git a/weights/mimolette/trainer_state.pt b/weights/mimolette/trainer_state.pt deleted file mode 100644 index 2ca46f6471..0000000000 Binary files a/weights/mimolette/trainer_state.pt and /dev/null differ