diff --git a/.github/workflows/cpu-tests.yaml b/.github/workflows/cpu-tests.yaml index 73e661b9..16847cf9 100644 --- a/.github/workflows/cpu-tests.yaml +++ b/.github/workflows/cpu-tests.yaml @@ -153,9 +153,39 @@ jobs: - uses: ./.github/actions/setup_repo with: python-version: '3.11' + - name: Install native validation dependencies + run: | + sudo apt-get update + packages=(pkg-config libfftw3-dev libopenmpi-dev openmpi-bin) + if ! command -v cmake >/dev/null; then packages+=(cmake); fi + sudo apt-get install -y --no-install-recommends "${packages[@]}" + uv sync --extra dev --extra native --frozen - name: Test with pytest + env: + OMPI_MCA_rmaps_base_oversubscribe: '1' run: uv run --locked pytest tests/test_vlasov1d --ignore=tests/test_vlasov1d/test_config_regression.py + test-native-vlasov1d: + needs: changes + if: needs.changes.outputs.run_all == 'true' || needs.changes.outputs.vlasov1d == 'true' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: ./.github/actions/setup_repo + with: + python-version: '3.11' + - name: Install native validation dependencies + run: | + sudo apt-get update + packages=(pkg-config libfftw3-dev libopenmpi-dev openmpi-bin) + if ! command -v cmake >/dev/null; then packages+=(cmake); fi + sudo apt-get install -y --no-install-recommends "${packages[@]}" + uv sync --extra dev --extra native --frozen + - name: Test native backend + env: + OMPI_MCA_rmaps_base_oversubscribe: '1' + run: uv run --locked pytest tests/test_vlasov1d/test_native.py tests/test_vlasov1d/test_observations.py + test-farsight1d: needs: changes if: needs.changes.outputs.run_all == 'true' || needs.changes.outputs.farsight1d == 'true' diff --git a/adept/_base_.py b/adept/_base_.py index 9d3405ed..fff3c1f9 100644 --- a/adept/_base_.py +++ b/adept/_base_.py @@ -53,6 +53,8 @@ class ADEPTModule: """ + jit_compile = True + def __init__(self, cfg) -> None: self.cfg = cfg self.state = None @@ -352,6 +354,10 @@ def _get_adept_module_(self, cfg: dict) -> ADEPTModule: else: raise NotImplementedError("This solver approach has not been implemented yet") + if cfg.get("backend", "jax") == "native": + from adept.vlasov1d import native_backend + + this_module = native_backend(this_module) return this_module(cfg) def _setup_(self, cfg: dict, td: str, adept_module: ADEPTModule = None, log: bool = True) -> dict[str, Module]: @@ -370,6 +376,10 @@ def _setup_(self, cfg: dict, td: str, adept_module: ADEPTModule = None, log: boo if adept_module is None: self.adept_module = self._get_adept_module_(cfg) else: + if cfg.get("backend", "jax") == "native": + from adept.vlasov1d import native_backend + + adept_module = native_backend(adept_module) self.adept_module = adept_module(cfg) # dump raw config @@ -401,11 +411,14 @@ def _setup_(self, cfg: dict, td: str, adept_module: ADEPTModule = None, log: boo self.adept_module.init_diffeqsolve() modules = self.adept_module.init_modules() - self._compat_execution, self.compatibility_fallback_reason = LegacyPreparedExecution.try_create( - original_cfg, - legacy_module=self.adept_module, - custom_module=adept_module is not None, - ) + if not self.adept_module.jit_compile: + self.compatibility_fallback_reason = "native execution owns its worker lifecycle" + else: + self._compat_execution, self.compatibility_fallback_reason = LegacyPreparedExecution.try_create( + original_cfg, + legacy_module=self.adept_module, + custom_module=adept_module is not None, + ) if self._compat_execution is not None: self.execution_backend = "prepared" @@ -414,6 +427,9 @@ def _setup_(self, cfg: dict, td: str, adept_module: ADEPTModule = None, log: boo return modules def _execute_simulation(self, modules: dict | None, args: dict | None) -> dict: + if not self.adept_module.jit_compile: + self.execution_backend = "native" + return self.adept_module(modules, args) if self._compat_execution is not None: reason = self._compat_execution.fallback_reason( state=self.adept_module.state, diff --git a/adept/_vlasov1d/builder.py b/adept/_vlasov1d/builder.py index bbf678fb..48fcdc34 100644 --- a/adept/_vlasov1d/builder.py +++ b/adept/_vlasov1d/builder.py @@ -83,6 +83,11 @@ def prepare(self, spec: SimulationSpec, *, key: int | jax.Array) -> PreparedSimu raise ValueError(f"Vlasov1DBuilder cannot prepare solver {spec.solver!r}") if spec.schema_version != "1": raise ValueError(f"Unsupported Vlasov1D specification schema version {spec.schema_version!r}") + if spec.config_dict().get("backend", "jax") == "native": + raise NotImplementedError( + "backend: native runs through ergoExo (including custom BaseVlasov1D subclasses); " + "Vlasov1DBuilder prepares a JAX program" + ) if not jax.config.jax_enable_x64: raise RuntimeError("vlasov-1d requires float64; enable jax_enable_x64 before preparation") normalized_key, seed, provenance = normalize_key(key) diff --git a/adept/_vlasov1d/datamodel.py b/adept/_vlasov1d/datamodel.py index 3cfa2a17..1dae5c54 100644 --- a/adept/_vlasov1d/datamodel.py +++ b/adept/_vlasov1d/datamodel.py @@ -340,8 +340,29 @@ class Vlasov1DConfig(BaseModel): grid: GridConfig save: SaveConfig solver: str + backend: Literal["jax", "native"] = "jax" + native: "NativeSolverConfig | None" = None mlflow: MLFlowConfig drivers: EMDriverSetConfig terms: TermsConfig diagnostics: DiagnosticsConfig = DiagnosticsConfig() iaw_diagnostics: IAWDiagnosticsConfig | None = None + + +class NativeSolverConfig(BaseModel): + """Execution options for the optional CUDA/MPI Vlasov backend.""" + + model_config = ConfigDict(extra="forbid") + library: str | None = None + device: Literal["cuda", "cpu"] = "cuda" + ranks: int = Field(default=1, ge=1) + launcher: list[str] = Field(default_factory=list) + workdir: str | None = None + halo_capacity: int = Field(default=64, ge=2) + scalar_stride: int = Field(default=1, ge=1) + custom_save_times: Literal["preserve", "steps"] = "preserve" + host_mpi: bool = False + timeout: float | None = Field(default=None, gt=0) + + +Vlasov1DConfig.model_rebuild() diff --git a/adept/_vlasov1d/native/CMakeLists.txt b/adept/_vlasov1d/native/CMakeLists.txt new file mode 100644 index 00000000..aa6e9108 --- /dev/null +++ b/adept/_vlasov1d/native/CMakeLists.txt @@ -0,0 +1,21 @@ +cmake_minimum_required(VERSION 3.24) +project(adept_vlasov_native LANGUAGES CXX) +option(VLASOV_CUDA "Use CUDA/cuFFT (OFF builds the FFTW validation backend)" ON) +find_package(MPI REQUIRED COMPONENTS CXX) +add_library(adept_vlasov SHARED api.cpp core/backend.cpp) +target_compile_features(adept_vlasov PRIVATE cxx_std_17) +target_link_libraries(adept_vlasov PRIVATE MPI::MPI_CXX) +set_target_properties(adept_vlasov PROPERTIES POSITION_INDEPENDENT_CODE ON) +if(VLASOV_CUDA) + enable_language(CUDA) + find_package(CUDAToolkit REQUIRED) + set_source_files_properties(core/backend.cpp PROPERTIES LANGUAGE CUDA) + set_target_properties(adept_vlasov PROPERTIES CUDA_STANDARD 17 CUDA_ARCHITECTURES 80) + target_compile_definitions(adept_vlasov PRIVATE VLASOV_CUDA) + target_compile_options(adept_vlasov PRIVATE $<$:--extended-lambda>) + target_link_libraries(adept_vlasov PRIVATE CUDA::cudart CUDA::cufft) +else() + find_package(PkgConfig REQUIRED) + pkg_check_modules(FFTW REQUIRED IMPORTED_TARGET fftw3) + target_link_libraries(adept_vlasov PRIVATE PkgConfig::FFTW) +endif() diff --git a/adept/_vlasov1d/native/README.md b/adept/_vlasov1d/native/README.md new file mode 100644 index 00000000..112ca765 --- /dev/null +++ b/adept/_vlasov1d/native/README.md @@ -0,0 +1,49 @@ +# Native Vlasov backend + +Select `backend: native` in the usual `solver: vlasov-1d` configuration and run +through `ergoExo`. See the [configuration and build instructions](../../../docs/source/solvers/vlasov1d/config.md#execution-backend). +An existing `BaseVlasov1D` subclass can still supply initialization, save +callbacks and postprocessing. C++/CUDA/MPI advances the distribution; JAX runs +the original save callbacks, without automatic differentiation of the solve. + +`core/` is the canonical numerical implementation. The standalone program in +`benchmarks/vlasov_cuda` includes these same sources. Build the shared library +on the target system with CMake; `VLASOV_CUDA=OFF` builds the FFTW validation +implementation. The library is loaded only by MPI workers, not the outer +Python driver. + +The outer driver performs initialization on the CPU and stages the initial +state in bounded blocks. One persistent worker owns the solver on each GPU. +At a save point, DLPack exposes synchronized native buffers as JAX arrays on +a velocity-slab mesh. CUDA views require zero-copy import. The CPU validation +build permits a copy when FFTW's host alignment is insufficient for JAX. +Callbacks complete and saved outputs reach the host before the next native +mutation. No borrowed state view may survive `av_close`; the library checks +this lifetime rule explicitly. + +Workers stream saved leaves into host `.npy` memory maps. The driver assembles +distributed saved outputs and returns the usual Diffrax `Solution` containing +NumPy arrays. Existing xarray/NetCDF writers consume it. The module retains +the backing workspace while its returned saves are in use. MLflow artifacts +include actual save schedules, worker statistics/logs and the library hash. + +Standard saves round to actual timestep boundaries and merge duplicates. +Scalars default to every step; `native.scalar_stride` can reduce the cadence. +Custom callbacks preserve their requested times by default because some +project postprocessors reconstruct those axes. Off-step custom saves require +a previous-state copy and interpolation. `custom_save_times: steps` avoids +that work when the postprocessor consumes actual result times. Postprocessors +that reconstruct uniform scalar axes also need actual result times if the +last scalar interval is shorter than the requested stride. + +The validated physics is spectral spatial streaming, ADEPT's local cubic C1 +velocity interpolation, sixth-order splitting, Poisson fields, and optional +constant-rate Chang-Cooper Dougherty collisions. Kinetic ions use the existing +`terms.species` entries, with independent velocity grids, charges and masses. +Each stage sums every species' charge density into one MPI field reduction, +then advances every species under that shared field. Collisions act on the +reference species, as in ADEPT's JAX path; inter-species collisional exchange +is not included. Rebuild the library when updating ADEPT (current ABI: 3). +Unsupported physics and +trainable modules fail explicitly. Performance and reproduction evidence is +under [the benchmark results](../../../benchmarks/vlasov_cuda/results/2026-10-24-adept). diff --git a/adept/_vlasov1d/native/__init__.py b/adept/_vlasov1d/native/__init__.py new file mode 100644 index 00000000..ca9c4720 --- /dev/null +++ b/adept/_vlasov1d/native/__init__.py @@ -0,0 +1 @@ +"""Optional native Vlasov timestepping with ADEPT preprocessing and saves.""" diff --git a/adept/_vlasov1d/native/api.cpp b/adept/_vlasov1d/native/api.cpp new file mode 100644 index 00000000..81ef2de3 --- /dev/null +++ b/adept/_vlasov1d/native/api.cpp @@ -0,0 +1,241 @@ +// Narrow C ABI for Python orchestration. Physics is shared with the benchmark. +#include "core/solver.hpp" +#include +#include +#include +#include + +using namespace adept_native; +namespace { +thread_local std::string last_error; +bool own_mpi = false; +std::atomic live_engines{0}; +struct SpeciesState { + Options options; + std::unique_ptr backend; + std::unique_ptr stepper; + int local_rank; + SpeciesState(Options opts, int local) : options(std::move(opts)), local_rank(local) { + backend = std::make_unique(options.grid, local); + stepper = std::make_unique(*backend, options); + if (!options.coordinates.empty()) backend->set_coordinates(read_binary(options.coordinates, options.grid.nv)); + if (!options.background.empty()) backend->set_background(read_binary(options.background, options.grid.nx)); + initialize(*backend, options); + } +}; +struct Engine : SpeciesState { + std::vector> additional; + Engine(Options opts, int local) : SpeciesState(std::move(opts), local) { ++live_engines; } + ~Engine() { --live_engines; } + SpeciesState& species(int index) { + if (index < 0 || index > int(additional.size())) throw std::runtime_error("invalid species index"); + return index == 0 ? *this : *additional[index - 1]; + } + template void each(F action) { + action(*this); + for (auto& state : additional) action(*state); + } + void solve_field() { + backend->density(options.grid.charge); + for (auto& state : additional) { + state->backend->density(state->options.grid.charge); + backend->add_density(*state->backend); + } + backend->synchronize(); + if (options.host_mpi) { + auto local = backend->download_density(); + std::vector global(local.size()); + mpi_check(MPI_Allreduce(local.data(), global.data(), options.grid.nx, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD)); + backend->upload_density(global); + } else { + mpi_check(MPI_Allreduce(backend->density_buffer(), backend->reduced_density_buffer(), + options.grid.nx, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD)); + } + backend->field(true); + for (auto& state : additional) state->backend->copy_field(*backend); + } + void step() { + if (additional.empty()) { stepper->step(); return; } + backend->synchronize(); + double started = MPI_Wtime(); + const SixthCoefficients coefficients(options.dt); + for (int k = 0; k < 6; ++k) { + stepper->timed(Field, [&] { solve_field(); }); + each([&](SpeciesState& state) { state.stepper->velocity_step(coefficients.d[k] * options.dt, true); }); + if (k < 5) each([&](SpeciesState& state) { + state.stepper->timed(Stream, [&] { state.backend->stream(coefficients.a[k] * options.dt); }); + }); + } + each([](SpeciesState& state) { state.stepper->collision_step(); }); + backend->synchronize(); + stepper->times[Total] += MPI_Wtime() - started; + } +}; +using Handle = std::shared_ptr; +template int guarded(F action) { + try { action(); return 0; } + catch (const std::exception& error) { last_error = error.what(); return -1; } +} +// Legacy DLPack 0.8 ABI, exported as a "dltensor" capsule by Python. +// https://dmlc.github.io/dlpack/latest/c_api.html +struct DLDevice { int32_t device_type, device_id; }; +struct DLDataType { uint8_t code, bits; uint16_t lanes; }; +struct DLTensor { + void* data; DLDevice device; int32_t ndim; DLDataType dtype; + int64_t* shape; int64_t* strides; uint64_t byte_offset; +}; +struct DLManagedTensor { + DLTensor dl_tensor; void* manager_ctx; void (*deleter)(DLManagedTensor*); +}; +struct Export { + DLManagedTensor managed; + Handle owner; + int64_t shape[2], strides[2]; +}; +void release_tensor(DLManagedTensor* tensor) { delete static_cast(tensor->manager_ctx); } +int device_type() { +#ifdef VLASOV_CUDA + return 2; // kDLCUDA +#else + return 1; // kDLCPU +#endif +} +void validate_options(Options& opts) { + Grid& g = opts.grid; + mpi_check(MPI_Comm_rank(MPI_COMM_WORLD, &g.rank)); + mpi_check(MPI_Comm_size(MPI_COMM_WORLD, &g.ranks)); + g.collisions = opts.nu > 0; + if (g.nx < 8 || g.nv < 4 || g.nx % g.ranks || g.nv % g.ranks || + g.count() > std::size_t(std::numeric_limits::max())) + throw std::runtime_error("native grids must divide across ranks and fit MPI int counts"); + if (!g.local_cubic || !g.halos || !g.cell_centred || opts.dt <= 0 || opts.nu < 0 || + g.mass <= 0 || !std::isfinite(g.mass) || !std::isfinite(g.charge) || + g.length <= 0 || g.vmax <= 0 || g.halo_capacity < 2 || + std::size_t(g.halo_capacity) * g.nx > std::size_t(std::numeric_limits::max()) || + (opts.integrator != "sixth" && opts.integrator != "strang")) + throw std::runtime_error("unsupported native solver options"); +} +} +extern "C" { +int av_abi_version() { return 3; } +const char* av_error() { return last_error.c_str(); } +void* av_create(int argc, const char** argv) { + void* result = nullptr; + if (guarded([&] { + prepare_device_from_environment(); + int initialized; + mpi_check(MPI_Initialized(&initialized)); + if (!initialized) { mpi_check(MPI_Init(nullptr, nullptr)); own_mpi = true; } + MPI_Comm_set_errhandler(MPI_COMM_WORLD, MPI_ERRORS_RETURN); + int rank; + mpi_check(MPI_Comm_rank(MPI_COMM_WORLD, &rank)); + Options opts = parse(argc, const_cast(argv), rank); + validate_options(opts); + MPI_Comm local; + mpi_check(MPI_Comm_split_type(MPI_COMM_WORLD, MPI_COMM_TYPE_SHARED, rank, MPI_INFO_NULL, &local)); + int local_rank; + mpi_check(MPI_Comm_rank(local, &local_rank)); + mpi_check(MPI_Comm_free(&local)); + result = new Handle(std::make_shared(std::move(opts), local_rank)); + })) return nullptr; + return result; +} +int av_add_species(void* handle, int argc, const char** argv) { + return guarded([&] { + auto& e = **static_cast(handle); + Options opts = parse(argc, const_cast(argv), e.options.grid.rank); + validate_options(opts); + if (opts.grid.nx != e.options.grid.nx || opts.grid.length != e.options.grid.length || + opts.dt != e.options.dt || opts.integrator != "sixth" || e.options.integrator != "sixth") + throw std::runtime_error("coupled species require the same x grid, timestep and sixth-order integrator"); + e.additional.push_back(std::make_unique(std::move(opts), e.local_rank)); + }); +} +int av_species_info(void* handle, int index, int* output) { + return guarded([&] { + auto& s = (**static_cast(handle)).species(index); + const Grid& g = s.options.grid; + const int values[] = {g.rank, g.ranks, s.local_rank, device_type(), g.nx, g.nv, g.local_v()}; + std::copy_n(values, 7, output); + }); +} +int av_info(void* handle, int* output) { + return guarded([&] { + auto& e = **static_cast(handle); + const Grid& g = e.options.grid; + const int values[] = {g.rank, g.ranks, e.local_rank, device_type(), g.nx, g.nv, g.local_v()}; + std::copy_n(values, 7, output); + }); +} +int av_device_ordinal(void* handle, int* output) { + return guarded([&] { *output = (**static_cast(handle)).backend->device_ordinal(); }); +} +int av_broadcast(char* value, int capacity) { + return guarded([&] { mpi_check(MPI_Bcast(value, capacity, MPI_CHAR, 0, MPI_COMM_WORLD)); }); +} +int av_barrier() { return guarded([] { mpi_check(MPI_Barrier(MPI_COMM_WORLD)); }); } +int av_step(void* handle, int count) { + return guarded([&] { + if (count < 0) throw std::runtime_error("negative native step count"); + auto& e = **static_cast(handle); + for (int i = 0; i < count; ++i) e.step(); + e.backend->synchronize(); + }); +} +int av_set_field(void* handle, const char* path) { + return guarded([&] { + auto& e = **static_cast(handle); + e.backend->set_field(read_binary(path, e.options.grid.nx)); + }); +} +void* av_species_tensor(void* handle, int species, int field) { + void* result = nullptr; + if (guarded([&] { + auto owner = *static_cast(handle); + auto& e = owner->species(species); + e.backend->synchronize(); // Producer completion before a consumer stream reads. + auto* tensor = new Export{}; + tensor->owner = std::move(owner); + tensor->shape[0] = e.options.grid.nx; + tensor->shape[1] = e.options.grid.local_v(); + tensor->strides[0] = 1; + tensor->strides[1] = e.options.grid.nx; + tensor->managed = {{field ? e.backend->electric_buffer() : e.backend->velocity_buffer(), + {device_type(), e.backend->device_ordinal()}, field ? 1 : 2, + {2, 64, 1}, tensor->shape, tensor->strides, 0}, tensor, release_tensor}; + result = &tensor->managed; + })) return nullptr; + return result; +} +void* av_tensor(void* handle, int field) { return av_species_tensor(handle, 0, field); } +int av_stats(void* handle, double* output) { + return guarded([&] { + auto& e = **static_cast(handle); + std::array times{}; + e.each([&](SpeciesState& state) { + for (int i = 0; i < StageCount; ++i) times[i] += state.stepper->times[i]; + }); + mpi_check(MPI_Allreduce(times.data(), output, StageCount, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD)); + }); +} +int av_close(void* handle) { + return guarded([&] { + auto* owner = static_cast(handle); + if (owner->use_count() != 1) throw std::runtime_error("native buffers still have live DLPack views"); + delete owner; + }); +} +int av_finalize() { + return guarded([] { + if (live_engines) throw std::runtime_error("native engines must close before MPI finalization"); + if (own_mpi) { mpi_check(MPI_Finalize()); own_mpi = false; } + }); +} +void av_abort(int code) { + int initialized = 0, finalized = 0; + MPI_Initialized(&initialized); + if (initialized) MPI_Finalized(&finalized); + if (initialized && !finalized) MPI_Abort(MPI_COMM_WORLD, code); + std::_Exit(code); +} +} diff --git a/adept/_vlasov1d/native/bridge.py b/adept/_vlasov1d/native/bridge.py new file mode 100644 index 00000000..19030988 --- /dev/null +++ b/adept/_vlasov1d/native/bridge.py @@ -0,0 +1,115 @@ +"""ctypes and DLPack boundary; importing this module does not initialize JAX.""" + +import ctypes as ct +import sys + + +class NativeCore: + """Own one persistent MPI/native solver inside a worker process.""" + + def __init__(self, library, options): + self.lib = ct.CDLL(str(library)) + version = getattr(self.lib, "av_abi_version", None) + if version is None or version() != 3: + raise RuntimeError( + "native library ABI mismatch; rebuild adept/_vlasov1d/native from the installed ADEPT source" + ) + signatures = { + "av_create": ([ct.c_int, ct.POINTER(ct.c_char_p)], ct.c_void_p), + "av_error": ([], ct.c_char_p), + "av_info": ([ct.c_void_p, ct.POINTER(ct.c_int)], ct.c_int), + "av_device_ordinal": ([ct.c_void_p, ct.POINTER(ct.c_int)], ct.c_int), + "av_add_species": ([ct.c_void_p, ct.c_int, ct.POINTER(ct.c_char_p)], ct.c_int), + "av_species_info": ([ct.c_void_p, ct.c_int, ct.POINTER(ct.c_int)], ct.c_int), + "av_species_tensor": ([ct.c_void_p, ct.c_int, ct.c_int], ct.c_void_p), + "av_step": ([ct.c_void_p, ct.c_int], ct.c_int), + "av_tensor": ([ct.c_void_p, ct.c_int], ct.c_void_p), + "av_set_field": ([ct.c_void_p, ct.c_char_p], ct.c_int), + "av_broadcast": ([ct.c_char_p, ct.c_int], ct.c_int), + "av_barrier": ([], ct.c_int), + "av_stats": ([ct.c_void_p, ct.POINTER(ct.c_double)], ct.c_int), + "av_close": ([ct.c_void_p], ct.c_int), + "av_finalize": ([], ct.c_int), + "av_abort": ([ct.c_int], None), + } + for name, (arguments, result) in signatures.items(): + function = getattr(self.lib, name) + function.argtypes, function.restype = arguments, result + argv = [b"adept-native", *(str(value).encode() for value in options)] + self.handle = self.lib.av_create(len(argv), (ct.c_char_p * len(argv))(*argv)) + if not self.handle: + print(self.lib.av_error().decode(), file=sys.stderr, flush=True) + self.lib.av_abort(1) + values = (ct.c_int * 7)() + self.check(self.lib.av_info(self.handle, values)) + self.rank, self.size, self.local_rank, self.device_type, self.nx, self.nv, self.local_nv = values + ordinal = ct.c_int() + self.check(self.lib.av_device_ordinal(self.handle, ct.byref(ordinal))) + self.device_ordinal = ordinal.value + + def check(self, result): + if result: + raise RuntimeError(self.lib.av_error().decode()) + + def step(self, count): + self.check(self.lib.av_step(self.handle, count)) + + def broadcast(self, value): + buffer = ct.create_string_buffer(value.encode(), 512) + self.check(self.lib.av_broadcast(buffer, len(buffer))) + return buffer.value.decode() + + def barrier(self): + self.check(self.lib.av_barrier()) + + def set_field(self, path): + self.check(self.lib.av_set_field(self.handle, str(path).encode())) + + def add_species(self, options): + argv = [b"adept-native", *(str(value).encode() for value in options)] + self.check(self.lib.av_add_species(self.handle, len(argv), (ct.c_char_p * len(argv))(*argv))) + + def species_info(self, index): + values = (ct.c_int * 7)() + self.check(self.lib.av_species_info(self.handle, index, values)) + return tuple(values) + + def view(self, field=False, species=0): + return _View(self, field, species) + + def stats(self): + values = (ct.c_double * 7)() + self.check(self.lib.av_stats(self.handle, values)) + names = ["stream", "velocity", "pack", "mpi", "field", "collision", "total"] + return dict(zip(names, values, strict=True)) + + def close(self): + if self.handle: + self.check(self.lib.av_close(self.handle)) + self.handle = None + self.check(self.lib.av_finalize()) + + +class _View: + """Synchronized, read-only-by-convention view valid until the next native step.""" + + def __init__(self, owner, field, species): + self.owner, self.field, self.species = owner, field, species + + def __dlpack_device__(self): + return self.owner.device_type, self.owner.device_ordinal + + def __dlpack__(self, stream=None, *, max_version=None, dl_device=None, copy=None): + if copy: + raise BufferError("the native exporter provides views; the consumer may copy") + if dl_device is not None and tuple(dl_device) != self.__dlpack_device__(): + raise BufferError("native DLPack export cannot change devices") + pointer = self.owner.lib.av_species_tensor(self.owner.handle, self.species, int(self.field)) + if not pointer: + raise RuntimeError(self.owner.lib.av_error().decode()) + # The consumer owns the tensor's C++ deleter after renaming this capsule + # to used_dltensor. Its manager keeps the native allocation alive. + capsule = ct.pythonapi.PyCapsule_New + capsule.restype = ct.py_object + capsule.argtypes = [ct.c_void_p, ct.c_char_p, ct.c_void_p] + return capsule(pointer, b"dltensor", None) diff --git a/adept/_vlasov1d/native/core/backend.cpp b/adept/_vlasov1d/native/core/backend.cpp new file mode 100644 index 00000000..7e7023bc --- /dev/null +++ b/adept/_vlasov1d/native/core/backend.cpp @@ -0,0 +1,525 @@ +#include "backend.hpp" +#include "local_cubic.hpp" + +#include +#include +#include +#include +#include +#include + +#ifdef VLASOV_CUDA +#include +#include +#define DEVICE __device__ +static void check_cuda(cudaError_t error) { + if (error != cudaSuccess) throw std::runtime_error(cudaGetErrorString(error)); +} +static void check_fft(cufftResult error) { + if (error != CUFFT_SUCCESS) throw std::runtime_error("cuFFT error " + std::to_string(error)); +} +template __global__ void kernel(std::size_t n, F fn) { + for (std::size_t i = blockIdx.x * std::size_t(blockDim.x) + threadIdx.x; + i < n; i += std::size_t(blockDim.x) * gridDim.x) fn(i); +} +template void parallel_for(std::size_t n, F fn) { + kernel<<((n + 255) / 256, 65535), 256>>>(n, fn); + check_cuda(cudaGetLastError()); +} +__global__ void maximum_absolute(const double* values, double* output, int n) { + __shared__ double partial[256]; + double maximum = 0; + for (int i = blockIdx.x * blockDim.x + threadIdx.x; i < n; i += blockDim.x * gridDim.x) + maximum = fmax(maximum, fabs(values[i])); + partial[threadIdx.x] = maximum; + __syncthreads(); + for (int stride = 128; stride; stride /= 2) { + if (threadIdx.x < stride) partial[threadIdx.x] = fmax(partial[threadIdx.x], partial[threadIdx.x + stride]); + __syncthreads(); + } + if (threadIdx.x == 0) + atomicMax(reinterpret_cast(output), + static_cast(__double_as_longlong(partial[0]))); +} +#else +#include +#define DEVICE +template void parallel_for(std::size_t n, F fn) { + for (std::size_t i = 0; i < n; ++i) fn(i); +} +#endif + +namespace { +constexpr double pi = 3.14159265358979323846; +class Buffer { +public: + explicit Buffer(std::size_t n) : size(n) { + if (!n) return; +#ifdef VLASOV_CUDA + check_cuda(cudaMalloc(reinterpret_cast(&ptr), n * sizeof(double))); +#else + ptr = static_cast(fftw_malloc(n * sizeof(double))); + if (!ptr) throw std::bad_alloc(); +#endif + } + ~Buffer() { +#ifdef VLASOV_CUDA + cudaFree(ptr); +#else + fftw_free(ptr); +#endif + } + Buffer(const Buffer&) = delete; + Buffer& operator=(const Buffer&) = delete; + std::vector download() const { + std::vector result(size); +#ifdef VLASOV_CUDA + check_cuda(cudaMemcpy(result.data(), ptr, size * sizeof(double), cudaMemcpyDeviceToHost)); +#else + std::copy_n(ptr, size, result.data()); +#endif + return result; + } + void upload(const std::vector& values) { + if (values.size() != size) throw std::runtime_error("buffer size mismatch"); +#ifdef VLASOV_CUDA + check_cuda(cudaMemcpy(ptr, values.data(), size * sizeof(double), cudaMemcpyHostToDevice)); +#else + std::copy(values.begin(), values.end(), ptr); +#endif + } + double* ptr = nullptr; + std::size_t size; +}; + +// Plans and workspaces persist for the full run. Complex storage is interleaved. +class FFT { +public: + FFT(int n, int batch, double* real, double* complex) { +#ifdef VLASOV_CUDA + check_fft(cufftPlan1d(&forward, n, CUFFT_D2Z, batch)); + check_fft(cufftPlan1d(&backward, n, CUFFT_Z2D, batch)); + std::size_t a = 0, b = 0; + check_fft(cufftGetSize(forward, &a)); + check_fft(cufftGetSize(backward, &b)); + workspace_bytes = a + b; +#else + int dims[] = {n}; + forward = fftw_plan_many_dft_r2c(1, dims, batch, real, nullptr, 1, n, + reinterpret_cast(complex), nullptr, 1, n / 2 + 1, FFTW_ESTIMATE); + backward = fftw_plan_many_dft_c2r(1, dims, batch, + reinterpret_cast(complex), nullptr, 1, n / 2 + 1, + real, nullptr, 1, n, FFTW_ESTIMATE); + if (!forward || !backward) throw std::runtime_error("FFTW plan creation failed"); +#endif + } + ~FFT() { +#ifdef VLASOV_CUDA + cufftDestroy(forward); + cufftDestroy(backward); +#else + fftw_destroy_plan(forward); + fftw_destroy_plan(backward); +#endif + } + void r2c(double* real, double* complex) { +#ifdef VLASOV_CUDA + check_fft(cufftExecD2Z(forward, real, reinterpret_cast(complex))); +#else + fftw_execute_dft_r2c(forward, real, reinterpret_cast(complex)); +#endif + } + void c2r(double* complex, double* real) { +#ifdef VLASOV_CUDA + check_fft(cufftExecZ2D(backward, reinterpret_cast(complex), real)); +#else + fftw_execute_dft_c2r(backward, reinterpret_cast(complex), real); +#endif + } + std::size_t workspace_bytes = 0; +private: +#ifdef VLASOV_CUDA + cufftHandle forward{}, backward{}; +#else + fftw_plan forward{}, backward{}; +#endif +}; + +std::string select_device(int local_rank) { +#ifdef VLASOV_CUDA + int count = 0; + check_cuda(cudaGetDeviceCount(&count)); + if (!count) throw std::runtime_error("no CUDA devices visible"); + // Slurm may expose one device per rank, or all devices on the node. + if (count != 1 && local_rank >= count) + throw std::runtime_error("more local ranks than visible GPUs"); + int device = count == 1 ? 0 : local_rank; + check_cuda(cudaSetDevice(device)); + cudaDeviceProp props{}; + check_cuda(cudaGetDeviceProperties(&props, device)); + return props.name; +#else + (void)local_rank; + return "CPU/FFTW (validation only)"; +#endif +} +int current_device_ordinal() { +#ifdef VLASOV_CUDA + int device = 0; + check_cuda(cudaGetDevice(&device)); + return device; +#else + return 0; +#endif +} +} // namespace + +void prepare_device_from_environment() { + int local_rank = 0; + for (const char* name : {"SLURM_LOCALID", "OMPI_COMM_WORLD_LOCAL_RANK", "MPI_LOCALRANKID", + "PMI_LOCAL_RANK", "MV2_COMM_WORLD_LOCAL_RANK"}) { + if (const char* value = std::getenv(name)) { + local_rank = std::stoi(value); + break; + } + } + select_device(local_rank); +} + +struct Backend::Impl { + Grid grid; + std::string device; + int ordinal; + Buffer fv, fx, send, recv, second, rhs, output, inverse; + Buffer spectrum, density_local, density_global, electric, field_spectrum; + Buffer coordinates, background, ghosts, field_maximum; + FFT streaming_fft, field_fft; + Impl(Grid g, int local_rank) + : grid(g), device(select_device(local_rank)), ordinal(current_device_ordinal()), + fv(g.count()), fx(!g.halos || g.collisions ? g.count() : 0), + send((!g.halos || g.collisions) && g.ranks > 1 ? g.count() : 0), + recv((!g.halos || g.collisions) && g.ranks > 1 ? g.count() : 0), + second(!g.local_cubic || g.collisions ? g.count() : 0), + rhs(!g.local_cubic || g.collisions ? g.count() : 0), + output(g.count()), inverse(!g.local_cubic ? g.nv : 0), + spectrum(2 * std::size_t(g.nx / 2 + 1) * g.local_v()), + density_local(g.nx), density_global(g.nx), electric(g.nx), + field_spectrum(2 * (g.nx / 2 + 1)), + coordinates(g.nv), background(g.nx), + ghosts(g.halos ? 2 * std::size_t(g.halo_capacity) * g.nx : 0), + field_maximum(1), + streaming_fft(g.nx, g.local_v(), fv.ptr, spectrum.ptr), + field_fft(g.nx, 1, electric.ptr, field_spectrum.ptr) { + std::vector inv(g.nv, 0.0); + // LU factors for natural spline: m[i-1] + 4m[i] + m[i+1] = 6*d2f/dv^2. + for (int i = 1; i < g.nv - 1; ++i) inv[i] = 1.0 / (4.0 - inv[i - 1]); + if (!g.local_cubic) inverse.upload(inv); + std::vector v(g.nv), bg(g.nx, 1.0); + for (int j = 0; j < g.nv; ++j) v[j] = g.velocity(j); + coordinates.upload(v); + background.upload(bg); + electric.upload(std::vector(g.nx, 0.0)); + } +}; + +Backend::Backend(Grid grid, int local_rank) : p(std::make_unique(grid, local_rank)) {} +Backend::~Backend() = default; +int Backend::device_ordinal() const { + return p->ordinal; +} +void Backend::synchronize() { +#ifdef VLASOV_CUDA + check_cuda(cudaSetDevice(p->ordinal)); + check_cuda(cudaDeviceSynchronize()); +#endif +} +void Backend::collide(double dt, double nu) { + const Grid g = p->grid; + const int nx = g.local_x(), nv = g.nv; + const double dv = g.dv(), factor = dt * nu; + const double* f = p->fx.ptr; + const double* v = p->coordinates.ptr; + double* upper = p->second.ptr; + double* rhs = p->rhs.ptr; + double* out = p->output.ptr; + parallel_for(nx, [=] DEVICE (std::size_t x) { + double sum = 0, momentum = 0; + for (int j = 0; j < nv; ++j) { + double value = f[std::size_t(j) * nx + x]; + sum += value; momentum += value * v[j]; + } + double mean = momentum / sum, variance = 0; + for (int j = 0; j < nv; ++j) { + double delta = v[j] - mean; + variance += f[std::size_t(j) * nx + x] * delta * delta * dv; + } + double temperature = variance / (sum * dv); + // Production self_consistent_beta is disabled: beta=1/(2*T). + double beta = 1.0 / (2.0 * temperature), D = 1.0 / (2.0 * beta); + double safe_D = D > 1e-30 ? D : 1e-30; + auto edge = [=] DEVICE (int j, double& alpha, double& beta_edge) { + if (j < 0 || j >= nv - 1) { alpha = beta_edge = 0; return; } + double drift = 2.0 * beta * D * (0.5 * (v[j] + v[j + 1]) - mean); + double w = drift * dv / safe_D; + double delta = fabs(w) < 1e-8 ? 0.5 - w / 12.0 + w * w * w / 720.0 + : 1.0 / w - 1.0 / expm1(w); + alpha = -drift * delta + safe_D / dv; + beta_edge = -drift * (1 - delta) - safe_D / dv; + }; + double previous_upper = 0, previous_rhs = 0; + for (int j = 0; j < nv; ++j) { + double al, bl, ar, br; + edge(j - 1, al, bl); edge(j, ar, br); + double diag = 1.0 + factor * (ar - bl) / dv; + double lower = -factor * al / dv, higher = factor * br / dv; + std::size_t i = std::size_t(j) * nx + x; + double applied = diag * f[i]; + if (j > 0) applied += lower * f[i - nx]; + if (j < nv - 1) applied += higher * f[i + nx]; + // Match ADEPT's delta formulation rather than solving for f directly. + double residual = f[i] - applied; + double denominator = diag - lower * previous_upper; + previous_upper = higher / denominator; + previous_rhs = (residual - lower * previous_rhs) / denominator; + upper[i] = previous_upper; rhs[i] = previous_rhs; + } + double next = 0; + for (int j = nv - 1; j >= 0; --j) { + std::size_t i = std::size_t(j) * nx + x; + next = rhs[i] - upper[i] * next; + out[i] = f[i] + next; + } + }); + std::swap(p->fx.ptr, p->output.ptr); +} +void Backend::initialize() { + const Grid g = p->grid; + double norm = 0.0; + for (int j = 0; j < g.nv; ++j) { + double v = g.velocity(j); + norm += std::exp(-0.5 * v * v) * (!g.cell_centred && (j == 0 || j == g.nv - 1) ? 0.5 : 1.0) * g.dv(); + } + double* f = p->fv.ptr; + const int nx = g.nx, nv_local = g.local_v(), v0 = g.rank * nv_local; + const double* vgrid = p->coordinates.ptr; + const double amp = g.amplitude; + parallel_for(g.count(), [=] DEVICE (std::size_t i) { + int x = i % nx, vindex = i / nx + v0; + double v = vgrid[vindex]; + f[i] = exp(-0.5 * v * v) / norm * (1.0 + amp * cos(2.0 * pi * x / nx)); + }); +} +void Backend::stream(double dt) { + const Grid g = p->grid; + const int nk = g.nx / 2 + 1, nx = g.nx, v0 = g.rank * g.local_v(); + const double k0 = 2 * pi / g.length; + const double* vgrid = p->coordinates.ptr; + double* fk = p->spectrum.ptr; + double* f = p->fv.ptr; + p->streaming_fft.r2c(f, fk); + parallel_for(std::size_t(nk) * g.local_v(), [=] DEVICE (std::size_t i) { + int k = i % nk; + double v = vgrid[i / nk + v0]; + double angle = -k0 * k * v * dt; + double c = cos(angle), s = sin(angle), re = fk[2 * i], im = fk[2 * i + 1]; + fk[2 * i] = re * c - im * s; + fk[2 * i + 1] = (k == 0 || (nx % 2 == 0 && k == nx / 2)) ? 0.0 : re * s + im * c; + }); + p->streaming_fft.c2r(fk, f); + parallel_for(g.count(), [=] DEVICE (std::size_t i) { f[i] /= nx; }); +} +void Backend::density(double charge) { + const Grid g = p->grid; + const int nx = g.nx, nv = g.nv, nvl = g.local_v(), v0 = g.rank * nvl; + const double dv = g.dv(); + const bool cells = g.cell_centred; + const double* f = p->fv.ptr; + double* rho = p->density_local.ptr; + parallel_for(nx, [=] DEVICE (std::size_t x) { + double sum = 0.0; + for (int j = 0; j < nvl; ++j) + sum += f[std::size_t(j) * nx + x] * (!cells && (j + v0 == 0 || j + v0 == nv - 1) ? 0.5 : 1.0); + rho[x] = charge * (sum * dv); + }); +} +void Backend::add_density(const Backend& other) { + double* target = p->density_local.ptr; + const double* source = other.p->density_local.ptr; + parallel_for(p->grid.nx, [=] DEVICE (std::size_t x) { target[x] += source[x]; }); +} +void Backend::copy_field(const Backend& other) { +#ifdef VLASOV_CUDA + check_cuda(cudaMemcpy(p->electric.ptr, other.p->electric.ptr, p->grid.nx * sizeof(double), cudaMemcpyDeviceToDevice)); +#else + std::copy_n(other.p->electric.ptr, p->grid.nx, p->electric.ptr); +#endif +} +void Backend::field(bool charge_density) { + const Grid g = p->grid; + const int nx = g.nx, nk = nx / 2 + 1; + const double k0 = 2 * pi / g.length; + const double* density = p->density_global.ptr; + const double* bg = p->background.ptr; + double* e = p->electric.ptr; + double* ek = p->field_spectrum.ptr; + parallel_for(nx, [=] DEVICE (std::size_t i) { e[i] = bg[i] + (charge_density ? density[i] : -density[i]); }); + p->field_fft.r2c(e, ek); + parallel_for(nk, [=] DEVICE (std::size_t i) { + if (i == 0 || (nx % 2 == 0 && i == std::size_t(nx / 2))) { + ek[2 * i] = ek[2 * i + 1] = 0.0; + } else { + double re = ek[2 * i], im = ek[2 * i + 1], k = k0 * i; + ek[2 * i] = im / k; + ek[2 * i + 1] = -re / k; + } + }); + p->field_fft.c2r(ek, e); + parallel_for(nx, [=] DEVICE (std::size_t i) { e[i] /= nx; }); +} +void Backend::velocity(double dt, double acceleration, bool self_consistent) { + if (p->grid.local_cubic) { local_velocity(dt, acceleration, self_consistent); return; } + const Grid g = p->grid; + const int nxl = g.local_x(), nv = g.nv, x0 = g.rank * nxl; + const double dv = g.dv(); + const double* f = p->fx.ptr; + const double* inv = p->inverse.ptr; + double* m = p->second.ptr; + double* r = p->rhs.ptr; + // One thread per spatial line. Keeping x contiguous coalesces each sweep + // across a warp. This baseline has a serial recurrence along each v line. + parallel_for(nxl, [=] DEVICE (std::size_t x) { + m[x] = m[std::size_t(nv - 1) * nxl + x] = 0.0; + double previous = 0.0; + for (int j = 1; j < nv - 1; ++j) { + std::size_t i = std::size_t(j) * nxl + x; + double b = 6.0 * (f[i - nxl] - 2.0 * f[i] + f[i + nxl]) / (dv * dv); + previous = (b - previous) * inv[j]; + r[i] = previous; + } + double next = 0.0; + for (int j = nv - 2; j >= 1; --j) { + std::size_t i = std::size_t(j) * nxl + x; + next = r[i] - inv[j] * next; + m[i] = next; + } + }); + double* out = p->output.ptr; + const double* e = p->electric.ptr; + parallel_for(g.count(), [=] DEVICE (std::size_t i) { + int x = i % nxl, j = i / nxl; + double a = self_consistent ? (g.charge / g.mass) * e[x0 + x] : acceleration; + double q = j - a * dt / dv; + if (q < 0.0 || q > nv - 1) { out[i] = 0.0; return; } + int lo = int(floor(q)); + if (lo == nv - 1) lo = nv - 2; + double b = q - lo, a0 = 1.0 - b; + std::size_t l = std::size_t(lo) * nxl + x, h = l + nxl; + out[i] = a0 * f[l] + b * f[h] + + ((a0 * a0 * a0 - a0) * m[l] + (b * b * b - b) * m[h]) * dv * dv / 6.0; + }); + std::swap(p->fx.ptr, p->output.ptr); +} +void Backend::local_velocity(double dt, double acceleration, bool self_consistent) { + const Grid g = p->grid; + const bool halo = g.halos; + const int width = halo ? g.nx : g.local_x(); + const int nv = g.nv, nvl = g.local_v(), v0 = halo ? g.rank * nvl : 0; + const int x0 = halo ? 0 : g.rank * g.local_x(), cap = g.halo_capacity; + const double dv = g.dv(); + const double* f = halo ? p->fv.ptr : p->fx.ptr; + const double* ghosts = p->ghosts.ptr; + const double* v = p->coordinates.ptr; + const double* e = p->electric.ptr; + double* out = p->output.ptr; + parallel_for(g.count(), [=] DEVICE (std::size_t i) { + int x = i % width, j = i / width + v0; + double a = self_consistent ? (g.charge / g.mass) * e[x0 + x] : acceleration; + double query = v[j] - a * dt; + int guess = int(floor(j - a * dt / dv)); + auto sample = [=] DEVICE (int index) { + int local = index - v0; + if (!halo || (local >= 0 && local < nvl)) return f[std::size_t(local) * width + x]; + int row = local < 0 ? cap + local : cap + local - nvl; + return ghosts[std::size_t(row) * width + x]; + }; + out[i] = local_cubic_value(query, guess, v, nv, sample, 1e-30); + }); + if (halo) std::swap(p->fv.ptr, p->output.ptr); + else std::swap(p->fx.ptr, p->output.ptr); +} +double Backend::max_field() { +#ifdef VLASOV_CUDA + check_cuda(cudaMemset(p->field_maximum.ptr, 0, sizeof(double))); + maximum_absolute<<grid.nx + 255) / 256, 65535), 256>>>( + p->electric.ptr, p->field_maximum.ptr, p->grid.nx); + check_cuda(cudaGetLastError()); + return p->field_maximum.download()[0]; +#else + double maximum = 0; + for (int x = 0; x < p->grid.nx; ++x) maximum = std::max(maximum, std::abs(p->electric.ptr[x])); + return maximum; +#endif +} +double* Backend::velocity_buffer() { return p->fv.ptr; } +double* Backend::electric_buffer() { return p->electric.ptr; } +void Backend::set_field(const std::vector& values) { p->electric.upload(values); } +double* Backend::halo_buffer() { return p->ghosts.ptr; } +void Backend::upload_halos(const std::vector& values) { p->ghosts.upload(values); } +void Backend::upload_state(const std::vector& values) { p->fv.upload(values); } +void Backend::set_coordinates(const std::vector& values) { + if (values.size() != std::size_t(p->grid.nv)) throw std::runtime_error("incorrect velocity coordinate count"); + for (std::size_t i = 1; i < values.size(); ++i) { + double spacing = values[i] - values[i - 1]; + if (!std::isfinite(spacing) || spacing <= 0 || std::abs(spacing / p->grid.dv() - 1) > 1e-10) + throw std::runtime_error("velocity halos require uniform coordinates matching --nv/--vmax/--cells"); + } + p->coordinates.upload(values); +} +void Backend::set_background(const std::vector& values) { p->background.upload(values); } +void Backend::pack(bool to_x) { + const Grid g = p->grid; + const int nxl = g.local_x(), nvl = g.local_v(), nx = g.nx; + const std::size_t tile = std::size_t(nxl) * nvl; + const double* input = to_x ? p->fv.ptr : p->fx.ptr; + double* output = p->send.ptr; + parallel_for(g.count(), [=] DEVICE (std::size_t i) { + int dst = i / tile; + std::size_t in_tile = i % tile; + int v = in_tile / nxl, x = in_tile % nxl; + output[i] = to_x ? input[std::size_t(v) * nx + dst * nxl + x] + : input[std::size_t(dst * nvl + v) * nxl + x]; + }); +} +void Backend::unpack(bool to_x) { + const Grid g = p->grid; + const int nxl = g.local_x(), nvl = g.local_v(), nx = g.nx; + const std::size_t tile = std::size_t(nxl) * nvl; + const double* input = p->recv.ptr; + double* output = to_x ? p->fx.ptr : p->fv.ptr; + parallel_for(g.count(), [=] DEVICE (std::size_t i) { + int src = i / tile; + std::size_t in_tile = i % tile; + int v = in_tile / nxl, x = in_tile % nxl; + if (to_x) output[std::size_t(src * nvl + v) * nxl + x] = input[i]; + else output[std::size_t(v) * nx + src * nxl + x] = input[i]; + }); +} +double* Backend::send_buffer() { return p->send.ptr; } +void Backend::local_layout() { std::swap(p->fv.ptr, p->fx.ptr); } +double* Backend::recv_buffer() { return p->recv.ptr; } +double* Backend::density_buffer() { return p->density_local.ptr; } +double* Backend::reduced_density_buffer() { return p->density_global.ptr; } +std::vector Backend::download() { return p->fv.download(); } +std::vector Backend::download_field() { return p->electric.download(); } +std::vector Backend::download_send() { return p->send.download(); } +std::vector Backend::download_density() { return p->density_local.download(); } +void Backend::upload_received(const std::vector& values) { p->recv.upload(values); } +void Backend::upload_density(const std::vector& values) { p->density_global.upload(values); } +std::string Backend::device_name() const { return p->device; } +std::size_t Backend::allocated_bytes() const { + return (p->fv.size + p->fx.size + p->send.size + p->recv.size + p->second.size + p->rhs.size + + p->output.size + p->inverse.size + p->spectrum.size + + 3 * p->grid.nx + p->field_spectrum.size) * sizeof(double) + + p->streaming_fft.workspace_bytes + p->field_fft.workspace_bytes + + (p->coordinates.size + p->background.size + p->ghosts.size + p->field_maximum.size) * sizeof(double); +} diff --git a/adept/_vlasov1d/native/core/backend.hpp b/adept/_vlasov1d/native/core/backend.hpp new file mode 100644 index 00000000..4ae2e694 --- /dev/null +++ b/adept/_vlasov1d/native/core/backend.hpp @@ -0,0 +1,70 @@ +#pragma once + +#include +#include +#include +#include + +// Cray MPI initializes its GPU transport during MPI_Init, so bind first. +void prepare_device_from_environment(); + +struct Grid { + int nx = 4096, nv = 1024, ranks = 1, rank = 0; + double length = 12.566370614359172, vmax = 8.0, amplitude = 0.01; + double charge = -1.0, mass = 1.0; + bool cell_centred = false, local_cubic = false, halos = false, collisions = false; + int halo_capacity = 64; + int local_x() const { return nx / ranks; } + int local_v() const { return nv / ranks; } + std::size_t count() const { return std::size_t(nx) * local_v(); } + double dv() const { return 2.0 * vmax / (cell_centred ? nv : nv - 1); } + double velocity(int j) const { return -vmax + (j + (cell_centred ? 0.5 : 0.0)) * dv(); } +}; + +// Buffers are GPU-resident in the CUDA build; MPI must support device pointers. +// Both layouts keep the spatial index contiguous, including the spline batches. +class Backend { +public: + explicit Backend(Grid grid, int local_rank); + ~Backend(); + Backend(const Backend&) = delete; + Backend& operator=(const Backend&) = delete; + void initialize(); + void synchronize(); + void stream(double dt); + void density(double charge = 1.0); + void add_density(const Backend& other); + void field(bool charge_density = false); + void copy_field(const Backend& other); + void velocity(double dt, double acceleration, bool self_consistent); + void local_velocity(double dt, double acceleration, bool self_consistent); + void collide(double dt, double nu); + double max_field(); + double* velocity_buffer(); + double* electric_buffer(); + void set_field(const std::vector& values); + double* halo_buffer(); + void upload_halos(const std::vector& values); + void upload_state(const std::vector& values); + void set_coordinates(const std::vector& values); + void set_background(const std::vector& values); + void pack(bool to_x); + void unpack(bool to_x); + void local_layout(); + double* send_buffer(); + double* recv_buffer(); + double* density_buffer(); + double* reduced_density_buffer(); + std::vector download(); + std::vector download_field(); + void upload_received(const std::vector& values); + std::vector download_send(); + std::vector download_density(); + void upload_density(const std::vector& values); + std::string device_name() const; + int device_ordinal() const; + std::size_t allocated_bytes() const; +private: + struct Impl; + std::unique_ptr p; +}; diff --git a/adept/_vlasov1d/native/core/local_cubic.hpp b/adept/_vlasov1d/native/core/local_cubic.hpp new file mode 100644 index 00000000..b78570a5 --- /dev/null +++ b/adept/_vlasov1d/native/core/local_cubic.hpp @@ -0,0 +1,30 @@ +#pragma once +#include + +#ifdef __CUDACC__ +#define CUBIC_HD __host__ __device__ +#else +#define CUBIC_HD +#endif + +// interpax method="cubic": C1 Hermite interpolation, arithmetic mean of +// neighbouring secant slopes, one-sided slopes at physical endpoints. +template +CUBIC_HD double local_cubic_value(double query, int guess, const double* v, + int nv, Sample sample, double extrap) { + if (query < v[0] || query > v[nv - 1]) return extrap; + int lo = guess < 0 ? 0 : (guess >= nv - 1 ? nv - 2 : guess); + while (lo > 0 && query < v[lo]) --lo; + while (lo < nv - 2 && query >= v[lo + 1]) ++lo; + double f0 = sample(lo), f1 = sample(lo + 1), h = v[lo + 1] - v[lo]; + double secant = (f1 - f0) / h; + double d0 = lo == 0 ? secant : + 0.5 * ((f0 - sample(lo - 1)) / (v[lo] - v[lo - 1]) + secant); + double d1 = lo + 1 == nv - 1 ? secant : + 0.5 * (secant + (sample(lo + 2) - f1) / (v[lo + 2] - v[lo + 1])); + double t = (query - v[lo]) / h; + double s0 = d0 * h, s1 = d1 * h; + double c3 = 2.0 * f0 - 2.0 * f1 + s0 + s1; + double c2 = -3.0 * f0 + 3.0 * f1 - 2.0 * s0 - s1; + return ((c3 * t + c2) * t + s0) * t + f0; +} diff --git a/adept/_vlasov1d/native/core/solver.hpp b/adept/_vlasov1d/native/core/solver.hpp new file mode 100644 index 00000000..d4c51059 --- /dev/null +++ b/adept/_vlasov1d/native/core/solver.hpp @@ -0,0 +1,332 @@ +#pragma once +#include "backend.hpp" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace adept_native { +void mpi_check(int error) { + if (error == MPI_SUCCESS) return; + char message[MPI_MAX_ERROR_STRING]; + int length = 0; + MPI_Error_string(error, message, &length); + throw std::runtime_error(std::string(message, length)); +} +struct Options { + Grid grid; + int steps = 20, warmup = 3; + double dt = 0.1, acceleration = 0.2, nu = 0; + bool host_mpi = false; + std::string mode = "landau", dump, integrator = "strang", input, coordinates, background; +}; +int integer(const std::string& text) { + std::size_t end; + int result = std::stoi(text, &end); + if (end != text.size()) throw std::runtime_error("invalid integer: " + text); + return result; +} +double real(const std::string& text) { + std::size_t end; + double result = std::stod(text, &end); + if (end != text.size() || !std::isfinite(result)) throw std::runtime_error("invalid real: " + text); + return result; +} +Options parse(int argc, char** argv, int rank) { + Options o; + for (int i = 1; i < argc; ++i) { + std::string key = argv[i]; + if (key == "--host-mpi") { o.host_mpi = true; continue; } + if (key == "--cells") { o.grid.cell_centred = true; continue; } + if (key == "--help") { + if (rank == 0) std::cout + << "vlasov_bench [--nx N] [--nv N] [--steps N] [--warmup N] [--dt D]\n" + << " [--mode landau|stream|accelerate|transpose] [--acceleration A]\n" + << " [--length L] [--vmax V] [--amplitude A] [--host-mpi] [--dump FILE]\n"; + if (rank == 0) std::cout << " [--velocity natural|cubic] [--layout transpose|halos] [--halo-capacity N]\n" + << " [--integrator strang|sixth] [--cells] [--input FILE] [--v-grid FILE] [--background FILE]\n"; + if (rank == 0) std::cout << " [--nu D] (Chang-Cooper Dougherty; requires --cells)\n"; + // All ranks take the same path; finalize normally in main. + o.mode = "help"; + return o; + } + if (i + 1 == argc) throw std::runtime_error("missing value for " + key); + std::string value = argv[++i]; + if (key == "--nx") o.grid.nx = integer(value); + else if (key == "--nv") o.grid.nv = integer(value); + else if (key == "--steps") o.steps = integer(value); + else if (key == "--warmup") o.warmup = integer(value); + else if (key == "--dt") o.dt = real(value); + else if (key == "--acceleration") o.acceleration = real(value); + else if (key == "--nu") o.nu = real(value); + else if (key == "--charge") o.grid.charge = real(value); + else if (key == "--mass") o.grid.mass = real(value); + else if (key == "--length") o.grid.length = real(value); + else if (key == "--vmax") o.grid.vmax = real(value); + else if (key == "--amplitude") o.grid.amplitude = real(value); + else if (key == "--mode") o.mode = value; + else if (key == "--integrator") o.integrator = value; + else if (key == "--input") o.input = value; + else if (key == "--v-grid") o.coordinates = value; + else if (key == "--background") o.background = value; + else if (key == "--halo-capacity") o.grid.halo_capacity = integer(value); + else if (key == "--velocity") { + if (value != "natural" && value != "cubic") throw std::runtime_error("unknown velocity method"); + o.grid.local_cubic = value == "cubic"; + } else if (key == "--layout") { + if (value != "transpose" && value != "halos") throw std::runtime_error("unknown layout"); + o.grid.halos = value == "halos"; + } + else if (key == "--dump") o.dump = value; + else throw std::runtime_error("unknown option: " + key); + } + return o; +} +// Times include explicit device completion, with no computation/communication +// overlap. Reduce each rank's accumulated component times with MPI_MAX later. +enum Stage { Stream, Spline, Pack, Exchange, Field, Collision, Total, StageCount }; +struct SixthCoefficients { + double a[5] = {0.168735950563437422448196, 0.377851589220928303880766, + -0.093175079568731452657924, 0.377851589220928303880766, 0.168735950563437422448196}; + double d[6]; + explicit SixthCoefficients(double dt) { + double dt2 = dt * dt, dt4 = dt2 * dt2, dt6 = dt4 * dt2; + d[0] = d[5] = 0.049086460976116245491441 - 2 * 0.000069728715055305084099 * dt2; + d[1] = d[4] = 0.264177609888976700200146 - 2 * 0.000625704827430047189169 * dt2 + - 4 * 2.916600457689847816445691e-6 * dt4; + d[2] = d[3] = 0.186735929134907054308413 - 2 * 0.002213085124045325561636 * dt2 + + 4 * 3.048480261700038788680723e-5 * dt4 - 8 * 4.985549387875068121593988e-7 * dt6; + } +}; +class Stepper { +public: + Stepper(Backend& backend, const Options& opts) : b(backend), o(opts) {} + void collision_step() { + if (o.nu == 0) return; + // One transpose pair remains for the implicit collision solve; the + // six Vlasov kicks use halos. Count its communication in MPI/pack times. + redistribute(true); + timed(Collision, [&] { b.collide(o.dt, o.nu); }); + redistribute(false); + } + void velocity_step(double dt, bool self_consistent) { + if (!o.grid.halos) { + redistribute(true); + timed(Spline, [&] { b.velocity(dt, o.acceleration, self_consistent); }); + redistribute(false); + return; + } + timed(Exchange, [&] { + double maximum = self_consistent ? std::abs(o.grid.charge / o.grid.mass) * b.max_field() : std::abs(o.acceleration); + double distance = maximum * std::abs(dt) / o.grid.dv(); + if (!std::isfinite(distance) || distance > o.grid.halo_capacity - 2) + throw std::runtime_error("departure point exceeds halo capacity; increase --halo-capacity"); + int width = int(std::ceil(distance)) + 2, global_width; + mpi_check(MPI_Allreduce(&width, &global_width, 1, MPI_INT, MPI_MAX, MPI_COMM_WORLD)); + width = global_width; + max_halo_width = std::max(max_halo_width, width); + exchange_halos(width); + }); + timed(Spline, [&] { b.local_velocity(dt, o.acceleration, self_consistent); }); + } + void exchange_halos(int width) { + const Grid& g = o.grid; + if (g.ranks == 1) return; + const int start = g.rank * g.local_v(), stop = start + g.local_v(); + std::vector requests; + std::vector host_send, host_recv; + double* send = b.velocity_buffer(); + double* recv = b.halo_buffer(); + if (o.host_mpi) { + host_send = b.download(); + host_recv.resize(2 * std::size_t(g.halo_capacity) * g.nx); + send = host_send.data(); recv = host_recv.data(); + } + b.synchronize(); + auto post = [&](bool receiving, int peer, int count, double* ptr) { + requests.push_back(MPI_REQUEST_NULL); + if (receiving) mpi_check(MPI_Irecv(ptr, count, MPI_DOUBLE, peer, 713, MPI_COMM_WORLD, &requests.back())); + else mpi_check(MPI_Isend(ptr, count, MPI_DOUBLE, peer, 713, MPI_COMM_WORLD, &requests.back())); + }; + for (int peer = 0; peer < g.ranks; ++peer) { + if (peer == g.rank) continue; + int ps = peer * g.local_v(), pe = ps + g.local_v(); + int lo = std::max(ps, peer < g.rank ? start - width : stop); + int hi = std::min(pe, peer < g.rank ? start : stop + width); + if (hi > lo) { + std::size_t offset = std::size_t(g.halo_capacity + lo - (peer < g.rank ? start : stop)) * g.nx; + post(true, peer, (hi - lo) * g.nx, recv + offset); + } + lo = std::max(start, g.rank < peer ? ps - width : pe); + hi = std::min(stop, g.rank < peer ? ps : pe + width); + if (hi > lo) { + int count = (hi - lo) * g.nx; + post(false, peer, count, send + std::size_t(lo - start) * g.nx); + halo_sent_bytes += std::uint64_t(count) * sizeof(double); + } + } + mpi_check(MPI_Waitall(int(requests.size()), requests.data(), MPI_STATUSES_IGNORE)); + if (o.host_mpi) b.upload_halos(host_recv); + } + template void timed(Stage stage, F fn) { + b.synchronize(); + double start = MPI_Wtime(); + fn(); + b.synchronize(); + times[stage] += MPI_Wtime() - start; + } + void redistribute(bool to_x) { + if (o.grid.ranks == 1) { b.local_layout(); return; } + timed(Pack, [&] { b.pack(to_x); }); + int count = o.grid.count() / o.grid.ranks; + timed(Exchange, [&] { + slab_sent_bytes += std::uint64_t(count) * (o.grid.ranks - 1) * sizeof(double); + if (o.host_mpi) { + auto send = b.download_send(); + std::vector recv(send.size()); + mpi_check(MPI_Alltoall(send.data(), count, MPI_DOUBLE, recv.data(), count, + MPI_DOUBLE, MPI_COMM_WORLD)); + b.upload_received(recv); + } else { + mpi_check(MPI_Alltoall(b.send_buffer(), count, MPI_DOUBLE, b.recv_buffer(), + count, MPI_DOUBLE, MPI_COMM_WORLD)); + } + }); + timed(Pack, [&] { b.unpack(to_x); }); + } + void solve_field() { + b.density(o.grid.charge); + b.synchronize(); + if (o.host_mpi) { + auto density = b.download_density(); + std::vector global(density.size()); + mpi_check(MPI_Allreduce(density.data(), global.data(), o.grid.nx, + MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD)); + b.upload_density(global); + } else { + mpi_check(MPI_Allreduce(b.density_buffer(), b.reduced_density_buffer(), + o.grid.nx, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD)); + } + b.field(true); + } + void step() { + b.synchronize(); + double start = MPI_Wtime(); + if (o.mode == "stream") timed(Stream, [&] { b.stream(o.dt); }); + else if (o.mode == "transpose") { redistribute(true); redistribute(false); } + else if (o.mode == "collide") collision_step(); + else { + if (o.mode == "landau" && o.integrator == "sixth") { + // Same force-gradient Hamiltonian coefficients and ordering as + // ADEPT SixthOrderHamIntegrator, including its dt corrections. + const SixthCoefficients coefficients(o.dt); + for (int k = 0; k < 6; ++k) { + timed(Field, [&] { solve_field(); }); + velocity_step(coefficients.d[k] * o.dt, true); + if (k < 5) timed(Stream, [&] { b.stream(coefficients.a[k] * o.dt); }); + } + collision_step(); + b.synchronize(); + times[Total] += MPI_Wtime() - start; + return; + } + if (o.mode == "landau") { + timed(Stream, [&] { b.stream(0.5 * o.dt); }); + timed(Field, [&] { solve_field(); }); + } + velocity_step(o.dt, o.mode == "landau"); + if (o.mode == "landau") timed(Stream, [&] { b.stream(0.5 * o.dt); }); + collision_step(); + } + b.synchronize(); + times[Total] += MPI_Wtime() - start; + } + std::array times{}; + std::uint64_t halo_sent_bytes = 0; + std::uint64_t slab_sent_bytes = 0; + int max_halo_width = 0; +private: + Backend& b; + const Options& o; +}; +struct Diagnostics { double mass, kinetic, field, minimum; bool finite; }; +Diagnostics diagnostics(Backend& b, Stepper& stepper, const Grid& g) { + auto f = b.download(); + double mass = 0, kinetic = 0, minimum = std::numeric_limits::infinity(); + int finite = 1; + for (std::size_t i = 0; i < f.size(); ++i) { + int j = i / g.nx + g.rank * g.local_v(); + double v = g.velocity(j); + double weight = (!g.cell_centred && (j == 0 || j == g.nv - 1) ? 0.5 : 1.0) * g.dv() * g.length / g.nx; + mass += f[i] * weight; + kinetic += 0.5 * v * v * f[i] * weight; + minimum = std::min(minimum, f[i]); + finite &= std::isfinite(f[i]); + } + double local[] = {mass, kinetic}, global[2]; + mpi_check(MPI_Allreduce(local, global, 2, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD)); + double global_min; + int global_finite; + mpi_check(MPI_Allreduce(&minimum, &global_min, 1, MPI_DOUBLE, MPI_MIN, MPI_COMM_WORLD)); + mpi_check(MPI_Allreduce(&finite, &global_finite, 1, MPI_INT, MPI_MIN, MPI_COMM_WORLD)); + stepper.solve_field(); + auto field = b.download_field(); + double energy = 0; + for (double e : field) energy += 0.5 * e * e * g.length / g.nx; + return {global[0], global[1], energy, global_min, bool(global_finite)}; +} +void dump(Backend& b, const Grid& g, const std::string& path) { + auto local = b.download(); + std::vector full(g.rank == 0 ? std::size_t(g.nx) * g.nv : 0); + mpi_check(MPI_Gather(local.data(), int(local.size()), MPI_DOUBLE, full.data(), + int(local.size()), MPI_DOUBLE, 0, MPI_COMM_WORLD)); + if (g.rank == 0) { + std::ofstream out(path, std::ios::binary); + out.write(reinterpret_cast(full.data()), full.size() * sizeof(double)); + if (!out) throw std::runtime_error("cannot write dump: " + path); + } +} +std::string quoted(const std::string& text) { + std::string out = "\""; + for (char c : text) { + if (c == '"' || c == '\\') out += '\\'; + if (c == '\n') out += "\\n"; + else out += c; + } + return out + '"'; +} +std::vector read_binary(const std::string& path, std::size_t count) { + std::vector values(count); + std::ifstream in(path, std::ios::binary | std::ios::ate); + if (!in || in.tellg() != std::streamoff(count * sizeof(double))) + throw std::runtime_error("wrong binary size: " + path); + in.seekg(0); + in.read(reinterpret_cast(values.data()), count * sizeof(double)); + if (!in) throw std::runtime_error("cannot read " + path); + return values; +} +void initialize(Backend& backend, const Options& o) { + if (o.input.empty()) { backend.initialize(); return; } + MPI_File file; + mpi_check(MPI_File_open(MPI_COMM_WORLD, o.input.c_str(), MPI_MODE_RDONLY, MPI_INFO_NULL, &file)); + MPI_Offset bytes; + mpi_check(MPI_File_get_size(file, &bytes)); + if (bytes != MPI_Offset(o.grid.nx) * o.grid.nv * sizeof(double)) + throw std::runtime_error("input state must be raw float64 with shape (nv,nx)"); + std::vector values(o.grid.count()); + MPI_Offset offset = MPI_Offset(o.grid.rank) * values.size() * sizeof(double); + mpi_check(MPI_File_read_at_all(file, offset, values.data(), int(values.size()), MPI_DOUBLE, MPI_STATUS_IGNORE)); + mpi_check(MPI_File_close(&file)); + backend.upload_state(values); +} +} // namespace adept_native diff --git a/adept/_vlasov1d/native/module.py b/adept/_vlasov1d/native/module.py new file mode 100644 index 00000000..d75ebf7f --- /dev/null +++ b/adept/_vlasov1d/native/module.py @@ -0,0 +1,407 @@ +"""Native execution of existing ADEPT modules with their original save functions.""" + +import gc +import hashlib +import json +import os +import shutil +import signal +import subprocess +import sys +import tempfile +from contextlib import contextmanager +from functools import cache +from pathlib import Path + +import jax +import numpy as np +from diffrax import RESULTS, Solution, SubSaveAt + +from adept._vlasov1d.datamodel import NativeSolverConfig +from adept._vlasov1d.modules import BaseVlasov1D + + +@contextmanager +def _host_preprocessing(): + with jax.default_device(jax.devices("cpu")[0]): + yield + + +def _restore_array(host): + """Recreate callback constants as replicated JAX arrays on the worker mesh.""" + from jax.sharding import Mesh, NamedSharding, PartitionSpec + + sharding = NamedSharding(Mesh(np.array(jax.devices()), ("device",)), PartitionSpec()) + return jax.make_array_from_callback(host.shape, sharding, lambda index: host[index]) + + +def _host_pickle(payload, stream): + import cloudpickle + + class HostPickler(cloudpickle.CloudPickler): + def reducer_override(self, value): + if isinstance(value, jax.Array): + return _restore_array, (np.asarray(value),) + return super().reducer_override(value) + + HostPickler(stream).dump(payload) + + +def _write_vx(path, values): + """Export a host x-v state with bounded transpose temporaries.""" + output = np.memmap(path, mode="w+", dtype=np.float64, shape=values.shape[::-1]) + for start in range(0, values.shape[1], 64): + output[start : start + 64] = np.asarray(values[:, start : start + 64]).T + output.flush() + del output + + +def _collect(root, times, ranks): + import cloudpickle + + with (root / "layout.pkl").open("rb") as stream: + layout = cloudpickle.load(stream) + indices = [json.loads((root / f"rank-{rank:05d}/index.json").read_text()) for rank in range(ranks)] + result = {} + directory = root / "collected" + directory.mkdir() + for key_id, (key, definition) in enumerate(layout.items()): + arrays = [] + for leaf_id, leaf in enumerate(definition["leaves"]): + shape = (len(times[key]), *leaf["shape"]) + parts = [index[key][leaf_id] for index in indices if index[key][leaf_id] is not None] + if len(parts) == 1 and all( + start == 0 and stop == size and step == 1 + for (start, stop, step), size in zip(parts[0]["index"], leaf["shape"], strict=True) + ): + array = np.load(root / parts[0]["file"], mmap_mode="r") + else: + array = np.lib.format.open_memmap( + directory / f"{key_id:04d}-{leaf_id:03d}.npy", mode="w+", dtype=np.dtype(leaf["dtype"]), shape=shape + ) + for part in parts: + local = np.load(root / part["file"], mmap_mode="r") + slices = tuple(slice(*axis) for axis in part["index"]) + # Bound host copies for large saved distributions. + for frame in range(len(times[key])): + array[(frame, *slices)] = local[frame] + array.flush() + if array.shape != shape: + raise RuntimeError(f"native save shape mismatch for {key}") + arrays.append(array) + result[key] = jax.tree.unflatten(definition["tree"], arrays) + return result + + +class NativeVlasovMixin: + """Swap execution while retaining a Vlasov module's lifecycle and callbacks.""" + + jit_compile = False + + def __init__(self, cfg): + if not jax.config.jax_enable_x64: + raise RuntimeError("native Vlasov requires jax_enable_x64 before preprocessing") + with _host_preprocessing(): + super().__init__(cfg) + self.native_options = NativeSolverConfig.model_validate(cfg.get("native") or {}) + self._validate_native_physics() + + def _validate_native_physics(self): + cfg, sim = self.cfg, self.simulation + names = [species.name for species in sim.species] + if not names or any(not name for name in names) or len(set(names)) != len(names): + raise ValueError("native species names must be nonempty and unique") + for species in sim.species: + if not np.isfinite(species.mass) or species.mass <= 0 or not np.isfinite(species.charge): + raise ValueError("native species require positive finite mass and finite charge") + if not np.isclose(species.vmin, -species.vmax, rtol=0, atol=1e-12): + raise NotImplementedError("native Vlasov requires symmetric velocity domains") + terms = cfg["terms"] + if terms["time"] != "sixth" or terms["edfdv"] != "cubic-spline" or terms["field"] != "poisson": + raise NotImplementedError( + "native Vlasov requires sixth-order time, cubic-spline velocity, and Poisson fields" + ) + if terms.get("vdfdx", "exponential") != "exponential": + raise NotImplementedError("native Vlasov requires exponential spatial streaming") + if ( + any(cfg["drivers"][key] for key in ["ex", "ey"]) + or cfg["drivers"].get("ex_stochastic") is not None + or terms["krook"]["is_on"] + or terms.get("hou_li_filter", {}).get("is_on", False) + ): + raise NotImplementedError("native Vlasov does not support external drivers, Krook relaxation, or filtering") + if any(cfg.get("diagnostics", {}).values()): + raise NotImplementedError("native Vlasov does not yet evolve df/dt diagnostic state") + fp = terms["fokker_planck"] + self.native_collision_frequency = 0.0 + if fp["is_on"]: + if fp["type"].casefold() not in {"chang_cooper_dougherty", "dougherty_chang_cooper"} or ( + fp.get("self_consistent_beta") or {} + ).get("enabled", False): + raise NotImplementedError("native collisions require Chang-Cooper Dougherty with analytic beta") + for axis in ["time", "space"]: + envelope = getattr(sim.nu_fp_prof, f"{axis}_envelope") + resolved_config = getattr(self.config_model.terms.fokker_planck, axis) + if envelope.bump_height != 0 or float(resolved_config.slope) != 0: + raise NotImplementedError("native collisions currently require constant frequency profiles") + self.native_collision_frequency = float( + sim.nu_fp_prof.time_envelope.baseline * sim.nu_fp_prof.space_envelope.baseline + ) + + def write_units(self): + with _host_preprocessing(): + return super().write_units() + + def get_derived_quantities(self): + with _host_preprocessing(): + return super().get_derived_quantities() + + def get_solver_quantities(self): + with _host_preprocessing(): + return super().get_solver_quantities() + + def init_state_and_args(self): + with _host_preprocessing(): + return super().init_state_and_args() + + def init_diffeqsolve(self): + with _host_preprocessing(): + super().init_diffeqsolve() + # Standard outputs use native steps, avoiding full-state copies for + # Diffrax's off-step interpolation. Keep sparse subclass callback times + # unless the caller opts into step rounding for those too; some existing + # project postprocessors reconstruct those axes from their own config. + t0, t1 = (float(self.time_quantities[key]) for key in ["t0", "t1"]) + dt = self.simulation.grid.dt + steps = round((t1 - t0) / dt) + subs = self.diffeqsolve_quants["saveat"]["subs"] + for key, save in list(subs.items()): + standard = key in self.cfg["save"] + if key == "default": + indices = np.arange(0, steps + 1, self.native_options.scalar_stride, dtype=int) + indices = np.union1d(indices, [steps]) + elif standard or self.native_options.custom_save_times == "steps": + requested = np.asarray(save.ts) + if ( + requested.size == 0 + or np.any(np.diff(requested) < 0) + or requested.min() < t0 - 1e-9 + or requested.max() > t1 + 1e-9 + ): + raise ValueError("native save times must be ordered and inside the solve interval") + indices = np.unique(np.clip(np.rint((requested - t0) / dt).astype(int), 0, steps)) + else: + continue + times = t0 + indices * dt + subs[key] = SubSaveAt(ts=times, fn=save.fn) + if standard: + self.cfg["save"][key]["t"].update(ax=times, nt=len(times), tmin=float(times[0]), tmax=float(times[-1])) + + def __call__(self, trainable_modules=None, args=None): + if jax.process_count() != 1: + raise RuntimeError("run one outer Python driver; the native backend launches its MPI workers") + if trainable_modules: + raise NotImplementedError("native Vlasov does not execute trainable modules") + if args is not None and args is not self.args: + raise NotImplementedError("native Vlasov currently uses the configured static arguments") + options = self.native_options + library = options.library or os.environ.get("ADEPT_VLASOV_LIBRARY") + if not library or not Path(library).expanduser().is_file(): + raise FileNotFoundError( + "Build adept/_vlasov1d/native with CMake and set native.library or ADEPT_VLASOV_LIBRARY" + ) + library = str(Path(library).expanduser().resolve()) + grid = self.simulation.grid + t0, t1 = (float(self.time_quantities[key]) for key in ["t0", "t1"]) + steps = round((t1 - t0) / grid.dt) + if steps < 1 or not np.isclose(steps * grid.dt, t1 - t0, rtol=0, atol=1e-9): + raise NotImplementedError("native solve endpoints must be separated by an integer number of timesteps") + species_list = sorted(self.simulation.species, key=lambda species: species.name != "electron") + if grid.nx % options.ranks or any(species.nv % options.ranks for species in species_list): + raise ValueError("native ranks must divide nx and every species nv") + species_names = {species.name for species in species_list} + known = species_names | {"e", "de", "a", "da", "prev_a"} + if set(self.state) - known or any(np.any(np.asarray(self.state[key])) for key in ["de", "a", "da", "prev_a"]): + raise NotImplementedError( + "native Vlasov requires electrostatic state without additional evolved diagnostics" + ) + subs = self.diffeqsolve_quants["saveat"]["subs"] + times = {key: np.asarray(save.ts, dtype=float).tolist() for key, save in subs.items()} + if any( + not values or min(values) < t0 - 1e-9 or max(values) > t1 + 1e-9 or np.any(np.diff(values) < 0) + for values in times.values() + ): + raise ValueError("native save times must be ordered and inside the solve interval") + base = options.workdir or os.environ.get("BASE_TEMPDIR") + if options.ranks > 1 and not base: + raise ValueError("set native.workdir to a shared directory for MPI workers") + if base: + base = os.path.expandvars(os.path.expanduser(base)) + if "$" in base: + raise ValueError("native.workdir contains an undefined environment variable") + Path(base).mkdir(parents=True, exist_ok=True) + workspace = tempfile.TemporaryDirectory(prefix="adept-native-", dir=base) + root = Path(workspace.name) + # Retain backing files for the NumPy memmaps returned in solver result. + self._native_workspaces = getattr(self, "_native_workspaces", []) + [workspace] + for name, values in [ + ("background.bin", self.cfg["grid"]["ion_charge"]), + ("field.bin", self.state["e"]), + ]: + np.asarray(values, dtype=np.float64).tofile(root / name) + # Initializer cache is not used by save callbacks. Avoid serializing a + # second full distribution through a closure over the grid config. + cache = self.cfg["grid"].pop("species_distributions", None) + try: + with (root / "callbacks.pkl").open("wb") as stream: + _host_pickle( + dict( + callbacks={key: save.fn for key, save in subs.items()}, + args=self.args, + state={key: value for key, value in self.state.items() if key not in species_names | {"e"}}, + ), + stream, + ) + finally: + if cache is not None: + self.cfg["grid"]["species_distributions"] = cache + nu = self.native_collision_frequency + common_args = [ + "--nx", + str(grid.nx), + "--length", + str(grid.xmax - grid.xmin), + "--dt", + str(grid.dt), + "--cells", + "--integrator", + "sixth", + "--velocity", + "cubic", + "--layout", + "halos", + "--halo-capacity", + str(options.halo_capacity), + "--background", + str(root / "background.bin"), + ] + if options.host_mpi: + common_args.append("--host-mpi") + species_specs = [] + for index, species in enumerate(species_list): + species_grid = self.cfg["grid"]["species_grids"][species.name] + initial, coordinates = root / f"initial-{index}.bin", root / f"v-{index}.bin" + _write_vx(initial, self.state[species.name]) + np.asarray(species_grid["v"], dtype=np.float64).tofile(coordinates) + species_args = [ + *common_args, + "--nv", + str(species.nv), + "--vmax", + str(species.vmax), + "--charge", + str(species.charge), + "--mass", + str(species.mass), + "--nu", + str(nu if index == 0 else 0.0), + "--input", + str(initial), + "--v-grid", + str(coordinates), + ] + species_specs.append( + dict( + name=species.name, + nv=species.nv, + vmax=species.vmax, + charge=species.charge, + mass=species.mass, + options=species_args, + ) + ) + request = dict( + library=library, + options=species_specs[0]["options"], + species=species_specs, + times=times, + t0=t0, + dt=grid.dt, + steps=steps, + device=options.device, + ranks=options.ranks, + pythonpath=[str(Path(entry or os.getcwd()).resolve()) for entry in sys.path], + ) + (root / "request.json").write_text(json.dumps(request, indent=2)) + launcher = [part.format(ranks=options.ranks) for part in options.launcher] + if options.ranks > 1 and not launcher: + launcher = ["mpiexec", "-n", str(options.ranks)] + command = [*launcher, sys.executable, str(Path(__file__).with_name("worker.py")), str(root / "request.json")] + environment = dict(os.environ, XLA_PYTHON_CLIENT_PREALLOCATE="false") + environment.setdefault("MPICH_GPU_SUPPORT_ENABLED", "1") + log = root / "worker.log" + with log.open("w") as stream: + process = subprocess.Popen( + command, stdout=stream, stderr=subprocess.STDOUT, env=environment, start_new_session=True + ) + try: + returncode = process.wait(timeout=options.timeout) + except BaseException as error: + os.killpg(process.pid, signal.SIGTERM) + try: + process.wait(timeout=10) + except subprocess.TimeoutExpired: + os.killpg(process.pid, signal.SIGKILL) + process.wait() + raise RuntimeError( + f"native Vlasov workers interrupted; log: {log}\n{log.read_text()[-12000:]}" + ) from error + if returncode: + raise RuntimeError(f"native Vlasov workers failed ({returncode}); log: {log}\n{log.read_text()[-12000:]}") + ys = _collect(root, times, options.ranks) + stats = json.loads((root / "stats.json").read_text()) + result = Solution( + t0=t0, + t1=t1, + ts={key: np.asarray(value) for key, value in times.items()}, + ys=ys, + stats={"num_steps": steps, "num_accepted_steps": steps, "num_rejected_steps": 0}, + result=RESULTS.successful, + interpolation=None, + solver_state=None, + controller_state=None, + made_jump=None, + event_mask=None, + ) + gc.collect() + return {"solver result": result, "native stats": stats, "native workdir": str(root)} + + def post_process(self, run_output, td): + source = Path(run_output["native workdir"]) + directory = Path(td) / "native" + directory.mkdir(exist_ok=True) + for name in ["request.json", "stats.json", "worker.log"]: + shutil.copyfile(source / name, directory / name) + request = json.loads((source / "request.json").read_text()) + library = Path(request["library"]) + (directory / "library.sha256").write_text( + f"{hashlib.sha256(library.read_bytes()).hexdigest()} {library.name}\n" + ) + with _host_preprocessing(): + return super().post_process(run_output, td) + + def vg(self, trainable_modules, args): + raise NotImplementedError("the native Vlasov backend does not provide automatic differentiation") + + +@cache +def native_backend(module_class): + """Adapt a BaseVlasov1D subclass without changing its pre/postprocessing.""" + if not isinstance(module_class, type) or not issubclass(module_class, BaseVlasov1D): + raise TypeError("the native backend requires a BaseVlasov1D subclass") + if issubclass(module_class, NativeVlasovMixin): + return module_class + return type(f"Native{module_class.__name__}", (NativeVlasovMixin, module_class), {}) + + +NativeVlasov1D = native_backend(BaseVlasov1D) diff --git a/adept/_vlasov1d/native/worker.py b/adept/_vlasov1d/native/worker.py new file mode 100644 index 00000000..4d85a9ae --- /dev/null +++ b/adept/_vlasov1d/native/worker.py @@ -0,0 +1,221 @@ +"""Internal MPI worker. Run by filename so JAX initializes before importing ADEPT.""" + +import argparse +import gc +import importlib.metadata +import json +import os +import socket +import sys +import time +import traceback +from pathlib import Path + +from bridge import NativeCore + + +def run(request): + import numpy as np + + root = request.parent + spec = json.loads(request.read_text()) + os.environ["JAX_PLATFORMS"] = "cpu" if spec["device"] == "cpu" else "cuda,cpu" + os.environ.setdefault("XLA_PYTHON_CLIENT_PREALLOCATE", "false") + core = NativeCore(spec["library"], spec["options"]) + try: + for species in spec["species"][1:]: + core.add_species(species["options"]) + if core.size != spec["ranks"]: + raise RuntimeError("MPI launcher rank count differs from native.ranks") + if core.device_type != (2 if spec["device"] == "cuda" else 1): + raise RuntimeError("native library was built for a different device") + core.set_field(root / "field.bin") + import jax + + jax.config.update("jax_enable_x64", True) + if core.size > 1: + coordinator = "" + if core.rank == 0: + with socket.socket() as connection: + connection.bind(("", 0)) + coordinator = f"{socket.gethostname()}:{connection.getsockname()[1]}" + coordinator = core.broadcast(coordinator) + jax.distributed.initialize( + coordinator_address=coordinator, + num_processes=core.size, + process_id=core.rank, + local_device_ids=[core.device_ordinal], + ) + elif core.device_type == 2: + jax.config.update("jax_cuda_visible_devices", str(core.device_ordinal)) + sys.path[:0] = spec.get("pythonpath", []) + import cloudpickle + import jax.numpy as jnp + from jax.sharding import Mesh, NamedSharding + from jax.sharding import PartitionSpec as P + + if len(jax.devices()) != core.size or len(jax.local_devices()) != 1: + raise RuntimeError("native workers require exactly one JAX device per MPI rank") + mesh = Mesh(np.array(jax.devices()), ("device",)) + slab = NamedSharding(mesh, P(None, "device")) + replicated = NamedSharding(mesh, P()) + with (root / "callbacks.pkl").open("rb") as stream: + payload = cloudpickle.load(stream) + + def replicate(value): + host = np.asarray(value) + return jax.make_array_from_callback(host.shape, replicated, lambda index: host[index]) + + static = jax.tree.map(replicate, payload["state"]) + callbacks = { + key: jax.jit(lambda t, y, fn=fn: fn(t, y, payload["args"])) for key, fn in payload["callbacks"].items() + } + + def snapshot(): + # FFTW may align host allocations to 32 bytes; JAX CPU requires 64. + # CUDA allocations satisfy JAX's alignment, so GPU views stay strict. + copy = False if core.device_type == 2 else None + e = jax.dlpack.from_dlpack(core.view(field=True), copy=copy) + state = dict(static, e=jax.make_array_from_single_device_arrays((core.nx,), replicated, [e])) + for index, species in enumerate(spec["species"]): + f = jax.dlpack.from_dlpack(core.view(species=index), copy=copy) + state[species["name"]] = jax.make_array_from_single_device_arrays((core.nx, species["nv"]), slab, [f]) + return state + + directory = root / f"rank-{core.rank:05d}" + directory.mkdir() + (directory / "placement.json").write_text( + json.dumps( + dict( + rank=core.rank, + local_rank=core.local_rank, + device_ordinal=core.device_ordinal, + cuda_visible_devices=os.environ.get("CUDA_VISIBLE_DEVICES"), + host=socket.gethostname(), + ) + ) + ) + writers, metadata, layout = {}, {}, {} + + def save(key, frame, t, state): + value = callbacks[key](replicate(np.asarray(t)), state) + jax.block_until_ready(value) + leaves, tree = jax.tree.flatten(value) + if key not in layout: + layout[key] = {"tree": tree, "leaves": []} + metadata[key] = [] + for i, leaf in enumerate(leaves): + layout[key]["leaves"].append(dict(shape=leaf.shape, dtype=str(leaf.dtype))) + if leaf.is_fully_replicated and core.rank != 0: + metadata[key].append(None) + continue + shards = leaf.addressable_shards + if len(shards) != 1: + raise RuntimeError("unexpected native save sharding") + shard = shards[0] + index = [ + [part.start or 0, part.stop if part.stop is not None else leaf.shape[d], part.step or 1] + for d, part in enumerate(shard.index) + ] + filename = f"{len(layout) - 1:04d}-{i:03d}.npy" + shape = (len(spec["times"][key]), *shard.data.shape) + writers[key, i] = np.lib.format.open_memmap( + directory / filename, mode="w+", dtype=np.dtype(leaf.dtype), shape=shape + ) + metadata[key].append(dict(file=str((directory / filename).relative_to(root)), index=index)) + for i, leaf in enumerate(leaves): + if (key, i) in writers: + (shard,) = leaf.addressable_shards + expected = metadata[key][i]["index"] + index = [ + [part.start or 0, part.stop if part.stop is not None else leaf.shape[d], part.step or 1] + for d, part in enumerate(shard.index) + ] + if index != expected: + raise RuntimeError("save output sharding changed between frames") + writers[key, i][frame] = np.asarray(shard.data) + # No device save output survives a native mutation, including identity saves. + del leaves, value + + requests = {} + for key, times in spec["times"].items(): + for frame, t in enumerate(times): + position = (t - spec["t0"]) / spec["dt"] + nearest = round(position) + if abs(position - nearest) <= 1e-9: + position = float(nearest) + upper = int(np.ceil(position)) + requests.setdefault(upper, []).append((key, frame, t, position - (upper - 1))) + current = 0 + started = time.perf_counter() + for upper, saves in sorted(requests.items()): + off_grid = any(0 < alpha < 1 for _, _, _, alpha in saves) + previous = None + if off_grid: + core.step(upper - 1 - current) + source = snapshot() + previous = jax.tree.map(lambda value: jnp.array(value, copy=True), source) + jax.block_until_ready(previous) + del source + core.step(1) + else: + core.step(upper - current) + current = upper + state = snapshot() + for key, frame, t, alpha in saves: + if 0 < alpha < 1: + sample = jax.tree.map(lambda a, b, alpha=alpha: a + (b - a) * alpha, previous, state) + save(key, frame, t, sample) + del sample + else: + save(key, frame, t, state) + del state, previous + core.barrier() + core.step(spec["steps"] - current) + for writer in writers.values(): + writer.flush() + (directory / "index.json").write_text(json.dumps(metadata)) + if core.rank == 0: + with (root / "layout.pkl").open("wb") as stream: + cloudpickle.dump(layout, stream) + stats = core.stats() + if core.rank == 0: + (root / "stats.json").write_text( + json.dumps( + dict( + native_seconds=stats, + solve_and_save_seconds=time.perf_counter() - started, + steps=spec["steps"], + ranks=core.size, + device=spec["device"], + placement=[ + json.loads((root / f"rank-{rank:05d}/placement.json").read_text()) + for rank in range(core.size) + ], + species=[ + {key: value for key, value in species.items() if key != "options"} + for species in spec["species"] + ], + libraries={ + name: importlib.metadata.version(name) + for name in ["jax", "jaxlib", "cloudpickle", "diffrax"] + }, + ), + indent=2, + ) + ) + core.barrier() + gc.collect() + core.close() + if core.size > 1: + jax.distributed.shutdown() + except BaseException: + traceback.print_exc() + core.lib.av_abort(1) + raise + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("request", type=Path) + run(parser.parse_args().request) diff --git a/adept/_vlasov1d/observations.py b/adept/_vlasov1d/observations.py index 5285bb82..1dc995de 100644 --- a/adept/_vlasov1d/observations.py +++ b/adept/_vlasov1d/observations.py @@ -4,6 +4,7 @@ import jax from interpax import interp2d from jax import numpy as jnp +from jax.scipy.special import xlogy class FieldsObservation(eqx.Module): @@ -25,7 +26,7 @@ def __call__(self, t, y, args): "v": velocity, "p": jnp.sum(f * centered_v**2.0, axis=1) * dv, "q": jnp.sum(f * centered_v**3.0, axis=1) * dv, - "-flogf": jnp.sum(-jnp.abs(f) * jnp.log(jnp.abs(f)), axis=1) * dv, + "-flogf": jnp.sum(-xlogy(jnp.abs(f), jnp.abs(f)), axis=1) * dv, "f^2": jnp.sum(f * f, axis=1) * dv, } result.update({name: y[name] for name in ("e", "de", "a", "prev_a")}) @@ -80,7 +81,7 @@ def moment(value, spacing=dv): scalars[f"mean_j_{name}"] = moment(f * v) scalars[f"mean_n_{name}"] = moment(f) scalars[f"mean_q_{name}"] = moment(f * v**3.0) - scalars[f"mean_-flogf_{name}"] = moment(-jnp.log(jnp.abs(f)) * jnp.abs(f)) + scalars[f"mean_-flogf_{name}"] = moment(-xlogy(jnp.abs(f), jnp.abs(f))) scalars[f"mean_f2_{name}"] = moment(f * f) mean_kinetic_energy += 0.5 * self.species_params[name]["mass"] * scalars[f"mean_P_{name}"] scalars["mean_de2"] = jnp.mean(y["de"] ** 2.0) diff --git a/adept/vlasov1d.py b/adept/vlasov1d.py index 5fa4c3ca..ac713c1c 100644 --- a/adept/vlasov1d.py +++ b/adept/vlasov1d.py @@ -2,5 +2,6 @@ from ._vlasov1d.iaw import IAWTurbulence1D from ._vlasov1d.modules import BaseVlasov1D +from ._vlasov1d.native.module import NativeVlasov1D, native_backend -__all__ = ["BaseVlasov1D", "IAWTurbulence1D"] +__all__ = ["BaseVlasov1D", "IAWTurbulence1D", "NativeVlasov1D", "native_backend"] diff --git a/benchmarks/vlasov_cuda/CMakeLists.txt b/benchmarks/vlasov_cuda/CMakeLists.txt new file mode 100644 index 00000000..cebb6241 --- /dev/null +++ b/benchmarks/vlasov_cuda/CMakeLists.txt @@ -0,0 +1,21 @@ +cmake_minimum_required(VERSION 3.24) +project(vlasov_split_benchmark LANGUAGES CXX) + +option(VLASOV_CUDA "Build the CUDA/cuFFT backend (OFF uses FFTW for validation)" ON) +find_package(MPI REQUIRED COMPONENTS CXX) +add_executable(vlasov_bench main.cpp backend.cpp) +target_compile_features(vlasov_bench PRIVATE cxx_std_17) +target_link_libraries(vlasov_bench PRIVATE MPI::MPI_CXX) +if(VLASOV_CUDA) + enable_language(CUDA) + find_package(CUDAToolkit REQUIRED) + set_source_files_properties(backend.cpp PROPERTIES LANGUAGE CUDA) + set_target_properties(vlasov_bench PROPERTIES CUDA_STANDARD 17 CUDA_ARCHITECTURES 80) + target_compile_definitions(vlasov_bench PRIVATE VLASOV_CUDA) + target_compile_options(vlasov_bench PRIVATE $<$:--extended-lambda>) + target_link_libraries(vlasov_bench PRIVATE CUDA::cudart CUDA::cufft) +else() + find_package(PkgConfig REQUIRED) + pkg_check_modules(FFTW REQUIRED IMPORTED_TARGET fftw3) + target_link_libraries(vlasov_bench PRIVATE PkgConfig::FFTW) +endif() diff --git a/benchmarks/vlasov_cuda/README.md b/benchmarks/vlasov_cuda/README.md new file mode 100644 index 00000000..d95e7268 --- /dev/null +++ b/benchmarks/vlasov_cuda/README.md @@ -0,0 +1,308 @@ +# CUDA/MPI Vlasov split-step prototype + +A standalone, FP64, 1D1V Vlasov–Poisson benchmark targeting A100 GPUs. It does +not require AD or import ADEPT. The optional FFTW backend exists for numerical +validation; it is not a threaded CPU performance implementation. + +The numerical implementation now lives in +[`adept/_vlasov1d/native`](../../adept/_vlasov1d/native/README.md). Its optional +ADEPT backend retains Python initialization, custom diagnostics, MLflow and +xarray/NetCDF output. The standalone benchmark shares its C++ core. Controlled +ADEPT/JAX versus CUDA measurements are preserved in +[`results/2026-10-24-adept`](results/2026-10-24-adept/summary.json). + +## Latest turbulence target and halo mode + +The reproduction target is the **23 October Nv4096 campaign**, not the older +L8000 cases: Nx134400 and Nx268800, Nv4096, L12600, vmax15, dt0.02, sixth-order +Hamiltonian splitting, exponential/spectral x advection, `cubic-spline` velocity +advection, and Chang-Cooper Dougherty collisions at nu1e-12. The pinned runtime +is ADEPT `ddd70b957dc3df85f0b5ec84af927c25d09d24e0` and JAX 0.10.2. The exact +member configs and source identifiers are under +[reference/latest-nv4096](reference/latest-nv4096/target.json). + +**Correction to the original benchmark's method assumption:** ADEPT's +`VelocityCubicSpline` calls interpax's default `method="cubic"`, a **local C1 +Hermite interpolant**, with slopes averaged from adjacent secants and one-sided +endpoint slopes. It does not use the global natural C2 spline implemented in +the first benchmark. Consequently, preserving the actual velocity method does +not require full velocity lines to reside on one rank. + +`--velocity cubic --layout halos` keeps the velocity-slab layout for all six +Vlasov kicks. It communicates just the velocity rows required by the departure +points and the four-point local stencil. The active width is computed from the +maximum field, kick duration, and dv, with two extra rows for the stencil. +Requests may span several neighbouring ranks. If a departure exceeds the +allocated `--halo-capacity`, the program fails explicitly. Both signed values +and the original extrapolation value `1e-30` are retained; no positivity limiter +or mass normalization changes the interpolant. + +Each rank owns a velocity slab and the complete x axis, so the batched x FFTs +stay local. The Poisson solve reduces the density array across velocity ranks. + +`--integrator sixth` uses ADEPT's actual force-gradient coefficients and dt +corrections, with its six velocity/five streaming stages. `--cells` selects the +production cell-centred velocity grid and rectangular velocity quadrature. +`--nu` enables the production Dougherty moment closure, Chang-Cooper weights, +zero-flux endpoints, and delta-form implicit solve. Its self-consistent-beta +iteration is disabled, matching the target configs. **One transpose pair per +timestep remains for collisions**; the Vlasov halo path removes the other six +pairs. Spectral x streaming is retained. + +For an exact native-state comparison, import raw float64 arrays with +`--input` (v,x order), `--v-grid`, and `--background`. Import the original ion +background; replacing its seeded density profile with a constant would change +the simulation. `checkpoint_reference.py` verifies and exports a committed +native checkpoint and advances the pinned ADEPT operators. It writes only to +its own output directory. `compare_checkpoint.py` checks full-resolution f, +Poisson fields and the latest whole-box N observer (periodic Gaussian sigma3, +potential depths/vb², strict cuts0.02 and0.5). Neither script modifies an +existing production controller or MLflow parent. + +`distributed_reference.py` advances the same verified prepared state with +eight JAX processes on two nodes. This is needed for Nx268800 on 40 GB GPUs: +the pinned JAX collision solve could not allocate its gtsv2 workspace on four. +The eight-process reference uses one GPU per rank and the production NCCL +Libfabric plugin; its provenance and network log are preserved with the results. +This reference uses all numerical operators, including nu1e-12 collisions. + +Omitting `--checkpoint` from the reference exporter uses the pinned project's +original t0 initializer, including both seeded, band-limited beams. This provides +an exact initial-state input for a future full-duration reproduction; the C++ +benchmark's default Maxwellian initializer is not the turbulence initializer. + +`validate_local.py` checks interpax C1 parity, signed values, physical endpoints, +multi-neighbour shifts, the sixth-order step and collisions on both layouts: + +```sh +python benchmarks/vlasov_cuda/validate_local.py /path/to/vlasov_bench + +# Example latest-case invocation after checkpoint_reference.py exports input: +mpiexec -n 8 /path/to/vlasov_bench --nx 134400 --nv 4096 \ + --length 12600 --vmax 15 --dt .02 --cells --integrator sixth \ + --velocity cubic --layout halos --nu 1e-12 --steps 100 --warmup 0 \ + --input /path/to/reference/initial.bin --v-grid /path/to/reference/v.bin \ + --background /path/to/reference/background.bin +``` + +Short-window agreement is an implementation check. It does not establish the +full t0–2000 decay trajectory, resolution convergence, or equivalence of another +spatial/velocity discretization. The earlier natural-spline/Strang results below +are preserved as the initial experiment, not a matched turbulence performance +comparison. Their exact source is archived in +[baseline-source.tar.gz](results/2026-10-24/baseline-source.tar.gz). + +## Results using the latest turbulence operators + +On Perlmutter A100-SXM4-40GB GPUs, the halo path gives the following medians +over three repeats, each with 10 measured steps after 3 warmup steps. Both +layouts use the same native t600 input and all operators listed above. Timings +exclude initialization, diagnostics and I/O; communication/computation do not +overlap. These compare two CUDA layouts, not CUDA against JAX. + +| Nx Ă— Nv | Nodes | GPUs | Full transpose (ms/step) | Halos (ms/step) | Speedup | +| --- | ---: | ---: | ---: | ---: | ---: | +| 134400 Ă— 4096 | 1 | 4 | 323.5 | 174.3 | 1.86Ă— | +| 134400 Ă— 4096 | 2 | 8 | 490.5 | 140.0 | 3.50Ă— | +| 268800 Ă— 4096 | 1 | 4 | 649.6 | 342.5 | 1.90Ă— | +| 268800 Ă— 4096 | 2 | 8 | 965.0 | 275.0 | 3.51Ă— | + +Maximum per-rank distribution bytes sent fall by 85.1–85.3%. One full transpose +pair remains for collisions. On eight GPUs, MPI still takes about 60 ms on the +smaller grid and 119 ms on the larger grid, so that remaining communication +limits further strong scaling. + +All 20 native-state comparisons pass across both layouts and GPU placements. +Through 100 steps (t600→602), maximum absolute f error is `6.73e-13`, relative +L2 f error is at most `9.52e-14`, and relative L2 E error is at most `5.64e-14`. +The exact latest whole-box hole counts match at both thresholds in every case: +at t602, 129/47 for Nx134400 and 129/50 for Nx268800. Another 84 CPU/CUDA/MPI +operator and initial natural-spline regression checks pass. Conservation diagnostics +are recorded without clipping f or renormalizing mass. + +Reports, commands, runtime versions, source/executable hashes and the pinned +reference provenance are in +[results/2026-10-24-halos](results/2026-10-24-halos/summary.json). +`results/2026-10-24-halos/summarize.py` rebuilds the compact report from the +adjacent raw JSONL and comparison files. Native distributions stay on scratch; +their exact locations and manifest hashes are in the reference provenance. +The owned development allocations were released after saving the results. +This is a short-window implementation check; the full t0–2000 decay trajectory +has not been reproduced yet. + +## Initial natural-C2/Strang benchmark: 24 October 2026 + +On Perlmutter A100-SXM4-40GB GPUs, a `32768 x 8192` FP64 distribution +(2 GiB of state), with 20 timesteps after 3 warmup steps, gave the following +medians over three runs. All runs use the same global problem size. + +| Nodes | GPUs | Step (ms) | Streaming (ms) | Spline (ms) | Pack/unpack (ms) | Slab MPI (ms) | Speedup | +| ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| 1 | 1 | 68.91 | 49.02 | 17.81 | 0.00 | 0.00 | 1.00x | +| 1 | 2 | 58.70 | 24.54 | 12.88 | 6.34 | 13.78 | 1.17x | +| 1 | 4 | 36.95 | 12.33 | 10.98 | 3.17 | 9.75 | 1.87x | +| 2 | 2 | 121.00 | 24.54 | 12.88 | 6.34 | 76.10 | 0.57x | +| 2 | 4 | 65.88 | 12.32 | 10.97 | 3.17 | 38.55 | 1.05x | + +The field stage adds about 0.8–2.1 ms. Owned device allocations, including +cuFFT workspaces, are about 20 GiB on one GPU, 10 GiB per GPU on two, and +5 GiB per GPU on four. GPU runtime/MPI allocations are additional. + +The full distribution agrees with NumPy FFTs and SciPy natural splines to +less than `4e-15` maximum absolute error in 28 checks: 12 CPU/MPI checks, +12 CUDA checks on one node, and 4 CUDA checks spanning two nodes. For the +larger timing runs, maximum relative mass change is `5.1e-12` and maximum +relative total-energy change is `3.4e-7` over physical time 2.0. The smallest +f is `-2.4e-16` to the stated precision. + +This establishes a working initial implementation. It also shows that the +two full-state redistributions limit multinode strong scaling: they consume +about 59% of the two-node/four-GPU timestep. Streaming is the largest component +on a single GPU; the spline recurrence also scales less than ideally. These +measurements do not establish a performance comparison with ADEPT/JAX. + +Raw measurements, source hashes, and correctness reports are in +[results/2026-10-24](results/2026-10-24/summary.json). All final validation and +measurement Slurm steps completed with exit status `0:0`. The development +allocation was released after downloading the results. + +## Original natural-spline baseline + +The normalized model has electrons with acceleration `-E`, an immobile uniform +ion background, and `dE/dx = 1 - integral(f dv)`. The initial condition is a +discretely normalized Maxwellian times `1 + amplitude*cos(2*pi*x/length)`. +Defaults are `length=4*pi`, `vmax=8`, `amplitude=0.01`, and `dt=0.1`. + +Each Strang step performs: + +1. Half a streaming step using batched real FFTs in x and spectral phase shifts. +2. A velocity-moment MPI Allreduce and a replicated 1D FFT Poisson solve. +3. An MPI Alltoall from velocity slabs to spatial slabs. +4. A full velocity step using natural cubic splines, including coefficient + construction and evaluation at the backward characteristic. +5. An MPI Alltoall back to velocity slabs, then half a streaming step. + +Velocity slabs have shape `(nv/ranks, nx)`; spatial slabs have shape +`(nv, nx/ranks)`. Both keep x contiguous. Spline construction uses a Thomas +solve per spatial line with precomputed LU factors. Threads run across x so +memory accesses in each recurrence sweep are coalesced. The recurrence along +v is serial within each thread; this is a baseline to profile before investing +in a parallel tridiagonal solver. One-rank runs swap layout pointers and skip +the redistribution copies and MPI Alltoall. + +Distribution, packing buffers, coefficients, FFT plans, and FFT workspaces +remain allocated throughout the run. GPU-aware MPI is the default. `--host-mpi` +stages MPI buffers through host memory for comparison or systems without device +pointer support. FFT streaming and the spline solve still execute on the GPU. + +## Build + +CUDA, cuFFT, MPI, and CMake >= 3.24 are required. The CUDA target defaults to +SM80. Use the system's MPI compiler wrapper when required by its toolchain: + +```sh +cmake -S benchmarks/vlasov_cuda -B /tmp/vlasov-cuda-build \ + -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=mpicxx +cmake --build /tmp/vlasov-cuda-build -j 4 +mpiexec -n 4 /tmp/vlasov-cuda-build/vlasov_bench --nx 32768 --nv 8192 +``` + +Each local MPI rank selects its GPU by local rank before MPI initialization, +using the launcher's local-rank environment variable. This is needed because +Cray MPI initializes its GPU transport in `MPI_Init`. If the launcher exposes +just one GPU per rank, each rank uses its visible device 0. Do not launch +multiple ranks bound to the same physical GPU. + +On Perlmutter, use `CC` instead of `mpicxx`, load an available CMake module, +and set `MPICH_GPU_SUPPORT_ENABLED=1`. Expose peer GPUs with +`--gpus-per-node=4 --gpu-bind=none`; the program assigns one rank to each GPU. +The tested CUDA 13.2 / Cray MPICH 9.1 setup reported invalid CUDA IPC handles +when Slurm restricted visibility to one GPU per rank. A development-session +build is: + +```sh +~/.ergodic-agent-workflows/ops/session.sh exec -- bash -lc ' + set -euo pipefail + if [ "${SLURM_PROCID:-0}" -ne 0 ]; then exit 0; fi + module load cmake/3.30.2 + cmake -S benchmarks/vlasov_cuda -B "$EC_SESSION_WORKDIR/cuda-build" \ + -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=CC + cmake --build "$EC_SESSION_WORKDIR/cuda-build" -j 4 +' +``` + +Use an isolated development session and verify the account, modules, resource +counts, and launcher on the actual system before allocating. The benchmark +does not allocate or cancel jobs. + +## Validate + +The reference checks require NumPy >= 2 and SciPy. They compare the entire +distribution against NumPy FFTs and SciPy's natural `CubicSpline`, verify +analytic streaming, test exact redistribution, and check rank independence. +They also check mass and short-time energy error for the coupled step. + +```sh +# CPU build on a machine with FFTW and MPI: +cmake -S benchmarks/vlasov_cuda -B /tmp/vlasov-cpu-build \ + -DVLASOV_CUDA=OFF -DCMAKE_BUILD_TYPE=Release +cmake --build /tmp/vlasov-cpu-build -j 4 +python benchmarks/vlasov_cuda/validate.py /tmp/vlasov-cpu-build/vlasov_bench + +# CUDA build on an allocated GPU node: +python benchmarks/vlasov_cuda/validate.py /path/to/cuda-build/vlasov_bench \ + --launcher 'srun --jobid=JOBID --overlap --nodes=1 --ntasks={ranks} --ntasks-per-node={ranks} --gpus-per-node=4 --gpu-bind=none' +``` + +For validation across nodes, put `TMPDIR` on shared scratch so the driver can +read rank 0's dump, and use an appropriate explicit node/task layout. + +## Measure + +`measure.py` runs serial repeats of the same fixed problem on different MPI +placements. It verifies the reported placement and saves JSONL with timings, +diagnostics, commands, rank mapping, and optional source-state provenance: + +```sh +python benchmarks/vlasov_cuda/measure.py /path/to/cuda-build/vlasov_bench \ + --launcher 'srun --jobid=JOBID --overlap --nodes={nodes} --ntasks={ranks} --ntasks-per-node={ranks_per_node} --gpus-per-node=4 --gpu-bind=none' \ + --placements 1:1 1:2 1:4 2:2 2:4 --expect-cuda \ + --nx 32768 --nv 8192 --steps 20 --warmup 3 --repeats 3 \ + --output /path/to/outputs/scaling.jsonl +``` + +Run the lightweight driver on a compute node, with overlapping inner Slurm +steps if using a development-session outer step. The source directory remains +the local worktree; synchronize edits before rebuilding. Keep builds under +the session `workdir` and results under `outputs` so source sync preserves them. + +Each executable prints one JSON object. `step_ms` excludes initialization, +planning, warmup, diagnostics, downloads, and optional dumps. Warmup is followed +by reinitialization so reported physical time is exactly `steps*dt`. Component +timings explicitly synchronize the device and do not overlap communication +with computation. Times are the maximum across ranks of each rank's mean; +the component maxima need not sum to `step_ms`. `mpi_ms` covers the slab/halo +exchanges and halo-width coordination; `field_ms` includes density integration, +its Allreduce, and Poisson. `pack_ms` includes packing and unpacking for all +full-state redistributions. + +## Scope + +This is an initial solver/communication benchmark, not a replacement for ADEPT. +It supports one species, a uniform 1D1V grid and periodic x. The original natural +spline uses zero extrapolation; local cubic mode uses ADEPT's `1e-30`. Remapping +does not guarantee positivity or exact discrete conservation; no clipping or +mass correction hides that error. Keep velocity tails negligible and inspect +the reported mass, energy, and minimum f. The validation is short-time and +does not establish nonlinear or long-time fidelity. + +Both dimensions must divide evenly by the MPI rank count. Local arrays must +fit MPI's int count limit. Memory reporting includes owned device buffers and +cuFFT workspaces, but excludes CUDA/MPI runtime overhead. The current storage +uses roughly eight copies of the distribution per rank plus FFT workspace when +full-line solves are enabled. Unused transpose/coefficient buffers are omitted +for collisionless local-cubic halo runs. +Packing copies, scratch reuse, spline batching, and communication overlap are +deliberately available optimization targets once measurements identify the +limiting operation. No speedup over ADEPT/JAX is claimed without an equivalent +comparison run. diff --git a/benchmarks/vlasov_cuda/adept_timing.py b/benchmarks/vlasov_cuda/adept_timing.py new file mode 100644 index 00000000..a90b3d5d --- /dev/null +++ b/benchmarks/vlasov_cuda/adept_timing.py @@ -0,0 +1,152 @@ +"""Time pinned ADEPT operators on prepared latest-run checkpoints, without saves. + +Launch with one Python process per GPU using the pinned vp-turbulence venv. +Compilation, initial input, warmup and correctness reductions are outside timing. +""" + +import argparse +import copy +import importlib.metadata +import json +import os +import socket +import sys +import time +from pathlib import Path + +os.environ.setdefault("XLA_PYTHON_CLIENT_PREALLOCATE", "false") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--prepared", type=Path, required=True) + parser.add_argument("--config", type=Path, required=True) + parser.add_argument("--vp-source", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--steps", type=int, default=10) + parser.add_argument("--warmup", type=int, default=3) + parser.add_argument("--repeats", type=int, default=3) + args = parser.parse_args() + sys.path.insert(0, str(args.vp_source)) + import jax + + jax.config.update("jax_enable_x64", True) + jax.distributed.initialize(local_device_ids=[int(os.environ["SLURM_LOCALID"])]) + import numpy as np + import yaml + from checkpoint_reference import sha + from jax.experimental import multihost_utils as mh + from jax.sharding import Mesh, NamedSharding + from jax.sharding import PartitionSpec as P + from vp_turbulence.adeptmodules import VPTwoStream1D + from vp_turbulence.multinode import initialize_module_sharded + + rank, ranks = jax.process_index(), jax.process_count() + dist = importlib.metadata.distribution("adept") + commit = json.loads(dist.read_text("direct_url.json")).get("vcs_info", {}).get("commit_id") + if commit != "ddd70b957dc3df85f0b5ec84af927c25d09d24e0" or jax.__version__ != "0.10.2": + raise RuntimeError("incorrect pinned numerical environment") + metadata = json.loads((args.prepared / "provenance.json").read_text()) + if sha(args.config) != metadata["config_sha256"]: + raise RuntimeError("config differs from verified checkpoint") + nx, nv = metadata["nx"], metadata["nv"] + cfg = yaml.safe_load(args.config.read_text()) + module = VPTwoStream1D(copy.deepcopy(cfg)) + module.write_units() + module.get_derived_quantities() + initialize_module_sharded(module) + np.testing.assert_array_equal( + np.fromfile(args.prepared / "v.bin", dtype=np.float64), + np.asarray(module.cfg["grid"]["species_grids"]["electron"]["v"]), + ) + np.testing.assert_array_equal( + np.fromfile(args.prepared / "background.bin", dtype=np.float64), np.asarray(module.cfg["grid"]["ion_charge"]) + ) + source = np.memmap(args.prepared / "initial.bin", mode="r", dtype=np.float64, shape=(nv, nx)) + mesh = Mesh(np.array(jax.devices()), ("device",)) + sharding = NamedSharding(mesh, P("device", None)) + initial = jax.make_array_from_callback((nx, nv), sharding, lambda index: source.T[index]) + module.state = {"electron": initial} + module.init_diffeqsolve() + operator = module.diffeqsolve_quants["terms"].vector_field.vpfp + if operator.fp._sc_max_steps != 0: + raise RuntimeError("unexpected self-consistent beta") + replicated = NamedSharding(mesh, P()) + + def replicate(value): + host = np.asarray(value) + return jax.make_array_from_callback(host.shape, replicated, lambda index: host[index]) + + a, prev = replicate(np.zeros(nx + 2)), replicate(np.zeros(nx)) + external = replicate(np.zeros((6, nx))) + nu_fp, nu_k = replicate(np.full(nx, metadata["nu"])), replicate(np.zeros(nx)) + + @jax.jit + def advance(values, iterations): + def body(_, current): + _, result, _ = operator({"electron": current}, a, prev, external, nu_fp, nu_k) + return result["electron"] + + return jax.lax.fori_loop(0, iterations, body, values) + + iterations = replicate(np.asarray(args.steps, dtype=np.int32)) + start = time.perf_counter() + compiled = advance.lower(initial, iterations).compile() + compile_seconds = time.perf_counter() - start + warm = compiled(initial, replicate(np.asarray(args.warmup, dtype=np.int32))) + warm.block_until_ready() + del warm + rows = [] + for repeat in range(args.repeats): + # Reset to the same checkpoint each repeat; execution uses immutable input. + mh.sync_global_devices(f"before-{repeat}") + start = time.perf_counter() + result = compiled(initial, iterations) + result.block_until_ready() + seconds = time.perf_counter() - start + all_seconds = np.asarray(mh.process_allgather(np.asarray(seconds))) + local = np.asarray(result.addressable_shards[0].data) + if not np.isfinite(local).all(): + raise RuntimeError("nonfinite ADEPT output") + local_sums = np.asarray([local.sum(), np.square(local).sum(), local.min(), local.max()]) + sums = np.asarray(mh.process_allgather(local_sums)).reshape(ranks, 4) + if rank == 0: + row = dict( + backend="adept-jax", + nx=nx, + nv=nv, + ranks=ranks, + repeat=repeat, + steps=args.steps, + warmup=args.warmup, + step_ms=1000 * float(all_seconds.max()) / args.steps, + rank_seconds=all_seconds.tolist(), + sum=float(sums[:, 0].sum()), + sum_squares=float(sums[:, 1].sum()), + minimum=float(sums[:, 2].min()), + maximum=float(sums[:, 3].max()), + ) + rows.append(row) + print(json.dumps(row), flush=True) + del result, local + hosts = np.asarray( + mh.process_allgather(np.asarray(list(socket.gethostname().encode().ljust(64, b"\0")), dtype=np.uint8)) + ) + if rank == 0: + report = dict( + rows=rows, + checkpoint=metadata, + adept_commit=commit, + libraries={name: importlib.metadata.version(name) for name in ["jax", "jaxlib", "interpax", "diffrax"]}, + hosts=[bytes(host).rstrip(b"\0").decode() for host in hosts.reshape(ranks, 64)], + compile_seconds=compile_seconds, + devices=[str(d) for d in jax.devices()], + scope="Warmed operator loop; no SaveAt, diagnostics, I/O, initialization or compilation in timings", + ) + with args.output.open("x") as stream: + json.dump(report, stream, indent=2) + jax.distributed.shutdown() + + +if __name__ == "__main__": + main() diff --git a/benchmarks/vlasov_cuda/backend.cpp b/benchmarks/vlasov_cuda/backend.cpp new file mode 100644 index 00000000..293d6790 --- /dev/null +++ b/benchmarks/vlasov_cuda/backend.cpp @@ -0,0 +1,2 @@ +// Compatibility include; the implementation ships with ADEPT. +#include "../../adept/_vlasov1d/native/core/backend.cpp" diff --git a/benchmarks/vlasov_cuda/backend.hpp b/benchmarks/vlasov_cuda/backend.hpp new file mode 100644 index 00000000..b6d96321 --- /dev/null +++ b/benchmarks/vlasov_cuda/backend.hpp @@ -0,0 +1,2 @@ +// Compatibility include; the implementation ships with ADEPT. +#include "../../adept/_vlasov1d/native/core/backend.hpp" diff --git a/benchmarks/vlasov_cuda/benchmark_latest.py b/benchmarks/vlasov_cuda/benchmark_latest.py new file mode 100644 index 00000000..53d77eef --- /dev/null +++ b/benchmarks/vlasov_cuda/benchmark_latest.py @@ -0,0 +1,93 @@ +"""Paired strong-scaling measurements after latest native-window checks pass.""" + +import argparse +import json +import subprocess +import sys +from pathlib import Path + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("binary", type=Path) + parser.add_argument("--data", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--launcher", required=True) + parser.add_argument("--source-state", type=Path) + parser.add_argument("--nx", nargs="+", type=int, default=[134400, 268800]) + parser.add_argument("--placements", nargs="+", default=["1:4", "2:8"]) + parser.add_argument("--steps", type=int, default=10) + parser.add_argument("--repeats", type=int, default=3) + args = parser.parse_args() + args.output.mkdir(parents=True, exist_ok=False) + for nx in args.nx: + reference = args.data / f"native-nx{nx}" + checks = json.loads((args.data / f"comparison-nx{nx}-window/comparison.json").read_text()) + if not checks or not all(row["passed"] for row in checks): + raise RuntimeError("native-state comparison must pass before timing") + for placement in args.placements: + nodes, ranks = map(int, placement.split(":")) + for layout in ["transpose", "halos"]: + if not any( + row["nodes"] == nodes and row["ranks"] == ranks and row["layout"] == layout and row["steps"] == 100 + for row in checks + ): + raise RuntimeError(f"missing 100-step native comparison: {nx}, {placement}, {layout}") + metadata = json.loads((reference / "provenance.json").read_text()) + if metadata["nx"] != nx or metadata["nv"] != 4096 or metadata["dt"] != 0.02 or metadata["nu"] != 1e-12: + raise RuntimeError("not the pinned latest turbulence target") + for layout in ["transpose", "halos"]: + command = [ + sys.executable, + str(Path(__file__).with_name("measure.py")), + str(args.binary), + "--launcher", + args.launcher, + "--placements", + *args.placements, + "--nx", + str(nx), + "--nv", + "4096", + "--length", + "12600", + "--vmax", + "15", + "--dt", + ".02", + "--integrator", + "sixth", + "--velocity", + "cubic", + "--layout", + layout, + "--cells", + "--nu", + "1e-12", + "--steps", + str(args.steps), + "--warmup", + "3", + "--repeats", + str(args.repeats), + "--expect-cuda", + "--input", + str(reference / "initial.bin"), + "--v-grid", + str(reference / "v.bin"), + "--background", + str(reference / "background.bin"), + "--max-mass-change=-1", + "--output", + str(args.output / f"nx{nx}-{layout}.jsonl"), + ] + if args.source_state: + command.extend(["--source-state", str(args.source_state)]) + print(json.dumps({"nx": nx, "layout": layout, "status": "measuring"}), flush=True) + # Conservation diagnostics are recorded without changing the target + # operator. Full-state/field/N parity is gated independently above. + subprocess.run(command, check=True) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/vlasov_cuda/checkpoint_reference.py b/benchmarks/vlasov_cuda/checkpoint_reference.py new file mode 100644 index 00000000..a819383f --- /dev/null +++ b/benchmarks/vlasov_cuda/checkpoint_reference.py @@ -0,0 +1,178 @@ +"""Read a committed native checkpoint and advance its pinned ADEPT operators. + +Run with the production vp-turbulence environment on an allocated GPU node. +Only --output is written; the source campaign and checkpoint are read-only. +This deliberately tests numerical operators directly, without production +tracking, save buffers, or changes to either existing MLflow parent. +""" + +import argparse +import copy +import hashlib +import importlib.metadata +import json +import os +import sys +import time +from pathlib import Path + +os.environ.setdefault("XLA_PYTHON_CLIENT_PREALLOCATE", "false") +os.environ.setdefault("JAX_ENABLE_X64", "true") + + +def sha(path): + with Path(path).open("rb") as stream: + return hashlib.file_digest(stream, "sha256").hexdigest() + + +def save_vx(path, array): + """Write x,v data as a contiguous v,x file with bounded copy temporaries.""" + import numpy as np + + out = np.memmap(path, mode="w+", dtype=np.float64, shape=array.shape[::-1]) + for v0 in range(0, array.shape[1], 64): + out[v0 : v0 + 64] = array[:, v0 : v0 + 64].T + out.flush() + del out + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--checkpoint", type=Path, help="committed native input; omit to export the original seeded t0 initializer" + ) + parser.add_argument("--config", type=Path, required=True) + parser.add_argument("--vp-source", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--steps", nargs="+", type=int, default=[1, 10, 100]) + parser.add_argument("--expected-manifest") + args = parser.parse_args() + args.output.mkdir(parents=True, exist_ok=False) + if args.checkpoint and ( + not args.expected_manifest or sha(args.checkpoint / "manifest.json") != args.expected_manifest + ): + raise RuntimeError("native checkpoint manifest checksum mismatch") + distribution = importlib.metadata.distribution("adept") + direct = json.loads(distribution.read_text("direct_url.json")) + commit = direct.get("vcs_info", {}).get("commit_id") + if commit != "ddd70b957dc3df85f0b5ec84af927c25d09d24e0": + raise RuntimeError(f"wrong ADEPT environment: {commit}") + sys.path.insert(0, str(args.vp_source)) + import adept + + if Path(adept.__file__).resolve().parent != Path(distribution.locate_file("adept")).resolve(): + raise RuntimeError("PYTHONPATH shadows the pinned ADEPT installation") + import jax + import jax.numpy as jnp + import numpy as np + import yaml + from jax.sharding import Mesh, NamedSharding + from jax.sharding import PartitionSpec as P + from vp_turbulence.adeptmodules import VPTwoStream1D + from vp_turbulence.multinode import initialize_module_sharded + + jax.config.update("jax_enable_x64", True) + cfg = yaml.safe_load(args.config.read_text()) + if cfg["terms"]["time"] != "sixth" or cfg["terms"]["edfdv"] != "cubic-spline": + raise RuntimeError("expected the latest sixth-order/cubic target") + if cfg["drivers"]["ex"] or cfg["drivers"]["ey"]: + raise RuntimeError("external forcing is outside this comparison") + manifest = ( + json.loads((args.checkpoint / "manifest.json").read_text()) + if args.checkpoint + else {"actual_time": 0.0, "ranks": []} + ) + nx, nv = int(cfg["grid"]["nx"]), int(cfg["grid"]["nv"]) + state = np.empty((nx, nv), dtype=np.float64) if args.checkpoint else None + covered = np.zeros(nx, dtype=bool) + for rank in manifest["ranks"]: + leaf = rank["leaves"]["electron"] + path = args.checkpoint / leaf["file"] + if sha(path) != leaf["sha256"]: + raise RuntimeError(f"native shard checksum mismatch: {path}") + (start, stop, stride), vrange = leaf["index"] + if stride != 1 or vrange != [0, nv, 1] or np.any(covered[start:stop]): + raise RuntimeError("unexpected native shard coverage") + state[start:stop] = np.load(path, mmap_mode="r", allow_pickle=False) + covered[start:stop] = True + if args.checkpoint and (not covered.all() or not np.isfinite(state).all()): + raise RuntimeError("incomplete or nonfinite native input") + v = ( + np.load(args.checkpoint / manifest["coordinates"]["v_electron"]["file"], allow_pickle=False) + if args.checkpoint + else None + ) + # Rebuild the original ion background/geometry from the frozen initializer; + # the native perturbed background is not replaced with uniform density. + module = VPTwoStream1D(copy.deepcopy(cfg)) + module.write_units() + module.get_derived_quantities() + initialize_module_sharded(module) + native_v = np.asarray(module.cfg["grid"]["species_grids"]["electron"]["v"]) + if args.checkpoint is None: + state = np.asarray(module.state["electron"]) + v = native_v + if not np.array_equal(v, native_v): + raise RuntimeError("native velocity coordinates differ from pinned setup") + v.tofile(args.output / "v.bin") + np.asarray(module.cfg["grid"]["ion_charge"]).tofile(args.output / "background.bin") + save_vx(args.output / "initial.bin", state) + module.init_diffeqsolve() + operator = module.diffeqsolve_quants["terms"].vector_field.vpfp + if type(operator.vlasov_poisson).__name__ != "SixthOrderHamIntegrator": + raise RuntimeError("incorrect time integrator") + if operator.fp._sc_max_steps != 0: + raise RuntimeError("this prototype expects production self-consistent beta disabled") + sharding = NamedSharding(Mesh(np.array(jax.devices()), ("device",)), P("device", None)) + f = jax.make_array_from_callback(state.shape, sharding, lambda index: state[index]) + module.state = {"electron": f} + nu = float(cfg["terms"]["fokker_planck"]["time"]["baseline"]) + a, prev, external = jnp.zeros(nx + 2), jnp.zeros(nx), jnp.zeros((6, nx)) + nu_fp, nu_k = jnp.full(nx, nu), jnp.zeros(nx) + + @jax.jit + def advance(values, count): + def body(_, current): + _, result, _ = operator({"electron": current}, a, prev, external, nu_fp, nu_k) + return result["electron"] + + return jax.lax.fori_loop(0, count, body, values) + + compiled = advance.lower(f, jnp.asarray(1)).compile() + record = dict( + checkpoint=str(args.checkpoint) if args.checkpoint else None, + manifest_sha256=args.expected_manifest, + actual_time=manifest["actual_time"], + config_sha256=sha(args.config), + adept_commit=commit, + jax=jax.__version__, + nx=nx, + nv=nv, + dt=cfg["grid"]["dt"], + nu=nu, + devices=[str(d) for d in jax.devices()], + milestones=[], + ) + (args.output / "provenance.json").write_text(json.dumps(record, indent=2) + "\n") + print(json.dumps({"prepared": str(args.output), "nx": nx, "nv": nv, "time": manifest["actual_time"]}), flush=True) + previous = 0 + for stop in sorted(set(args.steps)): + if stop < 1: + raise ValueError("milestones must be positive") + start = time.perf_counter() + f = compiled(f, jnp.asarray(stop - previous)) + f.block_until_ready() + elapsed = time.perf_counter() - start + result = np.asarray(f) + if not np.isfinite(result).all(): + raise RuntimeError("nonfinite reference result") + save_vx(args.output / f"reference-{stop:04d}.bin", result) + row = dict(steps=stop, advanced_steps=stop - previous, seconds=elapsed) + record["milestones"].append(row) + (args.output / "provenance.json").write_text(json.dumps(record, indent=2) + "\n") + print(json.dumps(row), flush=True) + previous = stop + + +if __name__ == "__main__": + main() diff --git a/benchmarks/vlasov_cuda/compare_checkpoint.py b/benchmarks/vlasov_cuda/compare_checkpoint.py new file mode 100644 index 00000000..b2e387af --- /dev/null +++ b/benchmarks/vlasov_cuda/compare_checkpoint.py @@ -0,0 +1,150 @@ +"""Compare CUDA layouts at full native resolution with a pinned ADEPT window.""" + +import argparse +import json +import shlex +import subprocess +from pathlib import Path + +import numpy as np + + +def hole_counts(electric, k): + # Same whole-box observer as the latest Nv4096 comparison: Gaussian + # sigma_x=3, adjacent-minimum potential depths / vb^2, strict cuts. + kernel = np.zeros(len(k), dtype=complex) + kernel[1:] = 1j / k[1:] * np.exp(-0.5 * (3 * k[1:]) ** 2) + phi = np.fft.irfft(np.fft.rfft(electric) * kernel, n=len(electric)) + peaks = np.flatnonzero((phi > np.roll(phi, 1)) & (phi > np.roll(phi, -1))) + minima = np.flatnonzero((phi < np.roll(phi, 1)) & (phi < np.roll(phi, -1))) + if not len(minima): + return [0, 0] + i = np.searchsorted(minima, peaks) + depth = phi[peaks] - np.maximum(phi[minima[(i - 1) % len(minima)]], phi[minima[i % len(minima)]]) + return [int(np.count_nonzero(depth / 9 > cut)) for cut in [0.02, 0.5]] + + +def compare(candidate, reference, nx, nv, dv, length, background): + actual = np.memmap(candidate, mode="r", dtype=np.float64, shape=(nv, nx)) + expected = np.memmap(reference, mode="r", dtype=np.float64, shape=(nv, nx)) + error2, reference2, maximum = 0.0, 0.0, 0.0 + density = np.zeros(nx) + reference_density = np.zeros(nx) + for start in range(0, nv, 64): + a, b = actual[start : start + 64], expected[start : start + 64] + difference = a - b + if not np.isfinite(difference).all(): + raise RuntimeError("nonfinite checkpoint comparison") + maximum = max(maximum, float(np.max(np.abs(difference)))) + error2 += float(np.sum(difference * difference)) + reference2 += float(np.sum(b * b)) + density += np.sum(a, axis=0) * dv + reference_density += np.sum(b, axis=0) * dv + k = 2 * np.pi * np.fft.rfftfreq(nx, d=length / nx) + ef, rf = np.zeros(len(k), dtype=complex), np.zeros(len(k), dtype=complex) + ef[1:] = np.fft.rfft(background - density)[1:] / (1j * k[1:]) + rf[1:] = np.fft.rfft(background - reference_density)[1:] / (1j * k[1:]) + ef[-1] = rf[-1] = 0 + electric, reference_electric = np.fft.irfft(ef, n=nx), np.fft.irfft(rf, n=nx) + counts, reference_counts = hole_counts(electric, k), hole_counts(reference_electric, k) + return dict( + max_abs_f_error=maximum, + relative_l2_f_error=float(np.sqrt(error2 / reference2)), + relative_l2_e_error=float(np.linalg.norm(electric - reference_electric) / np.linalg.norm(reference_electric)), + max_abs_e_error=float(np.max(np.abs(electric - reference_electric))), + hole_counts=counts, + reference_hole_counts=reference_counts, + hole_counts_match=counts == reference_counts, + ) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("binary", type=Path) + parser.add_argument("--reference", required=True, type=Path) + parser.add_argument("--launcher", required=True) + parser.add_argument("--placements", nargs="+", default=["1:4", "2:8"]) + parser.add_argument("--steps", nargs="+", type=int, default=[1, 10, 100]) + parser.add_argument("--output", required=True, type=Path) + args = parser.parse_args() + source = json.loads((args.reference / "provenance.json").read_text()) + nx, nv, dt, nu = (source[key] for key in ["nx", "nv", "dt", "nu"]) + # Thresholds are implementation-matching checks for a short mature window, + # not criteria for long-time turbulent convergence or positivity. + limits = dict(max_abs_f_error=1e-7, relative_l2_f_error=1e-8, relative_l2_e_error=1e-7) + background = np.fromfile(args.reference / "background.bin", dtype=np.float64) + records = [] + args.output.mkdir(parents=True, exist_ok=False) + for placement in args.placements: + nodes, ranks = map(int, placement.split(":")) + for layout in ["transpose", "halos"]: + for steps in args.steps: + candidate = args.output / f"{nodes}n-{ranks}g-{layout}-{steps:04d}.bin" + command = [ + *shlex.split(args.launcher.format(nodes=nodes, ranks=ranks, ranks_per_node=ranks // nodes)), + str(args.binary.resolve()), + "--nx", + str(nx), + "--nv", + str(nv), + "--length", + "12600", + "--vmax", + "15", + "--dt", + str(dt), + "--nu", + str(nu), + "--cells", + "--velocity", + "cubic", + "--integrator", + "sixth", + "--layout", + layout, + "--steps", + str(steps), + "--warmup", + "0", + "--input", + str(args.reference / "initial.bin"), + "--v-grid", + str(args.reference / "v.bin"), + "--background", + str(args.reference / "background.bin"), + "--dump", + str(candidate), + ] + result = subprocess.run(command, text=True, capture_output=True, timeout=300) + if result.returncode: + raise RuntimeError(f"{' '.join(command)}\n{result.stdout}\n{result.stderr}") + metrics = compare( + candidate, args.reference / f"reference-{steps:04d}.bin", nx, nv, 30.0 / nv, 12600.0, background + ) + passed = ( + all(metrics[key] <= threshold for key, threshold in limits.items()) and metrics["hole_counts_match"] + ) + row = dict( + nodes=nodes, + ranks=ranks, + layout=layout, + steps=steps, + **metrics, + passed=passed, + limits=limits, + benchmark=json.loads(result.stdout.strip().splitlines()[-1]), + command=command, + rank_log=result.stderr, + ) + records.append(row) + (args.output / "comparison.json").write_text(json.dumps(records, indent=2) + "\n") + print( + json.dumps({key: row[key] for key in ["nodes", "ranks", "layout", "steps", *metrics, "passed"]}), + flush=True, + ) + if not passed: + raise RuntimeError("native checkpoint comparison did not meet predeclared matching limits") + + +if __name__ == "__main__": + main() diff --git a/benchmarks/vlasov_cuda/distributed_reference.py b/benchmarks/vlasov_cuda/distributed_reference.py new file mode 100644 index 00000000..62930ebc --- /dev/null +++ b/benchmarks/vlasov_cuda/distributed_reference.py @@ -0,0 +1,160 @@ +"""Advance a verified prepared native state with pinned ADEPT over GPU ranks. + +Used when the production collision workspace does not fit four 40GB GPUs. +All writes go under a new --output; the prepared input is read-only. +""" + +import argparse +import copy +import importlib.metadata +import json +import os +import socket +import sys +import time +from pathlib import Path + +os.environ.setdefault("XLA_PYTHON_CLIENT_PREALLOCATE", "false") +os.environ.setdefault("JAX_ENABLE_X64", "true") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--prepared", type=Path, required=True) + parser.add_argument("--config", type=Path, required=True) + parser.add_argument("--vp-source", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--steps", type=int, nargs="+", default=[1, 100]) + args = parser.parse_args() + sys.path.insert(0, str(args.vp_source)) + import jax + + jax.config.update("jax_enable_x64", True) + jax.distributed.initialize(local_device_ids=[int(os.environ["SLURM_LOCALID"])]) + import jax.numpy as jnp + import numpy as np + import yaml + from checkpoint_reference import save_vx, sha + from jax.experimental import multihost_utils + from jax.sharding import Mesh, NamedSharding + from jax.sharding import PartitionSpec as P + from vp_turbulence.adeptmodules import VPTwoStream1D + from vp_turbulence.multinode import initialize_module_sharded + + rank, count = jax.process_index(), jax.process_count() + if count != 8 or len(jax.local_devices()) != 1: + raise RuntimeError("expected eight ranks with one GPU each") + d = importlib.metadata.distribution("adept") + commit = json.loads(d.read_text("direct_url.json")).get("vcs_info", {}).get("commit_id") + if commit != "ddd70b957dc3df85f0b5ec84af927c25d09d24e0" or jax.__version__ != "0.10.2": + raise RuntimeError("incorrect pinned numerical environment") + metadata = json.loads((args.prepared / "provenance.json").read_text()) + if sha(args.config) != metadata["config_sha256"]: + raise RuntimeError("configuration differs from verified prepared input") + nx, nv = metadata["nx"], metadata["nv"] + if (args.prepared / "initial.bin").stat().st_size != nx * nv * 8: + raise RuntimeError("incorrect prepared state size") + if rank == 0: + args.output.mkdir(parents=True, exist_ok=False) + for name in ["initial.bin", "v.bin", "background.bin"]: + os.link(args.prepared / name, args.output / name) + metadata.update( + process_count=count, + devices=[str(d) for d in jax.devices()], + distributed=True, + milestones=[], + prepared_input=str(args.prepared), + ) + (args.output / "provenance.json").write_text(json.dumps(metadata, indent=2) + "\n") + multihost_utils.sync_global_devices("output-created") + (args.output / f"rank-{rank:02d}.json").write_text( + json.dumps( + dict( + rank=rank, + host=socket.gethostname(), + pid=os.getpid(), + local_devices=[str(d) for d in jax.local_devices()], + ) + ) + + "\n" + ) + cfg = yaml.safe_load(args.config.read_text()) + module = VPTwoStream1D(copy.deepcopy(cfg)) + module.write_units() + module.get_derived_quantities() + initialize_module_sharded(module) + v = np.fromfile(args.prepared / "v.bin", dtype=np.float64) + np.testing.assert_array_equal(v, np.asarray(module.cfg["grid"]["species_grids"]["electron"]["v"])) + np.testing.assert_array_equal( + np.fromfile(args.prepared / "background.bin", dtype=np.float64), np.asarray(module.cfg["grid"]["ion_charge"]) + ) + source = np.memmap(args.prepared / "initial.bin", mode="r", dtype=np.float64, shape=(nv, nx)) + mesh = Mesh(np.array(jax.devices()), ("device",)) + sharding = NamedSharding(mesh, P("device", None)) + f = jax.make_array_from_callback((nx, nv), sharding, lambda index: source.T[index]) + module.state = {"electron": f} + module.init_diffeqsolve() + operator = module.diffeqsolve_quants["terms"].vector_field.vpfp + if operator.fp._sc_max_steps != 0: + raise RuntimeError("unexpected self-consistent-beta iterations") + replicated = NamedSharding(mesh, P()) + + def replicate(value): + host = np.asarray(value) + return jax.make_array_from_callback(host.shape, replicated, lambda index: host[index]) + + a = replicate(np.zeros(nx + 2)) + prev = replicate(np.zeros(nx)) + external = replicate(np.zeros((6, nx))) + nu_fp = replicate(np.full(nx, metadata["nu"])) + nu_k = replicate(np.zeros(nx)) + + @jax.jit + def advance(values, iterations): + def body(_, current): + _, result, _ = operator({"electron": current}, a, prev, external, nu_fp, nu_k) + return result["electron"] + + return jax.lax.fori_loop(0, iterations, body, values) + + compiled = advance.lower(f, replicate(np.asarray(1, dtype=np.int32))).compile() + if rank == 0: + print(json.dumps(dict(status="compiled", nx=nx, nv=nv, ranks=count)), flush=True) + previous = 0 + for stop in sorted(set(args.steps)): + start = time.perf_counter() + f = compiled(f, replicate(np.asarray(stop - previous, dtype=np.int32))) + f.block_until_ready() + elapsed = time.perf_counter() - start + shards = f.addressable_shards + if len(shards) != 1: + raise RuntimeError("unexpected endpoint layout") + shard = shards[0] + values = np.asarray(shard.data) + if not np.isfinite(values).all(): + raise RuntimeError("nonfinite reference result") + np.save(args.output / f"step-{stop:04d}-rank-{rank:02d}.npy", values, allow_pickle=False) + (args.output / f"step-{stop:04d}-rank-{rank:02d}.json").write_text( + json.dumps(dict(start=shard.index[0].start or 0, stop=shard.index[0].stop or nx, seconds=elapsed)) + "\n" + ) + multihost_utils.sync_global_devices(f"shards-saved-{stop}") + if rank == 0: + whole = np.empty((nx, nv), dtype=np.float64) + for r in range(count): + part = json.loads((args.output / f"step-{stop:04d}-rank-{r:02d}.json").read_text()) + whole[part["start"] : part["stop"]] = np.load( + args.output / f"step-{stop:04d}-rank-{r:02d}.npy", mmap_mode="r" + ) + save_vx(args.output / f"reference-{stop:04d}.bin", whole) + del whole + row = dict(steps=stop, advanced_steps=stop - previous, rank0_seconds=elapsed) + metadata["milestones"].append(row) + (args.output / "provenance.json").write_text(json.dumps(metadata, indent=2) + "\n") + print(json.dumps(row), flush=True) + multihost_utils.sync_global_devices(f"reference-saved-{stop}") + previous = stop + jax.distributed.shutdown() + + +if __name__ == "__main__": + main() diff --git a/benchmarks/vlasov_cuda/integration_check.py b/benchmarks/vlasov_cuda/integration_check.py new file mode 100644 index 00000000..53e01f34 --- /dev/null +++ b/benchmarks/vlasov_cuda/integration_check.py @@ -0,0 +1,187 @@ +"""Exercise the native ADEPT/ergoExo path, original callbacks and NetCDF writers.""" + +import argparse +import copy +import json +import os +import sys +from pathlib import Path + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--library", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--ranks", nargs="+", type=int, default=[1, 4, 8]) + parser.add_argument("--vp-source", type=Path) + parser.add_argument("--module", choices=["base", "vp", "both"], default="both") + parser.add_argument("--ions", action="store_true", help="Validate kinetic protons using ADEPT's species config") + parser.add_argument("--steps", type=int, default=5) + parser.add_argument("--gpus-per-node", type=int, default=4) + parser.add_argument("--host-mpi", action="store_true") + parser.add_argument( + "--per-rank-gpu", action="store_true", help="Expose one GPU per worker; validate host-staged MPI" + ) + args = parser.parse_args() + if args.per_rank_gpu or args.host_mpi: + os.environ["MPICH_GPU_SUPPORT_ENABLED"] = "0" + os.environ["JAX_PLATFORMS"] = "cpu" + os.environ["JAX_ENABLE_X64"] = "true" + os.environ["MLFLOW_ALLOW_FILE_STORE"] = "true" + import numpy as np + import yaml + from diffrax import SubSaveAt + + from adept import ergoExo + from adept import patched_mlflow as mlflow + from adept._vlasov1d.storage import store_f, store_fields + from adept.vlasov1d import BaseVlasov1D + + args.output.mkdir(parents=True, exist_ok=False) + mlflow.set_tracking_uri((args.output / "mlruns").as_uri()) + classes = [] if args.module == "vp" else [BaseVlasov1D] + if args.vp_source: + sys.path.insert(0, str(args.vp_source)) + from vp_turbulence.adeptmodules import VPTwoStream1D + + if args.module != "base": + classes.append(VPTwoStream1D) + checks = [] + for cls in classes: + for ranks in args.ranks: + cfg = yaml.safe_load(Path("tests/test_base/configs/example.yaml").read_text()) + cfg["grid"].update(nx=32, nv=64, dt=0.02, tmax=0.02 * (args.steps - 1), vmax=8.0) + cfg["drivers"] = {"ex": {}, "ey": {}} + cfg["terms"]["time"] = "sixth" + cfg["terms"]["fokker_planck"].update(is_on=True, type="chang_cooper_dougherty") + cfg["terms"]["fokker_planck"]["time"]["baseline"] = 0.001 + cfg["terms"]["fokker_planck"]["space"]["baseline"] = 1.0 + cfg["terms"]["krook"]["is_on"] = False + cfg["save"] = {"fields": {"t": {"nt": 5}}, "electron": {"main": {"t": {"nt": 3}}}} + beam = cfg["density"]["species-background"] + beam.update(v0=3.0, T0=0.2, noise_val=0.001) + cfg["density"]["species-left"] = copy.deepcopy(beam) + cfg["density"]["species-left"].update(v0=-3.0, noise_seed=43) + if args.ions: + electron = copy.deepcopy(beam) + electron.update(v0=0.0, T0=1.0, noise_val=0.0) + ion = copy.deepcopy(electron) + ion.update(T0=0.1, basis="sine", baseline=1.0, amplitude=0.03, wavenumber=2 * np.pi / 20.94) + cfg["density"] = {"quasineutrality": True, "species-electrons": electron, "species-ions": ion} + cfg["terms"]["species"] = [ + dict( + name="ion", + charge=1.0, + mass=1836.0, + nv=32, + vmax=6 * np.sqrt(0.1 / 1836.0), + density_components=["species-ions"], + ), + dict( + name="electron", + charge=-1.0, + mass=1.0, + nv=64, + vmax=8.0, + density_components=["species-electrons"], + ), + ] + cfg["save"]["ion"] = {"main": {"t": {"nt": 3}}} + cfg["native"] = dict(workdir=str(args.output)) + if cls.__name__ == "VPTwoStream1D": + cfg["two_stream"] = dict( + v_b=3.0, + delta_v_b=0.5, + native_cascade_budget=True, + native_kv_spectrum=dict(is_on=True, t=dict(tmin=0.0, tmax=0.1, nt=3)), + ) + cfg["postprocess"] = { + key: False + for key in [ + "plot_scalars", + "plot_fields", + "plot_fullres_dists", + "plot_lowres_dists", + "plot_transformed_dists", + "plot_field_spectrum", + ] + } + original = cls(copy.deepcopy(cfg)) + original.write_units() + original.get_derived_quantities() + original.get_solver_quantities() + original.init_state_and_args() + original.init_diffeqsolve() + nodes = (ranks + args.gpus_per_node - 1) // args.gpus_per_node + cfg["backend"] = "native" + cfg["native"].update( + device="cuda", + library=str(args.library), + ranks=ranks, + timeout=180, + host_mpi=args.per_rank_gpu or args.host_mpi, + launcher=[ + "env", + "-u", + "CUDA_VISIBLE_DEVICES", + "srun", + f"--jobid={os.environ['SLURM_JOB_ID']}", + "--overlap", + f"--nodes={nodes}", + "--ntasks={ranks}", + f"--ntasks-per-node={ranks // nodes}", + f"--gpus-per-node={args.gpus_per_node}", + "--gpu-bind=single:1" if args.per_rank_gpu else "--gpu-bind=none", + "--cpu-bind=none", + ], + ) + cfg["mlflow"] = dict(experiment="native-adept-integration", run=f"{cls.__name__}-{ranks}") + exo = ergoExo() + modules = exo.setup(copy.deepcopy(cfg), adept_module=cls) + output, post, run_id = exo(modules) + for key, save in exo.adept_module.diffeqsolve_quants["saveat"]["subs"].items(): + subs = original.diffeqsolve_quants["saveat"]["subs"] + subs[key] = SubSaveAt(ts=save.ts, fn=subs[key].fn) + reference = original({})["solver result"] + root = args.output / f"{cls.__name__}-{ranks}" + (root / "binary").mkdir(parents=True) + sol = output["solver result"] + store_fields(exo.adept_module.cfg, str(root / "binary"), sol.ys["fields"], sol.ts["fields"], "fields") + store_f(exo.adept_module.cfg, sol.ts, str(root), sol.ys) + errors = [] + import jax + + for key in sol.ys: + np.testing.assert_allclose(reference.ts[key], sol.ts[key], atol=1e-14) + for a, b in zip(jax.tree.leaves(reference.ys[key]), jax.tree.leaves(sol.ys[key]), strict=True): + np.testing.assert_allclose(a, b, rtol=1e-10, atol=1e-11, equal_nan=True, err_msg=key) + delta = np.not_equal(a, b) if np.asarray(a).dtype.kind == "b" else np.asarray(a) - np.asarray(b) + errors.append(float(np.max(np.abs(delta))) if delta.size else 0.0) + row = dict( + module=cls.__name__, + ranks=ranks, + passed=True, + max_abs_error=max(errors), + mlflow_run_id=run_id, + stats=output["native stats"], + keys=list(sol.ys), + ) + if args.per_rank_gpu: + placement = output["native stats"]["placement"] + assert all(worker["device_ordinal"] == 0 for worker in placement), placement + assert all(len(worker["cuda_visible_devices"].split(",")) == 1 for worker in placement), placement + checks.append(row) + if args.ions: + changes = { + name: float(np.max(np.abs(sol.ys[f"{name}.main"][-1] - sol.ys[f"{name}.main"][0]))) + for name in ["electron", "ion"] + } + assert all(change > 1e-9 for change in changes.values()), changes + row["distribution_changes"] = changes + (args.output / "checks.json").write_text(json.dumps(checks, indent=2)) + print(json.dumps(row), flush=True) + (args.output / "checks.json").write_text(json.dumps(checks, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/vlasov_cuda/local_cubic.hpp b/benchmarks/vlasov_cuda/local_cubic.hpp new file mode 100644 index 00000000..0dbdc4e9 --- /dev/null +++ b/benchmarks/vlasov_cuda/local_cubic.hpp @@ -0,0 +1,2 @@ +// Compatibility include; the implementation ships with ADEPT. +#include "../../adept/_vlasov1d/native/core/local_cubic.hpp" diff --git a/benchmarks/vlasov_cuda/main.cpp b/benchmarks/vlasov_cuda/main.cpp new file mode 100644 index 00000000..c4b45a40 --- /dev/null +++ b/benchmarks/vlasov_cuda/main.cpp @@ -0,0 +1,122 @@ +#include "../../adept/_vlasov1d/native/core/solver.hpp" + +using namespace adept_native; + +int main(int argc, char** argv) { + try { + prepare_device_from_environment(); + } catch (const std::exception& error) { + std::cerr << "device binding before MPI_Init: " << error.what() << '\n'; + return 1; + } + MPI_Init(&argc, &argv); + MPI_Comm_set_errhandler(MPI_COMM_WORLD, MPI_ERRORS_RETURN); + int rank = 0; + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + try { + Options o = parse(argc, argv, rank); + if (o.mode == "help") { MPI_Finalize(); return 0; } + Grid& g = o.grid; + g.collisions = o.nu > 0; + g.rank = rank; + mpi_check(MPI_Comm_size(MPI_COMM_WORLD, &g.ranks)); + if (g.nx < 8 || g.nv < 4 || g.nx % g.ranks || g.nv % g.ranks) + throw std::runtime_error("nx >= 8 and nv >= 4 must both be divisible by MPI ranks"); + if (g.count() > std::size_t(std::numeric_limits::max())) + throw std::runtime_error("local state exceeds this prototype's MPI int count limit"); + if (o.steps < 1 || o.warmup < 0 || o.dt <= 0 || g.vmax <= 0 || g.length <= 0 || std::abs(g.amplitude) >= 1) + throw std::runtime_error("require steps>0, warmup>=0, dt/vmax/length>0 and |amplitude|<1"); + if (o.mode != "landau" && o.mode != "stream" && o.mode != "accelerate" && o.mode != "transpose" && o.mode != "collide") + throw std::runtime_error("unknown mode: " + o.mode); + if (o.integrator != "strang" && o.integrator != "sixth") throw std::runtime_error("unknown integrator"); + if (g.halos && (!g.local_cubic || g.halo_capacity < 2)) + throw std::runtime_error("halos require --velocity cubic and capacity >= 2"); + if (g.halos && o.mode == "transpose") + throw std::runtime_error("transpose-only mode requires --layout transpose"); + if (o.nu < 0 || (o.nu > 0 && !g.cell_centred)) + throw std::runtime_error("collisions require nonnegative --nu and --cells"); + if (std::size_t(g.halo_capacity) * g.nx > std::size_t(std::numeric_limits::max())) + throw std::runtime_error("halo exceeds MPI int count limit"); + MPI_Comm local; + mpi_check(MPI_Comm_split_type(MPI_COMM_WORLD, MPI_COMM_TYPE_SHARED, rank, MPI_INFO_NULL, &local)); + int local_rank; + mpi_check(MPI_Comm_rank(local, &local_rank)); + int is_head = local_rank == 0, nodes; + mpi_check(MPI_Allreduce(&is_head, &nodes, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD)); + char hostname[MPI_MAX_PROCESSOR_NAME]; + int host_length; + mpi_check(MPI_Get_processor_name(hostname, &host_length)); + { + Backend backend(g, local_rank); + std::cerr << "rank=" << rank << " host=" << std::string(hostname, host_length) + << " local_rank=" << local_rank << " device=" << backend.device_name() << '\n'; + Stepper stepper(backend, o); + if (!o.coordinates.empty()) backend.set_coordinates(read_binary(o.coordinates, g.nv)); + if (!o.background.empty()) backend.set_background(read_binary(o.background, g.nx)); + initialize(backend, o); + for (int i = 0; i < o.warmup; ++i) stepper.step(); + // Warmup is excluded from both timings and reported physical time. + initialize(backend, o); + auto initial = diagnostics(backend, stepper, g); + stepper.times.fill(0.0); + stepper.halo_sent_bytes = 0; + stepper.slab_sent_bytes = 0; + stepper.max_halo_width = 0; + backend.synchronize(); + mpi_check(MPI_Barrier(MPI_COMM_WORLD)); + for (int i = 0; i < o.steps; ++i) stepper.step(); + std::array times; + mpi_check(MPI_Reduce(stepper.times.data(), times.data(), StageCount, + MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD)); + auto final = diagnostics(backend, stepper, g); + if (!initial.finite || !final.finite) throw std::runtime_error("nonfinite distribution"); + if (!o.dump.empty()) dump(backend, g, o.dump); + std::uint64_t allocated = backend.allocated_bytes(), max_allocated; + std::uint64_t max_halo_bytes; + mpi_check(MPI_Reduce(&stepper.halo_sent_bytes, &max_halo_bytes, 1, MPI_UINT64_T, MPI_MAX, 0, MPI_COMM_WORLD)); + mpi_check(MPI_Reduce(&allocated, &max_allocated, 1, MPI_UINT64_T, MPI_MAX, 0, MPI_COMM_WORLD)); + if (rank == 0) { + const char* names[] = {"stream_ms", "spline_ms", "pack_ms", "mpi_ms", "field_ms", "collision_ms", "step_ms"}; + std::cout << std::setprecision(16) << "{\"backend\":" +#ifdef VLASOV_CUDA + << "\"cuda\"" +#else + << "\"fftw\"" +#endif + << ",\"device\":" << quoted(backend.device_name()) + << ",\"precision\":\"float64\",\"mode\":" << quoted(o.mode) + << ",\"integrator\":" << quoted(o.integrator) + << ",\"nu\":" << o.nu + << ",\"velocity\":" << quoted(g.local_cubic ? "cubic" : "natural") + << ",\"layout\":" << quoted(g.halos ? "halos" : "transpose") + << ",\"cell_centred\":" << (g.cell_centred ? "true" : "false") + << ",\"max_halo_width\":" << stepper.max_halo_width + << ",\"rank0_halo_sent_bytes\":" << stepper.halo_sent_bytes + << ",\"halo_sent_bytes_per_rank_max\":" << max_halo_bytes + << ",\"slab_sent_bytes_per_rank\":" << stepper.slab_sent_bytes + << ",\"input\":" << quoted(o.input) + << ",\"nx\":" << g.nx << ",\"nv\":" << g.nv << ",\"ranks\":" << g.ranks + << ",\"nodes\":" << nodes << ",\"steps\":" << o.steps << ",\"warmup\":" << o.warmup + << ",\"dt\":" << o.dt << ",\"length\":" << g.length << ",\"vmax\":" << g.vmax + << ",\"amplitude\":" << g.amplitude << ",\"acceleration\":" << o.acceleration + << ",\"host_mpi\":" << (o.host_mpi ? "true" : "false") + << ",\"state_bytes\":" << std::uint64_t(g.nx) * g.nv * sizeof(double) + << ",\"max_allocated_bytes_per_rank\":" << max_allocated; + for (int i = 0; i < StageCount; ++i) + std::cout << ",\"" << names[i] << "\":" << times[i] * 1000.0 / o.steps; + std::cout << ",\"initial_mass\":" << initial.mass << ",\"final_mass\":" << final.mass + << ",\"mass_relative_change\":" << (final.mass - initial.mass) / initial.mass + << ",\"initial_kinetic_energy\":" << initial.kinetic << ",\"final_kinetic_energy\":" << final.kinetic + << ",\"initial_field_energy\":" << initial.field << ",\"final_field_energy\":" << final.field + << ",\"minimum_f\":" << final.minimum << ",\"finite\":true}\n"; + } + } // Destroy CUDA resources before MPI finalization. + mpi_check(MPI_Comm_free(&local)); + MPI_Finalize(); + return 0; + } catch (const std::exception& error) { + std::cerr << "rank " << rank << ": " << error.what() << '\n'; + MPI_Abort(MPI_COMM_WORLD, 1); + return 1; + } +} diff --git a/benchmarks/vlasov_cuda/measure.py b/benchmarks/vlasov_cuda/measure.py new file mode 100644 index 00000000..ed7d6c85 --- /dev/null +++ b/benchmarks/vlasov_cuda/measure.py @@ -0,0 +1,124 @@ +"""Run repeated strong-scaling measurements with an MPI launcher template. + +Placements have the form nodes:ranks. For Slurm, pass an explicit job ID, +--overlap, --nodes={nodes}, --ntasks={ranks}, and +--ntasks-per-node={ranks_per_node}. This script never allocates resources. +""" + +import argparse +import json +import shlex +import subprocess +from pathlib import Path + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("binary", type=Path) + parser.add_argument("--launcher", default="mpiexec -n {ranks}") + parser.add_argument("--placements", nargs="+", default=["1:1", "1:2", "1:4"]) + parser.add_argument("--nx", type=int, default=32768) + parser.add_argument("--nv", type=int, default=8192) + parser.add_argument("--steps", type=int, default=20) + parser.add_argument("--warmup", type=int, default=3) + parser.add_argument("--repeats", type=int, default=3) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--source-state", type=Path) + parser.add_argument("--host-mpi", action="store_true") + parser.add_argument("--expect-cuda", action="store_true") + parser.add_argument("--velocity", choices=["natural", "cubic"], default="natural") + parser.add_argument("--layout", choices=["transpose", "halos"], default="transpose") + parser.add_argument("--integrator", choices=["strang", "sixth"], default="strang") + parser.add_argument("--cells", action="store_true") + parser.add_argument("--input", type=Path) + parser.add_argument("--v-grid", type=Path) + parser.add_argument("--background", type=Path) + parser.add_argument("--nu", type=float, default=0.0) + parser.add_argument("--length", type=float, default=12.566370614359172) + parser.add_argument("--vmax", type=float, default=8.0) + parser.add_argument("--dt", type=float, default=0.1) + parser.add_argument( + "--max-mass-change", + type=float, + default=1e-8, + help="negative records conservation diagnostically without an absolute cutoff", + ) + args = parser.parse_args() + placements = [] + for placement in args.placements: + nodes, ranks = map(int, placement.split(":")) + if nodes < 1 or ranks < 1 or ranks % nodes: + parser.error("placements must have positive nodes/ranks and equal ranks per node") + placements.append((nodes, ranks)) + if args.repeats < 1: + parser.error("repeats must be positive") + provenance = args.source_state.read_text() if args.source_state else None + # Exclusive create prevents accidentally replacing an earlier measurement. + with args.output.open("x") as output: + for nodes, ranks in placements: + for repeat in range(args.repeats): + command = [ + *shlex.split(args.launcher.format(nodes=nodes, ranks=ranks, ranks_per_node=ranks // nodes)), + str(args.binary.resolve()), + "--nx", + str(args.nx), + "--nv", + str(args.nv), + "--steps", + str(args.steps), + "--warmup", + str(args.warmup), + "--mode", + "landau", + ] + if args.host_mpi: + command.append("--host-mpi") + command.extend( + [ + "--velocity", + args.velocity, + "--layout", + args.layout, + "--integrator", + args.integrator, + "--nu", + str(args.nu), + "--length", + str(args.length), + "--vmax", + str(args.vmax), + "--dt", + str(args.dt), + ] + ) + if args.cells: + command.append("--cells") + for key in ["input", "v_grid", "background"]: + value = getattr(args, key) + if value: + command.extend(["--" + key.replace("_", "-"), str(value)]) + completed = subprocess.run(command, capture_output=True, text=True, timeout=300) + if completed.returncode: + raise RuntimeError(f"{' '.join(command)}\n{completed.stdout}\n{completed.stderr}") + row = json.loads(completed.stdout.strip().splitlines()[-1]) + if row["nodes"] != nodes or row["ranks"] != ranks: + raise RuntimeError(f"placement mismatch: {row}") + if args.expect_cuda and row["backend"] != "cuda": + raise RuntimeError("expected the CUDA backend") + if (args.max_mass_change >= 0 and abs(row["mass_relative_change"]) > args.max_mass_change) or not row[ + "finite" + ]: + raise RuntimeError(f"invalid physical output: {row}") + row.update(repeat=repeat, command=command, rank_log=completed.stderr, source_state=provenance) + output.write(json.dumps(row) + "\n") + output.flush() + print( + json.dumps( + {key: row[key] for key in ["nodes", "ranks", "repeat", "step_ms", "spline_ms", "mpi_ms"]} + ), + flush=True, + ) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/vlasov_cuda/reference/latest-nv4096/nx134400.yaml b/benchmarks/vlasov_cuda/reference/latest-nv4096/nx134400.yaml new file mode 100644 index 00000000..16753567 --- /dev/null +++ b/benchmarks/vlasov_cuda/reference/latest-nv4096/nx134400.yaml @@ -0,0 +1,168 @@ +solver: vp-two-stream-1d +mlflow: + experiment: vp-turbulence + run: nv4096-fine-20261023-nx134400-nv4096-t600-seed42 +units: + normalizing_density: 1.0e20/cc + normalizing_temperature: 1000eV +grid: + nx: 134400 + nv: 4096 + xmin: 0.0 + xmax: 12600.0 + tmin: 0.0 + tmax: 600.0 + vmax: 15.0 + dt: 0.02 + parallel: + - x + - v +two_stream: + v_b: 3.0 + delta_v_b: 0.03 + noise_val: 1.0e-06 + save_e_only: true + native_cascade_budget: true + native_kv_spectrum: + is_on: true + t: + tmin: 0.0 + tmax: 600.0 + nt: 61 +density: + quasineutrality: true + beam-right: + basis: uniform + noise_seed: 42 + noise_type: band_limited_uniform + noise_reference_nx: 51610 + noise_val: 1.0e-06 + v0: 3.0 + T0: 0.00405 + m: 100.0 + baseline: 0.5 + bump_or_trough: bump + center: 0.0 + rise: 25.0 + bump_height: 0.0 + width: 100000.0 + noise_max_mode: 25804 + beam-left: + basis: uniform + noise_seed: 43 + noise_type: band_limited_uniform + noise_reference_nx: 51610 + noise_val: 1.0e-06 + v0: -3.0 + T0: 0.00405 + m: 100.0 + baseline: 0.5 + bump_or_trough: bump + center: 0.0 + rise: 25.0 + bump_height: 0.0 + width: 100000.0 + noise_max_mode: 25804 +terms: + field: poisson + edfdv: cubic-spline + time: sixth + hou_li_filter: + is_on: false + alpha: 36.0 + order: 36 + dimensions: + - x + species: + - name: electron + charge: -1.0 + mass: 1.0 + vmax: 15.0 + nv: 4096 + density_components: + - beam-right + - beam-left + fokker_planck: + is_on: true + type: chang_cooper_dougherty + time: + baseline: 1.0e-12 + bump_or_trough: bump + center: 0.0 + rise: 25.0 + slope: 0.0 + bump_height: 0.0 + width: 100000.0 + space: + baseline: 1.0 + bump_or_trough: bump + center: 0.0 + rise: 25.0 + slope: 0.0 + bump_height: 0.0 + width: 100000.0 + krook: + is_on: false + time: + baseline: 0.0 + bump_or_trough: bump + center: 0.0 + rise: 25.0 + slope: 0.0 + bump_height: 0.0 + width: 100000.0 + space: + baseline: 1.0 + bump_or_trough: bump + center: 0.0 + rise: 25.0 + slope: 0.0 + bump_height: 0.0 + width: 100000.0 + vdfdx: exponential +drivers: + ex: {} + ey: {} +diagnostics: + diag-vlasov-dfdt: false + diag-fp-dfdt: false +postprocess: + plot_scalars: false + plot_fields: false + plot_field_spectrum: false + plot_lowres_dists: false + plot_fullres_dists: false + plot_transformed_dists: false +save: + fields: + t: + tmin: 0.0 + tmax: 600.0 + nt: 301 + electron: + lowres: + t: + tmin: 0.0 + tmax: 600.0 + nt: 61 + x: + xmin: 0.046875 + xmax: 12599.953125 + nx: 2048 + v: + vmin: -14.996337890625 + vmax: 14.996337890625 + nv: 256 + full: + t: + tmin: 0.0 + tmax: 600.0 + nt: 7 + x: + xmin: 0.046875 + xmax: 12599.953125 + nx: 8192 + v: + vmin: -14.996337890625 + vmax: 14.996337890625 + nv: 1024 diff --git a/benchmarks/vlasov_cuda/reference/latest-nv4096/nx268800.yaml b/benchmarks/vlasov_cuda/reference/latest-nv4096/nx268800.yaml new file mode 100644 index 00000000..5c956dd6 --- /dev/null +++ b/benchmarks/vlasov_cuda/reference/latest-nv4096/nx268800.yaml @@ -0,0 +1,166 @@ +solver: vp-two-stream-1d +mlflow: + experiment: vp-turbulence + run: nv4096-fine-20261023-nx268800-nv4096-t600-seed42 +units: + normalizing_density: 1.0e20/cc + normalizing_temperature: 1000eV +grid: + nx: 268800 + nv: 4096 + xmin: 0.0 + xmax: 12600.0 + tmin: 0.0 + tmax: 600.0 + vmax: 15.0 + dt: 0.02 + parallel: + - x + - v +two_stream: + v_b: 3.0 + delta_v_b: 0.03 + noise_val: 1.0e-06 + save_e_only: true + native_cascade_budget: true + native_kv_spectrum: + is_on: true + t: + tmin: 0.0 + tmax: 600.0 + nt: 61 +density: + quasineutrality: true + beam-right: + basis: uniform + noise_seed: 42 + noise_type: band_limited_uniform + noise_reference_nx: 51610 + noise_val: 1.0e-06 + v0: 3.0 + T0: 0.00405 + m: 100.0 + baseline: 0.5 + bump_or_trough: bump + center: 0.0 + rise: 25.0 + bump_height: 0.0 + width: 100000.0 + beam-left: + basis: uniform + noise_seed: 43 + noise_type: band_limited_uniform + noise_reference_nx: 51610 + noise_val: 1.0e-06 + v0: -3.0 + T0: 0.00405 + m: 100.0 + baseline: 0.5 + bump_or_trough: bump + center: 0.0 + rise: 25.0 + bump_height: 0.0 + width: 100000.0 +terms: + field: poisson + edfdv: cubic-spline + time: sixth + hou_li_filter: + is_on: false + alpha: 36.0 + order: 36 + dimensions: + - x + species: + - name: electron + charge: -1.0 + mass: 1.0 + vmax: 15.0 + nv: 4096 + density_components: + - beam-right + - beam-left + fokker_planck: + is_on: true + type: chang_cooper_dougherty + time: + baseline: 1.0e-12 + bump_or_trough: bump + center: 0.0 + rise: 25.0 + slope: 0.0 + bump_height: 0.0 + width: 100000.0 + space: + baseline: 1.0 + bump_or_trough: bump + center: 0.0 + rise: 25.0 + slope: 0.0 + bump_height: 0.0 + width: 100000.0 + krook: + is_on: false + time: + baseline: 0.0 + bump_or_trough: bump + center: 0.0 + rise: 25.0 + slope: 0.0 + bump_height: 0.0 + width: 100000.0 + space: + baseline: 1.0 + bump_or_trough: bump + center: 0.0 + rise: 25.0 + slope: 0.0 + bump_height: 0.0 + width: 100000.0 + vdfdx: exponential +drivers: + ex: {} + ey: {} +diagnostics: + diag-vlasov-dfdt: false + diag-fp-dfdt: false +postprocess: + plot_scalars: false + plot_fields: false + plot_field_spectrum: false + plot_lowres_dists: false + plot_fullres_dists: false + plot_transformed_dists: false +save: + fields: + t: + tmin: 0.0 + tmax: 600.0 + nt: 301 + electron: + lowres: + t: + tmin: 0.0 + tmax: 600.0 + nt: 61 + x: + xmin: 0.0234375 + xmax: 12599.9765625 + nx: 2048 + v: + vmin: -14.996337890625 + vmax: 14.996337890625 + nv: 256 + full: + t: + tmin: 0.0 + tmax: 600.0 + nt: 7 + x: + xmin: 0.0234375 + xmax: 12599.9765625 + nx: 8192 + v: + vmin: -14.996337890625 + vmax: 14.996337890625 + nv: 1024 diff --git a/benchmarks/vlasov_cuda/reference/latest-nv4096/target.json b/benchmarks/vlasov_cuda/reference/latest-nv4096/target.json new file mode 100644 index 00000000..92f78b07 --- /dev/null +++ b/benchmarks/vlasov_cuda/reference/latest-nv4096/target.json @@ -0,0 +1,39 @@ +{ + "campaign": "nv4096-t2000-20261023-nx134400", + "git_commit": "2c4904968009d695e2625fd264f057dd9c846039", + "config_sha256": "5a9f5347e6253a3d4201745318843857c568b3787cf91e9fe81a623d0cdbb214", + "pinned_adept_commit": "ddd70b957dc3df85f0b5ec84af927c25d09d24e0", + "pinned_jax": "0.10.2", + "matched_ladder": { + "nx": 134400, + "L": 12600.0, + "nv": 4096, + "dt": 0.02, + "beam_seeds": [ + 42, + 43 + ], + "noise_reference_nx": 51610, + "noise_max_mode_both_beams": 25804, + "noise_value": 1e-06, + "noise_rms_renormalized": false, + "nyquist_mode_dropped": true, + "projection": "Same seeded reference Fourier coefficients; truncate to representable conjugate pairs, retain cell-centre phase and amplitudes." + }, + "members": [ + { + "nx": 134400, + "nv": 4096, + "parent_run_id": "aa042900211640d9a90f20001772f957", + "config": "nx134400.yaml", + "config_sha256": "5a9f5347e6253a3d4201745318843857c568b3787cf91e9fe81a623d0cdbb214" + }, + { + "nx": 268800, + "nv": 4096, + "parent_run_id": "916dc390224d4c14a417d772991a2812", + "config": "nx268800.yaml", + "config_sha256": "6adb42855432b79a74f63290678103c17c53842074d77bf092df38483f634a83" + } + ] +} diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-adept/README.md b/benchmarks/vlasov_cuda/results/2026-10-24-adept/README.md new file mode 100644 index 00000000..357ca7ef --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-adept/README.md @@ -0,0 +1,48 @@ +# ADEPT/JAX versus native CUDA on A100s + +The native backend is integrated into ADEPT (`backend: native`). It retains +`ergoExo`, ordinary/custom module initialization, original save callbacks and +xarray/NetCDF postprocessing. Standard saves use actual timestep boundaries; +outputs are streamed to host memory maps. The C++ physics core is shared with +the standalone timing executable. + +| Grid | Nodes / A100s | ADEPT/JAX ms/step | Native ms/step | Speedup | +|---|---:|---:|---:|---:| +| 134400 Ă— 4096 | 1 / 4 | 223.54 | 174.24 | 1.283Ă— | +| 134400 Ă— 4096 | 2 / 8 | 383.94 | 140.07 | 2.741Ă— | +| 268800 Ă— 4096 | 2 / 8 | 756.70 | 275.99 | 2.742Ă— | + +FP64, spectral x, local cubic C1 v, sixth-order splitting, dt=0.02, +Chang-Cooper Dougherty nu=1e-12. Both paths reset to the same mature t600 +checkpoint before each repeat. Median of three repeats, ten timed steps, +three warmup steps. Timings exclude compilation, preprocessing, save callbacks, +I/O and postprocessing; these are physics-step speedups, not complete-run +wall-clock speedups. No concurrent benchmark workloads were run. + +Allocation 60221398 used nid001600 and nid003745, four A100-SXM4-40GB cards +per node. ADEPT was pinned to ddd70b957dc3df85f0b5ec84af927c25d09d24e0, +JAX/JAXlib 0.10.2, interpax 0.3.14+10.g84aaf30, Diffrax 0.7.2. +The four-GPU pair was rerun with both solvers pinned to nid001600 after an +audit found the initial pair used different nodes. Excluded measurements are +retained under `unmatched-node/`; the final pair's provenance is under +`pinned-one-node/`. Both eight-GPU pairs used the same physical nodes/GPUs. + +Five native CPU integration tests and 33 existing configuration/save/logging +regressions passed. Seven GPU integration cases passed: ordinary ADEPT and +VPTwoStream1D at 1/4/8 ranks, plus the latest native velocity-spectrum and +cascade callbacks at eight ranks. Each compares original save callbacks at +the actual native save times (rtol=1e-10, atol=1e-11). The largest absolute +error was 1.16e-11 across all saved diagnostics; ordinary ADEPT saves were +within 1.32e-13. `observer-netcdf/` contains outputs from the unchanged +VPTwoStream1D postprocessor; `observer-native/` contains its MLflow provenance. +The initial `integration-v3.log` also records a subsequently fixed callback +constant serialization failure; its three ordinary ADEPT cases passed. + +`summary.json` is rebuilt by `python summarize.py`, which verifies source +archive hashes, matched hosts, finite outputs, endpoint aggregate checks and +all seven integration results. Raw repeated measurements, software versions, +checkpoint provenance, command lines and executable hashes are adjacent. +Pointwise full-grid t600-to-t602 validation of the shared numerical core is +in `../2026-10-24-halos/`; a full t0-to-2000 trajectory has not been tested. +The NERSC session's files remain under +`/pscratch/sd/a/archis/adept-sessions/20261024T190159Z-99199-22637/`. diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-adept/binaries.sha256 b/benchmarks/vlasov_cuda/results/2026-10-24-adept/binaries.sha256 new file mode 100644 index 00000000..9bfe9c31 --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-adept/binaries.sha256 @@ -0,0 +1,2 @@ +13429e0e2873ab774bdb695874836b4133750a4f36bc96d20bd22ab73aa104a4 /pscratch/sd/a/archis/adept-sessions/20261024T190159Z-99199-22637/workdir/cuda-build/vlasov_bench +8ea49ba7a81a1e0074e35e9f63ae73680f38005adc7b49acd12343b7e7951dd7 /pscratch/sd/a/archis/adept-sessions/20261024T190159Z-99199-22637/workdir/native-build/libadept_vlasov.so diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-adept/cuda-nx134400-g4.jsonl b/benchmarks/vlasov_cuda/results/2026-10-24-adept/cuda-nx134400-g4.jsonl new file mode 100644 index 00000000..8dfcec5a --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-adept/cuda-nx134400-g4.jsonl @@ -0,0 +1,3 @@ +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "integrator": "sixth", "nu": 1e-12, "velocity": "cubic", "layout": "halos", "cell_centred": true, "max_halo_width": 4, "rank0_halo_sent_bytes": 215040000, "halo_sent_bytes_per_rank_max": 430080000, "slab_sent_bytes_per_rank": 16515072000, "input": "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/initial.bin", "nx": 134400, "nv": 4096, "ranks": 4, "nodes": 1, "steps": 10, "warmup": 3, "dt": 0.02, "length": 12600, "vmax": 15, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 4404019200, "max_allocated_bytes_per_rank": 11155281976, "stream_ms": 96.76214629996593, "spline_ms": 29.39172449987382, "pack_ms": 6.529895999938162, "mpi_ms": 21.04299519983215, "field_ms": 6.631393699944965, "collision_ms": 14.49398829993243, "step_ms": 174.2776944999605, "initial_mass": 12597.6196507818, "final_mass": 12597.62016522545, "mass_relative_change": 4.083657516783715e-08, "initial_kinetic_energy": 55713.43281125277, "final_kinetic_energy": 55714.15660222915, "initial_field_energy": 787.6538181371966, "final_field_energy": 786.9554651665927, "minimum_f": -0.1916308738890627, "finite": true, "repeat": 0, "command": ["srun", "--jobid=60221398", "--overlap", "--nodes=1", "--nodelist=nid001600", "--ntasks=4", "--ntasks-per-node=4", "--gpus-per-node=4", "--gpu-bind=none", "--cpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T190159Z-99199-22637/workdir/cuda-build/vlasov_bench", "--nx", "134400", "--nv", "4096", "--steps", "10", "--warmup", "3", "--mode", "landau", "--velocity", "cubic", "--layout", "halos", "--integrator", "sixth", "--nu", "1e-12", "--length", "12600.0", "--vmax", "15.0", "--dt", "0.02", "--cells", "--input", "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/initial.bin", "--v-grid", "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/v.bin", "--background", "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/background.bin"], "rank_log": "rank=0 host=nid001600 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001600 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001600 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001600 local_rank=1 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=c0a2abe12f3d288f697c8b183b00313aa4c6ba063d5122a570dfae4a37751380\nsynced_at=2026-10-24T19:31:46Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "integrator": "sixth", "nu": 1e-12, "velocity": "cubic", "layout": "halos", "cell_centred": true, "max_halo_width": 4, "rank0_halo_sent_bytes": 215040000, "halo_sent_bytes_per_rank_max": 430080000, "slab_sent_bytes_per_rank": 16515072000, "input": "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/initial.bin", "nx": 134400, "nv": 4096, "ranks": 4, "nodes": 1, "steps": 10, "warmup": 3, "dt": 0.02, "length": 12600, "vmax": 15, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 4404019200, "max_allocated_bytes_per_rank": 11155281976, "stream_ms": 96.7626505999533, "spline_ms": 29.38959880000311, "pack_ms": 6.514031399979103, "mpi_ms": 21.03211520004606, "field_ms": 6.66575620000458, "collision_ms": 14.49633700001414, "step_ms": 174.2364576000227, "initial_mass": 12597.6196507818, "final_mass": 12597.62016522545, "mass_relative_change": 4.083657516783715e-08, "initial_kinetic_energy": 55713.43281125277, "final_kinetic_energy": 55714.15660222915, "initial_field_energy": 787.6538181371966, "final_field_energy": 786.9554651665927, "minimum_f": -0.1916308738890627, "finite": true, "repeat": 1, "command": ["srun", "--jobid=60221398", "--overlap", "--nodes=1", "--nodelist=nid001600", "--ntasks=4", "--ntasks-per-node=4", "--gpus-per-node=4", "--gpu-bind=none", "--cpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T190159Z-99199-22637/workdir/cuda-build/vlasov_bench", "--nx", "134400", "--nv", "4096", "--steps", "10", "--warmup", "3", "--mode", "landau", "--velocity", "cubic", "--layout", "halos", "--integrator", "sixth", "--nu", "1e-12", "--length", "12600.0", "--vmax", "15.0", "--dt", "0.02", "--cells", "--input", "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/initial.bin", "--v-grid", "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/v.bin", "--background", "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/background.bin"], "rank_log": "rank=1 host=nid001600 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001600 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001600 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001600 local_rank=0 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=c0a2abe12f3d288f697c8b183b00313aa4c6ba063d5122a570dfae4a37751380\nsynced_at=2026-10-24T19:31:46Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "integrator": "sixth", "nu": 1e-12, "velocity": "cubic", "layout": "halos", "cell_centred": true, "max_halo_width": 4, "rank0_halo_sent_bytes": 215040000, "halo_sent_bytes_per_rank_max": 430080000, "slab_sent_bytes_per_rank": 16515072000, "input": "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/initial.bin", "nx": 134400, "nv": 4096, "ranks": 4, "nodes": 1, "steps": 10, "warmup": 3, "dt": 0.02, "length": 12600, "vmax": 15, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 4404019200, "max_allocated_bytes_per_rank": 11155281976, "stream_ms": 96.76016480002545, "spline_ms": 29.38630270000431, "pack_ms": 6.511509500023749, "mpi_ms": 21.00970499996038, "field_ms": 6.653158599988274, "collision_ms": 14.5029157999943, "step_ms": 174.2416196000249, "initial_mass": 12597.6196507818, "final_mass": 12597.62016522545, "mass_relative_change": 4.083657516783715e-08, "initial_kinetic_energy": 55713.43281125277, "final_kinetic_energy": 55714.15660222915, "initial_field_energy": 787.6538181371966, "final_field_energy": 786.9554651665927, "minimum_f": -0.1916308738890627, "finite": true, "repeat": 2, "command": ["srun", "--jobid=60221398", "--overlap", "--nodes=1", "--nodelist=nid001600", "--ntasks=4", "--ntasks-per-node=4", "--gpus-per-node=4", "--gpu-bind=none", "--cpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T190159Z-99199-22637/workdir/cuda-build/vlasov_bench", "--nx", "134400", "--nv", "4096", "--steps", "10", "--warmup", "3", "--mode", "landau", "--velocity", "cubic", "--layout", "halos", "--integrator", "sixth", "--nu", "1e-12", "--length", "12600.0", "--vmax", "15.0", "--dt", "0.02", "--cells", "--input", "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/initial.bin", "--v-grid", "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/v.bin", "--background", "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/background.bin"], "rank_log": "rank=1 host=nid001600 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001600 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001600 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001600 local_rank=2 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=c0a2abe12f3d288f697c8b183b00313aa4c6ba063d5122a570dfae4a37751380\nsynced_at=2026-10-24T19:31:46Z\n"} diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-adept/cuda-nx134400-g8.jsonl b/benchmarks/vlasov_cuda/results/2026-10-24-adept/cuda-nx134400-g8.jsonl new file mode 100644 index 00000000..a86d7ad8 --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-adept/cuda-nx134400-g8.jsonl @@ -0,0 +1,3 @@ +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "integrator": "sixth", "nu": 1e-12, "velocity": "cubic", "layout": "halos", "cell_centred": true, "max_halo_width": 4, "rank0_halo_sent_bytes": 215040000, "halo_sent_bytes_per_rank_max": 430080000, "slab_sent_bytes_per_rank": 9633792000, "input": "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/initial.bin", "nx": 134400, "nv": 4096, "ranks": 8, "nodes": 2, "steps": 10, "warmup": 3, "dt": 0.02, "length": 12600, "vmax": 15, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 4404019200, "max_allocated_bytes_per_rank": 5650233400, "stream_ms": 48.50647130010657, "spline_ms": 12.24466279991248, "pack_ms": 3.281769600107509, "mpi_ms": 60.21116610008903, "field_ms": 6.163560300092286, "collision_ms": 11.23915750001743, "step_ms": 140.0714161999531, "initial_mass": 12597.61965078436, "final_mass": 12597.62016522508, "mass_relative_change": 4.083634327505363e-08, "initial_kinetic_energy": 55713.43281126581, "final_kinetic_energy": 55714.15660220217, "initial_field_energy": 787.6538181371961, "final_field_energy": 786.9554651665925, "minimum_f": -0.1916308738890627, "finite": true, "repeat": 0, "command": ["srun", "--jobid=60221398", "--overlap", "--nodes=2", "--ntasks=8", "--ntasks-per-node=4", "--gpus-per-node=4", "--gpu-bind=none", "--cpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T190159Z-99199-22637/workdir/cuda-build/vlasov_bench", "--nx", "134400", "--nv", "4096", "--steps", "10", "--warmup", "3", "--mode", "landau", "--velocity", "cubic", "--layout", "halos", "--integrator", "sixth", "--nu", "1e-12", "--length", "12600.0", "--vmax", "15.0", "--dt", "0.02", "--cells", "--input", "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/initial.bin", "--v-grid", "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/v.bin", "--background", "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/background.bin"], "rank_log": "rank=3 host=nid003745 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=5 host=nid003745 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid003745 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=7 host=nid003745 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001600 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001600 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=6 host=nid001600 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=4 host=nid001600 local_rank=2 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=afdc248c4553148cd3ca5a63cb0431b8e474acd835d818af386e898beb75eb2d\nsynced_at=2026-10-24T19:13:57Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "integrator": "sixth", "nu": 1e-12, "velocity": "cubic", "layout": "halos", "cell_centred": true, "max_halo_width": 4, "rank0_halo_sent_bytes": 215040000, "halo_sent_bytes_per_rank_max": 430080000, "slab_sent_bytes_per_rank": 9633792000, "input": "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/initial.bin", "nx": 134400, "nv": 4096, "ranks": 8, "nodes": 2, "steps": 10, "warmup": 3, "dt": 0.02, "length": 12600, "vmax": 15, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 4404019200, "max_allocated_bytes_per_rank": 5650233400, "stream_ms": 48.50949399988167, "spline_ms": 12.24919390003834, "pack_ms": 3.278222700009792, "mpi_ms": 60.14856769998005, "field_ms": 5.799373300033039, "collision_ms": 11.23889549996875, "step_ms": 140.015343999994, "initial_mass": 12597.61965078436, "final_mass": 12597.62016522508, "mass_relative_change": 4.083634327505363e-08, "initial_kinetic_energy": 55713.43281126581, "final_kinetic_energy": 55714.15660220217, "initial_field_energy": 787.6538181371961, "final_field_energy": 786.9554651665925, "minimum_f": -0.1916308738890627, "finite": true, "repeat": 1, "command": ["srun", "--jobid=60221398", "--overlap", "--nodes=2", "--ntasks=8", "--ntasks-per-node=4", "--gpus-per-node=4", "--gpu-bind=none", "--cpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T190159Z-99199-22637/workdir/cuda-build/vlasov_bench", "--nx", "134400", "--nv", "4096", "--steps", "10", "--warmup", "3", "--mode", "landau", "--velocity", "cubic", "--layout", "halos", "--integrator", "sixth", "--nu", "1e-12", "--length", "12600.0", "--vmax", "15.0", "--dt", "0.02", "--cells", "--input", "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/initial.bin", "--v-grid", "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/v.bin", "--background", "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/background.bin"], "rank_log": "rank=2 host=nid001600 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001600 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid003745 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=6 host=nid001600 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=4 host=nid001600 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid003745 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=5 host=nid003745 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=7 host=nid003745 local_rank=3 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=afdc248c4553148cd3ca5a63cb0431b8e474acd835d818af386e898beb75eb2d\nsynced_at=2026-10-24T19:13:57Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "integrator": "sixth", "nu": 1e-12, "velocity": "cubic", "layout": "halos", "cell_centred": true, "max_halo_width": 4, "rank0_halo_sent_bytes": 215040000, "halo_sent_bytes_per_rank_max": 430080000, "slab_sent_bytes_per_rank": 9633792000, "input": "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/initial.bin", "nx": 134400, "nv": 4096, "ranks": 8, "nodes": 2, "steps": 10, "warmup": 3, "dt": 0.02, "length": 12600, "vmax": 15, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 4404019200, "max_allocated_bytes_per_rank": 5650233400, "stream_ms": 48.49759509997966, "spline_ms": 12.24273310001536, "pack_ms": 3.282368400095947, "mpi_ms": 60.46683179993124, "field_ms": 6.560649900029603, "collision_ms": 11.23777200000404, "step_ms": 140.3418015999932, "initial_mass": 12597.61965078436, "final_mass": 12597.62016522508, "mass_relative_change": 4.083634327505363e-08, "initial_kinetic_energy": 55713.43281126581, "final_kinetic_energy": 55714.15660220217, "initial_field_energy": 787.6538181371961, "final_field_energy": 786.9554651665925, "minimum_f": -0.1916308738890627, "finite": true, "repeat": 2, "command": ["srun", "--jobid=60221398", "--overlap", "--nodes=2", "--ntasks=8", "--ntasks-per-node=4", "--gpus-per-node=4", "--gpu-bind=none", "--cpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T190159Z-99199-22637/workdir/cuda-build/vlasov_bench", "--nx", "134400", "--nv", "4096", "--steps", "10", "--warmup", "3", "--mode", "landau", "--velocity", "cubic", "--layout", "halos", "--integrator", "sixth", "--nu", "1e-12", "--length", "12600.0", "--vmax", "15.0", "--dt", "0.02", "--cells", "--input", "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/initial.bin", "--v-grid", "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/v.bin", "--background", "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/background.bin"], "rank_log": "rank=0 host=nid001600 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=6 host=nid001600 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001600 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid003745 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=5 host=nid003745 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=4 host=nid001600 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid003745 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=7 host=nid003745 local_rank=3 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=afdc248c4553148cd3ca5a63cb0431b8e474acd835d818af386e898beb75eb2d\nsynced_at=2026-10-24T19:13:57Z\n"} diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-adept/cuda-nx268800-g8.jsonl b/benchmarks/vlasov_cuda/results/2026-10-24-adept/cuda-nx268800-g8.jsonl new file mode 100644 index 00000000..705db02d --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-adept/cuda-nx268800-g8.jsonl @@ -0,0 +1,3 @@ +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "integrator": "sixth", "nu": 1e-12, "velocity": "cubic", "layout": "halos", "cell_centred": true, "max_halo_width": 4, "rank0_halo_sent_bytes": 438681600, "halo_sent_bytes_per_rank_max": 877363200, "slab_sent_bytes_per_rank": 19267584000, "input": "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/initial.bin", "nx": 268800, "nv": 4096, "ranks": 8, "nodes": 2, "steps": 10, "warmup": 3, "dt": 0.02, "length": 12600, "vmax": 15, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 8808038400, "max_allocated_bytes_per_rank": 11300409400, "stream_ms": 97.32425760009846, "spline_ms": 30.11911129995042, "pack_ms": 6.521739400091064, "mpi_ms": 119.9787687000708, "field_ms": 10.82064670004002, "collision_ms": 14.49070120004308, "step_ms": 276.0648734999677, "initial_mass": 12597.62207265221, "final_mass": 12597.62136207489, "mass_relative_change": -5.640567068345301e-08, "initial_kinetic_energy": 55701.51067942529, "final_kinetic_energy": 55701.00688716807, "initial_field_energy": 791.8606180599771, "final_field_energy": 792.3077830250799, "minimum_f": -0.09580882788649095, "finite": true, "repeat": 0, "command": ["srun", "--jobid=60221398", "--overlap", "--nodes=2", "--ntasks=8", "--ntasks-per-node=4", "--gpus-per-node=4", "--gpu-bind=none", "--cpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T190159Z-99199-22637/workdir/cuda-build/vlasov_bench", "--nx", "268800", "--nv", "4096", "--steps", "10", "--warmup", "3", "--mode", "landau", "--velocity", "cubic", "--layout", "halos", "--integrator", "sixth", "--nu", "1e-12", "--length", "12600.0", "--vmax", "15.0", "--dt", "0.02", "--cells", "--input", "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/initial.bin", "--v-grid", "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/v.bin", "--background", "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/background.bin"], "rank_log": "rank=7 host=nid003745 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=6 host=nid001600 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001600 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=4 host=nid001600 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001600 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid003745 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=5 host=nid003745 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid003745 local_rank=1 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=37396dd0852f9e6ff790f730c645eaa6681006a0a135e7039737d9472112a7d3\nsynced_at=2026-10-24T19:19:12Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "integrator": "sixth", "nu": 1e-12, "velocity": "cubic", "layout": "halos", "cell_centred": true, "max_halo_width": 4, "rank0_halo_sent_bytes": 438681600, "halo_sent_bytes_per_rank_max": 877363200, "slab_sent_bytes_per_rank": 19267584000, "input": "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/initial.bin", "nx": 268800, "nv": 4096, "ranks": 8, "nodes": 2, "steps": 10, "warmup": 3, "dt": 0.02, "length": 12600, "vmax": 15, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 8808038400, "max_allocated_bytes_per_rank": 11300409400, "stream_ms": 97.33070519996545, "spline_ms": 30.12465930000872, "pack_ms": 6.524045600053796, "mpi_ms": 119.7810640999251, "field_ms": 10.77050900000813, "collision_ms": 14.49306849997356, "step_ms": 275.9853768999847, "initial_mass": 12597.62207265221, "final_mass": 12597.62136207489, "mass_relative_change": -5.640567068345301e-08, "initial_kinetic_energy": 55701.51067942529, "final_kinetic_energy": 55701.00688716807, "initial_field_energy": 791.8606180599771, "final_field_energy": 792.3077830250799, "minimum_f": -0.09580882788649095, "finite": true, "repeat": 1, "command": ["srun", "--jobid=60221398", "--overlap", "--nodes=2", "--ntasks=8", "--ntasks-per-node=4", "--gpus-per-node=4", "--gpu-bind=none", "--cpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T190159Z-99199-22637/workdir/cuda-build/vlasov_bench", "--nx", "268800", "--nv", "4096", "--steps", "10", "--warmup", "3", "--mode", "landau", "--velocity", "cubic", "--layout", "halos", "--integrator", "sixth", "--nu", "1e-12", "--length", "12600.0", "--vmax", "15.0", "--dt", "0.02", "--cells", "--input", "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/initial.bin", "--v-grid", "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/v.bin", "--background", "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/background.bin"], "rank_log": "rank=0 host=nid001600 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid003745 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid003745 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=7 host=nid003745 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=5 host=nid003745 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001600 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=6 host=nid001600 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=4 host=nid001600 local_rank=2 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=37396dd0852f9e6ff790f730c645eaa6681006a0a135e7039737d9472112a7d3\nsynced_at=2026-10-24T19:19:12Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "integrator": "sixth", "nu": 1e-12, "velocity": "cubic", "layout": "halos", "cell_centred": true, "max_halo_width": 4, "rank0_halo_sent_bytes": 438681600, "halo_sent_bytes_per_rank_max": 877363200, "slab_sent_bytes_per_rank": 19267584000, "input": "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/initial.bin", "nx": 268800, "nv": 4096, "ranks": 8, "nodes": 2, "steps": 10, "warmup": 3, "dt": 0.02, "length": 12600, "vmax": 15, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 8808038400, "max_allocated_bytes_per_rank": 11300409400, "stream_ms": 97.31154520000018, "spline_ms": 30.11697609997555, "pack_ms": 6.524896600012653, "mpi_ms": 119.2875023999704, "field_ms": 10.904615400068, "collision_ms": 14.49597959997391, "step_ms": 275.6066252999744, "initial_mass": 12597.62207265221, "final_mass": 12597.62136207489, "mass_relative_change": -5.640567068345301e-08, "initial_kinetic_energy": 55701.51067942529, "final_kinetic_energy": 55701.00688716807, "initial_field_energy": 791.8606180599771, "final_field_energy": 792.3077830250799, "minimum_f": -0.09580882788649095, "finite": true, "repeat": 2, "command": ["srun", "--jobid=60221398", "--overlap", "--nodes=2", "--ntasks=8", "--ntasks-per-node=4", "--gpus-per-node=4", "--gpu-bind=none", "--cpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T190159Z-99199-22637/workdir/cuda-build/vlasov_bench", "--nx", "268800", "--nv", "4096", "--steps", "10", "--warmup", "3", "--mode", "landau", "--velocity", "cubic", "--layout", "halos", "--integrator", "sixth", "--nu", "1e-12", "--length", "12600.0", "--vmax", "15.0", "--dt", "0.02", "--cells", "--input", "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/initial.bin", "--v-grid", "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/v.bin", "--background", "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/background.bin"], "rank_log": "rank=3 host=nid003745 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid003745 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=7 host=nid003745 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=5 host=nid003745 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001600 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001600 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=4 host=nid001600 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=6 host=nid001600 local_rank=3 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=37396dd0852f9e6ff790f730c645eaa6681006a0a135e7039737d9472112a7d3\nsynced_at=2026-10-24T19:19:12Z\n"} diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-adept/integration-checks.json b/benchmarks/vlasov_cuda/results/2026-10-24-adept/integration-checks.json new file mode 100644 index 00000000..6c396f06 --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-adept/integration-checks.json @@ -0,0 +1,210 @@ +[ + { + "module": "BaseVlasov1D", + "ranks": 1, + "passed": true, + "max_abs_error": 1.0302869668521453e-13, + "mlflow_run_id": "10db23e28ed8428d9849bce2dea93d3f", + "stats": { + "native_seconds": { + "stream": 0.000619517000018277, + "velocity": 0.0003302759998859983, + "pack": 0.0, + "mpi": 0.0008770139997977822, + "field": 0.0017125090001854915, + "collision": 0.0005039860001261331, + "total": 0.004283324999960314 + }, + "solve_and_save_seconds": 0.7859312820073683, + "steps": 5, + "ranks": 1, + "device": "cuda" + }, + "keys": [ + "fields", + "electron.main", + "default" + ], + "log": "integration-v3.log" + }, + { + "module": "BaseVlasov1D", + "ranks": 4, + "passed": true, + "max_abs_error": 1.2434497875801753e-13, + "mlflow_run_id": "c15bc81503a44c02bb7f0d22043aa3df", + "stats": { + "native_seconds": { + "stream": 0.0005902340002421624, + "velocity": 0.0003807449994610579, + "pack": 0.00023884999984602473, + "mpi": 0.005609786999912103, + "field": 0.0067621719998669505, + "collision": 0.000502029000017501, + "total": 0.013536288000068453 + }, + "solve_and_save_seconds": 1.881096900004195, + "steps": 5, + "ranks": 4, + "device": "cuda" + }, + "keys": [ + "fields", + "electron.main", + "default" + ], + "log": "integration-v3.log" + }, + { + "module": "BaseVlasov1D", + "ranks": 8, + "passed": true, + "max_abs_error": 1.3145040611561853e-13, + "mlflow_run_id": "fb178ca1d165436c98457fd4ba0db80f", + "stats": { + "native_seconds": { + "stream": 0.0006939400003602714, + "velocity": 0.0005676090001998091, + "pack": 0.00025299700007508363, + "mpi": 0.01576393800007736, + "field": 0.013597956999888083, + "collision": 0.0005164489999742727, + "total": 0.02774611500001356 + }, + "solve_and_save_seconds": 3.1865445940056816, + "steps": 5, + "ranks": 8, + "device": "cuda" + }, + "keys": [ + "fields", + "electron.main", + "default" + ], + "log": "integration-v3.log" + }, + { + "module": "VPTwoStream1D", + "ranks": 1, + "passed": true, + "max_abs_error": 1.0457412713549274e-11, + "mlflow_run_id": "ac6e44af25584f74add8c11f040061c7", + "stats": { + "native_seconds": { + "stream": 0.0005986490004943334, + "velocity": 0.0003280549998407878, + "pack": 0.0, + "mpi": 0.0008833609997509484, + "field": 0.0016982180000013614, + "collision": 0.000500530999829607, + "total": 0.004253136000102131 + }, + "solve_and_save_seconds": 0.8431226150132716, + "steps": 5, + "ranks": 1, + "device": "cuda" + }, + "keys": [ + "fields", + "electron.main", + "default", + "electron.f0" + ], + "log": "integration-vp.log" + }, + { + "module": "VPTwoStream1D", + "ranks": 4, + "passed": true, + "max_abs_error": 6.3788974102862994e-12, + "mlflow_run_id": "d16748ce2f364431bb4fa17896661f44", + "stats": { + "native_seconds": { + "stream": 0.0006106250001494118, + "velocity": 0.000375073000213888, + "pack": 0.00023151799985043908, + "mpi": 0.005738596999663059, + "field": 0.006173489000275367, + "collision": 0.0005022090000466051, + "total": 0.012953435000099489 + }, + "solve_and_save_seconds": 1.350150696001947, + "steps": 5, + "ranks": 4, + "device": "cuda" + }, + "keys": [ + "fields", + "electron.main", + "default", + "electron.f0" + ], + "log": "integration-vp.log" + }, + { + "module": "VPTwoStream1D", + "ranks": 8, + "passed": true, + "max_abs_error": 1.1592504733926035e-11, + "mlflow_run_id": "e3e58e103a6747489eadadb244806865", + "stats": { + "native_seconds": { + "stream": 0.0006200880002324993, + "velocity": 0.0004060329994217682, + "pack": 0.0002525769999692784, + "mpi": 0.008024029000125665, + "field": 0.007057047000103012, + "collision": 0.000508114000012938, + "total": 0.01669021899999734 + }, + "solve_and_save_seconds": 7.212047922017518, + "steps": 5, + "ranks": 8, + "device": "cuda" + }, + "keys": [ + "fields", + "electron.main", + "default", + "electron.f0" + ], + "log": "integration-vp.log" + }, + { + "module": "VPTwoStream1D", + "ranks": 8, + "passed": true, + "max_abs_error": 1.1592504733926035e-11, + "mlflow_run_id": "a31a0009b9a440c79965f3e70332ccde", + "stats": { + "native_seconds": { + "stream": 0.000619704999962778, + "velocity": 0.0004217190003146243, + "pack": 0.00027711899952009844, + "mpi": 0.008916388000216102, + "field": 0.010889343000826557, + "collision": 0.0005323509999470843, + "total": 0.02131122699961452 + }, + "solve_and_save_seconds": 3.074192091007717, + "steps": 5, + "ranks": 8, + "device": "cuda", + "libraries": { + "jax": "0.10.2", + "jaxlib": "0.10.2", + "cloudpickle": "3.1.2", + "diffrax": "0.7.2" + } + }, + "keys": [ + "fields", + "electron.main", + "default", + "electron.f0", + "electron.pkv", + "electron.cascade" + ], + "log": "integration-observers-v2.log" + } +] diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-adept/integration-observers-v2.log b/benchmarks/vlasov_cuda/results/2026-10-24-adept/integration-observers-v2.log new file mode 100644 index 00000000..8e7fe526 --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-adept/integration-observers-v2.log @@ -0,0 +1,12 @@ +2026/10/24 12:27:01 INFO mlflow.tracking.fluent: Experiment with name 'native-adept-integration' does not exist. Creating a new experiment. +Successfully removed /pscratch/sd/a/archis/tmp/tmp2g3h194q +2026/10/24 12:27:04 INFO mlflow.system_metrics.system_metrics_monitor: Started monitoring system metrics. +/pscratch/sd/a/archis/vp-turbulence-sessions/20261023T081121Z-nv4096-nx134400-5ce4fe14/src/vp_turbulence/adeptmodules.py:811: UserWarning: Saving distribution for 'electron.main' at full resolution. + f_result = store_f( +Successfully removed /pscratch/sd/a/archis/tmp/tmp_gtq6flt +2026/10/24 12:27:23 INFO mlflow.system_metrics.system_metrics_monitor: Stopping system metrics monitoring... +2026/10/24 12:27:23 INFO mlflow.system_metrics.system_metrics_monitor: Successfully terminated system metrics monitoring! + 0.00%| | [00:00 + main() + ~~~~^^ + File "/pscratch/sd/a/archis/adept-sessions/20261024T190159Z-99199-22637/src/benchmarks/vlasov_cuda/integration_check.py", line 96, in main + output, post, run_id = exo(modules) + ~~~^^^^^^^^^ + File "/pscratch/sd/a/archis/adept-sessions/20261024T190159Z-99199-22637/src/adept/_base_.py", line 423, in __call__ + run_output = solve(modules, args) + File "/pscratch/sd/a/archis/adept-sessions/20261024T190159Z-99199-22637/src/adept/_vlasov1d/native/module.py", line 317, in __call__ + raise RuntimeError(f"native Vlasov workers failed ({returncode}); log: {log}\n{log.read_text()[-12000:]}") +RuntimeError: native Vlasov workers failed (1); log: /pscratch/sd/a/archis/adept-sessions/20261024T190159Z-99199-22637/outputs/integration-v3/adept-native-5ofo1tiv/worker.log +Traceback (most recent call last): + File "/pscratch/sd/a/archis/adept-sessions/20261024T190159Z-99199-22637/src/adept/_vlasov1d/native/worker.py", line 153, in run + save(key, frame, t, state) + ~~~~^^^^^^^^^^^^^^^^^^^^^^ + File "/pscratch/sd/a/archis/adept-sessions/20261024T190159Z-99199-22637/src/adept/_vlasov1d/native/worker.py", line 83, in save + value = callbacks[key](replicate(np.asarray(t)), state) + File "/pscratch/sd/a/archis/adept-sessions/20261024T190159Z-99199-22637/src/adept/_vlasov1d/native/worker.py", line 66, in + key: jax.jit(lambda t, y, fn=fn: fn(t, y, payload["args"])) for key, fn in payload["callbacks"].items() + ~~^^^^^^^^^^^^^^^^^^^^^^^ + File "/pscratch/sd/a/archis/vp-turbulence-sessions/20261023T081121Z-nv4096-nx134400-5ce4fe14/src/vp_turbulence/adeptmodules.py", line 118, in save + k_peak = k_x[jnp.argmax(e_pow)] + ~~~^^^^^^^^^^^^^^^^^^^ +jax.errors.TracerArrayConversionError: The numpy.ndarray conversion method __array__() was called on traced array with shape int64[] +The error occurred while tracing the function at /pscratch/sd/a/archis/adept-sessions/20261024T190159Z-99199-22637/src/adept/_vlasov1d/native/worker.py:66 for jit. This concrete value was not available in Python because it depends on the value of the argument y['e']. +See https://docs.jax.dev/en/latest/errors.html#jax.errors.TracerArrayConversionError +-------------------- +For simplicity, JAX has removed its internal frames from the traceback of the following exception. Set JAX_TRACEBACK_FILTERING=off to include these. +MPICH ERROR [Rank 0] [job id 60221398.6] [Sat Oct 24 12:12:01 2026] [nid001600] - Abort(1) (rank 0 in comm 0): application called MPI_Abort(MPI_COMM_WORLD, 1) - process 0 + +srun: error: nid001600: task 0: Exited with exit code 1 +srun: Terminating StepId=60221398.6 diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-adept/integration-vp.log b/benchmarks/vlasov_cuda/results/2026-10-24-adept/integration-vp.log new file mode 100644 index 00000000..17de03a6 --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-adept/integration-vp.log @@ -0,0 +1,34 @@ +2026/10/24 12:15:01 INFO mlflow.tracking.fluent: Experiment with name 'native-adept-integration' does not exist. Creating a new experiment. +Successfully removed /pscratch/sd/a/archis/tmp/tmp73h5_i9i +2026/10/24 12:15:04 INFO mlflow.system_metrics.system_metrics_monitor: Started monitoring system metrics. +/pscratch/sd/a/archis/vp-turbulence-sessions/20261023T081121Z-nv4096-nx134400-5ce4fe14/src/vp_turbulence/adeptmodules.py:811: UserWarning: Saving distribution for 'electron.main' at full resolution. + f_result = store_f( +Successfully removed /pscratch/sd/a/archis/tmp/tmpu5zif7m1 +2026/10/24 12:15:18 INFO mlflow.system_metrics.system_metrics_monitor: Stopping system metrics monitoring... +2026/10/24 12:15:18 INFO mlflow.system_metrics.system_metrics_monitor: Successfully terminated system metrics monitoring! + 0.00%| | [00:00/dev/null || true + source /global/common/software/m4490/archis/venvs/vp-turbulence/bin/activate + export PYTHONPATH="$vp_source" + srun --jobid="$SLURM_JOB_ID" --overlap --nodes="$nodes" --ntasks="$ranks" \ + --ntasks-per-node=$((ranks / nodes)) --gpus-per-node=4 --gpu-bind=none \ + --cpu-bind=none --mpi=none --kill-on-bad-exit=1 \ + python -u "$source_root/benchmarks/vlasov_cuda/adept_timing.py" \ + --prepared "$prepared" --config "$source_root/benchmarks/vlasov_cuda/reference/latest-nv4096/nx$nx.yaml" \ + --vp-source "$vp_source" --output "$report_root/jax-nx$nx-g$ranks.json" \ + > "$report_root/jax-nx$nx-g$ranks.log" 2>&1 + ) + python -u benchmarks/vlasov_cuda/measure.py "$EC_SESSION_WORKDIR/cuda-build/vlasov_bench" \ + --launcher "srun --jobid=$SLURM_JOB_ID --overlap --nodes={nodes} --ntasks={ranks} --ntasks-per-node={ranks_per_node} --gpus-per-node=4 --gpu-bind=none --cpu-bind=none" \ + --placements "$nodes:$ranks" --nx "$nx" --nv 4096 --steps 10 --warmup 3 --repeats 3 \ + --expect-cuda --velocity cubic --layout halos --integrator sixth --cells --nu 1e-12 \ + --length 12600 --vmax 15 --dt .02 --input "$prepared/initial.bin" \ + --v-grid "$prepared/v.bin" --background "$prepared/background.bin" --max-mass-change -1 \ + --source-state "$EC_SESSION_ROOT/source-state" --output "$report_root/cuda-nx$nx-g$ranks.jsonl" +done +sha256sum "$EC_SESSION_WORKDIR/cuda-build/vlasov_bench" "$EC_SESSION_WORKDIR/native-build/libadept_vlasov.so" > "$report_root/binaries.sha256" +sha256sum adept/_vlasov1d/native/api.cpp adept/_vlasov1d/native/core/* benchmarks/vlasov_cuda/adept_timing.py > "$report_root/source.sha256" +cp "$EC_SESSION_ROOT/source-state" "$report_root/source-state" +cp benchmarks/vlasov_cuda/run_adept_comparison.sh "$report_root/launch.sh" diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-adept/observer-native/library.sha256 b/benchmarks/vlasov_cuda/results/2026-10-24-adept/observer-native/library.sha256 new file mode 100644 index 00000000..e871ac63 --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-adept/observer-native/library.sha256 @@ -0,0 +1 @@ +8ea49ba7a81a1e0074e35e9f63ae73680f38005adc7b49acd12343b7e7951dd7 libadept_vlasov.so diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-adept/observer-native/request.json b/benchmarks/vlasov_cuda/results/2026-10-24-adept/observer-native/request.json new file mode 100644 index 00000000..fcf91621 --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-adept/observer-native/request.json @@ -0,0 +1,84 @@ +{ + "library": "/pscratch/sd/a/archis/adept-sessions/20261024T190159Z-99199-22637/workdir/native-build/libadept_vlasov.so", + "options": [ + "--nx", + "32", + "--nv", + "64", + "--length", + "20.94", + "--vmax", + "8.0", + "--dt", + "0.02", + "--nu", + "0.001", + "--cells", + "--integrator", + "sixth", + "--velocity", + "cubic", + "--layout", + "halos", + "--halo-capacity", + "64", + "--input", + "/pscratch/sd/a/archis/adept-sessions/20261024T190159Z-99199-22637/outputs/integration-observers-v2/adept-native-gz4p09y5/initial.bin", + "--v-grid", + "/pscratch/sd/a/archis/adept-sessions/20261024T190159Z-99199-22637/outputs/integration-observers-v2/adept-native-gz4p09y5/v.bin", + "--background", + "/pscratch/sd/a/archis/adept-sessions/20261024T190159Z-99199-22637/outputs/integration-observers-v2/adept-native-gz4p09y5/background.bin" + ], + "times": { + "fields": [ + 0.0, + 0.02, + 0.04, + 0.08, + 0.1 + ], + "electron.main": [ + 0.0, + 0.04, + 0.1 + ], + "default": [ + 0.0, + 0.02, + 0.04, + 0.06, + 0.08, + 0.1 + ], + "electron.f0": [ + 0.0 + ], + "electron.pkv": [ + 0.0, + 0.05, + 0.1 + ], + "electron.cascade": [ + 0.0, + 0.05, + 0.1 + ] + }, + "t0": 0.0, + "dt": 0.02, + "steps": 5, + "device": "cuda", + "ranks": 8, + "pythonpath": [ + "/pscratch/sd/a/archis/vp-turbulence-sessions/20261023T081121Z-nv4096-nx134400-5ce4fe14/src", + "/pscratch/sd/a/archis/adept-sessions/20261024T190159Z-99199-22637/src/benchmarks/vlasov_cuda", + "/pscratch/sd/a/archis/adept-sessions/20261024T190159Z-99199-22637/src", + "/pscratch/sd/a/archis/adept-sessions/20261024T190159Z-99199-22637/src/src", + "/opt/nersc/pymon", + "/global/common/software/m4490/archis/uv-python/cpython-3.13.7-linux-x86_64-gnu/lib/python313.zip", + "/global/common/software/m4490/archis/uv-python/cpython-3.13.7-linux-x86_64-gnu/lib/python3.13", + "/global/common/software/m4490/archis/uv-python/cpython-3.13.7-linux-x86_64-gnu/lib/python3.13/lib-dynload", + "/global/common/software/m4490/archis/venvs/adept/lib/python3.13/site-packages", + "/pscratch/sd/a/archis/adept" + ] +} diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-adept/observer-native/stats.json b/benchmarks/vlasov_cuda/results/2026-10-24-adept/observer-native/stats.json new file mode 100644 index 00000000..00632af3 --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-adept/observer-native/stats.json @@ -0,0 +1,21 @@ +{ + "native_seconds": { + "stream": 0.000619704999962778, + "velocity": 0.0004217190003146243, + "pack": 0.00027711899952009844, + "mpi": 0.008916388000216102, + "field": 0.010889343000826557, + "collision": 0.0005323509999470843, + "total": 0.02131122699961452 + }, + "solve_and_save_seconds": 3.074192091007717, + "steps": 5, + "ranks": 8, + "device": "cuda", + "libraries": { + "jax": "0.10.2", + "jaxlib": "0.10.2", + "cloudpickle": "3.1.2", + "diffrax": "0.7.2" + } +} diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-adept/observer-native/worker.log b/benchmarks/vlasov_cuda/results/2026-10-24-adept/observer-native/worker.log new file mode 100644 index 00000000..baa144a6 --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-adept/observer-native/worker.log @@ -0,0 +1,24 @@ +W1024 12:27:21.218968 1217676 pjrt_client.cc:1615] WatchTasksAsync failed for task 0: UNAVAILABLE: failed to connect to all addresses; last error: UNKNOWN: ipv4:128.55.65.28:48901: Failed to connect to remote host: Connection refused +Additional GRPC error information from remote target coordination_service while calling /xla.coordination.CoordinationService/WatchTasks: +:UNKNOWN:Error received from peer {grpc_status:14, grpc_message:"failed to connect to all addresses; last error: UNKNOWN: ipv4:128.55.65.28:48901: Failed to connect to remote host: Connection refused"} +W1024 12:27:21.219021 1217768 pjrt_client.cc:1615] WatchTasksAsync failed for task 2: UNAVAILABLE: failed to connect to all addresses; last error: UNKNOWN: ipv4:128.55.65.28:48901: Failed to connect to remote host: Connection refused +Additional GRPC error information from remote target coordination_service while calling /xla.coordination.CoordinationService/WatchTasks: +:UNKNOWN:Error received from peer {grpc_status:14, grpc_message:"failed to connect to all addresses; last error: UNKNOWN: ipv4:128.55.65.28:48901: Failed to connect to remote host: Connection refused"} +W1024 12:27:21.219259 1219476 pjrt_client.cc:1615] WatchTasksAsync failed for task 5: UNAVAILABLE: failed to connect to all addresses; last error: UNKNOWN: ipv4:128.55.65.28:48901: Failed to connect to remote host: Connection refused +Additional GRPC error information from remote target coordination_service while calling /xla.coordination.CoordinationService/WatchTasks: +:UNKNOWN:Error received from peer {grpc_status:14, grpc_message:"failed to connect to all addresses; last error: UNKNOWN: ipv4:128.55.65.28:48901: Failed to connect to remote host: Connection refused"} +W1024 12:27:21.219087 1217766 pjrt_client.cc:1615] WatchTasksAsync failed for task 6: UNAVAILABLE: failed to connect to all addresses; last error: UNKNOWN: ipv4:128.55.65.28:48901: Failed to connect to remote host: Connection refused +Additional GRPC error information from remote target coordination_service while calling /xla.coordination.CoordinationService/WatchTasks: +:UNKNOWN:Error received from peer {grpc_status:14, grpc_message:"failed to connect to all addresses; last error: UNKNOWN: ipv4:128.55.65.28:48901: Failed to connect to remote host: Connection refused"} +W1024 12:27:21.219457 1219427 pjrt_client.cc:1615] WatchTasksAsync failed for task 7: UNAVAILABLE: failed to connect to all addresses; last error: UNKNOWN: ipv4:128.55.65.28:48901: Failed to connect to remote host: Connection refused +Additional GRPC error information from remote target coordination_service while calling /xla.coordination.CoordinationService/WatchTasks: +:UNKNOWN:Error received from peer {grpc_message:"failed to connect to all addresses; last error: UNKNOWN: ipv4:128.55.65.28:48901: Failed to connect to remote host: Connection refused", grpc_status:14} +W1024 12:27:21.219140 1217770 pjrt_client.cc:1615] WatchTasksAsync failed for task 4: UNAVAILABLE: failed to connect to all addresses; last error: UNKNOWN: ipv4:128.55.65.28:48901: Failed to connect to remote host: Connection refused +Additional GRPC error information from remote target coordination_service while calling /xla.coordination.CoordinationService/WatchTasks: +:UNKNOWN:Error received from peer {grpc_status:14, grpc_message:"failed to connect to all addresses; last error: UNKNOWN: ipv4:128.55.65.28:48901: Failed to connect to remote host: Connection refused"} +W1024 12:27:21.219482 1219470 pjrt_client.cc:1615] WatchTasksAsync failed for task 1: UNAVAILABLE: failed to connect to all addresses; last error: UNKNOWN: ipv4:128.55.65.28:48901: Failed to connect to remote host: Connection refused +Additional GRPC error information from remote target coordination_service while calling /xla.coordination.CoordinationService/WatchTasks: +:UNKNOWN:Error received from peer {grpc_message:"failed to connect to all addresses; last error: UNKNOWN: ipv4:128.55.65.28:48901: Failed to connect to remote host: Connection refused", grpc_status:14} +W1024 12:27:21.219480 1219473 pjrt_client.cc:1615] WatchTasksAsync failed for task 3: UNAVAILABLE: failed to connect to all addresses; last error: UNKNOWN: ipv4:128.55.65.28:48901: Failed to connect to remote host: Connection refused +Additional GRPC error information from remote target coordination_service while calling /xla.coordination.CoordinationService/WatchTasks: +:UNKNOWN:Error received from peer {grpc_message:"failed to connect to all addresses; last error: UNKNOWN: ipv4:128.55.65.28:48901: Failed to connect to remote host: Connection refused", grpc_status:14} diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-adept/observer-netcdf-check.json b/benchmarks/vlasov_cuda/results/2026-10-24-adept/observer-netcdf-check.json new file mode 100644 index 00000000..053112b6 --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-adept/observer-netcdf-check.json @@ -0,0 +1,144 @@ +{ + "dist-electron.f0.nc": { + "sizes": { + "t": 1, + "v_electron": 64 + }, + "variables": [ + "electron.f0" + ], + "times": [ + 0.0 + ] + }, + "dist-electron.main.nc": { + "sizes": { + "t": 3, + "x": 32, + "v_electron": 64 + }, + "variables": [ + "electron.main" + ], + "times": [ + 0.0, + 0.04, + 0.1 + ] + }, + "fields-electron-t=0.1.nc": { + "sizes": { + "t": 5, + "x": 32 + }, + "variables": [ + "fields--flogf", + "fields-f^2", + "fields-j", + "fields-n", + "fields-p", + "fields-q", + "fields-v" + ], + "times": [ + 0.0, + 0.02, + 0.04, + 0.08, + 0.1 + ] + }, + "fields-shared-t=0.1.nc": { + "sizes": { + "t": 5, + "x": 32 + }, + "variables": [ + "fields-e", + "fields-de", + "fields-a", + "fields-prev_a", + "fields-pond" + ], + "times": [ + 0.0, + 0.02, + 0.04, + 0.08, + 0.1 + ] + }, + "native-cascade.nc": { + "sizes": { + "t": 3, + "kx": 17 + }, + "variables": [ + "c2", + "c2_background", + "c2_fluctuation", + "collision_loss_rate_estimate", + "collision_moments_valid", + "density_min", + "edge_absolute_mass", + "f_min", + "invalid_moment_cell_fraction", + "mass", + "negative_mass", + "pkx", + "temperature_min" + ], + "times": [ + 0.0, + 0.05, + 0.1 + ] + }, + "scalars-t=0.1.nc": { + "sizes": { + "t": 6 + }, + "variables": [ + "c2_0", + "delta_c2", + "gamma2", + "mean_-flogf_electron", + "mean_P_electron", + "mean_de2", + "mean_e2", + "mean_f2_electron", + "mean_field_energy", + "mean_j_electron", + "mean_kinetic_energy", + "mean_n_electron", + "mean_pond", + "mean_q_electron", + "mean_total_energy", + "n_holes_kc", + "n_holes_kpeak", + "v_the" + ], + "times": [ + 0.0, + 0.02, + 0.04, + 0.06, + 0.08, + 0.1 + ] + }, + "spectrum-electron.pkv.nc": { + "sizes": { + "t": 3, + "kv_electron": 33 + }, + "variables": [ + "electron.pkv" + ], + "times": [ + 0.0, + 0.05, + 0.1 + ] + } +} diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-adept/observer-netcdf/dist-electron.f0.nc b/benchmarks/vlasov_cuda/results/2026-10-24-adept/observer-netcdf/dist-electron.f0.nc new file mode 100644 index 00000000..85854528 Binary files /dev/null and b/benchmarks/vlasov_cuda/results/2026-10-24-adept/observer-netcdf/dist-electron.f0.nc differ diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-adept/observer-netcdf/dist-electron.main.nc b/benchmarks/vlasov_cuda/results/2026-10-24-adept/observer-netcdf/dist-electron.main.nc new file mode 100644 index 00000000..dcd89f3f Binary files /dev/null and b/benchmarks/vlasov_cuda/results/2026-10-24-adept/observer-netcdf/dist-electron.main.nc differ diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-adept/observer-netcdf/fields-electron-t=0.1.nc b/benchmarks/vlasov_cuda/results/2026-10-24-adept/observer-netcdf/fields-electron-t=0.1.nc new file mode 100644 index 00000000..0175327b Binary files /dev/null and b/benchmarks/vlasov_cuda/results/2026-10-24-adept/observer-netcdf/fields-electron-t=0.1.nc differ diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-adept/observer-netcdf/fields-shared-t=0.1.nc b/benchmarks/vlasov_cuda/results/2026-10-24-adept/observer-netcdf/fields-shared-t=0.1.nc new file mode 100644 index 00000000..cbc273c9 Binary files /dev/null and b/benchmarks/vlasov_cuda/results/2026-10-24-adept/observer-netcdf/fields-shared-t=0.1.nc differ diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-adept/observer-netcdf/native-cascade.nc b/benchmarks/vlasov_cuda/results/2026-10-24-adept/observer-netcdf/native-cascade.nc new file mode 100644 index 00000000..9399ca42 Binary files /dev/null and b/benchmarks/vlasov_cuda/results/2026-10-24-adept/observer-netcdf/native-cascade.nc differ diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-adept/observer-netcdf/scalars-t=0.1.nc b/benchmarks/vlasov_cuda/results/2026-10-24-adept/observer-netcdf/scalars-t=0.1.nc new file mode 100644 index 00000000..5cbbb671 Binary files /dev/null and b/benchmarks/vlasov_cuda/results/2026-10-24-adept/observer-netcdf/scalars-t=0.1.nc differ diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-adept/observer-netcdf/spectrum-electron.pkv.nc b/benchmarks/vlasov_cuda/results/2026-10-24-adept/observer-netcdf/spectrum-electron.pkv.nc new file mode 100644 index 00000000..48145efe Binary files /dev/null and b/benchmarks/vlasov_cuda/results/2026-10-24-adept/observer-netcdf/spectrum-electron.pkv.nc differ diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-adept/pinned-one-node/binaries.sha256 b/benchmarks/vlasov_cuda/results/2026-10-24-adept/pinned-one-node/binaries.sha256 new file mode 100644 index 00000000..9bfe9c31 --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-adept/pinned-one-node/binaries.sha256 @@ -0,0 +1,2 @@ +13429e0e2873ab774bdb695874836b4133750a4f36bc96d20bd22ab73aa104a4 /pscratch/sd/a/archis/adept-sessions/20261024T190159Z-99199-22637/workdir/cuda-build/vlasov_bench +8ea49ba7a81a1e0074e35e9f63ae73680f38005adc7b49acd12343b7e7951dd7 /pscratch/sd/a/archis/adept-sessions/20261024T190159Z-99199-22637/workdir/native-build/libadept_vlasov.so diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-adept/pinned-one-node/cuda-nx134400-g4.jsonl b/benchmarks/vlasov_cuda/results/2026-10-24-adept/pinned-one-node/cuda-nx134400-g4.jsonl new file mode 100644 index 00000000..8dfcec5a --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-adept/pinned-one-node/cuda-nx134400-g4.jsonl @@ -0,0 +1,3 @@ +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "integrator": "sixth", "nu": 1e-12, "velocity": "cubic", "layout": "halos", "cell_centred": true, "max_halo_width": 4, "rank0_halo_sent_bytes": 215040000, "halo_sent_bytes_per_rank_max": 430080000, "slab_sent_bytes_per_rank": 16515072000, "input": "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/initial.bin", "nx": 134400, "nv": 4096, "ranks": 4, "nodes": 1, "steps": 10, "warmup": 3, "dt": 0.02, "length": 12600, "vmax": 15, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 4404019200, "max_allocated_bytes_per_rank": 11155281976, "stream_ms": 96.76214629996593, "spline_ms": 29.39172449987382, "pack_ms": 6.529895999938162, "mpi_ms": 21.04299519983215, "field_ms": 6.631393699944965, "collision_ms": 14.49398829993243, "step_ms": 174.2776944999605, "initial_mass": 12597.6196507818, "final_mass": 12597.62016522545, "mass_relative_change": 4.083657516783715e-08, "initial_kinetic_energy": 55713.43281125277, "final_kinetic_energy": 55714.15660222915, "initial_field_energy": 787.6538181371966, "final_field_energy": 786.9554651665927, "minimum_f": -0.1916308738890627, "finite": true, "repeat": 0, "command": ["srun", "--jobid=60221398", "--overlap", "--nodes=1", "--nodelist=nid001600", "--ntasks=4", "--ntasks-per-node=4", "--gpus-per-node=4", "--gpu-bind=none", "--cpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T190159Z-99199-22637/workdir/cuda-build/vlasov_bench", "--nx", "134400", "--nv", "4096", "--steps", "10", "--warmup", "3", "--mode", "landau", "--velocity", "cubic", "--layout", "halos", "--integrator", "sixth", "--nu", "1e-12", "--length", "12600.0", "--vmax", "15.0", "--dt", "0.02", "--cells", "--input", "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/initial.bin", "--v-grid", "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/v.bin", "--background", "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/background.bin"], "rank_log": "rank=0 host=nid001600 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001600 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001600 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001600 local_rank=1 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=c0a2abe12f3d288f697c8b183b00313aa4c6ba063d5122a570dfae4a37751380\nsynced_at=2026-10-24T19:31:46Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "integrator": "sixth", "nu": 1e-12, "velocity": "cubic", "layout": "halos", "cell_centred": true, "max_halo_width": 4, "rank0_halo_sent_bytes": 215040000, "halo_sent_bytes_per_rank_max": 430080000, "slab_sent_bytes_per_rank": 16515072000, "input": "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/initial.bin", "nx": 134400, "nv": 4096, "ranks": 4, "nodes": 1, "steps": 10, "warmup": 3, "dt": 0.02, "length": 12600, "vmax": 15, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 4404019200, "max_allocated_bytes_per_rank": 11155281976, "stream_ms": 96.7626505999533, "spline_ms": 29.38959880000311, "pack_ms": 6.514031399979103, "mpi_ms": 21.03211520004606, "field_ms": 6.66575620000458, "collision_ms": 14.49633700001414, "step_ms": 174.2364576000227, "initial_mass": 12597.6196507818, "final_mass": 12597.62016522545, "mass_relative_change": 4.083657516783715e-08, "initial_kinetic_energy": 55713.43281125277, "final_kinetic_energy": 55714.15660222915, "initial_field_energy": 787.6538181371966, "final_field_energy": 786.9554651665927, "minimum_f": -0.1916308738890627, "finite": true, "repeat": 1, "command": ["srun", "--jobid=60221398", "--overlap", "--nodes=1", "--nodelist=nid001600", "--ntasks=4", "--ntasks-per-node=4", "--gpus-per-node=4", "--gpu-bind=none", "--cpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T190159Z-99199-22637/workdir/cuda-build/vlasov_bench", "--nx", "134400", "--nv", "4096", "--steps", "10", "--warmup", "3", "--mode", "landau", "--velocity", "cubic", "--layout", "halos", "--integrator", "sixth", "--nu", "1e-12", "--length", "12600.0", "--vmax", "15.0", "--dt", "0.02", "--cells", "--input", "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/initial.bin", "--v-grid", "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/v.bin", "--background", "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/background.bin"], "rank_log": "rank=1 host=nid001600 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001600 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001600 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001600 local_rank=0 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=c0a2abe12f3d288f697c8b183b00313aa4c6ba063d5122a570dfae4a37751380\nsynced_at=2026-10-24T19:31:46Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "integrator": "sixth", "nu": 1e-12, "velocity": "cubic", "layout": "halos", "cell_centred": true, "max_halo_width": 4, "rank0_halo_sent_bytes": 215040000, "halo_sent_bytes_per_rank_max": 430080000, "slab_sent_bytes_per_rank": 16515072000, "input": "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/initial.bin", "nx": 134400, "nv": 4096, "ranks": 4, "nodes": 1, "steps": 10, "warmup": 3, "dt": 0.02, "length": 12600, "vmax": 15, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 4404019200, "max_allocated_bytes_per_rank": 11155281976, "stream_ms": 96.76016480002545, "spline_ms": 29.38630270000431, "pack_ms": 6.511509500023749, "mpi_ms": 21.00970499996038, "field_ms": 6.653158599988274, "collision_ms": 14.5029157999943, "step_ms": 174.2416196000249, "initial_mass": 12597.6196507818, "final_mass": 12597.62016522545, "mass_relative_change": 4.083657516783715e-08, "initial_kinetic_energy": 55713.43281125277, "final_kinetic_energy": 55714.15660222915, "initial_field_energy": 787.6538181371966, "final_field_energy": 786.9554651665927, "minimum_f": -0.1916308738890627, "finite": true, "repeat": 2, "command": ["srun", "--jobid=60221398", "--overlap", "--nodes=1", "--nodelist=nid001600", "--ntasks=4", "--ntasks-per-node=4", "--gpus-per-node=4", "--gpu-bind=none", "--cpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T190159Z-99199-22637/workdir/cuda-build/vlasov_bench", "--nx", "134400", "--nv", "4096", "--steps", "10", "--warmup", "3", "--mode", "landau", "--velocity", "cubic", "--layout", "halos", "--integrator", "sixth", "--nu", "1e-12", "--length", "12600.0", "--vmax", "15.0", "--dt", "0.02", "--cells", "--input", "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/initial.bin", "--v-grid", "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/v.bin", "--background", "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/background.bin"], "rank_log": "rank=1 host=nid001600 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001600 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001600 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001600 local_rank=2 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=c0a2abe12f3d288f697c8b183b00313aa4c6ba063d5122a570dfae4a37751380\nsynced_at=2026-10-24T19:31:46Z\n"} diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-adept/pinned-one-node/jax-nx134400-g4.json b/benchmarks/vlasov_cuda/results/2026-10-24-adept/pinned-one-node/jax-nx134400-g4.json new file mode 100644 index 00000000..049b894d --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-adept/pinned-one-node/jax-nx134400-g4.json @@ -0,0 +1,120 @@ +{ + "rows": [ + { + "backend": "adept-jax", + "nx": 134400, + "nv": 4096, + "ranks": 4, + "repeat": 0, + "steps": 10, + "warmup": 3, + "step_ms": 223.41581310029142, + "rank_seconds": [ + 2.233701307006413, + 2.234158131002914, + 2.233267256000545, + 2.23381844398682 + ], + "sum": 18346614.114405707, + "sum_squares": 2101439.806997545, + "minimum": -0.19163087388906355, + "maximum": 0.3687389216845132 + }, + { + "backend": "adept-jax", + "nx": 134400, + "nv": 4096, + "ranks": 4, + "repeat": 1, + "steps": 10, + "warmup": 3, + "step_ms": 223.62807510071434, + "rank_seconds": [ + 2.2362807510071434, + 2.2362757319933735, + 2.235863984999014, + 2.2360537410131656 + ], + "sum": 18346614.114405707, + "sum_squares": 2101439.806997545, + "minimum": -0.19163087388906355, + "maximum": 0.3687389216845132 + }, + { + "backend": "adept-jax", + "nx": 134400, + "nv": 4096, + "ranks": 4, + "repeat": 2, + "steps": 10, + "warmup": 3, + "step_ms": 223.53556799935177, + "rank_seconds": [ + 2.235278229985852, + 2.2352399160154164, + 2.2353556799935177, + 2.235204117023386 + ], + "sum": 18346614.114405707, + "sum_squares": 2101439.806997545, + "minimum": -0.19163087388906355, + "maximum": 0.3687389216845132 + } + ], + "checkpoint": { + "checkpoint": "/pscratch/sd/a/archis/vp-turbulence-sessions/20261023T081121Z-nv4096-nx134400-5ce4fe14/workdir/campaigns/production-nx134400/checkpoint-state/step-030000-1a19a5ae15f8", + "manifest_sha256": "9ab7e963d2699777bd935ca84fad446b89846e64c416d10db6d3c14829af8627", + "actual_time": 600.0, + "config_sha256": "5a9f5347e6253a3d4201745318843857c568b3787cf91e9fe81a623d0cdbb214", + "adept_commit": "ddd70b957dc3df85f0b5ec84af927c25d09d24e0", + "jax": "0.10.2", + "nx": 134400, + "nv": 4096, + "dt": 0.02, + "nu": 1e-12, + "devices": [ + "cuda:0", + "cuda:1", + "cuda:2", + "cuda:3" + ], + "milestones": [ + { + "steps": 1, + "advanced_steps": 1, + "seconds": 3.020330819010269 + }, + { + "steps": 10, + "advanced_steps": 9, + "seconds": 2.0284883639833424 + }, + { + "steps": 100, + "advanced_steps": 90, + "seconds": 20.115412370010745 + } + ] + }, + "adept_commit": "ddd70b957dc3df85f0b5ec84af927c25d09d24e0", + "libraries": { + "jax": "0.10.2", + "jaxlib": "0.10.2", + "interpax": "0.3.14+10.g84aaf30", + "diffrax": "0.7.2" + }, + "hosts": [ + "nid001600", + "nid001600", + "nid001600", + "nid001600" + ], + "compile_seconds": 0.9581184719863813, + "devices": [ + "cuda:0", + "cuda:1", + "cuda:2", + "cuda:3" + ], + "scope": "Warmed operator loop; no SaveAt, diagnostics, I/O, initialization or compilation in timings" +} diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-adept/pinned-one-node/jax-nx134400-g4.log b/benchmarks/vlasov_cuda/results/2026-10-24-adept/pinned-one-node/jax-nx134400-g4.log new file mode 100644 index 00000000..dfb1cf6a --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-adept/pinned-one-node/jax-nx134400-g4.log @@ -0,0 +1,15 @@ +{"backend": "adept-jax", "nx": 134400, "nv": 4096, "ranks": 4, "repeat": 0, "steps": 10, "warmup": 3, "step_ms": 223.41581310029142, "rank_seconds": [2.233701307006413, 2.234158131002914, 2.233267256000545, 2.23381844398682], "sum": 18346614.114405707, "sum_squares": 2101439.806997545, "minimum": -0.19163087388906355, "maximum": 0.3687389216845132} +{"backend": "adept-jax", "nx": 134400, "nv": 4096, "ranks": 4, "repeat": 1, "steps": 10, "warmup": 3, "step_ms": 223.62807510071434, "rank_seconds": [2.2362807510071434, 2.2362757319933735, 2.235863984999014, 2.2360537410131656], "sum": 18346614.114405707, "sum_squares": 2101439.806997545, "minimum": -0.19163087388906355, "maximum": 0.3687389216845132} +{"backend": "adept-jax", "nx": 134400, "nv": 4096, "ranks": 4, "repeat": 2, "steps": 10, "warmup": 3, "step_ms": 223.53556799935177, "rank_seconds": [2.235278229985852, 2.2352399160154164, 2.2353556799935177, 2.235204117023386], "sum": 18346614.114405707, "sum_squares": 2101439.806997545, "minimum": -0.19163087388906355, "maximum": 0.3687389216845132} +W1024 12:32:59.281476 1223266 pjrt_client.cc:1615] WatchTasksAsync failed for task 0: CANCELLED: CANCELLED +Additional GRPC error information from remote target coordination_service while calling /xla.coordination.CoordinationService/WatchTasks: +:UNKNOWN:Error received from peer {grpc_message:"CANCELLED", grpc_status:1} [type.googleapis.com/tensorflow.DerivedStatus=''] +W1024 12:32:59.281587 1223402 pjrt_client.cc:1615] WatchTasksAsync failed for task 2: CANCELLED: CANCELLED +Additional GRPC error information from remote target coordination_service while calling /xla.coordination.CoordinationService/WatchTasks: +:UNKNOWN:Error received from peer {grpc_status:1, grpc_message:"CANCELLED"} [type.googleapis.com/tensorflow.DerivedStatus=''] +W1024 12:32:59.281591 1223397 pjrt_client.cc:1615] WatchTasksAsync failed for task 3: CANCELLED: CANCELLED +Additional GRPC error information from remote target coordination_service while calling /xla.coordination.CoordinationService/WatchTasks: +:UNKNOWN:Error received from peer {grpc_message:"CANCELLED", grpc_status:1} [type.googleapis.com/tensorflow.DerivedStatus=''] +W1024 12:32:59.281792 1223403 pjrt_client.cc:1615] WatchTasksAsync failed for task 1: UNAVAILABLE: failed to connect to all addresses; last error: UNKNOWN: ipv4:128.55.65.28:63446: Failed to connect to remote host: Connection refused +Additional GRPC error information from remote target coordination_service while calling /xla.coordination.CoordinationService/WatchTasks: +:UNKNOWN:Error received from peer {grpc_message:"failed to connect to all addresses; last error: UNKNOWN: ipv4:128.55.65.28:63446: Failed to connect to remote host: Connection refused", grpc_status:14} diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-adept/pinned-one-node/launch.sh b/benchmarks/vlasov_cuda/results/2026-10-24-adept/pinned-one-node/launch.sh new file mode 100644 index 00000000..2408c3e0 --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-adept/pinned-one-node/launch.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# Run only inside an owned two-node allocation, after building both CUDA targets. +set -euo pipefail +if [ "${SLURM_PROCID:-0}" -ne 0 ]; then exit 0; fi +: "${EC_SESSION_ROOT:?}" "${SLURM_JOB_ID:?}" +source_root="$EC_SESSION_ROOT/src" +report_root="$EC_SESSION_OUTPUTS/${ADEPT_COMPARISON_OUTPUT:-adept-comparison}" +mkdir "$report_root" +prepared_root=/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs +vp_small=/pscratch/sd/a/archis/vp-turbulence-sessions/20261023T081121Z-nv4096-nx134400-5ce4fe14/src +vp_large=/pscratch/sd/a/archis/vp-turbulence-sessions/20261023T081121Z-nv4096-nx268800-93a418d7/src +export MPICH_GPU_SUPPORT_ENABLED=1 XLA_PYTHON_CLIENT_PREALLOCATE=false JAX_ENABLE_X64=true +export LD_LIBRARY_PATH="/global/common/software/nersc9/nccl/2.29.2-cu13/plugin/lib:/opt/nvidia/hpc_sdk/Linux_x86_64/26.5/cuda/13.2/lib64${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" +export NCCL_SOCKET_IFNAME=hsn NCCL_CROSS_NIC=2 NCCL_NET="AWS Libfabric" NCCL_NET_GDR_LEVEL=LOC FI_CXI_DISABLE_HOST_REGISTER=1 +export XLA_FLAGS="${XLA_FLAGS:+$XLA_FLAGS }--xla_gpu_autotune_level=0" +mapfile -t allocated_nodes < <(scontrol show hostnames "$SLURM_NODELIST") +for case in ${ADEPT_COMPARISON_CASES:-134400:1:4 134400:2:8 268800:2:8}; do + IFS=: read -r nx nodes ranks <<< "$case" + node_list=$(IFS=,; echo "${allocated_nodes[*]:0:nodes}") + prepared="$prepared_root/native-nx$nx" + vp_source="$vp_small" + if [ "$nx" = 268800 ]; then + prepared="$prepared_root/native-nx268800-distributed" + vp_source="$vp_large" + fi + # The pinned environment's own ADEPT must win over this development checkout. + # JAX CUDA wheels supply their runtime; unloading cudatoolkit matches the reference runs. + ( + module unload cudatoolkit 2>/dev/null || true + source /global/common/software/m4490/archis/venvs/vp-turbulence/bin/activate + export PYTHONPATH="$vp_source" + srun --jobid="$SLURM_JOB_ID" --overlap --nodes="$nodes" --nodelist="$node_list" --ntasks="$ranks" \ + --ntasks-per-node=$((ranks / nodes)) --gpus-per-node=4 --gpu-bind=none \ + --cpu-bind=none --mpi=none --kill-on-bad-exit=1 \ + python -u "$source_root/benchmarks/vlasov_cuda/adept_timing.py" \ + --prepared "$prepared" --config "$source_root/benchmarks/vlasov_cuda/reference/latest-nv4096/nx$nx.yaml" \ + --vp-source "$vp_source" --output "$report_root/jax-nx$nx-g$ranks.json" \ + > "$report_root/jax-nx$nx-g$ranks.log" 2>&1 + ) + python -u benchmarks/vlasov_cuda/measure.py "$EC_SESSION_WORKDIR/cuda-build/vlasov_bench" \ + --launcher "srun --jobid=$SLURM_JOB_ID --overlap --nodes={nodes} --nodelist=$node_list --ntasks={ranks} --ntasks-per-node={ranks_per_node} --gpus-per-node=4 --gpu-bind=none --cpu-bind=none" \ + --placements "$nodes:$ranks" --nx "$nx" --nv 4096 --steps 10 --warmup 3 --repeats 3 \ + --expect-cuda --velocity cubic --layout halos --integrator sixth --cells --nu 1e-12 \ + --length 12600 --vmax 15 --dt .02 --input "$prepared/initial.bin" \ + --v-grid "$prepared/v.bin" --background "$prepared/background.bin" --max-mass-change -1 \ + --source-state "$EC_SESSION_ROOT/source-state" --output "$report_root/cuda-nx$nx-g$ranks.jsonl" +done +sha256sum "$EC_SESSION_WORKDIR/cuda-build/vlasov_bench" "$EC_SESSION_WORKDIR/native-build/libadept_vlasov.so" > "$report_root/binaries.sha256" +sha256sum adept/_vlasov1d/native/api.cpp adept/_vlasov1d/native/core/* benchmarks/vlasov_cuda/adept_timing.py > "$report_root/source.sha256" +cp "$EC_SESSION_ROOT/source-state" "$report_root/source-state" +cp benchmarks/vlasov_cuda/run_adept_comparison.sh "$report_root/launch.sh" diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-adept/pinned-one-node/source-state b/benchmarks/vlasov_cuda/results/2026-10-24-adept/pinned-one-node/source-state new file mode 100644 index 00000000..84e90391 --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-adept/pinned-one-node/source-state @@ -0,0 +1,5 @@ +version=1 +base_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4 +dirty=true +workspace_fingerprint=c0a2abe12f3d288f697c8b183b00313aa4c6ba063d5122a570dfae4a37751380 +synced_at=2026-10-24T19:31:46Z diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-adept/pinned-one-node/source.sha256 b/benchmarks/vlasov_cuda/results/2026-10-24-adept/pinned-one-node/source.sha256 new file mode 100644 index 00000000..1ce74b5f --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-adept/pinned-one-node/source.sha256 @@ -0,0 +1,6 @@ +11ec35339aa737893312c0b2618097a8035d8bfa6bdd3debb900bf07d546f59c adept/_vlasov1d/native/api.cpp +39956b49c55c68572f85fda5b7bea073dbc96bafe708c7c8abcd0a29ba6a54fc adept/_vlasov1d/native/core/backend.cpp +06dac8cb9d13f2850fdc404aa27c42035d657e2c9320e6b42e711424a76d0e97 adept/_vlasov1d/native/core/backend.hpp +f87f6aa9731f64e501b94e95c20a53f45ae7d1ea5ec39c7e859c6b1a4b8c40c3 adept/_vlasov1d/native/core/local_cubic.hpp +a13890d45196df9a0f3da8f309fc5024c0f574174701a645e9558c3df08dcb66 adept/_vlasov1d/native/core/solver.hpp +3470b5039a1f4ee5b21930de121325e909b52ee7c8100836d880b51181993f00 benchmarks/vlasov_cuda/adept_timing.py diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-adept/slurm-accounting.txt b/benchmarks/vlasov_cuda/results/2026-10-24-adept/slurm-accounting.txt new file mode 100644 index 00000000..9dbd68e2 --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-adept/slurm-accounting.txt @@ -0,0 +1,35 @@ +JobID|JobName|State|ExitCode|Elapsed|NodeList +60221398|ec-adept-20261024T190159Z-99199-22637|CANCELLED by 71501|0:0|00:32:24|nid[001600,003745] +60221398.extern|extern|COMPLETED|0:0|00:32:24|nid[001600,003745] +60221398.0|bash|FAILED|1:0|00:00:32|nid[001600,003745] +60221398.1|bash|FAILED|1:0|00:00:25|nid[001600,003745] +60221398.2|bash|FAILED|1:0|00:02:48|nid[001600,003745] +60221398.3|python|COMPLETED|0:0|00:00:12|nid001600 +60221398.4|python|COMPLETED|0:0|00:00:37|nid003745 +60221398.5|python|COMPLETED|0:0|00:00:17|nid[001600,003745] +60221398.6|python|FAILED|1:0|00:00:10|nid001600 +60221398.7|bash|COMPLETED|0:0|00:07:08|nid[001600,003745] +60221398.8|python|COMPLETED|0:0|00:00:12|nid001600 +60221398.9|python|COMPLETED|0:0|00:00:15|nid003745 +60221398.10|python|COMPLETED|0:0|00:00:19|nid[001600,003745] +60221398.11|python|COMPLETED|0:0|00:01:05|nid001600 +60221398.12|vlasov_bench|COMPLETED|0:0|00:00:15|nid003745 +60221398.13|vlasov_bench|COMPLETED|0:0|00:00:13|nid001600 +60221398.14|vlasov_bench|COMPLETED|0:0|00:00:13|nid003745 +60221398.15|python|COMPLETED|0:0|00:00:47|nid[001600,003745] +60221398.16|vlasov_bench|COMPLETED|0:0|00:00:11|nid[001600,003745] +60221398.17|vlasov_bench|COMPLETED|0:0|00:00:10|nid[001600,003745] +60221398.18|vlasov_bench|COMPLETED|0:0|00:00:11|nid[001600,003745] +60221398.19|python|COMPLETED|0:0|00:01:47|nid[001600,003745] +60221398.20|vlasov_bench|COMPLETED|0:0|00:00:18|nid[001600,003745] +60221398.21|vlasov_bench|COMPLETED|0:0|00:00:17|nid[001600,003745] +60221398.22|vlasov_bench|COMPLETED|0:0|00:00:17|nid[001600,003745] +60221398.23|bash|FAILED|1:0|00:00:37|nid[001600,003745] +60221398.24|python|COMPLETED|0:0|00:00:20|nid[001600,003745] +60221398.25|bash|COMPLETED|0:0|00:00:34|nid[001600,003745] +60221398.26|python|COMPLETED|0:0|00:00:16|nid[001600,003745] +60221398.27|bash|COMPLETED|0:0|00:01:09|nid[001600,003745] +60221398.28|python|COMPLETED|0:0|00:00:30|nid001600 +60221398.29|vlasov_bench|COMPLETED|0:0|00:00:12|nid001600 +60221398.30|vlasov_bench|COMPLETED|0:0|00:00:13|nid001600 +60221398.31|vlasov_bench|COMPLETED|0:0|00:00:12|nid001600 diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-adept/source-state b/benchmarks/vlasov_cuda/results/2026-10-24-adept/source-state new file mode 100644 index 00000000..35ccb776 --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-adept/source-state @@ -0,0 +1,5 @@ +version=1 +base_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4 +dirty=true +workspace_fingerprint=37396dd0852f9e6ff790f730c645eaa6681006a0a135e7039737d9472112a7d3 +synced_at=2026-10-24T19:19:12Z diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-adept/source.sha256 b/benchmarks/vlasov_cuda/results/2026-10-24-adept/source.sha256 new file mode 100644 index 00000000..1ce74b5f --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-adept/source.sha256 @@ -0,0 +1,6 @@ +11ec35339aa737893312c0b2618097a8035d8bfa6bdd3debb900bf07d546f59c adept/_vlasov1d/native/api.cpp +39956b49c55c68572f85fda5b7bea073dbc96bafe708c7c8abcd0a29ba6a54fc adept/_vlasov1d/native/core/backend.cpp +06dac8cb9d13f2850fdc404aa27c42035d657e2c9320e6b42e711424a76d0e97 adept/_vlasov1d/native/core/backend.hpp +f87f6aa9731f64e501b94e95c20a53f45ae7d1ea5ec39c7e859c6b1a4b8c40c3 adept/_vlasov1d/native/core/local_cubic.hpp +a13890d45196df9a0f3da8f309fc5024c0f574174701a645e9558c3df08dcb66 adept/_vlasov1d/native/core/solver.hpp +3470b5039a1f4ee5b21930de121325e909b52ee7c8100836d880b51181993f00 benchmarks/vlasov_cuda/adept_timing.py diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-adept/source.tar.gz b/benchmarks/vlasov_cuda/results/2026-10-24-adept/source.tar.gz new file mode 100644 index 00000000..4ee63244 Binary files /dev/null and b/benchmarks/vlasov_cuda/results/2026-10-24-adept/source.tar.gz differ diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-adept/summarize.py b/benchmarks/vlasov_cuda/results/2026-10-24-adept/summarize.py new file mode 100644 index 00000000..69a582ad --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-adept/summarize.py @@ -0,0 +1,82 @@ +"""Rebuild paired ADEPT/CUDA timing comparisons from the preserved raw reports.""" + +import hashlib +import json +import math +import statistics +import tarfile +from pathlib import Path + +root = Path(__file__).resolve().parent +pairs = [] +for nx, ranks in [(134400, 4), (134400, 8), (268800, 8)]: + reference = json.loads((root / f"jax-nx{nx}-g{ranks}.json").read_text()) + jax_rows = reference["rows"] + cuda_rows = [json.loads(line) for line in (root / f"cuda-nx{nx}-g{ranks}.jsonl").read_text().splitlines()] + assert len(jax_rows) == len(cuda_rows) == 3 + assert reference["adept_commit"] == "ddd70b957dc3df85f0b5ec84af927c25d09d24e0" + assert reference["libraries"]["jax"] == "0.10.2" + assert len(reference["hosts"]) == ranks and len(set(reference["hosts"])) == ranks // 4 + for row in [*jax_rows, *cuda_rows]: + assert row["nx"] == nx and row["nv"] == 4096 and row["ranks"] == ranks + assert row["steps"] == 10 and row["warmup"] == 3 + for row in cuda_rows: + assert row["finite"] and row["nu"] == 1e-12 and row["integrator"] == "sixth" + assert row["velocity"] == "cubic" and row["layout"] == "halos" and row["cell_centred"] + assert row["dt"] == 0.02 and row["length"] == 12600 and row["vmax"] == 15 + for host in set(reference["hosts"]): + assert f"host={host} " in row["rank_log"] + jax_mass = jax_rows[0]["sum"] * (12600 / nx) * (30 / 4096) + assert math.isclose(jax_mass, row["final_mass"], rel_tol=3e-10) + assert math.isclose(jax_rows[0]["minimum"], row["minimum_f"], rel_tol=1e-10, abs_tol=1e-11) + a = statistics.median(row["step_ms"] for row in jax_rows) + b = statistics.median(row["step_ms"] for row in cuda_rows) + pairs.append( + dict( + nx=nx, + nv=4096, + nodes=ranks // 4, + gpus=ranks, + adept_jax_ms=a, + native_cuda_ms=b, + speedup=a / b, + adept_jax_ms_range=[min(row["step_ms"] for row in jax_rows), max(row["step_ms"] for row in jax_rows)], + native_cuda_ms_range=[min(row["step_ms"] for row in cuda_rows), max(row["step_ms"] for row in cuda_rows)], + hosts=sorted(set(reference["hosts"])), + repeats=3, + native_max_allocated_bytes_per_rank=max(row["max_allocated_bytes_per_rank"] for row in cuda_rows), + endpoint_aggregate_check="mass and minimum agree; pointwise validation is reported separately", + ) + ) +archive = root / "source.tar.gz" +with tarfile.open(archive) as snapshot: + for line in (root / "source.sha256").read_text().splitlines(): + expected, name = line.split() + assert hashlib.sha256(snapshot.extractfile(name).read()).hexdigest() == expected +checks = json.loads((root / "integration-checks.json").read_text()) +assert len(checks) == 7 and all(row["passed"] for row in checks) +report = dict( + date="2026-10-24", + system="NERSC Perlmutter", + allocation="60221398", + device="NVIDIA A100-SXM4-40GB", + precision="float64", + dt=0.02, + nu=1e-12, + libraries=reference["libraries"], + adept_commit=reference["adept_commit"], + timing=( + "Median of 3 repeats, 10 steps after 3 warmup steps, maximum rank elapsed time; " + "no concurrent benchmark workloads. Both paths reset to identical t600 input." + ), + scope=( + "Steady-state physics operators. Excludes compilation, preprocessing, save callbacks, I/O and " + "postprocessing. Not a full-run wall-clock speedup or a full t0-to-2000 reproduction." + ), + comparison=pairs, + integration_checks=checks, + local_tests=dict(native=5, existing_regressions=33), + source_archive_sha256=hashlib.sha256(archive.read_bytes()).hexdigest(), +) +(root / "summary.json").write_text(json.dumps(report, indent=2) + "\n") +print(json.dumps(pairs, indent=2)) diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-adept/summary.json b/benchmarks/vlasov_cuda/results/2026-10-24-adept/summary.json new file mode 100644 index 00000000..4c53a664 --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-adept/summary.json @@ -0,0 +1,306 @@ +{ + "date": "2026-10-24", + "system": "NERSC Perlmutter", + "allocation": "60221398", + "device": "NVIDIA A100-SXM4-40GB", + "precision": "float64", + "dt": 0.02, + "nu": 1e-12, + "libraries": { + "jax": "0.10.2", + "jaxlib": "0.10.2", + "interpax": "0.3.14+10.g84aaf30", + "diffrax": "0.7.2" + }, + "adept_commit": "ddd70b957dc3df85f0b5ec84af927c25d09d24e0", + "timing": "Median of 3 repeats, 10 steps after 3 warmup steps, maximum rank elapsed time; no concurrent benchmark workloads. Both paths reset to identical t600 input.", + "scope": "Steady-state physics operators. Excludes compilation, preprocessing, save callbacks, I/O and postprocessing. Not a full-run wall-clock speedup or a full t0-to-2000 reproduction.", + "comparison": [ + { + "nx": 134400, + "nv": 4096, + "nodes": 1, + "gpus": 4, + "adept_jax_ms": 223.53556799935177, + "native_cuda_ms": 174.2416196000249, + "speedup": 1.2829057059529296, + "adept_jax_ms_range": [ + 223.41581310029142, + 223.62807510071434 + ], + "native_cuda_ms_range": [ + 174.2364576000227, + 174.2776944999605 + ], + "hosts": [ + "nid001600" + ], + "repeats": 3, + "native_max_allocated_bytes_per_rank": 11155281976, + "endpoint_aggregate_check": "mass and minimum agree; pointwise validation is reported separately" + }, + { + "nx": 134400, + "nv": 4096, + "nodes": 2, + "gpus": 8, + "adept_jax_ms": 383.94048050104175, + "native_cuda_ms": 140.0714161999531, + "speedup": 2.74103375918584, + "adept_jax_ms_range": [ + 383.0053849000251, + 384.9415123986546 + ], + "native_cuda_ms_range": [ + 140.015343999994, + 140.3418015999932 + ], + "hosts": [ + "nid001600", + "nid003745" + ], + "repeats": 3, + "native_max_allocated_bytes_per_rank": 5650233400, + "endpoint_aggregate_check": "mass and minimum agree; pointwise validation is reported separately" + }, + { + "nx": 268800, + "nv": 4096, + "nodes": 2, + "gpus": 8, + "adept_jax_ms": 756.6989740997087, + "native_cuda_ms": 275.9853768999847, + "speedup": 2.741808216795238, + "adept_jax_ms_range": [ + 754.9265112000285, + 757.4678515986307 + ], + "native_cuda_ms_range": [ + 275.6066252999744, + 276.0648734999677 + ], + "hosts": [ + "nid001600", + "nid003745" + ], + "repeats": 3, + "native_max_allocated_bytes_per_rank": 11300409400, + "endpoint_aggregate_check": "mass and minimum agree; pointwise validation is reported separately" + } + ], + "integration_checks": [ + { + "module": "BaseVlasov1D", + "ranks": 1, + "passed": true, + "max_abs_error": 1.0302869668521453e-13, + "mlflow_run_id": "10db23e28ed8428d9849bce2dea93d3f", + "stats": { + "native_seconds": { + "stream": 0.000619517000018277, + "velocity": 0.0003302759998859983, + "pack": 0.0, + "mpi": 0.0008770139997977822, + "field": 0.0017125090001854915, + "collision": 0.0005039860001261331, + "total": 0.004283324999960314 + }, + "solve_and_save_seconds": 0.7859312820073683, + "steps": 5, + "ranks": 1, + "device": "cuda" + }, + "keys": [ + "fields", + "electron.main", + "default" + ], + "log": "integration-v3.log" + }, + { + "module": "BaseVlasov1D", + "ranks": 4, + "passed": true, + "max_abs_error": 1.2434497875801753e-13, + "mlflow_run_id": "c15bc81503a44c02bb7f0d22043aa3df", + "stats": { + "native_seconds": { + "stream": 0.0005902340002421624, + "velocity": 0.0003807449994610579, + "pack": 0.00023884999984602473, + "mpi": 0.005609786999912103, + "field": 0.0067621719998669505, + "collision": 0.000502029000017501, + "total": 0.013536288000068453 + }, + "solve_and_save_seconds": 1.881096900004195, + "steps": 5, + "ranks": 4, + "device": "cuda" + }, + "keys": [ + "fields", + "electron.main", + "default" + ], + "log": "integration-v3.log" + }, + { + "module": "BaseVlasov1D", + "ranks": 8, + "passed": true, + "max_abs_error": 1.3145040611561853e-13, + "mlflow_run_id": "fb178ca1d165436c98457fd4ba0db80f", + "stats": { + "native_seconds": { + "stream": 0.0006939400003602714, + "velocity": 0.0005676090001998091, + "pack": 0.00025299700007508363, + "mpi": 0.01576393800007736, + "field": 0.013597956999888083, + "collision": 0.0005164489999742727, + "total": 0.02774611500001356 + }, + "solve_and_save_seconds": 3.1865445940056816, + "steps": 5, + "ranks": 8, + "device": "cuda" + }, + "keys": [ + "fields", + "electron.main", + "default" + ], + "log": "integration-v3.log" + }, + { + "module": "VPTwoStream1D", + "ranks": 1, + "passed": true, + "max_abs_error": 1.0457412713549274e-11, + "mlflow_run_id": "ac6e44af25584f74add8c11f040061c7", + "stats": { + "native_seconds": { + "stream": 0.0005986490004943334, + "velocity": 0.0003280549998407878, + "pack": 0.0, + "mpi": 0.0008833609997509484, + "field": 0.0016982180000013614, + "collision": 0.000500530999829607, + "total": 0.004253136000102131 + }, + "solve_and_save_seconds": 0.8431226150132716, + "steps": 5, + "ranks": 1, + "device": "cuda" + }, + "keys": [ + "fields", + "electron.main", + "default", + "electron.f0" + ], + "log": "integration-vp.log" + }, + { + "module": "VPTwoStream1D", + "ranks": 4, + "passed": true, + "max_abs_error": 6.3788974102862994e-12, + "mlflow_run_id": "d16748ce2f364431bb4fa17896661f44", + "stats": { + "native_seconds": { + "stream": 0.0006106250001494118, + "velocity": 0.000375073000213888, + "pack": 0.00023151799985043908, + "mpi": 0.005738596999663059, + "field": 0.006173489000275367, + "collision": 0.0005022090000466051, + "total": 0.012953435000099489 + }, + "solve_and_save_seconds": 1.350150696001947, + "steps": 5, + "ranks": 4, + "device": "cuda" + }, + "keys": [ + "fields", + "electron.main", + "default", + "electron.f0" + ], + "log": "integration-vp.log" + }, + { + "module": "VPTwoStream1D", + "ranks": 8, + "passed": true, + "max_abs_error": 1.1592504733926035e-11, + "mlflow_run_id": "e3e58e103a6747489eadadb244806865", + "stats": { + "native_seconds": { + "stream": 0.0006200880002324993, + "velocity": 0.0004060329994217682, + "pack": 0.0002525769999692784, + "mpi": 0.008024029000125665, + "field": 0.007057047000103012, + "collision": 0.000508114000012938, + "total": 0.01669021899999734 + }, + "solve_and_save_seconds": 7.212047922017518, + "steps": 5, + "ranks": 8, + "device": "cuda" + }, + "keys": [ + "fields", + "electron.main", + "default", + "electron.f0" + ], + "log": "integration-vp.log" + }, + { + "module": "VPTwoStream1D", + "ranks": 8, + "passed": true, + "max_abs_error": 1.1592504733926035e-11, + "mlflow_run_id": "a31a0009b9a440c79965f3e70332ccde", + "stats": { + "native_seconds": { + "stream": 0.000619704999962778, + "velocity": 0.0004217190003146243, + "pack": 0.00027711899952009844, + "mpi": 0.008916388000216102, + "field": 0.010889343000826557, + "collision": 0.0005323509999470843, + "total": 0.02131122699961452 + }, + "solve_and_save_seconds": 3.074192091007717, + "steps": 5, + "ranks": 8, + "device": "cuda", + "libraries": { + "jax": "0.10.2", + "jaxlib": "0.10.2", + "cloudpickle": "3.1.2", + "diffrax": "0.7.2" + } + }, + "keys": [ + "fields", + "electron.main", + "default", + "electron.f0", + "electron.pkv", + "electron.cascade" + ], + "log": "integration-observers-v2.log" + } + ], + "local_tests": { + "native": 5, + "existing_regressions": 33 + }, + "source_archive_sha256": "a457bc1ff3919eb30c64ceba61a503c478aa94bbaa06cd0fef3428017380946e" +} diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-adept/unmatched-node/README.md b/benchmarks/vlasov_cuda/results/2026-10-24-adept/unmatched-node/README.md new file mode 100644 index 00000000..1acf8ac8 --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-adept/unmatched-node/README.md @@ -0,0 +1,4 @@ +These initial four-GPU measurements are excluded from the reported comparison. +Slurm placed ADEPT/JAX on nid001600 and CUDA on nid003745. The audit caught +this before publication; the final pair was repeated with an explicit node +list, using identical physical GPUs for both solvers. diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-adept/unmatched-node/cuda-nx134400-g4.jsonl b/benchmarks/vlasov_cuda/results/2026-10-24-adept/unmatched-node/cuda-nx134400-g4.jsonl new file mode 100644 index 00000000..d112a9c2 --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-adept/unmatched-node/cuda-nx134400-g4.jsonl @@ -0,0 +1,3 @@ +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "integrator": "sixth", "nu": 1e-12, "velocity": "cubic", "layout": "halos", "cell_centred": true, "max_halo_width": 4, "rank0_halo_sent_bytes": 215040000, "halo_sent_bytes_per_rank_max": 430080000, "slab_sent_bytes_per_rank": 16515072000, "input": "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/initial.bin", "nx": 134400, "nv": 4096, "ranks": 4, "nodes": 1, "steps": 10, "warmup": 3, "dt": 0.02, "length": 12600, "vmax": 15, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 4404019200, "max_allocated_bytes_per_rank": 11155281976, "stream_ms": 96.76722870008234, "spline_ms": 29.388790800067, "pack_ms": 6.512254699987352, "mpi_ms": 20.99453199996333, "field_ms": 6.691103699904488, "collision_ms": 14.49035960004039, "step_ms": 174.2564972000082, "initial_mass": 12597.6196507818, "final_mass": 12597.62016522545, "mass_relative_change": 4.083657516783715e-08, "initial_kinetic_energy": 55713.43281125277, "final_kinetic_energy": 55714.15660222915, "initial_field_energy": 787.6538181371966, "final_field_energy": 786.9554651665927, "minimum_f": -0.1916308738890627, "finite": true, "repeat": 0, "command": ["srun", "--jobid=60221398", "--overlap", "--nodes=1", "--ntasks=4", "--ntasks-per-node=4", "--gpus-per-node=4", "--gpu-bind=none", "--cpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T190159Z-99199-22637/workdir/cuda-build/vlasov_bench", "--nx", "134400", "--nv", "4096", "--steps", "10", "--warmup", "3", "--mode", "landau", "--velocity", "cubic", "--layout", "halos", "--integrator", "sixth", "--nu", "1e-12", "--length", "12600.0", "--vmax", "15.0", "--dt", "0.02", "--cells", "--input", "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/initial.bin", "--v-grid", "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/v.bin", "--background", "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/background.bin"], "rank_log": "rank=2 host=nid003745 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid003745 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid003745 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid003745 local_rank=0 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=afdc248c4553148cd3ca5a63cb0431b8e474acd835d818af386e898beb75eb2d\nsynced_at=2026-10-24T19:13:57Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "integrator": "sixth", "nu": 1e-12, "velocity": "cubic", "layout": "halos", "cell_centred": true, "max_halo_width": 4, "rank0_halo_sent_bytes": 215040000, "halo_sent_bytes_per_rank_max": 430080000, "slab_sent_bytes_per_rank": 16515072000, "input": "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/initial.bin", "nx": 134400, "nv": 4096, "ranks": 4, "nodes": 1, "steps": 10, "warmup": 3, "dt": 0.02, "length": 12600, "vmax": 15, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 4404019200, "max_allocated_bytes_per_rank": 11155281976, "stream_ms": 96.76718900000196, "spline_ms": 29.3890182000041, "pack_ms": 6.513456599805068, "mpi_ms": 21.01721300018653, "field_ms": 6.694112699869947, "collision_ms": 14.50337810001656, "step_ms": 174.2706530000532, "initial_mass": 12597.6196507818, "final_mass": 12597.62016522545, "mass_relative_change": 4.083657516783715e-08, "initial_kinetic_energy": 55713.43281125277, "final_kinetic_energy": 55714.15660222915, "initial_field_energy": 787.6538181371966, "final_field_energy": 786.9554651665927, "minimum_f": -0.1916308738890627, "finite": true, "repeat": 1, "command": ["srun", "--jobid=60221398", "--overlap", "--nodes=1", "--ntasks=4", "--ntasks-per-node=4", "--gpus-per-node=4", "--gpu-bind=none", "--cpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T190159Z-99199-22637/workdir/cuda-build/vlasov_bench", "--nx", "134400", "--nv", "4096", "--steps", "10", "--warmup", "3", "--mode", "landau", "--velocity", "cubic", "--layout", "halos", "--integrator", "sixth", "--nu", "1e-12", "--length", "12600.0", "--vmax", "15.0", "--dt", "0.02", "--cells", "--input", "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/initial.bin", "--v-grid", "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/v.bin", "--background", "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/background.bin"], "rank_log": "rank=2 host=nid001600 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001600 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001600 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001600 local_rank=0 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=afdc248c4553148cd3ca5a63cb0431b8e474acd835d818af386e898beb75eb2d\nsynced_at=2026-10-24T19:13:57Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "integrator": "sixth", "nu": 1e-12, "velocity": "cubic", "layout": "halos", "cell_centred": true, "max_halo_width": 4, "rank0_halo_sent_bytes": 215040000, "halo_sent_bytes_per_rank_max": 430080000, "slab_sent_bytes_per_rank": 16515072000, "input": "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/initial.bin", "nx": 134400, "nv": 4096, "ranks": 4, "nodes": 1, "steps": 10, "warmup": 3, "dt": 0.02, "length": 12600, "vmax": 15, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 4404019200, "max_allocated_bytes_per_rank": 11155281976, "stream_ms": 96.7677092000713, "spline_ms": 29.38927480004168, "pack_ms": 6.511935900039134, "mpi_ms": 20.97058130004825, "field_ms": 6.669697600000291, "collision_ms": 14.48755779995281, "step_ms": 174.236390100009, "initial_mass": 12597.6196507818, "final_mass": 12597.62016522545, "mass_relative_change": 4.083657516783715e-08, "initial_kinetic_energy": 55713.43281125277, "final_kinetic_energy": 55714.15660222915, "initial_field_energy": 787.6538181371966, "final_field_energy": 786.9554651665927, "minimum_f": -0.1916308738890627, "finite": true, "repeat": 2, "command": ["srun", "--jobid=60221398", "--overlap", "--nodes=1", "--ntasks=4", "--ntasks-per-node=4", "--gpus-per-node=4", "--gpu-bind=none", "--cpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T190159Z-99199-22637/workdir/cuda-build/vlasov_bench", "--nx", "134400", "--nv", "4096", "--steps", "10", "--warmup", "3", "--mode", "landau", "--velocity", "cubic", "--layout", "halos", "--integrator", "sixth", "--nu", "1e-12", "--length", "12600.0", "--vmax", "15.0", "--dt", "0.02", "--cells", "--input", "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/initial.bin", "--v-grid", "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/v.bin", "--background", "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/background.bin"], "rank_log": "rank=3 host=nid003745 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid003745 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid003745 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid003745 local_rank=0 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=afdc248c4553148cd3ca5a63cb0431b8e474acd835d818af386e898beb75eb2d\nsynced_at=2026-10-24T19:13:57Z\n"} diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-adept/unmatched-node/jax-nx134400-g4.json b/benchmarks/vlasov_cuda/results/2026-10-24-adept/unmatched-node/jax-nx134400-g4.json new file mode 100644 index 00000000..5c441e63 --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-adept/unmatched-node/jax-nx134400-g4.json @@ -0,0 +1,120 @@ +{ + "rows": [ + { + "backend": "adept-jax", + "nx": 134400, + "nv": 4096, + "ranks": 4, + "repeat": 0, + "steps": 10, + "warmup": 3, + "step_ms": 223.53141740022693, + "rank_seconds": [ + 2.2350591710128356, + 2.235074671014445, + 2.2353141740022693, + 2.2350723760027904 + ], + "sum": 18346614.114405707, + "sum_squares": 2101439.806997545, + "minimum": -0.19163087388906355, + "maximum": 0.3687389216845132 + }, + { + "backend": "adept-jax", + "nx": 134400, + "nv": 4096, + "ranks": 4, + "repeat": 1, + "steps": 10, + "warmup": 3, + "step_ms": 223.51721489976626, + "rank_seconds": [ + 2.2351721489976626, + 2.2348791630065534, + 2.2348948220023885, + 2.2348771590040997 + ], + "sum": 18346614.114405707, + "sum_squares": 2101439.806997545, + "minimum": -0.19163087388906355, + "maximum": 0.3687389216845132 + }, + { + "backend": "adept-jax", + "nx": 134400, + "nv": 4096, + "ranks": 4, + "repeat": 2, + "steps": 10, + "warmup": 3, + "step_ms": 223.62216420006007, + "rank_seconds": [ + 2.2362216420006007, + 2.2361008370062336, + 2.2359991599805653, + 2.236146445007762 + ], + "sum": 18346614.114405707, + "sum_squares": 2101439.806997545, + "minimum": -0.19163087388906355, + "maximum": 0.3687389216845132 + } + ], + "checkpoint": { + "checkpoint": "/pscratch/sd/a/archis/vp-turbulence-sessions/20261023T081121Z-nv4096-nx134400-5ce4fe14/workdir/campaigns/production-nx134400/checkpoint-state/step-030000-1a19a5ae15f8", + "manifest_sha256": "9ab7e963d2699777bd935ca84fad446b89846e64c416d10db6d3c14829af8627", + "actual_time": 600.0, + "config_sha256": "5a9f5347e6253a3d4201745318843857c568b3787cf91e9fe81a623d0cdbb214", + "adept_commit": "ddd70b957dc3df85f0b5ec84af927c25d09d24e0", + "jax": "0.10.2", + "nx": 134400, + "nv": 4096, + "dt": 0.02, + "nu": 1e-12, + "devices": [ + "cuda:0", + "cuda:1", + "cuda:2", + "cuda:3" + ], + "milestones": [ + { + "steps": 1, + "advanced_steps": 1, + "seconds": 3.020330819010269 + }, + { + "steps": 10, + "advanced_steps": 9, + "seconds": 2.0284883639833424 + }, + { + "steps": 100, + "advanced_steps": 90, + "seconds": 20.115412370010745 + } + ] + }, + "adept_commit": "ddd70b957dc3df85f0b5ec84af927c25d09d24e0", + "libraries": { + "jax": "0.10.2", + "jaxlib": "0.10.2", + "interpax": "0.3.14+10.g84aaf30", + "diffrax": "0.7.2" + }, + "hosts": [ + "nid001600", + "nid001600", + "nid001600", + "nid001600" + ], + "compile_seconds": 1.1488750469870865, + "devices": [ + "cuda:0", + "cuda:1", + "cuda:2", + "cuda:3" + ], + "scope": "Warmed operator loop; no SaveAt, diagnostics, I/O, initialization or compilation in timings" +} diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-adept/unmatched-node/jax-nx134400-g4.log b/benchmarks/vlasov_cuda/results/2026-10-24-adept/unmatched-node/jax-nx134400-g4.log new file mode 100644 index 00000000..2987bf5c --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-adept/unmatched-node/jax-nx134400-g4.log @@ -0,0 +1,15 @@ +{"backend": "adept-jax", "nx": 134400, "nv": 4096, "ranks": 4, "repeat": 0, "steps": 10, "warmup": 3, "step_ms": 223.53141740022693, "rank_seconds": [2.2350591710128356, 2.235074671014445, 2.2353141740022693, 2.2350723760027904], "sum": 18346614.114405707, "sum_squares": 2101439.806997545, "minimum": -0.19163087388906355, "maximum": 0.3687389216845132} +{"backend": "adept-jax", "nx": 134400, "nv": 4096, "ranks": 4, "repeat": 1, "steps": 10, "warmup": 3, "step_ms": 223.51721489976626, "rank_seconds": [2.2351721489976626, 2.2348791630065534, 2.2348948220023885, 2.2348771590040997], "sum": 18346614.114405707, "sum_squares": 2101439.806997545, "minimum": -0.19163087388906355, "maximum": 0.3687389216845132} +{"backend": "adept-jax", "nx": 134400, "nv": 4096, "ranks": 4, "repeat": 2, "steps": 10, "warmup": 3, "step_ms": 223.62216420006007, "rank_seconds": [2.2362216420006007, 2.2361008370062336, 2.2359991599805653, 2.236146445007762], "sum": 18346614.114405707, "sum_squares": 2101439.806997545, "minimum": -0.19163087388906355, "maximum": 0.3687389216845132} +W1024 12:17:08.935979 1200765 pjrt_client.cc:1615] WatchTasksAsync failed for task 2: CANCELLED: CANCELLED +Additional GRPC error information from remote target coordination_service while calling /xla.coordination.CoordinationService/WatchTasks: +:UNKNOWN:Error received from peer {grpc_status:1, grpc_message:"CANCELLED"} [type.googleapis.com/tensorflow.DerivedStatus=''] +W1024 12:17:08.936076 1200660 pjrt_client.cc:1615] WatchTasksAsync failed for task 0: UNAVAILABLE: failed to connect to all addresses; last error: UNKNOWN: ipv4:128.55.65.28:63446: Failed to connect to remote host: Connection refused +Additional GRPC error information from remote target coordination_service while calling /xla.coordination.CoordinationService/WatchTasks: +:UNKNOWN:Error received from peer {grpc_message:"failed to connect to all addresses; last error: UNKNOWN: ipv4:128.55.65.28:63446: Failed to connect to remote host: Connection refused", grpc_status:14} +W1024 12:17:08.936051 1200766 pjrt_client.cc:1615] WatchTasksAsync failed for task 3: CANCELLED: CANCELLED +Additional GRPC error information from remote target coordination_service while calling /xla.coordination.CoordinationService/WatchTasks: +:UNKNOWN:Error received from peer {grpc_status:1, grpc_message:"CANCELLED"} [type.googleapis.com/tensorflow.DerivedStatus=''] +W1024 12:17:08.936275 1200767 pjrt_client.cc:1615] WatchTasksAsync failed for task 1: UNAVAILABLE: failed to connect to all addresses; last error: UNKNOWN: ipv4:128.55.65.28:63446: Failed to connect to remote host: Connection refused +Additional GRPC error information from remote target coordination_service while calling /xla.coordination.CoordinationService/WatchTasks: +:UNKNOWN:Error received from peer {grpc_status:14, grpc_message:"failed to connect to all addresses; last error: UNKNOWN: ipv4:128.55.65.28:63446: Failed to connect to remote host: Connection refused"} diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-halos/comparison-nx134400-smoke.json b/benchmarks/vlasov_cuda/results/2026-10-24-halos/comparison-nx134400-smoke.json new file mode 100644 index 00000000..f1ff7afc --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-halos/comparison-nx134400-smoke.json @@ -0,0 +1,446 @@ +[ + { + "nodes": 1, + "ranks": 4, + "layout": "transpose", + "steps": 1, + "max_abs_f_error": 4.9779624866630456e-14, + "relative_l2_f_error": 5.802416626972201e-15, + "relative_l2_e_error": 8.618125582264429e-15, + "max_abs_e_error": 7.389922007661198e-15, + "hole_counts": [ + 132, + 48 + ], + "reference_hole_counts": [ + 132, + 48 + ], + "hole_counts_match": true, + "passed": true, + "limits": { + "max_abs_f_error": 1e-07, + "relative_l2_f_error": 1e-08, + "relative_l2_e_error": 1e-07 + }, + "benchmark": { + "backend": "cuda", + "device": "NVIDIA A100-SXM4-40GB", + "precision": "float64", + "mode": "landau", + "integrator": "sixth", + "nu": 1e-12, + "velocity": "cubic", + "layout": "transpose", + "cell_centred": true, + "max_halo_width": 0, + "rank0_halo_sent_bytes": 0, + "nx": 134400, + "nv": 4096, + "ranks": 4, + "nodes": 1, + "steps": 1, + "warmup": 0, + "dt": 0.02, + "length": 12600, + "vmax": 15, + "amplitude": 0.01, + "acceleration": 0.2, + "host_mpi": false, + "state_bytes": 4404019200, + "max_allocated_bytes_per_rank": 11017656376, + "stream_ms": 96.85286999956588, + "spline_ms": 23.19442900079594, + "pack_ms": 45.6734180006606, + "mpi_ms": 139.9384269998336, + "field_ms": 6.371654999838938, + "collision_ms": 14.55919499994707, + "step_ms": 326.5107229999558, + "initial_mass": 12597.6196507818, + "final_mass": 12597.61992816154, + "mass_relative_change": 2.201842476587016e-08, + "initial_kinetic_energy": 55713.43281125277, + "final_kinetic_energy": 55713.52390600385, + "initial_field_energy": 787.6538181371966, + "final_field_energy": 787.5827421915643, + "minimum_f": -0.1915654276559724, + "finite": true + }, + "command": [ + "srun", + "--jobid=60205333", + "--overlap", + "--nodes=1", + "--ntasks=4", + "--ntasks-per-node=4", + "--gpus-per-node=4", + "--gpu-bind=none", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/workdir/cuda-build/vlasov_bench", + "--nx", + "134400", + "--nv", + "4096", + "--length", + "12600", + "--vmax", + "15", + "--dt", + "0.02", + "--nu", + "1e-12", + "--cells", + "--velocity", + "cubic", + "--integrator", + "sixth", + "--layout", + "transpose", + "--steps", + "1", + "--warmup", + "0", + "--input", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/initial.bin", + "--v-grid", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/v.bin", + "--background", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/background.bin", + "--dump", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/comparison-nx134400-smoke/1n-4g-transpose-0001.bin" + ], + "rank_log": "rank=0 host=nid001024 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001024 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001024 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001024 local_rank=2 device=NVIDIA A100-SXM4-40GB\n" + }, + { + "nodes": 1, + "ranks": 4, + "layout": "halos", + "steps": 1, + "max_abs_f_error": 4.9779624866630456e-14, + "relative_l2_f_error": 5.802416626972201e-15, + "relative_l2_e_error": 8.618125582264429e-15, + "max_abs_e_error": 7.389922007661198e-15, + "hole_counts": [ + 132, + 48 + ], + "reference_hole_counts": [ + 132, + 48 + ], + "hole_counts_match": true, + "passed": true, + "limits": { + "max_abs_f_error": 1e-07, + "relative_l2_f_error": 1e-08, + "relative_l2_e_error": 1e-07 + }, + "benchmark": { + "backend": "cuda", + "device": "NVIDIA A100-SXM4-40GB", + "precision": "float64", + "mode": "landau", + "integrator": "sixth", + "nu": 1e-12, + "velocity": "cubic", + "layout": "halos", + "cell_centred": true, + "max_halo_width": 4, + "rank0_halo_sent_bytes": 21504000, + "nx": 134400, + "nv": 4096, + "ranks": 4, + "nodes": 1, + "steps": 1, + "warmup": 0, + "dt": 0.02, + "length": 12600, + "vmax": 15, + "amplitude": 0.01, + "acceleration": 0.2, + "host_mpi": false, + "state_bytes": 4404019200, + "max_allocated_bytes_per_rank": 11155281976, + "stream_ms": 96.81700500050283, + "spline_ms": 29.46111599885626, + "pack_ms": 6.616684999698919, + "mpi_ms": 43.50261599984151, + "field_ms": 6.872578000184149, + "collision_ms": 14.55399099995702, + "step_ms": 197.2929050002676, + "initial_mass": 12597.6196507818, + "final_mass": 12597.61992816154, + "mass_relative_change": 2.201842476587016e-08, + "initial_kinetic_energy": 55713.43281125277, + "final_kinetic_energy": 55713.52390600385, + "initial_field_energy": 787.6538181371966, + "final_field_energy": 787.5827421915643, + "minimum_f": -0.1915654276559724, + "finite": true + }, + "command": [ + "srun", + "--jobid=60205333", + "--overlap", + "--nodes=1", + "--ntasks=4", + "--ntasks-per-node=4", + "--gpus-per-node=4", + "--gpu-bind=none", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/workdir/cuda-build/vlasov_bench", + "--nx", + "134400", + "--nv", + "4096", + "--length", + "12600", + "--vmax", + "15", + "--dt", + "0.02", + "--nu", + "1e-12", + "--cells", + "--velocity", + "cubic", + "--integrator", + "sixth", + "--layout", + "halos", + "--steps", + "1", + "--warmup", + "0", + "--input", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/initial.bin", + "--v-grid", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/v.bin", + "--background", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/background.bin", + "--dump", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/comparison-nx134400-smoke/1n-4g-halos-0001.bin" + ], + "rank_log": "rank=2 host=nid001025 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001025 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001025 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001025 local_rank=0 device=NVIDIA A100-SXM4-40GB\n" + }, + { + "nodes": 2, + "ranks": 8, + "layout": "transpose", + "steps": 1, + "max_abs_f_error": 4.9710235927591384e-14, + "relative_l2_f_error": 5.7939964374065035e-15, + "relative_l2_e_error": 1.0312384960988757e-14, + "max_abs_e_error": 9.159339953157541e-15, + "hole_counts": [ + 132, + 48 + ], + "reference_hole_counts": [ + 132, + 48 + ], + "hole_counts_match": true, + "passed": true, + "limits": { + "max_abs_f_error": 1e-07, + "relative_l2_f_error": 1e-08, + "relative_l2_e_error": 1e-07 + }, + "benchmark": { + "backend": "cuda", + "device": "NVIDIA A100-SXM4-40GB", + "precision": "float64", + "mode": "landau", + "integrator": "sixth", + "nu": 1e-12, + "velocity": "cubic", + "layout": "transpose", + "cell_centred": true, + "max_halo_width": 0, + "rank0_halo_sent_bytes": 0, + "nx": 134400, + "nv": 4096, + "ranks": 8, + "nodes": 2, + "steps": 1, + "warmup": 0, + "dt": 0.02, + "length": 12600, + "vmax": 15, + "amplitude": 0.01, + "acceleration": 0.2, + "host_mpi": false, + "state_bytes": 4404019200, + "max_allocated_bytes_per_rank": 5512607800, + "stream_ms": 48.57247100017048, + "spline_ms": 11.7405529995267, + "pack_ms": 23.03244600125254, + "mpi_ms": 412.1097049996933, + "field_ms": 10.8350719997361, + "collision_ms": 11.40117900013138, + "step_ms": 511.398970999835, + "initial_mass": 12597.61965078436, + "final_mass": 12597.6199281584, + "mass_relative_change": 2.201797209846199e-08, + "initial_kinetic_energy": 55713.43281126581, + "final_kinetic_energy": 55713.52390601106, + "initial_field_energy": 787.6538181371961, + "final_field_energy": 787.5827421915658, + "minimum_f": -0.1915654276559724, + "finite": true + }, + "command": [ + "srun", + "--jobid=60205333", + "--overlap", + "--nodes=2", + "--ntasks=8", + "--ntasks-per-node=4", + "--gpus-per-node=4", + "--gpu-bind=none", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/workdir/cuda-build/vlasov_bench", + "--nx", + "134400", + "--nv", + "4096", + "--length", + "12600", + "--vmax", + "15", + "--dt", + "0.02", + "--nu", + "1e-12", + "--cells", + "--velocity", + "cubic", + "--integrator", + "sixth", + "--layout", + "transpose", + "--steps", + "1", + "--warmup", + "0", + "--input", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/initial.bin", + "--v-grid", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/v.bin", + "--background", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/background.bin", + "--dump", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/comparison-nx134400-smoke/2n-8g-transpose-0001.bin" + ], + "rank_log": "rank=7 host=nid001025 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001025 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=6 host=nid001024 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001024 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=5 host=nid001025 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001024 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=4 host=nid001024 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001025 local_rank=1 device=NVIDIA A100-SXM4-40GB\n" + }, + { + "nodes": 2, + "ranks": 8, + "layout": "halos", + "steps": 1, + "max_abs_f_error": 4.9710235927591384e-14, + "relative_l2_f_error": 5.7939964374065035e-15, + "relative_l2_e_error": 1.0312384960988757e-14, + "max_abs_e_error": 9.159339953157541e-15, + "hole_counts": [ + 132, + 48 + ], + "reference_hole_counts": [ + 132, + 48 + ], + "hole_counts_match": true, + "passed": true, + "limits": { + "max_abs_f_error": 1e-07, + "relative_l2_f_error": 1e-08, + "relative_l2_e_error": 1e-07 + }, + "benchmark": { + "backend": "cuda", + "device": "NVIDIA A100-SXM4-40GB", + "precision": "float64", + "mode": "landau", + "integrator": "sixth", + "nu": 1e-12, + "velocity": "cubic", + "layout": "halos", + "cell_centred": true, + "max_halo_width": 4, + "rank0_halo_sent_bytes": 21504000, + "nx": 134400, + "nv": 4096, + "ranks": 8, + "nodes": 2, + "steps": 1, + "warmup": 0, + "dt": 0.02, + "length": 12600, + "vmax": 15, + "amplitude": 0.01, + "acceleration": 0.2, + "host_mpi": false, + "state_bytes": 4404019200, + "max_allocated_bytes_per_rank": 5650233400, + "stream_ms": 48.52810200009117, + "spline_ms": 12.26694700017106, + "pack_ms": 3.326494999782881, + "mpi_ms": 106.7915769999672, + "field_ms": 5.192836999412975, + "collision_ms": 11.28559500011761, + "step_ms": 186.7928990000109, + "initial_mass": 12597.61965078436, + "final_mass": 12597.6199281584, + "mass_relative_change": 2.201797209846199e-08, + "initial_kinetic_energy": 55713.43281126581, + "final_kinetic_energy": 55713.52390601106, + "initial_field_energy": 787.6538181371961, + "final_field_energy": 787.5827421915658, + "minimum_f": -0.1915654276559724, + "finite": true + }, + "command": [ + "srun", + "--jobid=60205333", + "--overlap", + "--nodes=2", + "--ntasks=8", + "--ntasks-per-node=4", + "--gpus-per-node=4", + "--gpu-bind=none", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/workdir/cuda-build/vlasov_bench", + "--nx", + "134400", + "--nv", + "4096", + "--length", + "12600", + "--vmax", + "15", + "--dt", + "0.02", + "--nu", + "1e-12", + "--cells", + "--velocity", + "cubic", + "--integrator", + "sixth", + "--layout", + "halos", + "--steps", + "1", + "--warmup", + "0", + "--input", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/initial.bin", + "--v-grid", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/v.bin", + "--background", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/background.bin", + "--dump", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/comparison-nx134400-smoke/2n-8g-halos-0001.bin" + ], + "rank_log": "rank=0 host=nid001024 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=6 host=nid001024 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=5 host=nid001025 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001025 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001025 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=7 host=nid001025 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=4 host=nid001024 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001024 local_rank=1 device=NVIDIA A100-SXM4-40GB\n" + } +] diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-halos/comparison-nx134400-window.json b/benchmarks/vlasov_cuda/results/2026-10-24-halos/comparison-nx134400-window.json new file mode 100644 index 00000000..bb4408e2 --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-halos/comparison-nx134400-window.json @@ -0,0 +1,914 @@ +[ + { + "nodes": 1, + "ranks": 4, + "layout": "transpose", + "steps": 10, + "max_abs_f_error": 1.2218004385999848e-13, + "relative_l2_f_error": 1.8566744087723585e-14, + "relative_l2_e_error": 4.172291687802881e-14, + "max_abs_e_error": 3.424344141578217e-14, + "hole_counts": [ + 130, + 50 + ], + "reference_hole_counts": [ + 130, + 50 + ], + "hole_counts_match": true, + "passed": true, + "limits": { + "max_abs_f_error": 1e-07, + "relative_l2_f_error": 1e-08, + "relative_l2_e_error": 1e-07 + }, + "benchmark": { + "backend": "cuda", + "device": "NVIDIA A100-SXM4-40GB", + "precision": "float64", + "mode": "landau", + "integrator": "sixth", + "nu": 1e-12, + "velocity": "cubic", + "layout": "transpose", + "cell_centred": true, + "max_halo_width": 0, + "rank0_halo_sent_bytes": 0, + "halo_sent_bytes_per_rank_max": 0, + "slab_sent_bytes_per_rank": 115605504000, + "input": "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/initial.bin", + "nx": 134400, + "nv": 4096, + "ranks": 4, + "nodes": 1, + "steps": 10, + "warmup": 0, + "dt": 0.02, + "length": 12600, + "vmax": 15, + "amplitude": 0.01, + "acceleration": 0.2, + "host_mpi": false, + "state_bytes": 4404019200, + "max_allocated_bytes_per_rank": 11017656376, + "stream_ms": 96.79199040010644, + "spline_ms": 23.18394020021515, + "pack_ms": 45.59686090110517, + "mpi_ms": 137.6495783999417, + "field_ms": 6.097852600396436, + "collision_ms": 14.47472449995075, + "step_ms": 323.8134939000247, + "initial_mass": 12597.6196507818, + "final_mass": 12597.62016522545, + "mass_relative_change": 4.083657516783715e-08, + "initial_kinetic_energy": 55713.43281125277, + "final_kinetic_energy": 55714.15660222915, + "initial_field_energy": 787.6538181371966, + "final_field_energy": 786.9554651665927, + "minimum_f": -0.1916308738890627, + "finite": true + }, + "command": [ + "srun", + "--jobid=60205333", + "--overlap", + "--nodes=1", + "--ntasks=4", + "--ntasks-per-node=4", + "--gpus-per-node=4", + "--gpu-bind=none", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/workdir/cuda-build/vlasov_bench", + "--nx", + "134400", + "--nv", + "4096", + "--length", + "12600", + "--vmax", + "15", + "--dt", + "0.02", + "--nu", + "1e-12", + "--cells", + "--velocity", + "cubic", + "--integrator", + "sixth", + "--layout", + "transpose", + "--steps", + "10", + "--warmup", + "0", + "--input", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/initial.bin", + "--v-grid", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/v.bin", + "--background", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/background.bin", + "--dump", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/comparison-nx134400-window/1n-4g-transpose-0010.bin" + ], + "rank_log": "rank=0 host=nid001024 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001024 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001024 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001024 local_rank=2 device=NVIDIA A100-SXM4-40GB\n" + }, + { + "nodes": 1, + "ranks": 4, + "layout": "transpose", + "steps": 100, + "max_abs_f_error": 6.725869861057276e-13, + "relative_l2_f_error": 9.515283499791307e-14, + "relative_l2_e_error": 4.921388303823267e-14, + "max_abs_e_error": 4.907185768843192e-14, + "hole_counts": [ + 129, + 47 + ], + "reference_hole_counts": [ + 129, + 47 + ], + "hole_counts_match": true, + "passed": true, + "limits": { + "max_abs_f_error": 1e-07, + "relative_l2_f_error": 1e-08, + "relative_l2_e_error": 1e-07 + }, + "benchmark": { + "backend": "cuda", + "device": "NVIDIA A100-SXM4-40GB", + "precision": "float64", + "mode": "landau", + "integrator": "sixth", + "nu": 1e-12, + "velocity": "cubic", + "layout": "transpose", + "cell_centred": true, + "max_halo_width": 0, + "rank0_halo_sent_bytes": 0, + "halo_sent_bytes_per_rank_max": 0, + "slab_sent_bytes_per_rank": 1156055040000, + "input": "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/initial.bin", + "nx": 134400, + "nv": 4096, + "ranks": 4, + "nodes": 1, + "steps": 100, + "warmup": 0, + "dt": 0.02, + "length": 12600, + "vmax": 15, + "amplitude": 0.01, + "acceleration": 0.2, + "host_mpi": false, + "state_bytes": 4404019200, + "max_allocated_bytes_per_rank": 11017656376, + "stream_ms": 96.7996663899794, + "spline_ms": 23.2147768099685, + "pack_ms": 45.58691152003121, + "mpi_ms": 137.3947532199827, + "field_ms": 6.027191200023481, + "collision_ms": 14.49148772999251, + "step_ms": 323.5432447600033, + "initial_mass": 12597.6196507818, + "final_mass": 12597.61289309191, + "mass_relative_change": -5.364259347188723e-07, + "initial_kinetic_energy": 55713.43281125277, + "final_kinetic_energy": 55718.73788187712, + "initial_field_energy": 787.6538181371966, + "final_field_energy": 781.820420166659, + "minimum_f": -0.1919615815105848, + "finite": true + }, + "command": [ + "srun", + "--jobid=60205333", + "--overlap", + "--nodes=1", + "--ntasks=4", + "--ntasks-per-node=4", + "--gpus-per-node=4", + "--gpu-bind=none", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/workdir/cuda-build/vlasov_bench", + "--nx", + "134400", + "--nv", + "4096", + "--length", + "12600", + "--vmax", + "15", + "--dt", + "0.02", + "--nu", + "1e-12", + "--cells", + "--velocity", + "cubic", + "--integrator", + "sixth", + "--layout", + "transpose", + "--steps", + "100", + "--warmup", + "0", + "--input", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/initial.bin", + "--v-grid", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/v.bin", + "--background", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/background.bin", + "--dump", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/comparison-nx134400-window/1n-4g-transpose-0100.bin" + ], + "rank_log": "rank=1 host=nid001025 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001025 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001025 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001025 local_rank=2 device=NVIDIA A100-SXM4-40GB\n" + }, + { + "nodes": 1, + "ranks": 4, + "layout": "halos", + "steps": 10, + "max_abs_f_error": 1.2218004385999848e-13, + "relative_l2_f_error": 1.8566744087723585e-14, + "relative_l2_e_error": 4.172291687802881e-14, + "max_abs_e_error": 3.424344141578217e-14, + "hole_counts": [ + 130, + 50 + ], + "reference_hole_counts": [ + 130, + 50 + ], + "hole_counts_match": true, + "passed": true, + "limits": { + "max_abs_f_error": 1e-07, + "relative_l2_f_error": 1e-08, + "relative_l2_e_error": 1e-07 + }, + "benchmark": { + "backend": "cuda", + "device": "NVIDIA A100-SXM4-40GB", + "precision": "float64", + "mode": "landau", + "integrator": "sixth", + "nu": 1e-12, + "velocity": "cubic", + "layout": "halos", + "cell_centred": true, + "max_halo_width": 4, + "rank0_halo_sent_bytes": 215040000, + "halo_sent_bytes_per_rank_max": 430080000, + "slab_sent_bytes_per_rank": 16515072000, + "input": "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/initial.bin", + "nx": 134400, + "nv": 4096, + "ranks": 4, + "nodes": 1, + "steps": 10, + "warmup": 0, + "dt": 0.02, + "length": 12600, + "vmax": 15, + "amplitude": 0.01, + "acceleration": 0.2, + "host_mpi": false, + "state_bytes": 4404019200, + "max_allocated_bytes_per_rank": 11155281976, + "stream_ms": 96.76741280004535, + "spline_ms": 29.37165490020561, + "pack_ms": 6.529228900035378, + "mpi_ms": 26.30602839999483, + "field_ms": 7.745825100028014, + "collision_ms": 14.48409960007666, + "step_ms": 179.6518952999122, + "initial_mass": 12597.6196507818, + "final_mass": 12597.62016522545, + "mass_relative_change": 4.083657516783715e-08, + "initial_kinetic_energy": 55713.43281125277, + "final_kinetic_energy": 55714.15660222915, + "initial_field_energy": 787.6538181371966, + "final_field_energy": 786.9554651665927, + "minimum_f": -0.1916308738890627, + "finite": true + }, + "command": [ + "srun", + "--jobid=60205333", + "--overlap", + "--nodes=1", + "--ntasks=4", + "--ntasks-per-node=4", + "--gpus-per-node=4", + "--gpu-bind=none", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/workdir/cuda-build/vlasov_bench", + "--nx", + "134400", + "--nv", + "4096", + "--length", + "12600", + "--vmax", + "15", + "--dt", + "0.02", + "--nu", + "1e-12", + "--cells", + "--velocity", + "cubic", + "--integrator", + "sixth", + "--layout", + "halos", + "--steps", + "10", + "--warmup", + "0", + "--input", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/initial.bin", + "--v-grid", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/v.bin", + "--background", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/background.bin", + "--dump", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/comparison-nx134400-window/1n-4g-halos-0010.bin" + ], + "rank_log": "rank=2 host=nid001024 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001024 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001024 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001024 local_rank=0 device=NVIDIA A100-SXM4-40GB\n" + }, + { + "nodes": 1, + "ranks": 4, + "layout": "halos", + "steps": 100, + "max_abs_f_error": 6.725869861057276e-13, + "relative_l2_f_error": 9.515283499791307e-14, + "relative_l2_e_error": 4.921388303823267e-14, + "max_abs_e_error": 4.907185768843192e-14, + "hole_counts": [ + 129, + 47 + ], + "reference_hole_counts": [ + 129, + 47 + ], + "hole_counts_match": true, + "passed": true, + "limits": { + "max_abs_f_error": 1e-07, + "relative_l2_f_error": 1e-08, + "relative_l2_e_error": 1e-07 + }, + "benchmark": { + "backend": "cuda", + "device": "NVIDIA A100-SXM4-40GB", + "precision": "float64", + "mode": "landau", + "integrator": "sixth", + "nu": 1e-12, + "velocity": "cubic", + "layout": "halos", + "cell_centred": true, + "max_halo_width": 4, + "rank0_halo_sent_bytes": 2150400000, + "halo_sent_bytes_per_rank_max": 4300800000, + "slab_sent_bytes_per_rank": 165150720000, + "input": "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/initial.bin", + "nx": 134400, + "nv": 4096, + "ranks": 4, + "nodes": 1, + "steps": 100, + "warmup": 0, + "dt": 0.02, + "length": 12600, + "vmax": 15, + "amplitude": 0.01, + "acceleration": 0.2, + "host_mpi": false, + "state_bytes": 4404019200, + "max_allocated_bytes_per_rank": 11155281976, + "stream_ms": 96.76782525001272, + "spline_ms": 29.39105241994184, + "pack_ms": 6.511805909976829, + "mpi_ms": 21.13449154009231, + "field_ms": 6.609109270007139, + "collision_ms": 14.49049542000466, + "step_ms": 174.3556135900099, + "initial_mass": 12597.6196507818, + "final_mass": 12597.61289309191, + "mass_relative_change": -5.364259347188723e-07, + "initial_kinetic_energy": 55713.43281125277, + "final_kinetic_energy": 55718.73788187712, + "initial_field_energy": 787.6538181371966, + "final_field_energy": 781.820420166659, + "minimum_f": -0.1919615815105848, + "finite": true + }, + "command": [ + "srun", + "--jobid=60205333", + "--overlap", + "--nodes=1", + "--ntasks=4", + "--ntasks-per-node=4", + "--gpus-per-node=4", + "--gpu-bind=none", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/workdir/cuda-build/vlasov_bench", + "--nx", + "134400", + "--nv", + "4096", + "--length", + "12600", + "--vmax", + "15", + "--dt", + "0.02", + "--nu", + "1e-12", + "--cells", + "--velocity", + "cubic", + "--integrator", + "sixth", + "--layout", + "halos", + "--steps", + "100", + "--warmup", + "0", + "--input", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/initial.bin", + "--v-grid", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/v.bin", + "--background", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/background.bin", + "--dump", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/comparison-nx134400-window/1n-4g-halos-0100.bin" + ], + "rank_log": "rank=0 host=nid001025 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001025 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001025 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001025 local_rank=3 device=NVIDIA A100-SXM4-40GB\n" + }, + { + "nodes": 2, + "ranks": 8, + "layout": "transpose", + "steps": 10, + "max_abs_f_error": 1.1882161921050738e-13, + "relative_l2_f_error": 1.854096980472526e-14, + "relative_l2_e_error": 3.8024486397472564e-14, + "max_abs_e_error": 3.226585665316861e-14, + "hole_counts": [ + 130, + 50 + ], + "reference_hole_counts": [ + 130, + 50 + ], + "hole_counts_match": true, + "passed": true, + "limits": { + "max_abs_f_error": 1e-07, + "relative_l2_f_error": 1e-08, + "relative_l2_e_error": 1e-07 + }, + "benchmark": { + "backend": "cuda", + "device": "NVIDIA A100-SXM4-40GB", + "precision": "float64", + "mode": "landau", + "integrator": "sixth", + "nu": 1e-12, + "velocity": "cubic", + "layout": "transpose", + "cell_centred": true, + "max_halo_width": 0, + "rank0_halo_sent_bytes": 0, + "halo_sent_bytes_per_rank_max": 0, + "slab_sent_bytes_per_rank": 67436544000, + "input": "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/initial.bin", + "nx": 134400, + "nv": 4096, + "ranks": 8, + "nodes": 2, + "steps": 10, + "warmup": 0, + "dt": 0.02, + "length": 12600, + "vmax": 15, + "amplitude": 0.01, + "acceleration": 0.2, + "host_mpi": false, + "state_bytes": 4404019200, + "max_allocated_bytes_per_rank": 5512607800, + "stream_ms": 48.5531004999757, + "spline_ms": 11.72833750001701, + "pack_ms": 22.98895100011578, + "mpi_ms": 392.2544488003041, + "field_ms": 12.02276010003516, + "collision_ms": 11.25639949996184, + "step_ms": 491.9087526999647, + "initial_mass": 12597.61965078436, + "final_mass": 12597.62016522508, + "mass_relative_change": 4.083634327505363e-08, + "initial_kinetic_energy": 55713.43281126581, + "final_kinetic_energy": 55714.15660220217, + "initial_field_energy": 787.6538181371961, + "final_field_energy": 786.9554651665925, + "minimum_f": -0.1916308738890627, + "finite": true + }, + "command": [ + "srun", + "--jobid=60205333", + "--overlap", + "--nodes=2", + "--ntasks=8", + "--ntasks-per-node=4", + "--gpus-per-node=4", + "--gpu-bind=none", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/workdir/cuda-build/vlasov_bench", + "--nx", + "134400", + "--nv", + "4096", + "--length", + "12600", + "--vmax", + "15", + "--dt", + "0.02", + "--nu", + "1e-12", + "--cells", + "--velocity", + "cubic", + "--integrator", + "sixth", + "--layout", + "transpose", + "--steps", + "10", + "--warmup", + "0", + "--input", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/initial.bin", + "--v-grid", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/v.bin", + "--background", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/background.bin", + "--dump", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/comparison-nx134400-window/2n-8g-transpose-0010.bin" + ], + "rank_log": "rank=4 host=nid001024 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=6 host=nid001024 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001024 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001025 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=7 host=nid001025 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001024 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=5 host=nid001025 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001025 local_rank=0 device=NVIDIA A100-SXM4-40GB\n" + }, + { + "nodes": 2, + "ranks": 8, + "layout": "transpose", + "steps": 100, + "max_abs_f_error": 6.686595721561162e-13, + "relative_l2_f_error": 9.400228503663682e-14, + "relative_l2_e_error": 5.6328186846468416e-14, + "max_abs_e_error": 4.96269692007445e-14, + "hole_counts": [ + 129, + 47 + ], + "reference_hole_counts": [ + 129, + 47 + ], + "hole_counts_match": true, + "passed": true, + "limits": { + "max_abs_f_error": 1e-07, + "relative_l2_f_error": 1e-08, + "relative_l2_e_error": 1e-07 + }, + "benchmark": { + "backend": "cuda", + "device": "NVIDIA A100-SXM4-40GB", + "precision": "float64", + "mode": "landau", + "integrator": "sixth", + "nu": 1e-12, + "velocity": "cubic", + "layout": "transpose", + "cell_centred": true, + "max_halo_width": 0, + "rank0_halo_sent_bytes": 0, + "halo_sent_bytes_per_rank_max": 0, + "slab_sent_bytes_per_rank": 674365440000, + "input": "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/initial.bin", + "nx": 134400, + "nv": 4096, + "ranks": 8, + "nodes": 2, + "steps": 100, + "warmup": 0, + "dt": 0.02, + "length": 12600, + "vmax": 15, + "amplitude": 0.01, + "acceleration": 0.2, + "host_mpi": false, + "state_bytes": 4404019200, + "max_allocated_bytes_per_rank": 5512607800, + "stream_ms": 48.54172135000226, + "spline_ms": 11.71508251993146, + "pack_ms": 22.96030381008222, + "mpi_ms": 390.6149536599469, + "field_ms": 12.35562636996747, + "collision_ms": 11.23707952998757, + "step_ms": 490.0134217499863, + "initial_mass": 12597.61965078436, + "final_mass": 12597.61289309384, + "mass_relative_change": -5.364259848226194e-07, + "initial_kinetic_energy": 55713.43281126581, + "final_kinetic_energy": 55718.73788187286, + "initial_field_energy": 787.6538181371961, + "final_field_energy": 781.8204201666606, + "minimum_f": -0.1919615815105861, + "finite": true + }, + "command": [ + "srun", + "--jobid=60205333", + "--overlap", + "--nodes=2", + "--ntasks=8", + "--ntasks-per-node=4", + "--gpus-per-node=4", + "--gpu-bind=none", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/workdir/cuda-build/vlasov_bench", + "--nx", + "134400", + "--nv", + "4096", + "--length", + "12600", + "--vmax", + "15", + "--dt", + "0.02", + "--nu", + "1e-12", + "--cells", + "--velocity", + "cubic", + "--integrator", + "sixth", + "--layout", + "transpose", + "--steps", + "100", + "--warmup", + "0", + "--input", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/initial.bin", + "--v-grid", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/v.bin", + "--background", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/background.bin", + "--dump", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/comparison-nx134400-window/2n-8g-transpose-0100.bin" + ], + "rank_log": "rank=7 host=nid001025 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001025 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001025 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=5 host=nid001025 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001024 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001024 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=6 host=nid001024 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=4 host=nid001024 local_rank=2 device=NVIDIA A100-SXM4-40GB\n" + }, + { + "nodes": 2, + "ranks": 8, + "layout": "halos", + "steps": 10, + "max_abs_f_error": 1.1882161921050738e-13, + "relative_l2_f_error": 1.854096980472526e-14, + "relative_l2_e_error": 3.8024486397472564e-14, + "max_abs_e_error": 3.226585665316861e-14, + "hole_counts": [ + 130, + 50 + ], + "reference_hole_counts": [ + 130, + 50 + ], + "hole_counts_match": true, + "passed": true, + "limits": { + "max_abs_f_error": 1e-07, + "relative_l2_f_error": 1e-08, + "relative_l2_e_error": 1e-07 + }, + "benchmark": { + "backend": "cuda", + "device": "NVIDIA A100-SXM4-40GB", + "precision": "float64", + "mode": "landau", + "integrator": "sixth", + "nu": 1e-12, + "velocity": "cubic", + "layout": "halos", + "cell_centred": true, + "max_halo_width": 4, + "rank0_halo_sent_bytes": 215040000, + "halo_sent_bytes_per_rank_max": 430080000, + "slab_sent_bytes_per_rank": 9633792000, + "input": "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/initial.bin", + "nx": 134400, + "nv": 4096, + "ranks": 8, + "nodes": 2, + "steps": 10, + "warmup": 0, + "dt": 0.02, + "length": 12600, + "vmax": 15, + "amplitude": 0.01, + "acceleration": 0.2, + "host_mpi": false, + "state_bytes": 4404019200, + "max_allocated_bytes_per_rank": 5650233400, + "stream_ms": 48.50393130009252, + "spline_ms": 12.2415565999745, + "pack_ms": 3.332494700134703, + "mpi_ms": 64.20001480000792, + "field_ms": 6.318310100277813, + "collision_ms": 11.24460030000591, + "step_ms": 144.2658396000297, + "initial_mass": 12597.61965078436, + "final_mass": 12597.62016522508, + "mass_relative_change": 4.083634327505363e-08, + "initial_kinetic_energy": 55713.43281126581, + "final_kinetic_energy": 55714.15660220217, + "initial_field_energy": 787.6538181371961, + "final_field_energy": 786.9554651665925, + "minimum_f": -0.1916308738890627, + "finite": true + }, + "command": [ + "srun", + "--jobid=60205333", + "--overlap", + "--nodes=2", + "--ntasks=8", + "--ntasks-per-node=4", + "--gpus-per-node=4", + "--gpu-bind=none", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/workdir/cuda-build/vlasov_bench", + "--nx", + "134400", + "--nv", + "4096", + "--length", + "12600", + "--vmax", + "15", + "--dt", + "0.02", + "--nu", + "1e-12", + "--cells", + "--velocity", + "cubic", + "--integrator", + "sixth", + "--layout", + "halos", + "--steps", + "10", + "--warmup", + "0", + "--input", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/initial.bin", + "--v-grid", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/v.bin", + "--background", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/background.bin", + "--dump", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/comparison-nx134400-window/2n-8g-halos-0010.bin" + ], + "rank_log": "rank=7 host=nid001025 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001025 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001025 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=6 host=nid001024 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=5 host=nid001025 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001024 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001024 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=4 host=nid001024 local_rank=2 device=NVIDIA A100-SXM4-40GB\n" + }, + { + "nodes": 2, + "ranks": 8, + "layout": "halos", + "steps": 100, + "max_abs_f_error": 6.686595721561162e-13, + "relative_l2_f_error": 9.400228503663682e-14, + "relative_l2_e_error": 5.6328186846468416e-14, + "max_abs_e_error": 4.96269692007445e-14, + "hole_counts": [ + 129, + 47 + ], + "reference_hole_counts": [ + 129, + 47 + ], + "hole_counts_match": true, + "passed": true, + "limits": { + "max_abs_f_error": 1e-07, + "relative_l2_f_error": 1e-08, + "relative_l2_e_error": 1e-07 + }, + "benchmark": { + "backend": "cuda", + "device": "NVIDIA A100-SXM4-40GB", + "precision": "float64", + "mode": "landau", + "integrator": "sixth", + "nu": 1e-12, + "velocity": "cubic", + "layout": "halos", + "cell_centred": true, + "max_halo_width": 4, + "rank0_halo_sent_bytes": 2150400000, + "halo_sent_bytes_per_rank_max": 4300800000, + "slab_sent_bytes_per_rank": 96337920000, + "input": "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/initial.bin", + "nx": 134400, + "nv": 4096, + "ranks": 8, + "nodes": 2, + "steps": 100, + "warmup": 0, + "dt": 0.02, + "length": 12600, + "vmax": 15, + "amplitude": 0.01, + "acceleration": 0.2, + "host_mpi": false, + "state_bytes": 4404019200, + "max_allocated_bytes_per_rank": 5650233400, + "stream_ms": 48.49508005004736, + "spline_ms": 12.23889673997746, + "pack_ms": 3.278673720083134, + "mpi_ms": 60.91060809000737, + "field_ms": 6.122944579915384, + "collision_ms": 11.23616700002458, + "step_ms": 140.7295966299989, + "initial_mass": 12597.61965078436, + "final_mass": 12597.61289309384, + "mass_relative_change": -5.364259848226194e-07, + "initial_kinetic_energy": 55713.43281126581, + "final_kinetic_energy": 55718.73788187286, + "initial_field_energy": 787.6538181371961, + "final_field_energy": 781.8204201666606, + "minimum_f": -0.1919615815105861, + "finite": true + }, + "command": [ + "srun", + "--jobid=60205333", + "--overlap", + "--nodes=2", + "--ntasks=8", + "--ntasks-per-node=4", + "--gpus-per-node=4", + "--gpu-bind=none", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/workdir/cuda-build/vlasov_bench", + "--nx", + "134400", + "--nv", + "4096", + "--length", + "12600", + "--vmax", + "15", + "--dt", + "0.02", + "--nu", + "1e-12", + "--cells", + "--velocity", + "cubic", + "--integrator", + "sixth", + "--layout", + "halos", + "--steps", + "100", + "--warmup", + "0", + "--input", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/initial.bin", + "--v-grid", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/v.bin", + "--background", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/background.bin", + "--dump", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/comparison-nx134400-window/2n-8g-halos-0100.bin" + ], + "rank_log": "rank=6 host=nid001024 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=4 host=nid001024 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001024 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001024 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001025 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001025 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=5 host=nid001025 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=7 host=nid001025 local_rank=3 device=NVIDIA A100-SXM4-40GB\n" + } +] diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-halos/comparison-nx268800-1node.json b/benchmarks/vlasov_cuda/results/2026-10-24-halos/comparison-nx268800-1node.json new file mode 100644 index 00000000..28872c84 --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-halos/comparison-nx268800-1node.json @@ -0,0 +1,462 @@ +[ + { + "nodes": 1, + "ranks": 4, + "layout": "transpose", + "steps": 1, + "max_abs_f_error": 4.1300296516055823e-14, + "relative_l2_f_error": 5.6935482508837695e-15, + "relative_l2_e_error": 5.236955259389934e-15, + "max_abs_e_error": 5.578870698741412e-15, + "hole_counts": [ + 118, + 54 + ], + "reference_hole_counts": [ + 118, + 54 + ], + "hole_counts_match": true, + "passed": true, + "limits": { + "max_abs_f_error": 1e-07, + "relative_l2_f_error": 1e-08, + "relative_l2_e_error": 1e-07 + }, + "benchmark": { + "backend": "cuda", + "device": "NVIDIA A100-SXM4-40GB", + "precision": "float64", + "mode": "landau", + "integrator": "sixth", + "nu": 1e-12, + "velocity": "cubic", + "layout": "transpose", + "cell_centred": true, + "max_halo_width": 0, + "rank0_halo_sent_bytes": 0, + "halo_sent_bytes_per_rank_max": 0, + "slab_sent_bytes_per_rank": 23121100800, + "input": "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/initial.bin", + "nx": 268800, + "nv": 4096, + "ranks": 4, + "nodes": 1, + "steps": 1, + "warmup": 0, + "dt": 0.02, + "length": 12600, + "vmax": 15, + "amplitude": 0.01, + "acceleration": 0.2, + "host_mpi": false, + "state_bytes": 8808038400, + "max_allocated_bytes_per_rank": 22035230776, + "stream_ms": 194.5556020002641, + "spline_ms": 57.92932199983625, + "pack_ms": 91.67665799964198, + "mpi_ms": 276.4357230003043, + "field_ms": 11.46622900000693, + "collision_ms": 21.33707100006177, + "step_ms": 653.1722329998502, + "initial_mass": 12597.62207264873, + "final_mass": 12597.62199624369, + "mass_relative_change": -6.065036546636737e-09, + "initial_kinetic_energy": 55701.51067944818, + "final_kinetic_energy": 55701.4580005624, + "initial_field_energy": 791.8606180599772, + "final_field_energy": 791.9073734470658, + "minimum_f": -0.09528937374122746, + "finite": true + }, + "command": [ + "srun", + "--jobid=60205333", + "--overlap", + "--nodes=1", + "--ntasks=4", + "--ntasks-per-node=4", + "--gpus-per-node=4", + "--gpu-bind=none", + "--cpu-bind=none", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/workdir/cuda-build/vlasov_bench", + "--nx", + "268800", + "--nv", + "4096", + "--length", + "12600", + "--vmax", + "15", + "--dt", + "0.02", + "--nu", + "1e-12", + "--cells", + "--velocity", + "cubic", + "--integrator", + "sixth", + "--layout", + "transpose", + "--steps", + "1", + "--warmup", + "0", + "--input", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/initial.bin", + "--v-grid", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/v.bin", + "--background", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/background.bin", + "--dump", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/comparison-nx268800-window/1n-4g-transpose-0001.bin" + ], + "rank_log": "rank=1 host=nid001024 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001024 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001024 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001024 local_rank=0 device=NVIDIA A100-SXM4-40GB\n" + }, + { + "nodes": 1, + "ranks": 4, + "layout": "transpose", + "steps": 100, + "max_abs_f_error": 5.11674036474119e-13, + "relative_l2_f_error": 7.512052460545957e-14, + "relative_l2_e_error": 4.687552521565683e-14, + "max_abs_e_error": 4.2334191707738e-14, + "hole_counts": [ + 129, + 50 + ], + "reference_hole_counts": [ + 129, + 50 + ], + "hole_counts_match": true, + "passed": true, + "limits": { + "max_abs_f_error": 1e-07, + "relative_l2_f_error": 1e-08, + "relative_l2_e_error": 1e-07 + }, + "benchmark": { + "backend": "cuda", + "device": "NVIDIA A100-SXM4-40GB", + "precision": "float64", + "mode": "landau", + "integrator": "sixth", + "nu": 1e-12, + "velocity": "cubic", + "layout": "transpose", + "cell_centred": true, + "max_halo_width": 0, + "rank0_halo_sent_bytes": 0, + "halo_sent_bytes_per_rank_max": 0, + "slab_sent_bytes_per_rank": 2312110080000, + "input": "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/initial.bin", + "nx": 268800, + "nv": 4096, + "ranks": 4, + "nodes": 1, + "steps": 100, + "warmup": 0, + "dt": 0.02, + "length": 12600, + "vmax": 15, + "amplitude": 0.01, + "acceleration": 0.2, + "host_mpi": false, + "state_bytes": 8808038400, + "max_allocated_bytes_per_rank": 22035230776, + "stream_ms": 194.5113329900028, + "spline_ms": 58.0104210499826, + "pack_ms": 91.58659695997585, + "mpi_ms": 273.2179411500283, + "field_ms": 11.19875519001198, + "collision_ms": 21.1204347399962, + "step_ms": 649.6034333900138, + "initial_mass": 12597.62207264873, + "final_mass": 12597.6175104671, + "mass_relative_change": -3.621462523818079e-07, + "initial_kinetic_energy": 55701.51067944818, + "final_kinetic_energy": 55695.27553237718, + "initial_field_energy": 791.8606180599772, + "final_field_energy": 797.6672297926372, + "minimum_f": -0.09642050941611767, + "finite": true + }, + "command": [ + "srun", + "--jobid=60205333", + "--overlap", + "--nodes=1", + "--ntasks=4", + "--ntasks-per-node=4", + "--gpus-per-node=4", + "--gpu-bind=none", + "--cpu-bind=none", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/workdir/cuda-build/vlasov_bench", + "--nx", + "268800", + "--nv", + "4096", + "--length", + "12600", + "--vmax", + "15", + "--dt", + "0.02", + "--nu", + "1e-12", + "--cells", + "--velocity", + "cubic", + "--integrator", + "sixth", + "--layout", + "transpose", + "--steps", + "100", + "--warmup", + "0", + "--input", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/initial.bin", + "--v-grid", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/v.bin", + "--background", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/background.bin", + "--dump", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/comparison-nx268800-window/1n-4g-transpose-0100.bin" + ], + "rank_log": "rank=2 host=nid001025 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001025 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001025 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001025 local_rank=0 device=NVIDIA A100-SXM4-40GB\n" + }, + { + "nodes": 1, + "ranks": 4, + "layout": "halos", + "steps": 1, + "max_abs_f_error": 4.1300296516055823e-14, + "relative_l2_f_error": 5.6935482508837695e-15, + "relative_l2_e_error": 5.236955259389934e-15, + "max_abs_e_error": 5.578870698741412e-15, + "hole_counts": [ + 118, + 54 + ], + "reference_hole_counts": [ + 118, + 54 + ], + "hole_counts_match": true, + "passed": true, + "limits": { + "max_abs_f_error": 1e-07, + "relative_l2_f_error": 1e-08, + "relative_l2_e_error": 1e-07 + }, + "benchmark": { + "backend": "cuda", + "device": "NVIDIA A100-SXM4-40GB", + "precision": "float64", + "mode": "landau", + "integrator": "sixth", + "nu": 1e-12, + "velocity": "cubic", + "layout": "halos", + "cell_centred": true, + "max_halo_width": 4, + "rank0_halo_sent_bytes": 43008000, + "halo_sent_bytes_per_rank_max": 86016000, + "slab_sent_bytes_per_rank": 3303014400, + "input": "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/initial.bin", + "nx": 268800, + "nv": 4096, + "ranks": 4, + "nodes": 1, + "steps": 1, + "warmup": 0, + "dt": 0.02, + "length": 12600, + "vmax": 15, + "amplitude": 0.01, + "acceleration": 0.2, + "host_mpi": false, + "state_bytes": 8808038400, + "max_allocated_bytes_per_rank": 22310481976, + "stream_ms": 194.4458430000395, + "spline_ms": 61.24707199978729, + "pack_ms": 13.11971299992365, + "mpi_ms": 97.95531999975537, + "field_ms": 12.17473299948324, + "collision_ms": 21.21514399982516, + "step_ms": 399.1295580001406, + "initial_mass": 12597.62207264873, + "final_mass": 12597.62199624369, + "mass_relative_change": -6.065036546636737e-09, + "initial_kinetic_energy": 55701.51067944818, + "final_kinetic_energy": 55701.4580005624, + "initial_field_energy": 791.8606180599772, + "final_field_energy": 791.9073734470658, + "minimum_f": -0.09528937374122746, + "finite": true + }, + "command": [ + "srun", + "--jobid=60205333", + "--overlap", + "--nodes=1", + "--ntasks=4", + "--ntasks-per-node=4", + "--gpus-per-node=4", + "--gpu-bind=none", + "--cpu-bind=none", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/workdir/cuda-build/vlasov_bench", + "--nx", + "268800", + "--nv", + "4096", + "--length", + "12600", + "--vmax", + "15", + "--dt", + "0.02", + "--nu", + "1e-12", + "--cells", + "--velocity", + "cubic", + "--integrator", + "sixth", + "--layout", + "halos", + "--steps", + "1", + "--warmup", + "0", + "--input", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/initial.bin", + "--v-grid", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/v.bin", + "--background", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/background.bin", + "--dump", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/comparison-nx268800-window/1n-4g-halos-0001.bin" + ], + "rank_log": "rank=3 host=nid001024 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001024 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001024 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001024 local_rank=2 device=NVIDIA A100-SXM4-40GB\n" + }, + { + "nodes": 1, + "ranks": 4, + "layout": "halos", + "steps": 100, + "max_abs_f_error": 5.11674036474119e-13, + "relative_l2_f_error": 7.512052460545957e-14, + "relative_l2_e_error": 4.687552521565683e-14, + "max_abs_e_error": 4.2334191707738e-14, + "hole_counts": [ + 129, + 50 + ], + "reference_hole_counts": [ + 129, + 50 + ], + "hole_counts_match": true, + "passed": true, + "limits": { + "max_abs_f_error": 1e-07, + "relative_l2_f_error": 1e-08, + "relative_l2_e_error": 1e-07 + }, + "benchmark": { + "backend": "cuda", + "device": "NVIDIA A100-SXM4-40GB", + "precision": "float64", + "mode": "landau", + "integrator": "sixth", + "nu": 1e-12, + "velocity": "cubic", + "layout": "halos", + "cell_centred": true, + "max_halo_width": 4, + "rank0_halo_sent_bytes": 4584652800, + "halo_sent_bytes_per_rank_max": 9169305600, + "slab_sent_bytes_per_rank": 330301440000, + "input": "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/initial.bin", + "nx": 268800, + "nv": 4096, + "ranks": 4, + "nodes": 1, + "steps": 100, + "warmup": 0, + "dt": 0.02, + "length": 12600, + "vmax": 15, + "amplitude": 0.01, + "acceleration": 0.2, + "host_mpi": false, + "state_bytes": 8808038400, + "max_allocated_bytes_per_rank": 22310481976, + "stream_ms": 194.4447632700076, + "spline_ms": 61.47456958001158, + "pack_ms": 13.08143660998667, + "mpi_ms": 41.6358091600182, + "field_ms": 12.248491260018, + "collision_ms": 21.12072830000443, + "step_ms": 342.950239549989, + "initial_mass": 12597.62207264873, + "final_mass": 12597.6175104671, + "mass_relative_change": -3.621462523818079e-07, + "initial_kinetic_energy": 55701.51067944818, + "final_kinetic_energy": 55695.27553237718, + "initial_field_energy": 791.8606180599772, + "final_field_energy": 797.6672297926372, + "minimum_f": -0.09642050941611767, + "finite": true + }, + "command": [ + "srun", + "--jobid=60205333", + "--overlap", + "--nodes=1", + "--ntasks=4", + "--ntasks-per-node=4", + "--gpus-per-node=4", + "--gpu-bind=none", + "--cpu-bind=none", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/workdir/cuda-build/vlasov_bench", + "--nx", + "268800", + "--nv", + "4096", + "--length", + "12600", + "--vmax", + "15", + "--dt", + "0.02", + "--nu", + "1e-12", + "--cells", + "--velocity", + "cubic", + "--integrator", + "sixth", + "--layout", + "halos", + "--steps", + "100", + "--warmup", + "0", + "--input", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/initial.bin", + "--v-grid", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/v.bin", + "--background", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/background.bin", + "--dump", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/comparison-nx268800-window/1n-4g-halos-0100.bin" + ], + "rank_log": "rank=1 host=nid001025 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001025 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001025 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001025 local_rank=3 device=NVIDIA A100-SXM4-40GB\n" + } +] diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-halos/comparison-nx268800-window.json b/benchmarks/vlasov_cuda/results/2026-10-24-halos/comparison-nx268800-window.json new file mode 100644 index 00000000..6afd1f12 --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-halos/comparison-nx268800-window.json @@ -0,0 +1,922 @@ +[ + { + "nodes": 1, + "ranks": 4, + "layout": "transpose", + "steps": 1, + "max_abs_f_error": 4.1300296516055823e-14, + "relative_l2_f_error": 5.6935482508837695e-15, + "relative_l2_e_error": 5.236955259389934e-15, + "max_abs_e_error": 5.578870698741412e-15, + "hole_counts": [ + 118, + 54 + ], + "reference_hole_counts": [ + 118, + 54 + ], + "hole_counts_match": true, + "passed": true, + "limits": { + "max_abs_f_error": 1e-07, + "relative_l2_f_error": 1e-08, + "relative_l2_e_error": 1e-07 + }, + "benchmark": { + "backend": "cuda", + "device": "NVIDIA A100-SXM4-40GB", + "precision": "float64", + "mode": "landau", + "integrator": "sixth", + "nu": 1e-12, + "velocity": "cubic", + "layout": "transpose", + "cell_centred": true, + "max_halo_width": 0, + "rank0_halo_sent_bytes": 0, + "halo_sent_bytes_per_rank_max": 0, + "slab_sent_bytes_per_rank": 23121100800, + "input": "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/initial.bin", + "nx": 268800, + "nv": 4096, + "ranks": 4, + "nodes": 1, + "steps": 1, + "warmup": 0, + "dt": 0.02, + "length": 12600, + "vmax": 15, + "amplitude": 0.01, + "acceleration": 0.2, + "host_mpi": false, + "state_bytes": 8808038400, + "max_allocated_bytes_per_rank": 22035230776, + "stream_ms": 194.5556020002641, + "spline_ms": 57.92932199983625, + "pack_ms": 91.67665799964198, + "mpi_ms": 276.4357230003043, + "field_ms": 11.46622900000693, + "collision_ms": 21.33707100006177, + "step_ms": 653.1722329998502, + "initial_mass": 12597.62207264873, + "final_mass": 12597.62199624369, + "mass_relative_change": -6.065036546636737e-09, + "initial_kinetic_energy": 55701.51067944818, + "final_kinetic_energy": 55701.4580005624, + "initial_field_energy": 791.8606180599772, + "final_field_energy": 791.9073734470658, + "minimum_f": -0.09528937374122746, + "finite": true + }, + "command": [ + "srun", + "--jobid=60205333", + "--overlap", + "--nodes=1", + "--ntasks=4", + "--ntasks-per-node=4", + "--gpus-per-node=4", + "--gpu-bind=none", + "--cpu-bind=none", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/workdir/cuda-build/vlasov_bench", + "--nx", + "268800", + "--nv", + "4096", + "--length", + "12600", + "--vmax", + "15", + "--dt", + "0.02", + "--nu", + "1e-12", + "--cells", + "--velocity", + "cubic", + "--integrator", + "sixth", + "--layout", + "transpose", + "--steps", + "1", + "--warmup", + "0", + "--input", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/initial.bin", + "--v-grid", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/v.bin", + "--background", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/background.bin", + "--dump", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/comparison-nx268800-window/1n-4g-transpose-0001.bin" + ], + "rank_log": "rank=1 host=nid001024 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001024 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001024 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001024 local_rank=0 device=NVIDIA A100-SXM4-40GB\n" + }, + { + "nodes": 1, + "ranks": 4, + "layout": "transpose", + "steps": 100, + "max_abs_f_error": 5.11674036474119e-13, + "relative_l2_f_error": 7.512052460545957e-14, + "relative_l2_e_error": 4.687552521565683e-14, + "max_abs_e_error": 4.2334191707738e-14, + "hole_counts": [ + 129, + 50 + ], + "reference_hole_counts": [ + 129, + 50 + ], + "hole_counts_match": true, + "passed": true, + "limits": { + "max_abs_f_error": 1e-07, + "relative_l2_f_error": 1e-08, + "relative_l2_e_error": 1e-07 + }, + "benchmark": { + "backend": "cuda", + "device": "NVIDIA A100-SXM4-40GB", + "precision": "float64", + "mode": "landau", + "integrator": "sixth", + "nu": 1e-12, + "velocity": "cubic", + "layout": "transpose", + "cell_centred": true, + "max_halo_width": 0, + "rank0_halo_sent_bytes": 0, + "halo_sent_bytes_per_rank_max": 0, + "slab_sent_bytes_per_rank": 2312110080000, + "input": "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/initial.bin", + "nx": 268800, + "nv": 4096, + "ranks": 4, + "nodes": 1, + "steps": 100, + "warmup": 0, + "dt": 0.02, + "length": 12600, + "vmax": 15, + "amplitude": 0.01, + "acceleration": 0.2, + "host_mpi": false, + "state_bytes": 8808038400, + "max_allocated_bytes_per_rank": 22035230776, + "stream_ms": 194.5113329900028, + "spline_ms": 58.0104210499826, + "pack_ms": 91.58659695997585, + "mpi_ms": 273.2179411500283, + "field_ms": 11.19875519001198, + "collision_ms": 21.1204347399962, + "step_ms": 649.6034333900138, + "initial_mass": 12597.62207264873, + "final_mass": 12597.6175104671, + "mass_relative_change": -3.621462523818079e-07, + "initial_kinetic_energy": 55701.51067944818, + "final_kinetic_energy": 55695.27553237718, + "initial_field_energy": 791.8606180599772, + "final_field_energy": 797.6672297926372, + "minimum_f": -0.09642050941611767, + "finite": true + }, + "command": [ + "srun", + "--jobid=60205333", + "--overlap", + "--nodes=1", + "--ntasks=4", + "--ntasks-per-node=4", + "--gpus-per-node=4", + "--gpu-bind=none", + "--cpu-bind=none", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/workdir/cuda-build/vlasov_bench", + "--nx", + "268800", + "--nv", + "4096", + "--length", + "12600", + "--vmax", + "15", + "--dt", + "0.02", + "--nu", + "1e-12", + "--cells", + "--velocity", + "cubic", + "--integrator", + "sixth", + "--layout", + "transpose", + "--steps", + "100", + "--warmup", + "0", + "--input", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/initial.bin", + "--v-grid", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/v.bin", + "--background", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/background.bin", + "--dump", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/comparison-nx268800-window/1n-4g-transpose-0100.bin" + ], + "rank_log": "rank=2 host=nid001025 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001025 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001025 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001025 local_rank=0 device=NVIDIA A100-SXM4-40GB\n" + }, + { + "nodes": 1, + "ranks": 4, + "layout": "halos", + "steps": 1, + "max_abs_f_error": 4.1300296516055823e-14, + "relative_l2_f_error": 5.6935482508837695e-15, + "relative_l2_e_error": 5.236955259389934e-15, + "max_abs_e_error": 5.578870698741412e-15, + "hole_counts": [ + 118, + 54 + ], + "reference_hole_counts": [ + 118, + 54 + ], + "hole_counts_match": true, + "passed": true, + "limits": { + "max_abs_f_error": 1e-07, + "relative_l2_f_error": 1e-08, + "relative_l2_e_error": 1e-07 + }, + "benchmark": { + "backend": "cuda", + "device": "NVIDIA A100-SXM4-40GB", + "precision": "float64", + "mode": "landau", + "integrator": "sixth", + "nu": 1e-12, + "velocity": "cubic", + "layout": "halos", + "cell_centred": true, + "max_halo_width": 4, + "rank0_halo_sent_bytes": 43008000, + "halo_sent_bytes_per_rank_max": 86016000, + "slab_sent_bytes_per_rank": 3303014400, + "input": "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/initial.bin", + "nx": 268800, + "nv": 4096, + "ranks": 4, + "nodes": 1, + "steps": 1, + "warmup": 0, + "dt": 0.02, + "length": 12600, + "vmax": 15, + "amplitude": 0.01, + "acceleration": 0.2, + "host_mpi": false, + "state_bytes": 8808038400, + "max_allocated_bytes_per_rank": 22310481976, + "stream_ms": 194.4458430000395, + "spline_ms": 61.24707199978729, + "pack_ms": 13.11971299992365, + "mpi_ms": 97.95531999975537, + "field_ms": 12.17473299948324, + "collision_ms": 21.21514399982516, + "step_ms": 399.1295580001406, + "initial_mass": 12597.62207264873, + "final_mass": 12597.62199624369, + "mass_relative_change": -6.065036546636737e-09, + "initial_kinetic_energy": 55701.51067944818, + "final_kinetic_energy": 55701.4580005624, + "initial_field_energy": 791.8606180599772, + "final_field_energy": 791.9073734470658, + "minimum_f": -0.09528937374122746, + "finite": true + }, + "command": [ + "srun", + "--jobid=60205333", + "--overlap", + "--nodes=1", + "--ntasks=4", + "--ntasks-per-node=4", + "--gpus-per-node=4", + "--gpu-bind=none", + "--cpu-bind=none", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/workdir/cuda-build/vlasov_bench", + "--nx", + "268800", + "--nv", + "4096", + "--length", + "12600", + "--vmax", + "15", + "--dt", + "0.02", + "--nu", + "1e-12", + "--cells", + "--velocity", + "cubic", + "--integrator", + "sixth", + "--layout", + "halos", + "--steps", + "1", + "--warmup", + "0", + "--input", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/initial.bin", + "--v-grid", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/v.bin", + "--background", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/background.bin", + "--dump", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/comparison-nx268800-window/1n-4g-halos-0001.bin" + ], + "rank_log": "rank=3 host=nid001024 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001024 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001024 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001024 local_rank=2 device=NVIDIA A100-SXM4-40GB\n" + }, + { + "nodes": 1, + "ranks": 4, + "layout": "halos", + "steps": 100, + "max_abs_f_error": 5.11674036474119e-13, + "relative_l2_f_error": 7.512052460545957e-14, + "relative_l2_e_error": 4.687552521565683e-14, + "max_abs_e_error": 4.2334191707738e-14, + "hole_counts": [ + 129, + 50 + ], + "reference_hole_counts": [ + 129, + 50 + ], + "hole_counts_match": true, + "passed": true, + "limits": { + "max_abs_f_error": 1e-07, + "relative_l2_f_error": 1e-08, + "relative_l2_e_error": 1e-07 + }, + "benchmark": { + "backend": "cuda", + "device": "NVIDIA A100-SXM4-40GB", + "precision": "float64", + "mode": "landau", + "integrator": "sixth", + "nu": 1e-12, + "velocity": "cubic", + "layout": "halos", + "cell_centred": true, + "max_halo_width": 4, + "rank0_halo_sent_bytes": 4584652800, + "halo_sent_bytes_per_rank_max": 9169305600, + "slab_sent_bytes_per_rank": 330301440000, + "input": "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/initial.bin", + "nx": 268800, + "nv": 4096, + "ranks": 4, + "nodes": 1, + "steps": 100, + "warmup": 0, + "dt": 0.02, + "length": 12600, + "vmax": 15, + "amplitude": 0.01, + "acceleration": 0.2, + "host_mpi": false, + "state_bytes": 8808038400, + "max_allocated_bytes_per_rank": 22310481976, + "stream_ms": 194.4447632700076, + "spline_ms": 61.47456958001158, + "pack_ms": 13.08143660998667, + "mpi_ms": 41.6358091600182, + "field_ms": 12.248491260018, + "collision_ms": 21.12072830000443, + "step_ms": 342.950239549989, + "initial_mass": 12597.62207264873, + "final_mass": 12597.6175104671, + "mass_relative_change": -3.621462523818079e-07, + "initial_kinetic_energy": 55701.51067944818, + "final_kinetic_energy": 55695.27553237718, + "initial_field_energy": 791.8606180599772, + "final_field_energy": 797.6672297926372, + "minimum_f": -0.09642050941611767, + "finite": true + }, + "command": [ + "srun", + "--jobid=60205333", + "--overlap", + "--nodes=1", + "--ntasks=4", + "--ntasks-per-node=4", + "--gpus-per-node=4", + "--gpu-bind=none", + "--cpu-bind=none", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/workdir/cuda-build/vlasov_bench", + "--nx", + "268800", + "--nv", + "4096", + "--length", + "12600", + "--vmax", + "15", + "--dt", + "0.02", + "--nu", + "1e-12", + "--cells", + "--velocity", + "cubic", + "--integrator", + "sixth", + "--layout", + "halos", + "--steps", + "100", + "--warmup", + "0", + "--input", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/initial.bin", + "--v-grid", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/v.bin", + "--background", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/background.bin", + "--dump", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/comparison-nx268800-window/1n-4g-halos-0100.bin" + ], + "rank_log": "rank=1 host=nid001025 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001025 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001025 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001025 local_rank=3 device=NVIDIA A100-SXM4-40GB\n" + }, + { + "nodes": 2, + "ranks": 8, + "layout": "transpose", + "steps": 1, + "max_abs_f_error": 4.1272540940440194e-14, + "relative_l2_f_error": 5.693709254174748e-15, + "relative_l2_e_error": 5.726948268097046e-15, + "max_abs_e_error": 5.842548667089886e-15, + "hole_counts": [ + 118, + 54 + ], + "reference_hole_counts": [ + 118, + 54 + ], + "hole_counts_match": true, + "passed": true, + "limits": { + "max_abs_f_error": 1e-07, + "relative_l2_f_error": 1e-08, + "relative_l2_e_error": 1e-07 + }, + "benchmark": { + "backend": "cuda", + "device": "NVIDIA A100-SXM4-40GB", + "precision": "float64", + "mode": "landau", + "integrator": "sixth", + "nu": 1e-12, + "velocity": "cubic", + "layout": "transpose", + "cell_centred": true, + "max_halo_width": 0, + "rank0_halo_sent_bytes": 0, + "halo_sent_bytes_per_rank_max": 0, + "slab_sent_bytes_per_rank": 13487308800, + "input": "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/initial.bin", + "nx": 268800, + "nv": 4096, + "ranks": 8, + "nodes": 2, + "steps": 1, + "warmup": 0, + "dt": 0.02, + "length": 12600, + "vmax": 15, + "amplitude": 0.01, + "acceleration": 0.2, + "host_mpi": false, + "state_bytes": 8808038400, + "max_allocated_bytes_per_rank": 11025158200, + "stream_ms": 97.36238300001787, + "spline_ms": 23.15364699984457, + "pack_ms": 45.73831100060488, + "mpi_ms": 795.7124910005859, + "field_ms": 21.12155600025289, + "collision_ms": 14.52817399990636, + "step_ms": 983.8031960000535, + "initial_mass": 12597.62207265221, + "final_mass": 12597.62199624691, + "mass_relative_change": -6.065056761443461e-09, + "initial_kinetic_energy": 55701.51067942529, + "final_kinetic_energy": 55701.45800056863, + "initial_field_energy": 791.8606180599771, + "final_field_energy": 791.9073734470651, + "minimum_f": -0.0952893737412273, + "finite": true + }, + "command": [ + "srun", + "--jobid=60206959", + "--overlap", + "--nodes=2", + "--ntasks=8", + "--ntasks-per-node=4", + "--gpus-per-node=4", + "--gpu-bind=none", + "--cpu-bind=none", + "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/workdir/cuda-build/vlasov_bench", + "--nx", + "268800", + "--nv", + "4096", + "--length", + "12600", + "--vmax", + "15", + "--dt", + "0.02", + "--nu", + "1e-12", + "--cells", + "--velocity", + "cubic", + "--integrator", + "sixth", + "--layout", + "transpose", + "--steps", + "1", + "--warmup", + "0", + "--input", + "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/initial.bin", + "--v-grid", + "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/v.bin", + "--background", + "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/background.bin", + "--dump", + "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/comparison-nx268800-2node/2n-8g-transpose-0001.bin" + ], + "rank_log": "rank=6 host=nid001024 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=4 host=nid001024 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001024 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001024 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001025 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=7 host=nid001025 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001025 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=5 host=nid001025 local_rank=2 device=NVIDIA A100-SXM4-40GB\n" + }, + { + "nodes": 2, + "ranks": 8, + "layout": "transpose", + "steps": 100, + "max_abs_f_error": 5.144495940356819e-13, + "relative_l2_f_error": 7.864106656264876e-14, + "relative_l2_e_error": 4.782235170485902e-14, + "max_abs_e_error": 4.3395842475035806e-14, + "hole_counts": [ + 129, + 50 + ], + "reference_hole_counts": [ + 129, + 50 + ], + "hole_counts_match": true, + "passed": true, + "limits": { + "max_abs_f_error": 1e-07, + "relative_l2_f_error": 1e-08, + "relative_l2_e_error": 1e-07 + }, + "benchmark": { + "backend": "cuda", + "device": "NVIDIA A100-SXM4-40GB", + "precision": "float64", + "mode": "landau", + "integrator": "sixth", + "nu": 1e-12, + "velocity": "cubic", + "layout": "transpose", + "cell_centred": true, + "max_halo_width": 0, + "rank0_halo_sent_bytes": 0, + "halo_sent_bytes_per_rank_max": 0, + "slab_sent_bytes_per_rank": 1348730880000, + "input": "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/initial.bin", + "nx": 268800, + "nv": 4096, + "ranks": 8, + "nodes": 2, + "steps": 100, + "warmup": 0, + "dt": 0.02, + "length": 12600, + "vmax": 15, + "amplitude": 0.01, + "acceleration": 0.2, + "host_mpi": false, + "state_bytes": 8808038400, + "max_allocated_bytes_per_rank": 11025158200, + "stream_ms": 97.34923378998246, + "spline_ms": 23.1298673800211, + "pack_ms": 45.69342769000741, + "mpi_ms": 777.1080535100737, + "field_ms": 23.60504804998754, + "collision_ms": 14.4846032800001, + "step_ms": 965.6051862700042, + "initial_mass": 12597.62207265221, + "final_mass": 12597.61751046275, + "mass_relative_change": -3.621468742758491e-07, + "initial_kinetic_energy": 55701.51067942529, + "final_kinetic_energy": 55695.27553237438, + "initial_field_energy": 791.8606180599771, + "final_field_energy": 797.6672297926364, + "minimum_f": -0.09642050941611634, + "finite": true + }, + "command": [ + "srun", + "--jobid=60206959", + "--overlap", + "--nodes=2", + "--ntasks=8", + "--ntasks-per-node=4", + "--gpus-per-node=4", + "--gpu-bind=none", + "--cpu-bind=none", + "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/workdir/cuda-build/vlasov_bench", + "--nx", + "268800", + "--nv", + "4096", + "--length", + "12600", + "--vmax", + "15", + "--dt", + "0.02", + "--nu", + "1e-12", + "--cells", + "--velocity", + "cubic", + "--integrator", + "sixth", + "--layout", + "transpose", + "--steps", + "100", + "--warmup", + "0", + "--input", + "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/initial.bin", + "--v-grid", + "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/v.bin", + "--background", + "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/background.bin", + "--dump", + "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/comparison-nx268800-2node/2n-8g-transpose-0100.bin" + ], + "rank_log": "rank=2 host=nid001024 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001024 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=4 host=nid001024 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=6 host=nid001024 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=7 host=nid001025 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=5 host=nid001025 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001025 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001025 local_rank=1 device=NVIDIA A100-SXM4-40GB\n" + }, + { + "nodes": 2, + "ranks": 8, + "layout": "halos", + "steps": 1, + "max_abs_f_error": 4.1272540940440194e-14, + "relative_l2_f_error": 5.693709254174748e-15, + "relative_l2_e_error": 5.726948268097046e-15, + "max_abs_e_error": 5.842548667089886e-15, + "hole_counts": [ + 118, + 54 + ], + "reference_hole_counts": [ + 118, + 54 + ], + "hole_counts_match": true, + "passed": true, + "limits": { + "max_abs_f_error": 1e-07, + "relative_l2_f_error": 1e-08, + "relative_l2_e_error": 1e-07 + }, + "benchmark": { + "backend": "cuda", + "device": "NVIDIA A100-SXM4-40GB", + "precision": "float64", + "mode": "landau", + "integrator": "sixth", + "nu": 1e-12, + "velocity": "cubic", + "layout": "halos", + "cell_centred": true, + "max_halo_width": 4, + "rank0_halo_sent_bytes": 43008000, + "halo_sent_bytes_per_rank_max": 86016000, + "slab_sent_bytes_per_rank": 1926758400, + "input": "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/initial.bin", + "nx": 268800, + "nv": 4096, + "ranks": 8, + "nodes": 2, + "steps": 1, + "warmup": 0, + "dt": 0.02, + "length": 12600, + "vmax": 15, + "amplitude": 0.01, + "acceleration": 0.2, + "host_mpi": false, + "state_bytes": 8808038400, + "max_allocated_bytes_per_rank": 11300409400, + "stream_ms": 97.3275939998075, + "spline_ms": 30.09822499984693, + "pack_ms": 6.571679999979096, + "mpi_ms": 151.4998850000211, + "field_ms": 8.81993999973929, + "collision_ms": 14.53518300013457, + "step_ms": 308.1012009999995, + "initial_mass": 12597.62207265221, + "final_mass": 12597.62199624691, + "mass_relative_change": -6.065056761443461e-09, + "initial_kinetic_energy": 55701.51067942529, + "final_kinetic_energy": 55701.45800056863, + "initial_field_energy": 791.8606180599771, + "final_field_energy": 791.9073734470651, + "minimum_f": -0.0952893737412273, + "finite": true + }, + "command": [ + "srun", + "--jobid=60206959", + "--overlap", + "--nodes=2", + "--ntasks=8", + "--ntasks-per-node=4", + "--gpus-per-node=4", + "--gpu-bind=none", + "--cpu-bind=none", + "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/workdir/cuda-build/vlasov_bench", + "--nx", + "268800", + "--nv", + "4096", + "--length", + "12600", + "--vmax", + "15", + "--dt", + "0.02", + "--nu", + "1e-12", + "--cells", + "--velocity", + "cubic", + "--integrator", + "sixth", + "--layout", + "halos", + "--steps", + "1", + "--warmup", + "0", + "--input", + "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/initial.bin", + "--v-grid", + "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/v.bin", + "--background", + "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/background.bin", + "--dump", + "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/comparison-nx268800-2node/2n-8g-halos-0001.bin" + ], + "rank_log": "rank=4 host=nid001024 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001024 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=6 host=nid001024 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=7 host=nid001025 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001024 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=5 host=nid001025 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001025 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001025 local_rank=0 device=NVIDIA A100-SXM4-40GB\n" + }, + { + "nodes": 2, + "ranks": 8, + "layout": "halos", + "steps": 100, + "max_abs_f_error": 5.144495940356819e-13, + "relative_l2_f_error": 7.864106656264876e-14, + "relative_l2_e_error": 4.782235170485902e-14, + "max_abs_e_error": 4.3395842475035806e-14, + "hole_counts": [ + 129, + 50 + ], + "reference_hole_counts": [ + 129, + 50 + ], + "hole_counts_match": true, + "passed": true, + "limits": { + "max_abs_f_error": 1e-07, + "relative_l2_f_error": 1e-08, + "relative_l2_e_error": 1e-07 + }, + "benchmark": { + "backend": "cuda", + "device": "NVIDIA A100-SXM4-40GB", + "precision": "float64", + "mode": "landau", + "integrator": "sixth", + "nu": 1e-12, + "velocity": "cubic", + "layout": "halos", + "cell_centred": true, + "max_halo_width": 4, + "rank0_halo_sent_bytes": 4584652800, + "halo_sent_bytes_per_rank_max": 9169305600, + "slab_sent_bytes_per_rank": 192675840000, + "input": "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/initial.bin", + "nx": 268800, + "nv": 4096, + "ranks": 8, + "nodes": 2, + "steps": 100, + "warmup": 0, + "dt": 0.02, + "length": 12600, + "vmax": 15, + "amplitude": 0.01, + "acceleration": 0.2, + "host_mpi": false, + "state_bytes": 8808038400, + "max_allocated_bytes_per_rank": 11300409400, + "stream_ms": 97.31272944003194, + "spline_ms": 30.11586632999069, + "pack_ms": 6.524751499971444, + "mpi_ms": 120.3591580900047, + "field_ms": 10.87078800999961, + "collision_ms": 14.4880603899901, + "step_ms": 276.4345721300037, + "initial_mass": 12597.62207265221, + "final_mass": 12597.61751046275, + "mass_relative_change": -3.621468742758491e-07, + "initial_kinetic_energy": 55701.51067942529, + "final_kinetic_energy": 55695.27553237438, + "initial_field_energy": 791.8606180599771, + "final_field_energy": 797.6672297926364, + "minimum_f": -0.09642050941611634, + "finite": true + }, + "command": [ + "srun", + "--jobid=60206959", + "--overlap", + "--nodes=2", + "--ntasks=8", + "--ntasks-per-node=4", + "--gpus-per-node=4", + "--gpu-bind=none", + "--cpu-bind=none", + "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/workdir/cuda-build/vlasov_bench", + "--nx", + "268800", + "--nv", + "4096", + "--length", + "12600", + "--vmax", + "15", + "--dt", + "0.02", + "--nu", + "1e-12", + "--cells", + "--velocity", + "cubic", + "--integrator", + "sixth", + "--layout", + "halos", + "--steps", + "100", + "--warmup", + "0", + "--input", + "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/initial.bin", + "--v-grid", + "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/v.bin", + "--background", + "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/background.bin", + "--dump", + "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/comparison-nx268800-2node/2n-8g-halos-0100.bin" + ], + "rank_log": "rank=4 host=nid001024 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001024 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001025 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001024 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=6 host=nid001024 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=7 host=nid001025 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001025 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=5 host=nid001025 local_rank=2 device=NVIDIA A100-SXM4-40GB\n" + } +] diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-halos/compiled-source.sha256 b/benchmarks/vlasov_cuda/results/2026-10-24-halos/compiled-source.sha256 new file mode 100644 index 00000000..ba6ebaa9 --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-halos/compiled-source.sha256 @@ -0,0 +1,5 @@ +4dab281aa96caf3589f56bf0b6f313686e70c7f3b0c10788ea071b4edb6a1ef2 /pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/src/benchmarks/vlasov_cuda/backend.cpp +fdea36e5d804fd24df663e6c48194dba4e42dec3771cea8167b526db91b52927 /pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/src/benchmarks/vlasov_cuda/main.cpp +73ee9d9f649eebf405873f87c1eece597e8bd57a32636d2ec047eef156ea0cb3 /pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/src/benchmarks/vlasov_cuda/backend.hpp +f87f6aa9731f64e501b94e95c20a53f45ae7d1ea5ec39c7e859c6b1a4b8c40c3 /pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/src/benchmarks/vlasov_cuda/local_cubic.hpp +d5f11680925c336c63ea39e306230371f6dcb3e41ba423c8c09599c7c592664c /pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/src/benchmarks/vlasov_cuda/CMakeLists.txt diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-halos/executable.sha256 b/benchmarks/vlasov_cuda/results/2026-10-24-halos/executable.sha256 new file mode 100644 index 00000000..077a5f64 --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-halos/executable.sha256 @@ -0,0 +1 @@ +7d88657b9597e939f310c2c488d81842793ec361ad190b8f60595c47c8c6ea51 /pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/workdir/cuda-build/vlasov_bench diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-halos/nx134400-halos.jsonl b/benchmarks/vlasov_cuda/results/2026-10-24-halos/nx134400-halos.jsonl new file mode 100644 index 00000000..eaac1851 --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-halos/nx134400-halos.jsonl @@ -0,0 +1,6 @@ +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "integrator": "sixth", "nu": 1e-12, "velocity": "cubic", "layout": "halos", "cell_centred": true, "max_halo_width": 4, "rank0_halo_sent_bytes": 215040000, "halo_sent_bytes_per_rank_max": 430080000, "slab_sent_bytes_per_rank": 16515072000, "input": "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/initial.bin", "nx": 134400, "nv": 4096, "ranks": 4, "nodes": 1, "steps": 10, "warmup": 3, "dt": 0.02, "length": 12600, "vmax": 15, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 4404019200, "max_allocated_bytes_per_rank": 11155281976, "stream_ms": 96.75858740024523, "spline_ms": 29.37038770005529, "pack_ms": 6.511311200029013, "mpi_ms": 20.98856150023494, "field_ms": 6.72662069978287, "collision_ms": 14.47656629998164, "step_ms": 174.2806366000877, "initial_mass": 12597.6196507818, "final_mass": 12597.62016522545, "mass_relative_change": 4.083657516783715e-08, "initial_kinetic_energy": 55713.43281125277, "final_kinetic_energy": 55714.15660222915, "initial_field_energy": 787.6538181371966, "final_field_energy": 786.9554651665927, "minimum_f": -0.1916308738890627, "finite": true, "repeat": 0, "command": ["srun", "--jobid=60206959", "--overlap", "--nodes=1", "--ntasks=4", "--ntasks-per-node=4", "--gpus-per-node=4", "--gpu-bind=none", "--cpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/workdir/cuda-build/vlasov_bench", "--nx", "134400", "--nv", "4096", "--steps", "10", "--warmup", "3", "--mode", "landau", "--velocity", "cubic", "--layout", "halos", "--integrator", "sixth", "--nu", "1e-12", "--length", "12600.0", "--vmax", "15.0", "--dt", "0.02", "--cells", "--input", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/initial.bin", "--v-grid", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/v.bin", "--background", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/background.bin"], "rank_log": "rank=1 host=nid001024 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001024 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001024 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001024 local_rank=0 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=35a370ebe958e3df17ad798c54efedf85892f8029da920ff437ff54953bea6b6\nsynced_at=2026-10-24T11:25:39Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "integrator": "sixth", "nu": 1e-12, "velocity": "cubic", "layout": "halos", "cell_centred": true, "max_halo_width": 4, "rank0_halo_sent_bytes": 215040000, "halo_sent_bytes_per_rank_max": 430080000, "slab_sent_bytes_per_rank": 16515072000, "input": "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/initial.bin", "nx": 134400, "nv": 4096, "ranks": 4, "nodes": 1, "steps": 10, "warmup": 3, "dt": 0.02, "length": 12600, "vmax": 15, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 4404019200, "max_allocated_bytes_per_rank": 11155281976, "stream_ms": 96.7698514001313, "spline_ms": 29.39963990011165, "pack_ms": 6.513359599830437, "mpi_ms": 20.97919940019892, "field_ms": 6.686601499905009, "collision_ms": 14.49262569999519, "step_ms": 174.3251663999672, "initial_mass": 12597.6196507818, "final_mass": 12597.62016522545, "mass_relative_change": 4.083657516783715e-08, "initial_kinetic_energy": 55713.43281125277, "final_kinetic_energy": 55714.15660222915, "initial_field_energy": 787.6538181371966, "final_field_energy": 786.9554651665927, "minimum_f": -0.1916308738890627, "finite": true, "repeat": 1, "command": ["srun", "--jobid=60206959", "--overlap", "--nodes=1", "--ntasks=4", "--ntasks-per-node=4", "--gpus-per-node=4", "--gpu-bind=none", "--cpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/workdir/cuda-build/vlasov_bench", "--nx", "134400", "--nv", "4096", "--steps", "10", "--warmup", "3", "--mode", "landau", "--velocity", "cubic", "--layout", "halos", "--integrator", "sixth", "--nu", "1e-12", "--length", "12600.0", "--vmax", "15.0", "--dt", "0.02", "--cells", "--input", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/initial.bin", "--v-grid", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/v.bin", "--background", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/background.bin"], "rank_log": "rank=3 host=nid001025 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001025 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001025 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001025 local_rank=1 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=35a370ebe958e3df17ad798c54efedf85892f8029da920ff437ff54953bea6b6\nsynced_at=2026-10-24T11:25:39Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "integrator": "sixth", "nu": 1e-12, "velocity": "cubic", "layout": "halos", "cell_centred": true, "max_halo_width": 4, "rank0_halo_sent_bytes": 215040000, "halo_sent_bytes_per_rank_max": 430080000, "slab_sent_bytes_per_rank": 16515072000, "input": "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/initial.bin", "nx": 134400, "nv": 4096, "ranks": 4, "nodes": 1, "steps": 10, "warmup": 3, "dt": 0.02, "length": 12600, "vmax": 15, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 4404019200, "max_allocated_bytes_per_rank": 11155281976, "stream_ms": 96.7597968001428, "spline_ms": 29.36039309979606, "pack_ms": 6.510793699726491, "mpi_ms": 20.96922610003276, "field_ms": 6.668245399851003, "collision_ms": 14.47216869996737, "step_ms": 174.2041793000681, "initial_mass": 12597.6196507818, "final_mass": 12597.62016522545, "mass_relative_change": 4.083657516783715e-08, "initial_kinetic_energy": 55713.43281125277, "final_kinetic_energy": 55714.15660222915, "initial_field_energy": 787.6538181371966, "final_field_energy": 786.9554651665927, "minimum_f": -0.1916308738890627, "finite": true, "repeat": 2, "command": ["srun", "--jobid=60206959", "--overlap", "--nodes=1", "--ntasks=4", "--ntasks-per-node=4", "--gpus-per-node=4", "--gpu-bind=none", "--cpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/workdir/cuda-build/vlasov_bench", "--nx", "134400", "--nv", "4096", "--steps", "10", "--warmup", "3", "--mode", "landau", "--velocity", "cubic", "--layout", "halos", "--integrator", "sixth", "--nu", "1e-12", "--length", "12600.0", "--vmax", "15.0", "--dt", "0.02", "--cells", "--input", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/initial.bin", "--v-grid", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/v.bin", "--background", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/background.bin"], "rank_log": "rank=2 host=nid001024 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001024 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001024 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001024 local_rank=1 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=35a370ebe958e3df17ad798c54efedf85892f8029da920ff437ff54953bea6b6\nsynced_at=2026-10-24T11:25:39Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "integrator": "sixth", "nu": 1e-12, "velocity": "cubic", "layout": "halos", "cell_centred": true, "max_halo_width": 4, "rank0_halo_sent_bytes": 215040000, "halo_sent_bytes_per_rank_max": 430080000, "slab_sent_bytes_per_rank": 9633792000, "input": "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/initial.bin", "nx": 134400, "nv": 4096, "ranks": 8, "nodes": 2, "steps": 10, "warmup": 3, "dt": 0.02, "length": 12600, "vmax": 15, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 4404019200, "max_allocated_bytes_per_rank": 5650233400, "stream_ms": 48.49465079987567, "spline_ms": 12.44509260009181, "pack_ms": 3.281850700113864, "mpi_ms": 60.40269479999552, "field_ms": 5.99238590011737, "collision_ms": 11.23696820000077, "step_ms": 140.1605807999658, "initial_mass": 12597.61965078436, "final_mass": 12597.62016522508, "mass_relative_change": 4.083634327505363e-08, "initial_kinetic_energy": 55713.43281126581, "final_kinetic_energy": 55714.15660220217, "initial_field_energy": 787.6538181371961, "final_field_energy": 786.9554651665925, "minimum_f": -0.1916308738890627, "finite": true, "repeat": 0, "command": ["srun", "--jobid=60206959", "--overlap", "--nodes=2", "--ntasks=8", "--ntasks-per-node=4", "--gpus-per-node=4", "--gpu-bind=none", "--cpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/workdir/cuda-build/vlasov_bench", "--nx", "134400", "--nv", "4096", "--steps", "10", "--warmup", "3", "--mode", "landau", "--velocity", "cubic", "--layout", "halos", "--integrator", "sixth", "--nu", "1e-12", "--length", "12600.0", "--vmax", "15.0", "--dt", "0.02", "--cells", "--input", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/initial.bin", "--v-grid", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/v.bin", "--background", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/background.bin"], "rank_log": "rank=6 host=nid001024 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=4 host=nid001024 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001024 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001024 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=7 host=nid001025 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001025 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001025 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=5 host=nid001025 local_rank=2 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=35a370ebe958e3df17ad798c54efedf85892f8029da920ff437ff54953bea6b6\nsynced_at=2026-10-24T11:25:39Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "integrator": "sixth", "nu": 1e-12, "velocity": "cubic", "layout": "halos", "cell_centred": true, "max_halo_width": 4, "rank0_halo_sent_bytes": 215040000, "halo_sent_bytes_per_rank_max": 430080000, "slab_sent_bytes_per_rank": 9633792000, "input": "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/initial.bin", "nx": 134400, "nv": 4096, "ranks": 8, "nodes": 2, "steps": 10, "warmup": 3, "dt": 0.02, "length": 12600, "vmax": 15, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 4404019200, "max_allocated_bytes_per_rank": 5650233400, "stream_ms": 48.50223350013039, "spline_ms": 12.23678170017592, "pack_ms": 3.282552000064243, "mpi_ms": 59.93244790010976, "field_ms": 6.357290200094212, "collision_ms": 11.2380733999089, "step_ms": 139.8805341999378, "initial_mass": 12597.61965078436, "final_mass": 12597.62016522508, "mass_relative_change": 4.083634327505363e-08, "initial_kinetic_energy": 55713.43281126581, "final_kinetic_energy": 55714.15660220217, "initial_field_energy": 787.6538181371961, "final_field_energy": 786.9554651665925, "minimum_f": -0.1916308738890627, "finite": true, "repeat": 1, "command": ["srun", "--jobid=60206959", "--overlap", "--nodes=2", "--ntasks=8", "--ntasks-per-node=4", "--gpus-per-node=4", "--gpu-bind=none", "--cpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/workdir/cuda-build/vlasov_bench", "--nx", "134400", "--nv", "4096", "--steps", "10", "--warmup", "3", "--mode", "landau", "--velocity", "cubic", "--layout", "halos", "--integrator", "sixth", "--nu", "1e-12", "--length", "12600.0", "--vmax", "15.0", "--dt", "0.02", "--cells", "--input", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/initial.bin", "--v-grid", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/v.bin", "--background", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/background.bin"], "rank_log": "rank=6 host=nid001024 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=7 host=nid001025 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=4 host=nid001024 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=5 host=nid001025 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001025 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001024 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001024 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001025 local_rank=0 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=35a370ebe958e3df17ad798c54efedf85892f8029da920ff437ff54953bea6b6\nsynced_at=2026-10-24T11:25:39Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "integrator": "sixth", "nu": 1e-12, "velocity": "cubic", "layout": "halos", "cell_centred": true, "max_halo_width": 4, "rank0_halo_sent_bytes": 215040000, "halo_sent_bytes_per_rank_max": 430080000, "slab_sent_bytes_per_rank": 9633792000, "input": "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/initial.bin", "nx": 134400, "nv": 4096, "ranks": 8, "nodes": 2, "steps": 10, "warmup": 3, "dt": 0.02, "length": 12600, "vmax": 15, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 4404019200, "max_allocated_bytes_per_rank": 5650233400, "stream_ms": 48.50200009991568, "spline_ms": 12.23685909994856, "pack_ms": 3.279367700179137, "mpi_ms": 60.16711680003937, "field_ms": 6.355414500239931, "collision_ms": 11.24243100002786, "step_ms": 140.0107500999638, "initial_mass": 12597.61965078436, "final_mass": 12597.62016522508, "mass_relative_change": 4.083634327505363e-08, "initial_kinetic_energy": 55713.43281126581, "final_kinetic_energy": 55714.15660220217, "initial_field_energy": 787.6538181371961, "final_field_energy": 786.9554651665925, "minimum_f": -0.1916308738890627, "finite": true, "repeat": 2, "command": ["srun", "--jobid=60206959", "--overlap", "--nodes=2", "--ntasks=8", "--ntasks-per-node=4", "--gpus-per-node=4", "--gpu-bind=none", "--cpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/workdir/cuda-build/vlasov_bench", "--nx", "134400", "--nv", "4096", "--steps", "10", "--warmup", "3", "--mode", "landau", "--velocity", "cubic", "--layout", "halos", "--integrator", "sixth", "--nu", "1e-12", "--length", "12600.0", "--vmax", "15.0", "--dt", "0.02", "--cells", "--input", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/initial.bin", "--v-grid", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/v.bin", "--background", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/background.bin"], "rank_log": "rank=4 host=nid001024 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001024 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=5 host=nid001025 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001024 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001025 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=6 host=nid001024 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=7 host=nid001025 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001025 local_rank=0 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=35a370ebe958e3df17ad798c54efedf85892f8029da920ff437ff54953bea6b6\nsynced_at=2026-10-24T11:25:39Z\n"} diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-halos/nx134400-transpose.jsonl b/benchmarks/vlasov_cuda/results/2026-10-24-halos/nx134400-transpose.jsonl new file mode 100644 index 00000000..80c1f0b4 --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-halos/nx134400-transpose.jsonl @@ -0,0 +1,6 @@ +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "integrator": "sixth", "nu": 1e-12, "velocity": "cubic", "layout": "transpose", "cell_centred": true, "max_halo_width": 0, "rank0_halo_sent_bytes": 0, "halo_sent_bytes_per_rank_max": 0, "slab_sent_bytes_per_rank": 115605504000, "input": "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/initial.bin", "nx": 134400, "nv": 4096, "ranks": 4, "nodes": 1, "steps": 10, "warmup": 3, "dt": 0.02, "length": 12600, "vmax": 15, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 4404019200, "max_allocated_bytes_per_rank": 11017656376, "stream_ms": 96.80129259998012, "spline_ms": 23.18175969996901, "pack_ms": 45.59191170001213, "mpi_ms": 137.4090791999834, "field_ms": 6.102015500027846, "collision_ms": 14.47260410000126, "step_ms": 323.5450839000123, "initial_mass": 12597.6196507818, "final_mass": 12597.62016522545, "mass_relative_change": 4.083657516783715e-08, "initial_kinetic_energy": 55713.43281125277, "final_kinetic_energy": 55714.15660222915, "initial_field_energy": 787.6538181371966, "final_field_energy": 786.9554651665927, "minimum_f": -0.1916308738890627, "finite": true, "repeat": 0, "command": ["srun", "--jobid=60206959", "--overlap", "--nodes=1", "--ntasks=4", "--ntasks-per-node=4", "--gpus-per-node=4", "--gpu-bind=none", "--cpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/workdir/cuda-build/vlasov_bench", "--nx", "134400", "--nv", "4096", "--steps", "10", "--warmup", "3", "--mode", "landau", "--velocity", "cubic", "--layout", "transpose", "--integrator", "sixth", "--nu", "1e-12", "--length", "12600.0", "--vmax", "15.0", "--dt", "0.02", "--cells", "--input", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/initial.bin", "--v-grid", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/v.bin", "--background", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/background.bin"], "rank_log": "rank=2 host=nid001024 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001024 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001024 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001024 local_rank=3 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=35a370ebe958e3df17ad798c54efedf85892f8029da920ff437ff54953bea6b6\nsynced_at=2026-10-24T11:25:39Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "integrator": "sixth", "nu": 1e-12, "velocity": "cubic", "layout": "transpose", "cell_centred": true, "max_halo_width": 0, "rank0_halo_sent_bytes": 0, "halo_sent_bytes_per_rank_max": 0, "slab_sent_bytes_per_rank": 115605504000, "input": "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/initial.bin", "nx": 134400, "nv": 4096, "ranks": 4, "nodes": 1, "steps": 10, "warmup": 3, "dt": 0.02, "length": 12600, "vmax": 15, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 4404019200, "max_allocated_bytes_per_rank": 11017656376, "stream_ms": 96.80772089986931, "spline_ms": 23.22596530000283, "pack_ms": 45.59232870010419, "mpi_ms": 137.3818458999949, "field_ms": 6.081885599883208, "collision_ms": 14.48917420000271, "step_ms": 323.5631748999822, "initial_mass": 12597.6196507818, "final_mass": 12597.62016522545, "mass_relative_change": 4.083657516783715e-08, "initial_kinetic_energy": 55713.43281125277, "final_kinetic_energy": 55714.15660222915, "initial_field_energy": 787.6538181371966, "final_field_energy": 786.9554651665927, "minimum_f": -0.1916308738890627, "finite": true, "repeat": 1, "command": ["srun", "--jobid=60206959", "--overlap", "--nodes=1", "--ntasks=4", "--ntasks-per-node=4", "--gpus-per-node=4", "--gpu-bind=none", "--cpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/workdir/cuda-build/vlasov_bench", "--nx", "134400", "--nv", "4096", "--steps", "10", "--warmup", "3", "--mode", "landau", "--velocity", "cubic", "--layout", "transpose", "--integrator", "sixth", "--nu", "1e-12", "--length", "12600.0", "--vmax", "15.0", "--dt", "0.02", "--cells", "--input", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/initial.bin", "--v-grid", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/v.bin", "--background", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/background.bin"], "rank_log": "rank=2 host=nid001025 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001025 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001025 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001025 local_rank=0 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=35a370ebe958e3df17ad798c54efedf85892f8029da920ff437ff54953bea6b6\nsynced_at=2026-10-24T11:25:39Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "integrator": "sixth", "nu": 1e-12, "velocity": "cubic", "layout": "transpose", "cell_centred": true, "max_halo_width": 0, "rank0_halo_sent_bytes": 0, "halo_sent_bytes_per_rank_max": 0, "slab_sent_bytes_per_rank": 115605504000, "input": "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/initial.bin", "nx": 134400, "nv": 4096, "ranks": 4, "nodes": 1, "steps": 10, "warmup": 3, "dt": 0.02, "length": 12600, "vmax": 15, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 4404019200, "max_allocated_bytes_per_rank": 11017656376, "stream_ms": 96.79299130004893, "spline_ms": 23.18371020005543, "pack_ms": 45.59643329980645, "mpi_ms": 137.4355132999881, "field_ms": 6.073187200104257, "collision_ms": 14.47208049996789, "step_ms": 323.5294349000242, "initial_mass": 12597.6196507818, "final_mass": 12597.62016522545, "mass_relative_change": 4.083657516783715e-08, "initial_kinetic_energy": 55713.43281125277, "final_kinetic_energy": 55714.15660222915, "initial_field_energy": 787.6538181371966, "final_field_energy": 786.9554651665927, "minimum_f": -0.1916308738890627, "finite": true, "repeat": 2, "command": ["srun", "--jobid=60206959", "--overlap", "--nodes=1", "--ntasks=4", "--ntasks-per-node=4", "--gpus-per-node=4", "--gpu-bind=none", "--cpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/workdir/cuda-build/vlasov_bench", "--nx", "134400", "--nv", "4096", "--steps", "10", "--warmup", "3", "--mode", "landau", "--velocity", "cubic", "--layout", "transpose", "--integrator", "sixth", "--nu", "1e-12", "--length", "12600.0", "--vmax", "15.0", "--dt", "0.02", "--cells", "--input", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/initial.bin", "--v-grid", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/v.bin", "--background", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/background.bin"], "rank_log": "rank=0 host=nid001024 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001024 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001024 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001024 local_rank=3 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=35a370ebe958e3df17ad798c54efedf85892f8029da920ff437ff54953bea6b6\nsynced_at=2026-10-24T11:25:39Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "integrator": "sixth", "nu": 1e-12, "velocity": "cubic", "layout": "transpose", "cell_centred": true, "max_halo_width": 0, "rank0_halo_sent_bytes": 0, "halo_sent_bytes_per_rank_max": 0, "slab_sent_bytes_per_rank": 67436544000, "input": "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/initial.bin", "nx": 134400, "nv": 4096, "ranks": 8, "nodes": 2, "steps": 10, "warmup": 3, "dt": 0.02, "length": 12600, "vmax": 15, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 4404019200, "max_allocated_bytes_per_rank": 5512607800, "stream_ms": 48.55348729995512, "spline_ms": 11.72295309993387, "pack_ms": 22.98062660006508, "mpi_ms": 391.2288103000037, "field_ms": 13.02523969998219, "collision_ms": 11.2371266999844, "step_ms": 490.7239440000239, "initial_mass": 12597.61965078436, "final_mass": 12597.62016522508, "mass_relative_change": 4.083634327505363e-08, "initial_kinetic_energy": 55713.43281126581, "final_kinetic_energy": 55714.15660220217, "initial_field_energy": 787.6538181371961, "final_field_energy": 786.9554651665925, "minimum_f": -0.1916308738890627, "finite": true, "repeat": 0, "command": ["srun", "--jobid=60206959", "--overlap", "--nodes=2", "--ntasks=8", "--ntasks-per-node=4", "--gpus-per-node=4", "--gpu-bind=none", "--cpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/workdir/cuda-build/vlasov_bench", "--nx", "134400", "--nv", "4096", "--steps", "10", "--warmup", "3", "--mode", "landau", "--velocity", "cubic", "--layout", "transpose", "--integrator", "sixth", "--nu", "1e-12", "--length", "12600.0", "--vmax", "15.0", "--dt", "0.02", "--cells", "--input", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/initial.bin", "--v-grid", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/v.bin", "--background", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/background.bin"], "rank_log": "rank=3 host=nid001025 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=7 host=nid001025 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001025 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=5 host=nid001025 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=4 host=nid001024 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001024 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001024 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=6 host=nid001024 local_rank=3 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=35a370ebe958e3df17ad798c54efedf85892f8029da920ff437ff54953bea6b6\nsynced_at=2026-10-24T11:25:39Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "integrator": "sixth", "nu": 1e-12, "velocity": "cubic", "layout": "transpose", "cell_centred": true, "max_halo_width": 0, "rank0_halo_sent_bytes": 0, "halo_sent_bytes_per_rank_max": 0, "slab_sent_bytes_per_rank": 67436544000, "input": "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/initial.bin", "nx": 134400, "nv": 4096, "ranks": 8, "nodes": 2, "steps": 10, "warmup": 3, "dt": 0.02, "length": 12600, "vmax": 15, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 4404019200, "max_allocated_bytes_per_rank": 5512607800, "stream_ms": 48.54444679997414, "spline_ms": 11.72291799998675, "pack_ms": 22.9713284998752, "mpi_ms": 390.2375082999924, "field_ms": 11.86256170008164, "collision_ms": 11.23720970006161, "step_ms": 489.5213302999537, "initial_mass": 12597.61965078436, "final_mass": 12597.62016522508, "mass_relative_change": 4.083634327505363e-08, "initial_kinetic_energy": 55713.43281126581, "final_kinetic_energy": 55714.15660220217, "initial_field_energy": 787.6538181371961, "final_field_energy": 786.9554651665925, "minimum_f": -0.1916308738890627, "finite": true, "repeat": 1, "command": ["srun", "--jobid=60206959", "--overlap", "--nodes=2", "--ntasks=8", "--ntasks-per-node=4", "--gpus-per-node=4", "--gpu-bind=none", "--cpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/workdir/cuda-build/vlasov_bench", "--nx", "134400", "--nv", "4096", "--steps", "10", "--warmup", "3", "--mode", "landau", "--velocity", "cubic", "--layout", "transpose", "--integrator", "sixth", "--nu", "1e-12", "--length", "12600.0", "--vmax", "15.0", "--dt", "0.02", "--cells", "--input", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/initial.bin", "--v-grid", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/v.bin", "--background", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/background.bin"], "rank_log": "rank=2 host=nid001024 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001024 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=7 host=nid001025 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=5 host=nid001025 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001025 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=4 host=nid001024 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=6 host=nid001024 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001025 local_rank=1 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=35a370ebe958e3df17ad798c54efedf85892f8029da920ff437ff54953bea6b6\nsynced_at=2026-10-24T11:25:39Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "integrator": "sixth", "nu": 1e-12, "velocity": "cubic", "layout": "transpose", "cell_centred": true, "max_halo_width": 0, "rank0_halo_sent_bytes": 0, "halo_sent_bytes_per_rank_max": 0, "slab_sent_bytes_per_rank": 67436544000, "input": "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/initial.bin", "nx": 134400, "nv": 4096, "ranks": 8, "nodes": 2, "steps": 10, "warmup": 3, "dt": 0.02, "length": 12600, "vmax": 15, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 4404019200, "max_allocated_bytes_per_rank": 5512607800, "stream_ms": 48.55127589999029, "spline_ms": 11.72265499997138, "pack_ms": 22.97926750006809, "mpi_ms": 391.0327226998788, "field_ms": 12.39774919995398, "collision_ms": 11.23903609998251, "step_ms": 490.5058394000434, "initial_mass": 12597.61965078436, "final_mass": 12597.62016522508, "mass_relative_change": 4.083634327505363e-08, "initial_kinetic_energy": 55713.43281126581, "final_kinetic_energy": 55714.15660220217, "initial_field_energy": 787.6538181371961, "final_field_energy": 786.9554651665925, "minimum_f": -0.1916308738890627, "finite": true, "repeat": 2, "command": ["srun", "--jobid=60206959", "--overlap", "--nodes=2", "--ntasks=8", "--ntasks-per-node=4", "--gpus-per-node=4", "--gpu-bind=none", "--cpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/workdir/cuda-build/vlasov_bench", "--nx", "134400", "--nv", "4096", "--steps", "10", "--warmup", "3", "--mode", "landau", "--velocity", "cubic", "--layout", "transpose", "--integrator", "sixth", "--nu", "1e-12", "--length", "12600.0", "--vmax", "15.0", "--dt", "0.02", "--cells", "--input", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/initial.bin", "--v-grid", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/v.bin", "--background", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/background.bin"], "rank_log": "rank=3 host=nid001025 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=7 host=nid001025 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001025 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=6 host=nid001024 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=5 host=nid001025 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001024 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=4 host=nid001024 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001024 local_rank=1 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=35a370ebe958e3df17ad798c54efedf85892f8029da920ff437ff54953bea6b6\nsynced_at=2026-10-24T11:25:39Z\n"} diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-halos/nx268800-halos.jsonl b/benchmarks/vlasov_cuda/results/2026-10-24-halos/nx268800-halos.jsonl new file mode 100644 index 00000000..0bb5ec17 --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-halos/nx268800-halos.jsonl @@ -0,0 +1,6 @@ +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "integrator": "sixth", "nu": 1e-12, "velocity": "cubic", "layout": "halos", "cell_centred": true, "max_halo_width": 4, "rank0_halo_sent_bytes": 438681600, "halo_sent_bytes_per_rank_max": 877363200, "slab_sent_bytes_per_rank": 33030144000, "input": "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/initial.bin", "nx": 268800, "nv": 4096, "ranks": 4, "nodes": 1, "steps": 10, "warmup": 3, "dt": 0.02, "length": 12600, "vmax": 15, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 8808038400, "max_allocated_bytes_per_rank": 22310481976, "stream_ms": 194.4373317000554, "spline_ms": 61.42588710008567, "pack_ms": 13.08217789996888, "mpi_ms": 41.11955449998277, "field_ms": 12.09190620002119, "collision_ms": 21.10292339998523, "step_ms": 342.4020051000753, "initial_mass": 12597.62207264873, "final_mass": 12597.62136206869, "mass_relative_change": -5.640588611556955e-08, "initial_kinetic_energy": 55701.51067944818, "final_kinetic_energy": 55701.0068872211, "initial_field_energy": 791.8606180599772, "final_field_energy": 792.3077830250793, "minimum_f": -0.09580882788649099, "finite": true, "repeat": 0, "command": ["srun", "--jobid=60206959", "--overlap", "--nodes=1", "--ntasks=4", "--ntasks-per-node=4", "--gpus-per-node=4", "--gpu-bind=none", "--cpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/workdir/cuda-build/vlasov_bench", "--nx", "268800", "--nv", "4096", "--steps", "10", "--warmup", "3", "--mode", "landau", "--velocity", "cubic", "--layout", "halos", "--integrator", "sixth", "--nu", "1e-12", "--length", "12600.0", "--vmax", "15.0", "--dt", "0.02", "--cells", "--input", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/initial.bin", "--v-grid", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/v.bin", "--background", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/background.bin"], "rank_log": "rank=1 host=nid001024 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001024 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001024 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001024 local_rank=0 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=35a370ebe958e3df17ad798c54efedf85892f8029da920ff437ff54953bea6b6\nsynced_at=2026-10-24T11:25:39Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "integrator": "sixth", "nu": 1e-12, "velocity": "cubic", "layout": "halos", "cell_centred": true, "max_halo_width": 4, "rank0_halo_sent_bytes": 438681600, "halo_sent_bytes_per_rank_max": 877363200, "slab_sent_bytes_per_rank": 33030144000, "input": "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/initial.bin", "nx": 268800, "nv": 4096, "ranks": 4, "nodes": 1, "steps": 10, "warmup": 3, "dt": 0.02, "length": 12600, "vmax": 15, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 8808038400, "max_allocated_bytes_per_rank": 22310481976, "stream_ms": 194.4473017999371, "spline_ms": 61.46278019996316, "pack_ms": 13.08220940004503, "mpi_ms": 41.15909789966281, "field_ms": 12.18012080012159, "collision_ms": 21.11939120004536, "step_ms": 342.4947891999636, "initial_mass": 12597.62207264873, "final_mass": 12597.62136206869, "mass_relative_change": -5.640588611556955e-08, "initial_kinetic_energy": 55701.51067944818, "final_kinetic_energy": 55701.0068872211, "initial_field_energy": 791.8606180599772, "final_field_energy": 792.3077830250793, "minimum_f": -0.09580882788649099, "finite": true, "repeat": 1, "command": ["srun", "--jobid=60206959", "--overlap", "--nodes=1", "--ntasks=4", "--ntasks-per-node=4", "--gpus-per-node=4", "--gpu-bind=none", "--cpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/workdir/cuda-build/vlasov_bench", "--nx", "268800", "--nv", "4096", "--steps", "10", "--warmup", "3", "--mode", "landau", "--velocity", "cubic", "--layout", "halos", "--integrator", "sixth", "--nu", "1e-12", "--length", "12600.0", "--vmax", "15.0", "--dt", "0.02", "--cells", "--input", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/initial.bin", "--v-grid", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/v.bin", "--background", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/background.bin"], "rank_log": "rank=2 host=nid001025 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001025 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001025 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001025 local_rank=3 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=35a370ebe958e3df17ad798c54efedf85892f8029da920ff437ff54953bea6b6\nsynced_at=2026-10-24T11:25:39Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "integrator": "sixth", "nu": 1e-12, "velocity": "cubic", "layout": "halos", "cell_centred": true, "max_halo_width": 4, "rank0_halo_sent_bytes": 438681600, "halo_sent_bytes_per_rank_max": 877363200, "slab_sent_bytes_per_rank": 33030144000, "input": "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/initial.bin", "nx": 268800, "nv": 4096, "ranks": 4, "nodes": 1, "steps": 10, "warmup": 3, "dt": 0.02, "length": 12600, "vmax": 15, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 8808038400, "max_allocated_bytes_per_rank": 22310481976, "stream_ms": 194.4254820000424, "spline_ms": 61.42703189980239, "pack_ms": 13.08142329990005, "mpi_ms": 41.25540480013115, "field_ms": 12.30976880001435, "collision_ms": 21.11400799994954, "step_ms": 342.5728927000364, "initial_mass": 12597.62207264873, "final_mass": 12597.62136206869, "mass_relative_change": -5.640588611556955e-08, "initial_kinetic_energy": 55701.51067944818, "final_kinetic_energy": 55701.0068872211, "initial_field_energy": 791.8606180599772, "final_field_energy": 792.3077830250793, "minimum_f": -0.09580882788649099, "finite": true, "repeat": 2, "command": ["srun", "--jobid=60206959", "--overlap", "--nodes=1", "--ntasks=4", "--ntasks-per-node=4", "--gpus-per-node=4", "--gpu-bind=none", "--cpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/workdir/cuda-build/vlasov_bench", "--nx", "268800", "--nv", "4096", "--steps", "10", "--warmup", "3", "--mode", "landau", "--velocity", "cubic", "--layout", "halos", "--integrator", "sixth", "--nu", "1e-12", "--length", "12600.0", "--vmax", "15.0", "--dt", "0.02", "--cells", "--input", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/initial.bin", "--v-grid", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/v.bin", "--background", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/background.bin"], "rank_log": "rank=0 host=nid001024 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001024 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001024 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001024 local_rank=1 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=35a370ebe958e3df17ad798c54efedf85892f8029da920ff437ff54953bea6b6\nsynced_at=2026-10-24T11:25:39Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "integrator": "sixth", "nu": 1e-12, "velocity": "cubic", "layout": "halos", "cell_centred": true, "max_halo_width": 4, "rank0_halo_sent_bytes": 438681600, "halo_sent_bytes_per_rank_max": 877363200, "slab_sent_bytes_per_rank": 19267584000, "input": "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/initial.bin", "nx": 268800, "nv": 4096, "ranks": 8, "nodes": 2, "steps": 10, "warmup": 3, "dt": 0.02, "length": 12600, "vmax": 15, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 8808038400, "max_allocated_bytes_per_rank": 11300409400, "stream_ms": 97.31439470001533, "spline_ms": 30.11662399994748, "pack_ms": 6.547028900104124, "mpi_ms": 119.097262600053, "field_ms": 10.73557600002459, "collision_ms": 14.48864589997356, "step_ms": 275.1715809000871, "initial_mass": 12597.62207265221, "final_mass": 12597.62136207489, "mass_relative_change": -5.640567068345301e-08, "initial_kinetic_energy": 55701.51067942529, "final_kinetic_energy": 55701.00688716807, "initial_field_energy": 791.8606180599771, "final_field_energy": 792.3077830250799, "minimum_f": -0.09580882788649095, "finite": true, "repeat": 0, "command": ["srun", "--jobid=60206959", "--overlap", "--nodes=2", "--ntasks=8", "--ntasks-per-node=4", "--gpus-per-node=4", "--gpu-bind=none", "--cpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/workdir/cuda-build/vlasov_bench", "--nx", "268800", "--nv", "4096", "--steps", "10", "--warmup", "3", "--mode", "landau", "--velocity", "cubic", "--layout", "halos", "--integrator", "sixth", "--nu", "1e-12", "--length", "12600.0", "--vmax", "15.0", "--dt", "0.02", "--cells", "--input", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/initial.bin", "--v-grid", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/v.bin", "--background", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/background.bin"], "rank_log": "rank=4 host=nid001024 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=5 host=nid001025 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001025 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001024 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001024 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=6 host=nid001024 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=7 host=nid001025 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001025 local_rank=0 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=35a370ebe958e3df17ad798c54efedf85892f8029da920ff437ff54953bea6b6\nsynced_at=2026-10-24T11:25:39Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "integrator": "sixth", "nu": 1e-12, "velocity": "cubic", "layout": "halos", "cell_centred": true, "max_halo_width": 4, "rank0_halo_sent_bytes": 438681600, "halo_sent_bytes_per_rank_max": 877363200, "slab_sent_bytes_per_rank": 19267584000, "input": "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/initial.bin", "nx": 268800, "nv": 4096, "ranks": 8, "nodes": 2, "steps": 10, "warmup": 3, "dt": 0.02, "length": 12600, "vmax": 15, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 8808038400, "max_allocated_bytes_per_rank": 11300409400, "stream_ms": 97.31809059990155, "spline_ms": 30.11344969995662, "pack_ms": 6.52626839982986, "mpi_ms": 118.8481177000085, "field_ms": 11.04300180018072, "collision_ms": 14.49239670005227, "step_ms": 274.8684877000414, "initial_mass": 12597.62207265221, "final_mass": 12597.62136207489, "mass_relative_change": -5.640567068345301e-08, "initial_kinetic_energy": 55701.51067942529, "final_kinetic_energy": 55701.00688716807, "initial_field_energy": 791.8606180599771, "final_field_energy": 792.3077830250799, "minimum_f": -0.09580882788649095, "finite": true, "repeat": 1, "command": ["srun", "--jobid=60206959", "--overlap", "--nodes=2", "--ntasks=8", "--ntasks-per-node=4", "--gpus-per-node=4", "--gpu-bind=none", "--cpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/workdir/cuda-build/vlasov_bench", "--nx", "268800", "--nv", "4096", "--steps", "10", "--warmup", "3", "--mode", "landau", "--velocity", "cubic", "--layout", "halos", "--integrator", "sixth", "--nu", "1e-12", "--length", "12600.0", "--vmax", "15.0", "--dt", "0.02", "--cells", "--input", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/initial.bin", "--v-grid", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/v.bin", "--background", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/background.bin"], "rank_log": "rank=4 host=nid001024 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001024 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=6 host=nid001024 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001024 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=5 host=nid001025 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001025 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001025 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=7 host=nid001025 local_rank=3 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=35a370ebe958e3df17ad798c54efedf85892f8029da920ff437ff54953bea6b6\nsynced_at=2026-10-24T11:25:39Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "integrator": "sixth", "nu": 1e-12, "velocity": "cubic", "layout": "halos", "cell_centred": true, "max_halo_width": 4, "rank0_halo_sent_bytes": 438681600, "halo_sent_bytes_per_rank_max": 877363200, "slab_sent_bytes_per_rank": 19267584000, "input": "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/initial.bin", "nx": 268800, "nv": 4096, "ranks": 8, "nodes": 2, "steps": 10, "warmup": 3, "dt": 0.02, "length": 12600, "vmax": 15, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 8808038400, "max_allocated_bytes_per_rank": 11300409400, "stream_ms": 97.31957070011958, "spline_ms": 30.1135374000296, "pack_ms": 6.523464800011425, "mpi_ms": 119.0691504999904, "field_ms": 10.25784469979953, "collision_ms": 14.49078770006054, "step_ms": 275.0057376999848, "initial_mass": 12597.62207265221, "final_mass": 12597.62136207489, "mass_relative_change": -5.640567068345301e-08, "initial_kinetic_energy": 55701.51067942529, "final_kinetic_energy": 55701.00688716807, "initial_field_energy": 791.8606180599771, "final_field_energy": 792.3077830250799, "minimum_f": -0.09580882788649095, "finite": true, "repeat": 2, "command": ["srun", "--jobid=60206959", "--overlap", "--nodes=2", "--ntasks=8", "--ntasks-per-node=4", "--gpus-per-node=4", "--gpu-bind=none", "--cpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/workdir/cuda-build/vlasov_bench", "--nx", "268800", "--nv", "4096", "--steps", "10", "--warmup", "3", "--mode", "landau", "--velocity", "cubic", "--layout", "halos", "--integrator", "sixth", "--nu", "1e-12", "--length", "12600.0", "--vmax", "15.0", "--dt", "0.02", "--cells", "--input", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/initial.bin", "--v-grid", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/v.bin", "--background", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/background.bin"], "rank_log": "rank=3 host=nid001025 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=7 host=nid001025 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=5 host=nid001025 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001024 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=6 host=nid001024 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=4 host=nid001024 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001024 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001025 local_rank=0 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=35a370ebe958e3df17ad798c54efedf85892f8029da920ff437ff54953bea6b6\nsynced_at=2026-10-24T11:25:39Z\n"} diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-halos/nx268800-transpose.jsonl b/benchmarks/vlasov_cuda/results/2026-10-24-halos/nx268800-transpose.jsonl new file mode 100644 index 00000000..b6bb5075 --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-halos/nx268800-transpose.jsonl @@ -0,0 +1,6 @@ +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "integrator": "sixth", "nu": 1e-12, "velocity": "cubic", "layout": "transpose", "cell_centred": true, "max_halo_width": 0, "rank0_halo_sent_bytes": 0, "halo_sent_bytes_per_rank_max": 0, "slab_sent_bytes_per_rank": 231211008000, "input": "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/initial.bin", "nx": 268800, "nv": 4096, "ranks": 4, "nodes": 1, "steps": 10, "warmup": 3, "dt": 0.02, "length": 12600, "vmax": 15, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 8808038400, "max_allocated_bytes_per_rank": 22035230776, "stream_ms": 194.4949301001088, "spline_ms": 57.92510880005466, "pack_ms": 91.58833139995295, "mpi_ms": 273.2574189000388, "field_ms": 11.27814200003741, "collision_ms": 21.10967729986442, "step_ms": 649.6461631000784, "initial_mass": 12597.62207264873, "final_mass": 12597.62136206869, "mass_relative_change": -5.640588611556955e-08, "initial_kinetic_energy": 55701.51067944818, "final_kinetic_energy": 55701.0068872211, "initial_field_energy": 791.8606180599772, "final_field_energy": 792.3077830250793, "minimum_f": -0.09580882788649099, "finite": true, "repeat": 0, "command": ["srun", "--jobid=60206959", "--overlap", "--nodes=1", "--ntasks=4", "--ntasks-per-node=4", "--gpus-per-node=4", "--gpu-bind=none", "--cpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/workdir/cuda-build/vlasov_bench", "--nx", "268800", "--nv", "4096", "--steps", "10", "--warmup", "3", "--mode", "landau", "--velocity", "cubic", "--layout", "transpose", "--integrator", "sixth", "--nu", "1e-12", "--length", "12600.0", "--vmax", "15.0", "--dt", "0.02", "--cells", "--input", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/initial.bin", "--v-grid", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/v.bin", "--background", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/background.bin"], "rank_log": "rank=0 host=nid001024 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001024 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001024 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001024 local_rank=1 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=35a370ebe958e3df17ad798c54efedf85892f8029da920ff437ff54953bea6b6\nsynced_at=2026-10-24T11:25:39Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "integrator": "sixth", "nu": 1e-12, "velocity": "cubic", "layout": "transpose", "cell_centred": true, "max_halo_width": 0, "rank0_halo_sent_bytes": 0, "halo_sent_bytes_per_rank_max": 0, "slab_sent_bytes_per_rank": 231211008000, "input": "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/initial.bin", "nx": 268800, "nv": 4096, "ranks": 4, "nodes": 1, "steps": 10, "warmup": 3, "dt": 0.02, "length": 12600, "vmax": 15, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 8808038400, "max_allocated_bytes_per_rank": 22035230776, "stream_ms": 194.5085770000787, "spline_ms": 57.99670299984427, "pack_ms": 91.58881610005665, "mpi_ms": 273.2411245995536, "field_ms": 11.20944810036235, "collision_ms": 21.11824810003782, "step_ms": 649.6359549999852, "initial_mass": 12597.62207264873, "final_mass": 12597.62136206869, "mass_relative_change": -5.640588611556955e-08, "initial_kinetic_energy": 55701.51067944818, "final_kinetic_energy": 55701.0068872211, "initial_field_energy": 791.8606180599772, "final_field_energy": 792.3077830250793, "minimum_f": -0.09580882788649099, "finite": true, "repeat": 1, "command": ["srun", "--jobid=60206959", "--overlap", "--nodes=1", "--ntasks=4", "--ntasks-per-node=4", "--gpus-per-node=4", "--gpu-bind=none", "--cpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/workdir/cuda-build/vlasov_bench", "--nx", "268800", "--nv", "4096", "--steps", "10", "--warmup", "3", "--mode", "landau", "--velocity", "cubic", "--layout", "transpose", "--integrator", "sixth", "--nu", "1e-12", "--length", "12600.0", "--vmax", "15.0", "--dt", "0.02", "--cells", "--input", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/initial.bin", "--v-grid", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/v.bin", "--background", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/background.bin"], "rank_log": "rank=2 host=nid001025 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001025 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001025 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001025 local_rank=3 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=35a370ebe958e3df17ad798c54efedf85892f8029da920ff437ff54953bea6b6\nsynced_at=2026-10-24T11:25:39Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "integrator": "sixth", "nu": 1e-12, "velocity": "cubic", "layout": "transpose", "cell_centred": true, "max_halo_width": 0, "rank0_halo_sent_bytes": 0, "halo_sent_bytes_per_rank_max": 0, "slab_sent_bytes_per_rank": 231211008000, "input": "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/initial.bin", "nx": 268800, "nv": 4096, "ranks": 4, "nodes": 1, "steps": 10, "warmup": 3, "dt": 0.02, "length": 12600, "vmax": 15, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 8808038400, "max_allocated_bytes_per_rank": 22035230776, "stream_ms": 194.4883884001683, "spline_ms": 57.92602180003996, "pack_ms": 91.58432890003496, "mpi_ms": 273.2619063001039, "field_ms": 11.26907970001412, "collision_ms": 21.11227130003499, "step_ms": 649.6297856999718, "initial_mass": 12597.62207264873, "final_mass": 12597.62136206869, "mass_relative_change": -5.640588611556955e-08, "initial_kinetic_energy": 55701.51067944818, "final_kinetic_energy": 55701.0068872211, "initial_field_energy": 791.8606180599772, "final_field_energy": 792.3077830250793, "minimum_f": -0.09580882788649099, "finite": true, "repeat": 2, "command": ["srun", "--jobid=60206959", "--overlap", "--nodes=1", "--ntasks=4", "--ntasks-per-node=4", "--gpus-per-node=4", "--gpu-bind=none", "--cpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/workdir/cuda-build/vlasov_bench", "--nx", "268800", "--nv", "4096", "--steps", "10", "--warmup", "3", "--mode", "landau", "--velocity", "cubic", "--layout", "transpose", "--integrator", "sixth", "--nu", "1e-12", "--length", "12600.0", "--vmax", "15.0", "--dt", "0.02", "--cells", "--input", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/initial.bin", "--v-grid", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/v.bin", "--background", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/background.bin"], "rank_log": "rank=1 host=nid001024 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001024 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001024 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001024 local_rank=2 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=35a370ebe958e3df17ad798c54efedf85892f8029da920ff437ff54953bea6b6\nsynced_at=2026-10-24T11:25:39Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "integrator": "sixth", "nu": 1e-12, "velocity": "cubic", "layout": "transpose", "cell_centred": true, "max_halo_width": 0, "rank0_halo_sent_bytes": 0, "halo_sent_bytes_per_rank_max": 0, "slab_sent_bytes_per_rank": 134873088000, "input": "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/initial.bin", "nx": 268800, "nv": 4096, "ranks": 8, "nodes": 2, "steps": 10, "warmup": 3, "dt": 0.02, "length": 12600, "vmax": 15, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 8808038400, "max_allocated_bytes_per_rank": 11025158200, "stream_ms": 97.35810910015061, "spline_ms": 23.13156989971503, "pack_ms": 45.69633740002246, "mpi_ms": 775.7187433006493, "field_ms": 21.47743779996745, "collision_ms": 14.48875990008673, "step_ms": 964.4204498000363, "initial_mass": 12597.62207265221, "final_mass": 12597.62136207489, "mass_relative_change": -5.640567068345301e-08, "initial_kinetic_energy": 55701.51067942529, "final_kinetic_energy": 55701.00688716807, "initial_field_energy": 791.8606180599771, "final_field_energy": 792.3077830250799, "minimum_f": -0.09580882788649095, "finite": true, "repeat": 0, "command": ["srun", "--jobid=60206959", "--overlap", "--nodes=2", "--ntasks=8", "--ntasks-per-node=4", "--gpus-per-node=4", "--gpu-bind=none", "--cpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/workdir/cuda-build/vlasov_bench", "--nx", "268800", "--nv", "4096", "--steps", "10", "--warmup", "3", "--mode", "landau", "--velocity", "cubic", "--layout", "transpose", "--integrator", "sixth", "--nu", "1e-12", "--length", "12600.0", "--vmax", "15.0", "--dt", "0.02", "--cells", "--input", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/initial.bin", "--v-grid", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/v.bin", "--background", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/background.bin"], "rank_log": "rank=2 host=nid001024 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001025 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=4 host=nid001024 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=5 host=nid001025 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001025 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=7 host=nid001025 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001024 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=6 host=nid001024 local_rank=3 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=35a370ebe958e3df17ad798c54efedf85892f8029da920ff437ff54953bea6b6\nsynced_at=2026-10-24T11:25:39Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "integrator": "sixth", "nu": 1e-12, "velocity": "cubic", "layout": "transpose", "cell_centred": true, "max_halo_width": 0, "rank0_halo_sent_bytes": 0, "halo_sent_bytes_per_rank_max": 0, "slab_sent_bytes_per_rank": 134873088000, "input": "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/initial.bin", "nx": 268800, "nv": 4096, "ranks": 8, "nodes": 2, "steps": 10, "warmup": 3, "dt": 0.02, "length": 12600, "vmax": 15, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 8808038400, "max_allocated_bytes_per_rank": 11025158200, "stream_ms": 97.35549070010165, "spline_ms": 23.1311465998715, "pack_ms": 45.69765130004271, "mpi_ms": 777.1058046999769, "field_ms": 24.25418779998836, "collision_ms": 14.48958839991974, "step_ms": 965.7143639999504, "initial_mass": 12597.62207265221, "final_mass": 12597.62136207489, "mass_relative_change": -5.640567068345301e-08, "initial_kinetic_energy": 55701.51067942529, "final_kinetic_energy": 55701.00688716807, "initial_field_energy": 791.8606180599771, "final_field_energy": 792.3077830250799, "minimum_f": -0.09580882788649095, "finite": true, "repeat": 1, "command": ["srun", "--jobid=60206959", "--overlap", "--nodes=2", "--ntasks=8", "--ntasks-per-node=4", "--gpus-per-node=4", "--gpu-bind=none", "--cpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/workdir/cuda-build/vlasov_bench", "--nx", "268800", "--nv", "4096", "--steps", "10", "--warmup", "3", "--mode", "landau", "--velocity", "cubic", "--layout", "transpose", "--integrator", "sixth", "--nu", "1e-12", "--length", "12600.0", "--vmax", "15.0", "--dt", "0.02", "--cells", "--input", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/initial.bin", "--v-grid", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/v.bin", "--background", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/background.bin"], "rank_log": "rank=0 host=nid001024 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=5 host=nid001025 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001025 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001025 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=7 host=nid001025 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=6 host=nid001024 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=4 host=nid001024 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001024 local_rank=1 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=35a370ebe958e3df17ad798c54efedf85892f8029da920ff437ff54953bea6b6\nsynced_at=2026-10-24T11:25:39Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "integrator": "sixth", "nu": 1e-12, "velocity": "cubic", "layout": "transpose", "cell_centred": true, "max_halo_width": 0, "rank0_halo_sent_bytes": 0, "halo_sent_bytes_per_rank_max": 0, "slab_sent_bytes_per_rank": 134873088000, "input": "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/initial.bin", "nx": 268800, "nv": 4096, "ranks": 8, "nodes": 2, "steps": 10, "warmup": 3, "dt": 0.02, "length": 12600, "vmax": 15, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 8808038400, "max_allocated_bytes_per_rank": 11025158200, "stream_ms": 97.34848410021186, "spline_ms": 23.13205560012648, "pack_ms": 45.71225319996302, "mpi_ms": 776.3868526999431, "field_ms": 24.17435689994818, "collision_ms": 14.48879250001482, "step_ms": 964.9749842999427, "initial_mass": 12597.62207265221, "final_mass": 12597.62136207489, "mass_relative_change": -5.640567068345301e-08, "initial_kinetic_energy": 55701.51067942529, "final_kinetic_energy": 55701.00688716807, "initial_field_energy": 791.8606180599771, "final_field_energy": 792.3077830250799, "minimum_f": -0.09580882788649095, "finite": true, "repeat": 2, "command": ["srun", "--jobid=60206959", "--overlap", "--nodes=2", "--ntasks=8", "--ntasks-per-node=4", "--gpus-per-node=4", "--gpu-bind=none", "--cpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/workdir/cuda-build/vlasov_bench", "--nx", "268800", "--nv", "4096", "--steps", "10", "--warmup", "3", "--mode", "landau", "--velocity", "cubic", "--layout", "transpose", "--integrator", "sixth", "--nu", "1e-12", "--length", "12600.0", "--vmax", "15.0", "--dt", "0.02", "--cells", "--input", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/initial.bin", "--v-grid", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/v.bin", "--background", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/background.bin"], "rank_log": "rank=6 host=nid001024 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001025 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001024 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=7 host=nid001025 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001025 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=5 host=nid001025 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=4 host=nid001024 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001024 local_rank=0 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=35a370ebe958e3df17ad798c54efedf85892f8029da920ff437ff54953bea6b6\nsynced_at=2026-10-24T11:25:39Z\n"} diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-halos/reference-library-versions.json b/benchmarks/vlasov_cuda/results/2026-10-24-halos/reference-library-versions.json new file mode 100644 index 00000000..a8f88d36 --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-halos/reference-library-versions.json @@ -0,0 +1,9 @@ +{ + "adept": "0.1.0", + "jax": "0.10.2", + "jaxlib": "0.10.2", + "interpax": "0.3.14+10.g84aaf30", + "diffrax": "0.7.2", + "numpy": "2.4.6", + "scipy": "1.18.0" +} diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-halos/reference-nx134400.json b/benchmarks/vlasov_cuda/results/2026-10-24-halos/reference-nx134400.json new file mode 100644 index 00000000..772a04da --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-halos/reference-nx134400.json @@ -0,0 +1,35 @@ +{ + "checkpoint": "/pscratch/sd/a/archis/vp-turbulence-sessions/20261023T081121Z-nv4096-nx134400-5ce4fe14/workdir/campaigns/production-nx134400/checkpoint-state/step-030000-1a19a5ae15f8", + "manifest_sha256": "9ab7e963d2699777bd935ca84fad446b89846e64c416d10db6d3c14829af8627", + "actual_time": 600.0, + "config_sha256": "5a9f5347e6253a3d4201745318843857c568b3787cf91e9fe81a623d0cdbb214", + "adept_commit": "ddd70b957dc3df85f0b5ec84af927c25d09d24e0", + "jax": "0.10.2", + "nx": 134400, + "nv": 4096, + "dt": 0.02, + "nu": 1e-12, + "devices": [ + "cuda:0", + "cuda:1", + "cuda:2", + "cuda:3" + ], + "milestones": [ + { + "steps": 1, + "advanced_steps": 1, + "seconds": 3.020330819010269 + }, + { + "steps": 10, + "advanced_steps": 9, + "seconds": 2.0284883639833424 + }, + { + "steps": 100, + "advanced_steps": 90, + "seconds": 20.115412370010745 + } + ] +} diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-halos/reference-nx268800.json b/benchmarks/vlasov_cuda/results/2026-10-24-halos/reference-nx268800.json new file mode 100644 index 00000000..8df4fc30 --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-halos/reference-nx268800.json @@ -0,0 +1,37 @@ +{ + "checkpoint": "/pscratch/sd/a/archis/vp-turbulence-sessions/20261023T081121Z-nv4096-nx268800-93a418d7/workdir/campaigns/production-nx268800/checkpoint-state/step-030000-167819aeccca", + "manifest_sha256": "c9b9083f55cfcee8595556685d9e22eb9d87829fa6748726f7f44d5749fbec1f", + "actual_time": 600.0, + "config_sha256": "6adb42855432b79a74f63290678103c17c53842074d77bf092df38483f634a83", + "adept_commit": "ddd70b957dc3df85f0b5ec84af927c25d09d24e0", + "jax": "0.10.2", + "nx": 268800, + "nv": 4096, + "dt": 0.02, + "nu": 1e-12, + "devices": [ + "cuda:0", + "cuda:1", + "cuda:2", + "cuda:3", + "cuda:4", + "cuda:5", + "cuda:6", + "cuda:7" + ], + "milestones": [ + { + "steps": 1, + "advanced_steps": 1, + "rank0_seconds": 60.40102391698747 + }, + { + "steps": 100, + "advanced_steps": 99, + "rank0_seconds": 74.4436078449944 + } + ], + "process_count": 8, + "distributed": true, + "prepared_input": "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800" +} diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-halos/reference-source-state b/benchmarks/vlasov_cuda/results/2026-10-24-halos/reference-source-state new file mode 100644 index 00000000..48df1656 --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-halos/reference-source-state @@ -0,0 +1,5 @@ +version=1 +base_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4 +dirty=true +workspace_fingerprint=122fb1530212c155231e59f9d01d12c2566e24058627476aeee8c6310f3ab959 +synced_at=2026-10-24T11:08:15Z diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-halos/reference8-launch.sh b/benchmarks/vlasov_cuda/results/2026-10-24-halos/reference8-launch.sh new file mode 100644 index 00000000..a21ffb6b --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-halos/reference8-launch.sh @@ -0,0 +1,12 @@ +set -euo pipefail +source /global/common/software/m4490/archis/venvs/vp-turbulence/bin/activate +export PYTHONPATH=/pscratch/sd/a/archis/vp-turbulence-sessions/20261023T081121Z-nv4096-nx268800-93a418d7/src +export XLA_PYTHON_CLIENT_PREALLOCATE=false +export LD_LIBRARY_PATH="/global/common/software/nersc9/nccl/2.29.2-cu13/plugin/lib:/opt/nvidia/hpc_sdk/Linux_x86_64/26.5/cuda/13.2/lib64${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" +export NCCL_SOCKET_IFNAME=hsn NCCL_CROSS_NIC=2 NCCL_NET="AWS Libfabric" NCCL_NET_GDR_LEVEL=LOC FI_CXI_DISABLE_HOST_REGISTER=1 NCCL_DEBUG=INFO NCCL_DEBUG_SUBSYS=INIT,NET +export XLA_FLAGS="${XLA_FLAGS:+$XLA_FLAGS }--xla_gpu_autotune_level=0" +export JAX_ENABLE_X64=true +root=/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473 +cd "$root/outputs" +srun --jobid=60205333 --overlap --nodes=2 --ntasks=8 --ntasks-per-node=4 --cpus-per-task=16 --cpu-bind=none --gpus-per-node=4 --gpu-bind=none --mpi=none --kill-on-bad-exit=1 python "$root/src/benchmarks/vlasov_cuda/distributed_reference.py" --prepared "$root/outputs/native-nx268800" --config "$root/src/benchmarks/vlasov_cuda/reference/latest-nv4096/nx268800.yaml" --vp-source "$PYTHONPATH" --output "$root/outputs/native-nx268800-distributed" --steps 1 100 > "$root/workdir/reference8.log" 2>&1 +tail -n 8 "$root/workdir/reference8.log" diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-halos/slurm-accounting.txt b/benchmarks/vlasov_cuda/results/2026-10-24-halos/slurm-accounting.txt new file mode 100644 index 00000000..b06a1f3f --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-halos/slurm-accounting.txt @@ -0,0 +1,31 @@ +60206959|ec-adept-20261024T112505Z-77310-4969|RUNNING|00:15:16|0:0 +60206959.extern|extern|RUNNING|00:15:16|0:0 +60206959.0|bash|COMPLETED|00:12:45|0:0 +60206959.1|vlasov_bench|COMPLETED|00:00:29|0:0 +60206959.2|vlasov_bench|COMPLETED|00:01:59|0:0 +60206959.3|vlasov_bench|COMPLETED|00:00:24|0:0 +60206959.4|vlasov_bench|COMPLETED|00:00:50|0:0 +60206959.5|vlasov_bench|COMPLETED|00:00:16|0:0 +60206959.6|vlasov_bench|COMPLETED|00:00:17|0:0 +60206959.7|vlasov_bench|COMPLETED|00:00:14|0:0 +60206959.8|vlasov_bench|COMPLETED|00:00:14|0:0 +60206959.9|vlasov_bench|COMPLETED|00:00:15|0:0 +60206959.10|vlasov_bench|COMPLETED|00:00:15|0:0 +60206959.11|vlasov_bench|COMPLETED|00:00:13|0:0 +60206959.12|vlasov_bench|COMPLETED|00:00:13|0:0 +60206959.13|vlasov_bench|COMPLETED|00:00:12|0:0 +60206959.14|vlasov_bench|COMPLETED|00:00:10|0:0 +60206959.15|vlasov_bench|COMPLETED|00:00:10|0:0 +60206959.16|vlasov_bench|COMPLETED|00:00:09|0:0 +60206959.17|vlasov_bench|COMPLETED|00:00:26|0:0 +60206959.18|vlasov_bench|COMPLETED|00:00:29|0:0 +60206959.19|vlasov_bench|COMPLETED|00:00:26|0:0 +60206959.20|vlasov_bench|COMPLETED|00:00:25|0:0 +60206959.21|vlasov_bench|COMPLETED|00:00:25|0:0 +60206959.22|vlasov_bench|COMPLETED|00:00:24|0:0 +60206959.23|vlasov_bench|COMPLETED|00:00:21|0:0 +60206959.24|vlasov_bench|COMPLETED|00:00:23|0:0 +60206959.25|vlasov_bench|COMPLETED|00:00:22|0:0 +60206959.26|vlasov_bench|COMPLETED|00:00:16|0:0 +60206959.27|vlasov_bench|COMPLETED|00:00:15|0:0 +60206959.28|vlasov_bench|COMPLETED|00:00:15|0:0 diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-halos/source-archive.sha256 b/benchmarks/vlasov_cuda/results/2026-10-24-halos/source-archive.sha256 new file mode 100644 index 00000000..5aef810e --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-halos/source-archive.sha256 @@ -0,0 +1 @@ +df359db1f60990589a2fcac57643fc4770d51d636ecad7379d9bf777f4baf9b3 source.tar.gz diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-halos/source-state b/benchmarks/vlasov_cuda/results/2026-10-24-halos/source-state new file mode 100644 index 00000000..db22eba5 --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-halos/source-state @@ -0,0 +1,5 @@ +version=1 +base_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4 +dirty=true +workspace_fingerprint=35a370ebe958e3df17ad798c54efedf85892f8029da920ff437ff54953bea6b6 +synced_at=2026-10-24T11:25:39Z diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-halos/source.tar.gz b/benchmarks/vlasov_cuda/results/2026-10-24-halos/source.tar.gz new file mode 100644 index 00000000..c0ec9888 Binary files /dev/null and b/benchmarks/vlasov_cuda/results/2026-10-24-halos/source.tar.gz differ diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-halos/summarize.py b/benchmarks/vlasov_cuda/results/2026-10-24-halos/summarize.py new file mode 100644 index 00000000..028c7d63 --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-halos/summarize.py @@ -0,0 +1,129 @@ +"""Rebuild the compact report from the adjacent preserved measurements.""" + +import hashlib +import json +import statistics +import tarfile +from collections import defaultdict +from pathlib import Path + +root = Path(__file__).resolve().parent +groups = defaultdict(list) +for nx in [134400, 268800]: + for layout in ["transpose", "halos"]: + for line in (root / f"nx{nx}-{layout}.jsonl").read_text().splitlines(): + row = json.loads(line) + groups[nx, row["nodes"], row["ranks"], layout].append(row) +timings = [] +for (nx, nodes, ranks, layout), rows in sorted(groups.items()): + assert len(rows) == 3 and all( + row["finite"] + and row["steps"] == 10 + and row["warmup"] == 3 + and row["integrator"] == "sixth" + and row["nu"] == 1e-12 + and row["velocity"] == "cubic" + and row["cell_centred"] + for row in rows + ) + result = dict(nx=nx, nv=4096, nodes=nodes, gpus=ranks, layout=layout, repeats=len(rows)) + for key in [ + "step_ms", + "stream_ms", + "spline_ms", + "pack_ms", + "mpi_ms", + "field_ms", + "collision_ms", + "max_allocated_bytes_per_rank", + ]: + result[key] = statistics.median(row[key] for row in rows) + result["step_ms_range"] = [min(row["step_ms"] for row in rows), max(row["step_ms"] for row in rows)] + result["maximum_halo_width"] = max(row["max_halo_width"] for row in rows) + result["sent_bytes_per_step_max_rank"] = statistics.median( + (row["slab_sent_bytes_per_rank"] + row["halo_sent_bytes_per_rank_max"]) / row["steps"] for row in rows + ) + result["max_abs_relative_mass_change"] = max(abs(row["mass_relative_change"]) for row in rows) + timings.append(result) +pairs = [] +for nx in [134400, 268800]: + for nodes, ranks in [(1, 4), (2, 8)]: + match = {row["layout"]: row for row in timings if (row["nx"], row["nodes"], row["gpus"]) == (nx, nodes, ranks)} + original, halo = match["transpose"], match["halos"] + pairs.append( + dict( + nx=nx, + nodes=nodes, + gpus=ranks, + transpose_ms=original["step_ms"], + halo_ms=halo["step_ms"], + speedup=original["step_ms"] / halo["step_ms"], + sent_bytes_reduction=1 + - halo["sent_bytes_per_step_max_rank"] / original["sent_bytes_per_step_max_rank"], + ) + ) +checks = [] +for name in ["comparison-nx134400-smoke.json", "comparison-nx134400-window.json", "comparison-nx268800-window.json"]: + rows = json.loads((root / name).read_text()) + assert rows and all(row["passed"] for row in rows) + checks.extend(rows) +validation = [] +for name in ["validation-local-cpu.json", "validation-local-cuda.json", "validation-natural-cpu.json"]: + rows = json.loads((root / name).read_text()) + assert all(row["passed"] for row in rows) + validation.append(dict(report=name, checks=len(rows), max_abs_error=max(row["max_abs_error"] for row in rows))) +archive = root / "source.tar.gz" +# These measurements refer to the preserved source snapshot, even after the +# solver is promoted into ADEPT or the working tree changes. +with tarfile.open(archive) as snapshot: + sources = { + Path(member.name).name: snapshot.extractfile(member).read() + for member in snapshot.getmembers() + if member.isfile() + and len(Path(member.name).parts) == 2 + and (Path(member.name).suffix in [".cpp", ".hpp", ".py"] or Path(member.name).name == "CMakeLists.txt") + } +for line in (root / "compiled-source.sha256").read_text().splitlines(): + expected, name = line.split() + assert hashlib.sha256(sources[Path(name).name]).hexdigest() == expected +report = dict( + date="2026-10-24", + system="NERSC Perlmutter", + device="NVIDIA A100-SXM4-40GB", + allocation="60206959", + reference_allocation="60205333", + nodes=["nid001024", "nid001025"], + precision="float64", + dt=0.02, + nu=1e-12, + software=dict(cuda="13.2.78", gcc="14.3.0", cray_mpich="9.1.0", cmake="3.30.2"), + reference_libraries=json.loads((root / "reference-library-versions.json").read_text()), + integrator="sixth", + velocity="interpax cubic C1 Hermite", + streaming="spectral exponential", + timing=( + "Median of 3 repeats; 10 measured steps after 3 warmup steps; maximum rank mean; " + "explicit GPU synchronization; no overlap. Input/output excluded." + ), + reference_cuda_executable_sha256="c5db5fa71e9d1119e7cb581a9e67624f42155c006da45184cb55de6542255b9a", + measured_executable_sha256=(root / "executable.sha256").read_text().split()[0], + comparison_scope=( + "Full native grids, mature checkpoint t600 to t602. Not a full t0-to-2000 trajectory " + "reproduction or a CUDA-versus-JAX speed comparison." + ), + checks=dict( + native_cases=len(checks), + max_abs_f_error=max(row["max_abs_f_error"] for row in checks), + max_relative_l2_f_error=max(row["relative_l2_f_error"] for row in checks), + max_relative_l2_e_error=max(row["relative_l2_e_error"] for row in checks), + all_hole_counts_match=all(row["hole_counts_match"] for row in checks), + validation=validation, + ), + paired=pairs, + timing_rows=timings, + source_sha256={name: hashlib.sha256(content).hexdigest() for name, content in sorted(sources.items())}, +) +if archive.exists(): + report["source_archive"] = dict(file=archive.name, sha256=hashlib.sha256(archive.read_bytes()).hexdigest()) +(root / "summary.json").write_text(json.dumps(report, indent=2) + "\n") +print(json.dumps(dict(checks=report["checks"], paired=pairs), indent=2)) diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-halos/summary.json b/benchmarks/vlasov_cuda/results/2026-10-24-halos/summary.json new file mode 100644 index 00000000..46daf241 --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-halos/summary.json @@ -0,0 +1,302 @@ +{ + "date": "2026-10-24", + "system": "NERSC Perlmutter", + "device": "NVIDIA A100-SXM4-40GB", + "allocation": "60206959", + "reference_allocation": "60205333", + "nodes": [ + "nid001024", + "nid001025" + ], + "precision": "float64", + "dt": 0.02, + "nu": 1e-12, + "software": { + "cuda": "13.2.78", + "gcc": "14.3.0", + "cray_mpich": "9.1.0", + "cmake": "3.30.2" + }, + "reference_libraries": { + "adept": "0.1.0", + "jax": "0.10.2", + "jaxlib": "0.10.2", + "interpax": "0.3.14+10.g84aaf30", + "diffrax": "0.7.2", + "numpy": "2.4.6", + "scipy": "1.18.0" + }, + "integrator": "sixth", + "velocity": "interpax cubic C1 Hermite", + "streaming": "spectral exponential", + "timing": "Median of 3 repeats; 10 measured steps after 3 warmup steps; maximum rank mean; explicit GPU synchronization; no overlap. Input/output excluded.", + "reference_cuda_executable_sha256": "c5db5fa71e9d1119e7cb581a9e67624f42155c006da45184cb55de6542255b9a", + "measured_executable_sha256": "7d88657b9597e939f310c2c488d81842793ec361ad190b8f60595c47c8c6ea51", + "comparison_scope": "Full native grids, mature checkpoint t600 to t602. Not a full t0-to-2000 trajectory reproduction or a CUDA-versus-JAX speed comparison.", + "checks": { + "native_cases": 20, + "max_abs_f_error": 6.725869861057276e-13, + "max_relative_l2_f_error": 9.515283499791307e-14, + "max_relative_l2_e_error": 5.6328186846468416e-14, + "all_hole_counts_match": true, + "validation": [ + { + "report": "validation-local-cpu.json", + "checks": 36, + "max_abs_error": 9.103828801926284e-15 + }, + { + "report": "validation-local-cuda.json", + "checks": 36, + "max_abs_error": 3.774758283725532e-15 + }, + { + "report": "validation-natural-cpu.json", + "checks": 12, + "max_abs_error": 3.1839523243379595e-15 + } + ] + }, + "paired": [ + { + "nx": 134400, + "nodes": 1, + "gpus": 4, + "transpose_ms": 323.5450839000123, + "halo_ms": 174.2806366000877, + "speedup": 1.8564603056989837, + "sent_bytes_reduction": 0.8534226190476191 + }, + { + "nx": 134400, + "nodes": 2, + "gpus": 8, + "transpose_ms": 490.5058394000434, + "halo_ms": 140.0107500999638, + "speedup": 3.5033441292889, + "sent_bytes_reduction": 0.8507653061224489 + }, + { + "nx": 268800, + "nodes": 1, + "gpus": 4, + "transpose_ms": 649.6359549999852, + "halo_ms": 342.4947891999636, + "speedup": 1.8967761714491342, + "sent_bytes_reduction": 0.8533482142857143 + }, + { + "nx": 268800, + "nodes": 2, + "gpus": 8, + "transpose_ms": 964.9749842999427, + "halo_ms": 275.0057376999848, + "speedup": 3.5089267313857793, + "sent_bytes_reduction": 0.8506377551020408 + } + ], + "timing_rows": [ + { + "nx": 134400, + "nv": 4096, + "nodes": 1, + "gpus": 4, + "layout": "halos", + "repeats": 3, + "step_ms": 174.2806366000877, + "stream_ms": 96.7597968001428, + "spline_ms": 29.37038770005529, + "pack_ms": 6.511311200029013, + "mpi_ms": 20.97919940019892, + "field_ms": 6.686601499905009, + "collision_ms": 14.47656629998164, + "max_allocated_bytes_per_rank": 11155281976, + "step_ms_range": [ + 174.2041793000681, + 174.3251663999672 + ], + "maximum_halo_width": 4, + "sent_bytes_per_step_max_rank": 1694515200.0, + "max_abs_relative_mass_change": 4.083657516783715e-08 + }, + { + "nx": 134400, + "nv": 4096, + "nodes": 1, + "gpus": 4, + "layout": "transpose", + "repeats": 3, + "step_ms": 323.5450839000123, + "stream_ms": 96.80129259998012, + "spline_ms": 23.18371020005543, + "pack_ms": 45.59232870010419, + "mpi_ms": 137.4090791999834, + "field_ms": 6.081885599883208, + "collision_ms": 14.47260410000126, + "max_allocated_bytes_per_rank": 11017656376, + "step_ms_range": [ + 323.5294349000242, + 323.5631748999822 + ], + "maximum_halo_width": 0, + "sent_bytes_per_step_max_rank": 11560550400.0, + "max_abs_relative_mass_change": 4.083657516783715e-08 + }, + { + "nx": 134400, + "nv": 4096, + "nodes": 2, + "gpus": 8, + "layout": "halos", + "repeats": 3, + "step_ms": 140.0107500999638, + "stream_ms": 48.50200009991568, + "spline_ms": 12.23685909994856, + "pack_ms": 3.281850700113864, + "mpi_ms": 60.16711680003937, + "field_ms": 6.355414500239931, + "collision_ms": 11.2380733999089, + "max_allocated_bytes_per_rank": 5650233400, + "step_ms_range": [ + 139.8805341999378, + 140.1605807999658 + ], + "maximum_halo_width": 4, + "sent_bytes_per_step_max_rank": 1006387200.0, + "max_abs_relative_mass_change": 4.083634327505363e-08 + }, + { + "nx": 134400, + "nv": 4096, + "nodes": 2, + "gpus": 8, + "layout": "transpose", + "repeats": 3, + "step_ms": 490.5058394000434, + "stream_ms": 48.55127589999029, + "spline_ms": 11.72291799998675, + "pack_ms": 22.97926750006809, + "mpi_ms": 391.0327226998788, + "field_ms": 12.39774919995398, + "collision_ms": 11.23720970006161, + "max_allocated_bytes_per_rank": 5512607800, + "step_ms_range": [ + 489.5213302999537, + 490.7239440000239 + ], + "maximum_halo_width": 0, + "sent_bytes_per_step_max_rank": 6743654400.0, + "max_abs_relative_mass_change": 4.083634327505363e-08 + }, + { + "nx": 268800, + "nv": 4096, + "nodes": 1, + "gpus": 4, + "layout": "halos", + "repeats": 3, + "step_ms": 342.4947891999636, + "stream_ms": 194.4373317000554, + "spline_ms": 61.42703189980239, + "pack_ms": 13.08217789996888, + "mpi_ms": 41.15909789966281, + "field_ms": 12.18012080012159, + "collision_ms": 21.11400799994954, + "max_allocated_bytes_per_rank": 22310481976, + "step_ms_range": [ + 342.4020051000753, + 342.5728927000364 + ], + "maximum_halo_width": 4, + "sent_bytes_per_step_max_rank": 3390750720.0, + "max_abs_relative_mass_change": 5.640588611556955e-08 + }, + { + "nx": 268800, + "nv": 4096, + "nodes": 1, + "gpus": 4, + "layout": "transpose", + "repeats": 3, + "step_ms": 649.6359549999852, + "stream_ms": 194.4949301001088, + "spline_ms": 57.92602180003996, + "pack_ms": 91.58833139995295, + "mpi_ms": 273.2574189000388, + "field_ms": 11.26907970001412, + "collision_ms": 21.11227130003499, + "max_allocated_bytes_per_rank": 22035230776, + "step_ms_range": [ + 649.6297856999718, + 649.6461631000784 + ], + "maximum_halo_width": 0, + "sent_bytes_per_step_max_rank": 23121100800.0, + "max_abs_relative_mass_change": 5.640588611556955e-08 + }, + { + "nx": 268800, + "nv": 4096, + "nodes": 2, + "gpus": 8, + "layout": "halos", + "repeats": 3, + "step_ms": 275.0057376999848, + "stream_ms": 97.31809059990155, + "spline_ms": 30.1135374000296, + "pack_ms": 6.52626839982986, + "mpi_ms": 119.0691504999904, + "field_ms": 10.73557600002459, + "collision_ms": 14.49078770006054, + "max_allocated_bytes_per_rank": 11300409400, + "step_ms_range": [ + 274.8684877000414, + 275.1715809000871 + ], + "maximum_halo_width": 4, + "sent_bytes_per_step_max_rank": 2014494720.0, + "max_abs_relative_mass_change": 5.640567068345301e-08 + }, + { + "nx": 268800, + "nv": 4096, + "nodes": 2, + "gpus": 8, + "layout": "transpose", + "repeats": 3, + "step_ms": 964.9749842999427, + "stream_ms": 97.35549070010165, + "spline_ms": 23.13156989971503, + "pack_ms": 45.69765130004271, + "mpi_ms": 776.3868526999431, + "field_ms": 24.17435689994818, + "collision_ms": 14.48879250001482, + "max_allocated_bytes_per_rank": 11025158200, + "step_ms_range": [ + 964.4204498000363, + 965.7143639999504 + ], + "maximum_halo_width": 0, + "sent_bytes_per_step_max_rank": 13487308800.0, + "max_abs_relative_mass_change": 5.640567068345301e-08 + } + ], + "source_sha256": { + "CMakeLists.txt": "d5f11680925c336c63ea39e306230371f6dcb3e41ba423c8c09599c7c592664c", + "backend.cpp": "4dab281aa96caf3589f56bf0b6f313686e70c7f3b0c10788ea071b4edb6a1ef2", + "backend.hpp": "73ee9d9f649eebf405873f87c1eece597e8bd57a32636d2ec047eef156ea0cb3", + "benchmark_latest.py": "9ab3ba1fc7413167bdb880b3520b50a791616a7f86ca929a79910c544387399a", + "checkpoint_reference.py": "6387baa7e5e56c909fc255ed4c9c822b84562f80fffc5386d00570a0e81dddef", + "compare_checkpoint.py": "0b333e43047049965ae9bdb792ea4578a48e6fb7e55e29406d755d8d18b7cd2d", + "distributed_reference.py": "746fef8b0b093eb964c78df99a2dac895ae618d2757cca38e27b90d9617e42a6", + "local_cubic.hpp": "f87f6aa9731f64e501b94e95c20a53f45ae7d1ea5ec39c7e859c6b1a4b8c40c3", + "main.cpp": "fdea36e5d804fd24df663e6c48194dba4e42dec3771cea8167b526db91b52927", + "measure.py": "fa044122a74245abd65e917c886fd2e6a12a8d926183427a75b2dbefceea77a4", + "validate.py": "36fd3383185f1a3ddba754d587ca8d76f94c790597a86859200683330907955f", + "validate_local.py": "9fc625fb020693efd292e9ead2388cbc0320b7eb1ee5d5f56c326706771c6f6b" + }, + "source_archive": { + "file": "source.tar.gz", + "sha256": "df359db1f60990589a2fcac57643fc4770d51d636ecad7379d9bf777f4baf9b3" + } +} diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-halos/validation-local-cpu.json b/benchmarks/vlasov_cuda/results/2026-10-24-halos/validation-local-cpu.json new file mode 100644 index 00000000..61cabd52 --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-halos/validation-local-cpu.json @@ -0,0 +1,254 @@ +[ + { + "test": "signed-positive", + "layout": "transpose", + "ranks": 1, + "max_abs_error": 8.881784197001252e-16, + "passed": true + }, + { + "test": "signed-positive", + "layout": "transpose", + "ranks": 2, + "max_abs_error": 8.881784197001252e-16, + "passed": true + }, + { + "test": "signed-positive", + "layout": "transpose", + "ranks": 4, + "max_abs_error": 8.881784197001252e-16, + "passed": true + }, + { + "test": "signed-positive", + "layout": "halos", + "ranks": 1, + "max_abs_error": 8.881784197001252e-16, + "passed": true + }, + { + "test": "signed-positive", + "layout": "halos", + "ranks": 2, + "max_abs_error": 8.881784197001252e-16, + "passed": true + }, + { + "test": "signed-positive", + "layout": "halos", + "ranks": 4, + "max_abs_error": 8.881784197001252e-16, + "passed": true + }, + { + "test": "signed-negative", + "layout": "transpose", + "ranks": 1, + "max_abs_error": 4.884981308350689e-15, + "passed": true + }, + { + "test": "signed-negative", + "layout": "transpose", + "ranks": 2, + "max_abs_error": 4.884981308350689e-15, + "passed": true + }, + { + "test": "signed-negative", + "layout": "transpose", + "ranks": 4, + "max_abs_error": 4.884981308350689e-15, + "passed": true + }, + { + "test": "signed-negative", + "layout": "halos", + "ranks": 1, + "max_abs_error": 4.884981308350689e-15, + "passed": true + }, + { + "test": "signed-negative", + "layout": "halos", + "ranks": 2, + "max_abs_error": 4.884981308350689e-15, + "passed": true + }, + { + "test": "signed-negative", + "layout": "halos", + "ranks": 4, + "max_abs_error": 4.884981308350689e-15, + "passed": true + }, + { + "test": "signed-wide", + "layout": "transpose", + "ranks": 1, + "max_abs_error": 9.103828801926284e-15, + "passed": true + }, + { + "test": "signed-wide", + "layout": "transpose", + "ranks": 2, + "max_abs_error": 9.103828801926284e-15, + "passed": true + }, + { + "test": "signed-wide", + "layout": "transpose", + "ranks": 4, + "max_abs_error": 9.103828801926284e-15, + "passed": true + }, + { + "test": "signed-wide", + "layout": "halos", + "ranks": 1, + "max_abs_error": 9.103828801926284e-15, + "passed": true + }, + { + "test": "signed-wide", + "layout": "halos", + "ranks": 2, + "max_abs_error": 9.103828801926284e-15, + "passed": true + }, + { + "test": "signed-wide", + "layout": "halos", + "ranks": 4, + "max_abs_error": 9.103828801926284e-15, + "passed": true + }, + { + "test": "sixth", + "layout": "transpose", + "ranks": 1, + "max_abs_error": 1.0547118733938987e-15, + "passed": true + }, + { + "test": "sixth", + "layout": "transpose", + "ranks": 2, + "max_abs_error": 1.0547118733938987e-15, + "passed": true + }, + { + "test": "sixth", + "layout": "transpose", + "ranks": 4, + "max_abs_error": 1.0547118733938987e-15, + "passed": true + }, + { + "test": "sixth", + "layout": "halos", + "ranks": 1, + "max_abs_error": 1.0547118733938987e-15, + "passed": true + }, + { + "test": "sixth", + "layout": "halos", + "ranks": 2, + "max_abs_error": 1.0547118733938987e-15, + "passed": true + }, + { + "test": "sixth", + "layout": "halos", + "ranks": 4, + "max_abs_error": 1.0547118733938987e-15, + "passed": true + }, + { + "test": "collide", + "layout": "transpose", + "ranks": 1, + "max_abs_error": 1.1102230246251565e-16, + "passed": true + }, + { + "test": "collide", + "layout": "transpose", + "ranks": 2, + "max_abs_error": 1.1102230246251565e-16, + "passed": true + }, + { + "test": "collide", + "layout": "transpose", + "ranks": 4, + "max_abs_error": 1.1102230246251565e-16, + "passed": true + }, + { + "test": "collide", + "layout": "halos", + "ranks": 1, + "max_abs_error": 1.1102230246251565e-16, + "passed": true + }, + { + "test": "collide", + "layout": "halos", + "ranks": 2, + "max_abs_error": 1.1102230246251565e-16, + "passed": true + }, + { + "test": "collide", + "layout": "halos", + "ranks": 4, + "max_abs_error": 1.1102230246251565e-16, + "passed": true + }, + { + "test": "sixth-collide", + "layout": "transpose", + "ranks": 1, + "max_abs_error": 1.0547118733938987e-15, + "passed": true + }, + { + "test": "sixth-collide", + "layout": "transpose", + "ranks": 2, + "max_abs_error": 1.0547118733938987e-15, + "passed": true + }, + { + "test": "sixth-collide", + "layout": "transpose", + "ranks": 4, + "max_abs_error": 1.0547118733938987e-15, + "passed": true + }, + { + "test": "sixth-collide", + "layout": "halos", + "ranks": 1, + "max_abs_error": 1.0547118733938987e-15, + "passed": true + }, + { + "test": "sixth-collide", + "layout": "halos", + "ranks": 2, + "max_abs_error": 1.0547118733938987e-15, + "passed": true + }, + { + "test": "sixth-collide", + "layout": "halos", + "ranks": 4, + "max_abs_error": 1.0547118733938987e-15, + "passed": true + } +] diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-halos/validation-local-cuda.json b/benchmarks/vlasov_cuda/results/2026-10-24-halos/validation-local-cuda.json new file mode 100644 index 00000000..e149fb4e --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-halos/validation-local-cuda.json @@ -0,0 +1,254 @@ +[ + { + "test": "signed-positive", + "layout": "transpose", + "ranks": 1, + "max_abs_error": 4.440892098500626e-16, + "passed": true + }, + { + "test": "signed-positive", + "layout": "transpose", + "ranks": 2, + "max_abs_error": 4.440892098500626e-16, + "passed": true + }, + { + "test": "signed-positive", + "layout": "transpose", + "ranks": 4, + "max_abs_error": 4.440892098500626e-16, + "passed": true + }, + { + "test": "signed-positive", + "layout": "halos", + "ranks": 1, + "max_abs_error": 4.440892098500626e-16, + "passed": true + }, + { + "test": "signed-positive", + "layout": "halos", + "ranks": 2, + "max_abs_error": 4.440892098500626e-16, + "passed": true + }, + { + "test": "signed-positive", + "layout": "halos", + "ranks": 4, + "max_abs_error": 4.440892098500626e-16, + "passed": true + }, + { + "test": "signed-negative", + "layout": "transpose", + "ranks": 1, + "max_abs_error": 3.774758283725532e-15, + "passed": true + }, + { + "test": "signed-negative", + "layout": "transpose", + "ranks": 2, + "max_abs_error": 3.774758283725532e-15, + "passed": true + }, + { + "test": "signed-negative", + "layout": "transpose", + "ranks": 4, + "max_abs_error": 3.774758283725532e-15, + "passed": true + }, + { + "test": "signed-negative", + "layout": "halos", + "ranks": 1, + "max_abs_error": 3.774758283725532e-15, + "passed": true + }, + { + "test": "signed-negative", + "layout": "halos", + "ranks": 2, + "max_abs_error": 3.774758283725532e-15, + "passed": true + }, + { + "test": "signed-negative", + "layout": "halos", + "ranks": 4, + "max_abs_error": 3.774758283725532e-15, + "passed": true + }, + { + "test": "signed-wide", + "layout": "transpose", + "ranks": 1, + "max_abs_error": 0.0, + "passed": true + }, + { + "test": "signed-wide", + "layout": "transpose", + "ranks": 2, + "max_abs_error": 0.0, + "passed": true + }, + { + "test": "signed-wide", + "layout": "transpose", + "ranks": 4, + "max_abs_error": 0.0, + "passed": true + }, + { + "test": "signed-wide", + "layout": "halos", + "ranks": 1, + "max_abs_error": 0.0, + "passed": true + }, + { + "test": "signed-wide", + "layout": "halos", + "ranks": 2, + "max_abs_error": 0.0, + "passed": true + }, + { + "test": "signed-wide", + "layout": "halos", + "ranks": 4, + "max_abs_error": 0.0, + "passed": true + }, + { + "test": "sixth", + "layout": "transpose", + "ranks": 1, + "max_abs_error": 1.0547118733938987e-15, + "passed": true + }, + { + "test": "sixth", + "layout": "transpose", + "ranks": 2, + "max_abs_error": 9.992007221626409e-16, + "passed": true + }, + { + "test": "sixth", + "layout": "transpose", + "ranks": 4, + "max_abs_error": 1.0547118733938987e-15, + "passed": true + }, + { + "test": "sixth", + "layout": "halos", + "ranks": 1, + "max_abs_error": 1.0547118733938987e-15, + "passed": true + }, + { + "test": "sixth", + "layout": "halos", + "ranks": 2, + "max_abs_error": 9.992007221626409e-16, + "passed": true + }, + { + "test": "sixth", + "layout": "halos", + "ranks": 4, + "max_abs_error": 1.0547118733938987e-15, + "passed": true + }, + { + "test": "collide", + "layout": "transpose", + "ranks": 1, + "max_abs_error": 1.1102230246251565e-16, + "passed": true + }, + { + "test": "collide", + "layout": "transpose", + "ranks": 2, + "max_abs_error": 1.1102230246251565e-16, + "passed": true + }, + { + "test": "collide", + "layout": "transpose", + "ranks": 4, + "max_abs_error": 1.1102230246251565e-16, + "passed": true + }, + { + "test": "collide", + "layout": "halos", + "ranks": 1, + "max_abs_error": 1.1102230246251565e-16, + "passed": true + }, + { + "test": "collide", + "layout": "halos", + "ranks": 2, + "max_abs_error": 1.1102230246251565e-16, + "passed": true + }, + { + "test": "collide", + "layout": "halos", + "ranks": 4, + "max_abs_error": 1.1102230246251565e-16, + "passed": true + }, + { + "test": "sixth-collide", + "layout": "transpose", + "ranks": 1, + "max_abs_error": 1.0547118733938987e-15, + "passed": true + }, + { + "test": "sixth-collide", + "layout": "transpose", + "ranks": 2, + "max_abs_error": 9.992007221626409e-16, + "passed": true + }, + { + "test": "sixth-collide", + "layout": "transpose", + "ranks": 4, + "max_abs_error": 1.0547118733938987e-15, + "passed": true + }, + { + "test": "sixth-collide", + "layout": "halos", + "ranks": 1, + "max_abs_error": 1.0547118733938987e-15, + "passed": true + }, + { + "test": "sixth-collide", + "layout": "halos", + "ranks": 2, + "max_abs_error": 9.992007221626409e-16, + "passed": true + }, + { + "test": "sixth-collide", + "layout": "halos", + "ranks": 4, + "max_abs_error": 1.0547118733938987e-15, + "passed": true + } +] diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-halos/validation-natural-cpu.json b/benchmarks/vlasov_cuda/results/2026-10-24-halos/validation-natural-cpu.json new file mode 100644 index 00000000..7a9996cb --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-halos/validation-natural-cpu.json @@ -0,0 +1,74 @@ +[ + { + "mode": "transpose", + "ranks": 1, + "max_abs_error": 2.220446049250313e-16, + "passed": true + }, + { + "mode": "transpose", + "ranks": 2, + "max_abs_error": 2.220446049250313e-16, + "passed": true + }, + { + "mode": "transpose", + "ranks": 4, + "max_abs_error": 2.220446049250313e-16, + "passed": true + }, + { + "mode": "stream", + "ranks": 1, + "max_abs_error": 3.3306690738754696e-16, + "passed": true + }, + { + "mode": "stream", + "ranks": 2, + "max_abs_error": 3.3306690738754696e-16, + "passed": true + }, + { + "mode": "stream", + "ranks": 4, + "max_abs_error": 3.3306690738754696e-16, + "passed": true + }, + { + "mode": "accelerate", + "ranks": 1, + "max_abs_error": 5.551115123125783e-16, + "passed": true + }, + { + "mode": "accelerate", + "ranks": 2, + "max_abs_error": 5.551115123125783e-16, + "passed": true + }, + { + "mode": "accelerate", + "ranks": 4, + "max_abs_error": 5.551115123125783e-16, + "passed": true + }, + { + "mode": "landau", + "ranks": 1, + "max_abs_error": 3.1839523243379595e-15, + "passed": true + }, + { + "mode": "landau", + "ranks": 2, + "max_abs_error": 3.1713215548549073e-15, + "passed": true + }, + { + "mode": "landau", + "ranks": 4, + "max_abs_error": 3.1710355546772805e-15, + "passed": true + } +] diff --git a/benchmarks/vlasov_cuda/results/2026-10-24/baseline-source.tar.gz b/benchmarks/vlasov_cuda/results/2026-10-24/baseline-source.tar.gz new file mode 100644 index 00000000..e5467871 Binary files /dev/null and b/benchmarks/vlasov_cuda/results/2026-10-24/baseline-source.tar.gz differ diff --git a/benchmarks/vlasov_cuda/results/2026-10-24/scaling-32768x8192.jsonl b/benchmarks/vlasov_cuda/results/2026-10-24/scaling-32768x8192.jsonl new file mode 100644 index 00000000..516da6b8 --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24/scaling-32768x8192.jsonl @@ -0,0 +1,15 @@ +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "nx": 32768, "nv": 8192, "ranks": 1, "nodes": 1, "steps": 20, "warmup": 3, "dt": 0.1, "length": 12.56637061435917, "vmax": 8, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 2147483648, "max_allocated_bytes_per_rank": 21476605968, "stream_ms": 49.01655189999872, "spline_ms": 17.81408299999896, "pack_ms": 0, "mpi_ms": 0, "field_ms": 2.06822614998714, "step_ms": 68.90644464999696, "initial_mass": 12.56637061282561, "final_mass": 12.56637061276199, "mass_relative_change": -5.062453547606825e-12, "initial_kinetic_energy": 6.283185306434273, "final_kinetic_energy": 6.28427053267025, "initial_field_energy": 0.001256637061435577, "final_field_energy": 0.0001735421558638325, "minimum_f": -2.32438533550678e-16, "finite": true, "repeat": 0, "command": ["srun", "--jobid=60204067", "--overlap", "--nodes=1", "--ntasks=1", "--ntasks-per-node=1", "--gpus-per-node=4", "--gpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T095729Z-66315-25333/workdir/cuda-build/vlasov_bench", "--nx", "32768", "--nv", "8192", "--steps", "20", "--warmup", "3", "--mode", "landau"], "rank_log": "rank=0 host=nid001409 local_rank=0 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=937bea67c3242b0cb8313a6de50eef13ee7dcfa6b642142a129b277dd1ac56bb\nsynced_at=2026-10-24T10:06:08Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "nx": 32768, "nv": 8192, "ranks": 1, "nodes": 1, "steps": 20, "warmup": 3, "dt": 0.1, "length": 12.56637061435917, "vmax": 8, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 2147483648, "max_allocated_bytes_per_rank": 21476605968, "stream_ms": 49.02767665001591, "spline_ms": 17.8467182999924, "pack_ms": 0, "mpi_ms": 0, "field_ms": 2.070863450006755, "step_ms": 68.95362119999504, "initial_mass": 12.56637061282561, "final_mass": 12.56637061276199, "mass_relative_change": -5.062453547606825e-12, "initial_kinetic_energy": 6.283185306434273, "final_kinetic_energy": 6.28427053267025, "initial_field_energy": 0.001256637061435577, "final_field_energy": 0.0001735421558638325, "minimum_f": -2.32438533550678e-16, "finite": true, "repeat": 1, "command": ["srun", "--jobid=60204067", "--overlap", "--nodes=1", "--ntasks=1", "--ntasks-per-node=1", "--gpus-per-node=4", "--gpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T095729Z-66315-25333/workdir/cuda-build/vlasov_bench", "--nx", "32768", "--nv", "8192", "--steps", "20", "--warmup", "3", "--mode", "landau"], "rank_log": "rank=0 host=nid001412 local_rank=0 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=937bea67c3242b0cb8313a6de50eef13ee7dcfa6b642142a129b277dd1ac56bb\nsynced_at=2026-10-24T10:06:08Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "nx": 32768, "nv": 8192, "ranks": 1, "nodes": 1, "steps": 20, "warmup": 3, "dt": 0.1, "length": 12.56637061435917, "vmax": 8, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 2147483648, "max_allocated_bytes_per_rank": 21476605968, "stream_ms": 49.01497695000785, "spline_ms": 17.79829555000561, "pack_ms": 0, "mpi_ms": 0, "field_ms": 2.064744599999813, "step_ms": 68.88639965001175, "initial_mass": 12.56637061282561, "final_mass": 12.56637061276199, "mass_relative_change": -5.062453547606825e-12, "initial_kinetic_energy": 6.283185306434273, "final_kinetic_energy": 6.28427053267025, "initial_field_energy": 0.001256637061435577, "final_field_energy": 0.0001735421558638325, "minimum_f": -2.32438533550678e-16, "finite": true, "repeat": 2, "command": ["srun", "--jobid=60204067", "--overlap", "--nodes=1", "--ntasks=1", "--ntasks-per-node=1", "--gpus-per-node=4", "--gpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T095729Z-66315-25333/workdir/cuda-build/vlasov_bench", "--nx", "32768", "--nv", "8192", "--steps", "20", "--warmup", "3", "--mode", "landau"], "rank_log": "rank=0 host=nid001409 local_rank=0 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=937bea67c3242b0cb8313a6de50eef13ee7dcfa6b642142a129b277dd1ac56bb\nsynced_at=2026-10-24T10:06:08Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "nx": 32768, "nv": 8192, "ranks": 2, "nodes": 1, "steps": 20, "warmup": 3, "dt": 0.1, "length": 12.56637061435917, "vmax": 8, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 2147483648, "max_allocated_bytes_per_rank": 10739122192, "stream_ms": 24.53958674999512, "spline_ms": 12.87446425000098, "pack_ms": 6.339934000004632, "mpi_ms": 13.78156439998861, "field_ms": 1.197888349997811, "step_ms": 58.71698414999287, "initial_mass": 12.56637061355123, "final_mass": 12.56637061358818, "mass_relative_change": 2.939963389748054e-12, "initial_kinetic_energy": 6.283185306796558, "final_kinetic_energy": 6.284270533062034, "initial_field_energy": 0.00125663706143567, "final_field_energy": 0.0001735421558640621, "minimum_f": -2.324385335496638e-16, "finite": true, "repeat": 0, "command": ["srun", "--jobid=60204067", "--overlap", "--nodes=1", "--ntasks=2", "--ntasks-per-node=2", "--gpus-per-node=4", "--gpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T095729Z-66315-25333/workdir/cuda-build/vlasov_bench", "--nx", "32768", "--nv", "8192", "--steps", "20", "--warmup", "3", "--mode", "landau"], "rank_log": "rank=0 host=nid001412 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001412 local_rank=1 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=937bea67c3242b0cb8313a6de50eef13ee7dcfa6b642142a129b277dd1ac56bb\nsynced_at=2026-10-24T10:06:08Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "nx": 32768, "nv": 8192, "ranks": 2, "nodes": 1, "steps": 20, "warmup": 3, "dt": 0.1, "length": 12.56637061435917, "vmax": 8, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 2147483648, "max_allocated_bytes_per_rank": 10739122192, "stream_ms": 24.5278908499813, "spline_ms": 12.87684925000008, "pack_ms": 6.339425149968747, "mpi_ms": 13.802522550003, "field_ms": 1.183015450004632, "step_ms": 58.70064285001035, "initial_mass": 12.56637061355123, "final_mass": 12.56637061358818, "mass_relative_change": 2.939963389748054e-12, "initial_kinetic_energy": 6.283185306796558, "final_kinetic_energy": 6.284270533062034, "initial_field_energy": 0.00125663706143567, "final_field_energy": 0.0001735421558640621, "minimum_f": -2.324385335496638e-16, "finite": true, "repeat": 1, "command": ["srun", "--jobid=60204067", "--overlap", "--nodes=1", "--ntasks=2", "--ntasks-per-node=2", "--gpus-per-node=4", "--gpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T095729Z-66315-25333/workdir/cuda-build/vlasov_bench", "--nx", "32768", "--nv", "8192", "--steps", "20", "--warmup", "3", "--mode", "landau"], "rank_log": "rank=1 host=nid001409 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001409 local_rank=0 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=937bea67c3242b0cb8313a6de50eef13ee7dcfa6b642142a129b277dd1ac56bb\nsynced_at=2026-10-24T10:06:08Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "nx": 32768, "nv": 8192, "ranks": 2, "nodes": 1, "steps": 20, "warmup": 3, "dt": 0.1, "length": 12.56637061435917, "vmax": 8, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 2147483648, "max_allocated_bytes_per_rank": 10739122192, "stream_ms": 24.53943520000053, "spline_ms": 12.8782511000054, "pack_ms": 6.340501999989101, "mpi_ms": 13.7808962999884, "field_ms": 1.185581100014588, "step_ms": 58.69956045000322, "initial_mass": 12.56637061355123, "final_mass": 12.56637061358818, "mass_relative_change": 2.939963389748054e-12, "initial_kinetic_energy": 6.283185306796558, "final_kinetic_energy": 6.284270533062034, "initial_field_energy": 0.00125663706143567, "final_field_energy": 0.0001735421558640621, "minimum_f": -2.324385335496638e-16, "finite": true, "repeat": 2, "command": ["srun", "--jobid=60204067", "--overlap", "--nodes=1", "--ntasks=2", "--ntasks-per-node=2", "--gpus-per-node=4", "--gpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T095729Z-66315-25333/workdir/cuda-build/vlasov_bench", "--nx", "32768", "--nv", "8192", "--steps", "20", "--warmup", "3", "--mode", "landau"], "rank_log": "rank=1 host=nid001412 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001412 local_rank=0 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=937bea67c3242b0cb8313a6de50eef13ee7dcfa6b642142a129b277dd1ac56bb\nsynced_at=2026-10-24T10:06:08Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "nx": 32768, "nv": 8192, "ranks": 4, "nodes": 1, "steps": 20, "warmup": 3, "dt": 0.1, "length": 12.56637061435917, "vmax": 8, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 2147483648, "max_allocated_bytes_per_rank": 5370380304, "stream_ms": 12.33204280001132, "spline_ms": 10.97677384998974, "pack_ms": 3.176275849972399, "mpi_ms": 9.777910849993532, "field_ms": 0.7748922500013578, "step_ms": 36.97000044999754, "initial_mass": 12.56637061435844, "final_mass": 12.56637061435783, "mass_relative_change": -4.834443115824827e-14, "initial_kinetic_energy": 6.283185307178614, "final_kinetic_energy": 6.284270533430982, "initial_field_energy": 0.001256637061435906, "final_field_energy": 0.000173542155864126, "minimum_f": -2.324385335491534e-16, "finite": true, "repeat": 0, "command": ["srun", "--jobid=60204067", "--overlap", "--nodes=1", "--ntasks=4", "--ntasks-per-node=4", "--gpus-per-node=4", "--gpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T095729Z-66315-25333/workdir/cuda-build/vlasov_bench", "--nx", "32768", "--nv", "8192", "--steps", "20", "--warmup", "3", "--mode", "landau"], "rank_log": "rank=2 host=nid001409 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001409 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001409 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001409 local_rank=1 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=937bea67c3242b0cb8313a6de50eef13ee7dcfa6b642142a129b277dd1ac56bb\nsynced_at=2026-10-24T10:06:08Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "nx": 32768, "nv": 8192, "ranks": 4, "nodes": 1, "steps": 20, "warmup": 3, "dt": 0.1, "length": 12.56637061435917, "vmax": 8, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 2147483648, "max_allocated_bytes_per_rank": 5370380304, "stream_ms": 12.3410621499886, "spline_ms": 10.97290559999919, "pack_ms": 3.171595100008062, "mpi_ms": 9.744720099979531, "field_ms": 0.7689962499910052, "step_ms": 36.94513030001758, "initial_mass": 12.56637061435844, "final_mass": 12.56637061435783, "mass_relative_change": -4.834443115824827e-14, "initial_kinetic_energy": 6.283185307178614, "final_kinetic_energy": 6.284270533430982, "initial_field_energy": 0.001256637061435906, "final_field_energy": 0.000173542155864126, "minimum_f": -2.324385335491534e-16, "finite": true, "repeat": 1, "command": ["srun", "--jobid=60204067", "--overlap", "--nodes=1", "--ntasks=4", "--ntasks-per-node=4", "--gpus-per-node=4", "--gpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T095729Z-66315-25333/workdir/cuda-build/vlasov_bench", "--nx", "32768", "--nv", "8192", "--steps", "20", "--warmup", "3", "--mode", "landau"], "rank_log": "rank=3 host=nid001412 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001412 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001412 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001412 local_rank=2 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=937bea67c3242b0cb8313a6de50eef13ee7dcfa6b642142a129b277dd1ac56bb\nsynced_at=2026-10-24T10:06:08Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "nx": 32768, "nv": 8192, "ranks": 4, "nodes": 1, "steps": 20, "warmup": 3, "dt": 0.1, "length": 12.56637061435917, "vmax": 8, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 2147483648, "max_allocated_bytes_per_rank": 5370380304, "stream_ms": 12.33317455000815, "spline_ms": 10.97503700001994, "pack_ms": 3.170171850013048, "mpi_ms": 9.749371850006128, "field_ms": 0.7777887999907307, "step_ms": 36.94672150001566, "initial_mass": 12.56637061435844, "final_mass": 12.56637061435783, "mass_relative_change": -4.834443115824827e-14, "initial_kinetic_energy": 6.283185307178614, "final_kinetic_energy": 6.284270533430982, "initial_field_energy": 0.001256637061435906, "final_field_energy": 0.000173542155864126, "minimum_f": -2.324385335491534e-16, "finite": true, "repeat": 2, "command": ["srun", "--jobid=60204067", "--overlap", "--nodes=1", "--ntasks=4", "--ntasks-per-node=4", "--gpus-per-node=4", "--gpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T095729Z-66315-25333/workdir/cuda-build/vlasov_bench", "--nx", "32768", "--nv", "8192", "--steps", "20", "--warmup", "3", "--mode", "landau"], "rank_log": "rank=3 host=nid001409 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001409 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001409 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001409 local_rank=2 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=937bea67c3242b0cb8313a6de50eef13ee7dcfa6b642142a129b277dd1ac56bb\nsynced_at=2026-10-24T10:06:08Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "nx": 32768, "nv": 8192, "ranks": 2, "nodes": 2, "steps": 20, "warmup": 3, "dt": 0.1, "length": 12.56637061435917, "vmax": 8, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 2147483648, "max_allocated_bytes_per_rank": 10739122192, "stream_ms": 24.53902204998144, "spline_ms": 12.87798075001092, "pack_ms": 6.338750850022734, "mpi_ms": 76.12148224999373, "field_ms": 1.194125749987052, "step_ms": 121.0326773000133, "initial_mass": 12.56637061355123, "final_mass": 12.56637061358818, "mass_relative_change": 2.939963389748054e-12, "initial_kinetic_energy": 6.283185306796558, "final_kinetic_energy": 6.284270533062034, "initial_field_energy": 0.00125663706143567, "final_field_energy": 0.0001735421558640621, "minimum_f": -2.324385335496638e-16, "finite": true, "repeat": 0, "command": ["srun", "--jobid=60204067", "--overlap", "--nodes=2", "--ntasks=2", "--ntasks-per-node=1", "--gpus-per-node=4", "--gpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T095729Z-66315-25333/workdir/cuda-build/vlasov_bench", "--nx", "32768", "--nv", "8192", "--steps", "20", "--warmup", "3", "--mode", "landau"], "rank_log": "rank=1 host=nid001412 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001409 local_rank=0 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=937bea67c3242b0cb8313a6de50eef13ee7dcfa6b642142a129b277dd1ac56bb\nsynced_at=2026-10-24T10:06:08Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "nx": 32768, "nv": 8192, "ranks": 2, "nodes": 2, "steps": 20, "warmup": 3, "dt": 0.1, "length": 12.56637061435917, "vmax": 8, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 2147483648, "max_allocated_bytes_per_rank": 10739122192, "stream_ms": 24.53835345001494, "spline_ms": 12.87596610001174, "pack_ms": 6.339330200000859, "mpi_ms": 75.99124849999725, "field_ms": 1.182632650005644, "step_ms": 120.8955622999952, "initial_mass": 12.56637061355123, "final_mass": 12.56637061358818, "mass_relative_change": 2.939963389748054e-12, "initial_kinetic_energy": 6.283185306796558, "final_kinetic_energy": 6.284270533062034, "initial_field_energy": 0.00125663706143567, "final_field_energy": 0.0001735421558640621, "minimum_f": -2.324385335496638e-16, "finite": true, "repeat": 1, "command": ["srun", "--jobid=60204067", "--overlap", "--nodes=2", "--ntasks=2", "--ntasks-per-node=1", "--gpus-per-node=4", "--gpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T095729Z-66315-25333/workdir/cuda-build/vlasov_bench", "--nx", "32768", "--nv", "8192", "--steps", "20", "--warmup", "3", "--mode", "landau"], "rank_log": "rank=0 host=nid001409 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001412 local_rank=0 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=937bea67c3242b0cb8313a6de50eef13ee7dcfa6b642142a129b277dd1ac56bb\nsynced_at=2026-10-24T10:06:08Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "nx": 32768, "nv": 8192, "ranks": 2, "nodes": 2, "steps": 20, "warmup": 3, "dt": 0.1, "length": 12.56637061435917, "vmax": 8, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 2147483648, "max_allocated_bytes_per_rank": 10739122192, "stream_ms": 24.54099340000084, "spline_ms": 12.87833215000092, "pack_ms": 6.337992449959984, "mpi_ms": 76.09802300000297, "field_ms": 1.184742850011844, "step_ms": 121.0015570499991, "initial_mass": 12.56637061355123, "final_mass": 12.56637061358818, "mass_relative_change": 2.939963389748054e-12, "initial_kinetic_energy": 6.283185306796558, "final_kinetic_energy": 6.284270533062034, "initial_field_energy": 0.00125663706143567, "final_field_energy": 0.0001735421558640621, "minimum_f": -2.324385335496638e-16, "finite": true, "repeat": 2, "command": ["srun", "--jobid=60204067", "--overlap", "--nodes=2", "--ntasks=2", "--ntasks-per-node=1", "--gpus-per-node=4", "--gpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T095729Z-66315-25333/workdir/cuda-build/vlasov_bench", "--nx", "32768", "--nv", "8192", "--steps", "20", "--warmup", "3", "--mode", "landau"], "rank_log": "rank=0 host=nid001409 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001412 local_rank=0 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=937bea67c3242b0cb8313a6de50eef13ee7dcfa6b642142a129b277dd1ac56bb\nsynced_at=2026-10-24T10:06:08Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "nx": 32768, "nv": 8192, "ranks": 4, "nodes": 2, "steps": 20, "warmup": 3, "dt": 0.1, "length": 12.56637061435917, "vmax": 8, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 2147483648, "max_allocated_bytes_per_rank": 5370380304, "stream_ms": 12.31625059999146, "spline_ms": 10.97326685001008, "pack_ms": 3.172203949969798, "mpi_ms": 38.4375890499939, "field_ms": 0.9794631499858042, "step_ms": 65.72114285000339, "initial_mass": 12.56637061435844, "final_mass": 12.56637061435783, "mass_relative_change": -4.834443115824827e-14, "initial_kinetic_energy": 6.283185307178615, "final_kinetic_energy": 6.284270533430982, "initial_field_energy": 0.001256637061435906, "final_field_energy": 0.000173542155864126, "minimum_f": -2.324385335491534e-16, "finite": true, "repeat": 0, "command": ["srun", "--jobid=60204067", "--overlap", "--nodes=2", "--ntasks=4", "--ntasks-per-node=2", "--gpus-per-node=4", "--gpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T095729Z-66315-25333/workdir/cuda-build/vlasov_bench", "--nx", "32768", "--nv", "8192", "--steps", "20", "--warmup", "3", "--mode", "landau"], "rank_log": "rank=1 host=nid001412 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001409 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001409 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001412 local_rank=1 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=937bea67c3242b0cb8313a6de50eef13ee7dcfa6b642142a129b277dd1ac56bb\nsynced_at=2026-10-24T10:06:08Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "nx": 32768, "nv": 8192, "ranks": 4, "nodes": 2, "steps": 20, "warmup": 3, "dt": 0.1, "length": 12.56637061435917, "vmax": 8, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 2147483648, "max_allocated_bytes_per_rank": 5370380304, "stream_ms": 12.31640194998249, "spline_ms": 10.97109569999475, "pack_ms": 3.171158949965047, "mpi_ms": 38.57280864999666, "field_ms": 0.9742971999912697, "step_ms": 65.93730769999411, "initial_mass": 12.56637061435844, "final_mass": 12.56637061435783, "mass_relative_change": -4.834443115824827e-14, "initial_kinetic_energy": 6.283185307178615, "final_kinetic_energy": 6.284270533430982, "initial_field_energy": 0.001256637061435906, "final_field_energy": 0.000173542155864126, "minimum_f": -2.324385335491534e-16, "finite": true, "repeat": 1, "command": ["srun", "--jobid=60204067", "--overlap", "--nodes=2", "--ntasks=4", "--ntasks-per-node=2", "--gpus-per-node=4", "--gpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T095729Z-66315-25333/workdir/cuda-build/vlasov_bench", "--nx", "32768", "--nv", "8192", "--steps", "20", "--warmup", "3", "--mode", "landau"], "rank_log": "rank=2 host=nid001409 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001409 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001412 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001412 local_rank=1 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=937bea67c3242b0cb8313a6de50eef13ee7dcfa6b642142a129b277dd1ac56bb\nsynced_at=2026-10-24T10:06:08Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "nx": 32768, "nv": 8192, "ranks": 4, "nodes": 2, "steps": 20, "warmup": 3, "dt": 0.1, "length": 12.56637061435917, "vmax": 8, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 2147483648, "max_allocated_bytes_per_rank": 5370380304, "stream_ms": 12.31745714999875, "spline_ms": 10.97455590000322, "pack_ms": 3.173706649960195, "mpi_ms": 38.54746239999827, "field_ms": 0.9067888499942001, "step_ms": 65.88151769999513, "initial_mass": 12.56637061435844, "final_mass": 12.56637061435783, "mass_relative_change": -4.834443115824827e-14, "initial_kinetic_energy": 6.283185307178615, "final_kinetic_energy": 6.284270533430982, "initial_field_energy": 0.001256637061435906, "final_field_energy": 0.000173542155864126, "minimum_f": -2.324385335491534e-16, "finite": true, "repeat": 2, "command": ["srun", "--jobid=60204067", "--overlap", "--nodes=2", "--ntasks=4", "--ntasks-per-node=2", "--gpus-per-node=4", "--gpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T095729Z-66315-25333/workdir/cuda-build/vlasov_bench", "--nx", "32768", "--nv", "8192", "--steps", "20", "--warmup", "3", "--mode", "landau"], "rank_log": "rank=1 host=nid001412 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001412 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001409 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001409 local_rank=0 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=937bea67c3242b0cb8313a6de50eef13ee7dcfa6b642142a129b277dd1ac56bb\nsynced_at=2026-10-24T10:06:08Z\n"} diff --git a/benchmarks/vlasov_cuda/results/2026-10-24/summary.json b/benchmarks/vlasov_cuda/results/2026-10-24/summary.json new file mode 100644 index 00000000..8075005f --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24/summary.json @@ -0,0 +1,101 @@ +{ + "date": "2026-10-24", + "system": "NERSC Perlmutter", + "device": "NVIDIA A100-SXM4-40GB", + "software": { + "cuda": "13.2.78", + "gcc": "14.3.0", + "cray_mpich": "9.1.0", + "cmake": "3.30.2" + }, + "allocation_job_id": "60204067", + "nodes": [ + "nid001409", + "nid001412" + ], + "problem": { + "nx": 32768, + "nv": 8192, + "precision": "float64", + "dt": 0.1, + "warmup": 3, + "steps": 20, + "repeats": 3 + }, + "timing": "Median of three runs, each reporting the maximum rank mean over 20 steps; explicit device synchronization; no communication/computation overlap.", + "summary": [ + { + "nodes": 1, + "ranks": 1, + "step_ms": 68.90644464999696, + "stream_ms": 49.01655189999872, + "spline_ms": 17.81408299999896, + "pack_ms": 0, + "mpi_ms": 0, + "field_ms": 2.06822614998714, + "max_allocated_bytes_per_rank": 21476605968, + "speedup": 1.0 + }, + { + "nodes": 1, + "ranks": 2, + "step_ms": 58.70064285001035, + "stream_ms": 24.53943520000053, + "spline_ms": 12.87684925000008, + "pack_ms": 6.339934000004632, + "mpi_ms": 13.78156439998861, + "field_ms": 1.185581100014588, + "max_allocated_bytes_per_rank": 10739122192, + "speedup": 1.1738618404242027 + }, + { + "nodes": 1, + "ranks": 4, + "step_ms": 36.94672150001566, + "stream_ms": 12.33317455000815, + "spline_ms": 10.97503700001994, + "pack_ms": 3.171595100008062, + "mpi_ms": 9.749371850006128, + "field_ms": 0.7748922500013578, + "max_allocated_bytes_per_rank": 5370380304, + "speedup": 1.8650218978148776 + }, + { + "nodes": 2, + "ranks": 2, + "step_ms": 121.0015570499991, + "stream_ms": 24.53902204998144, + "spline_ms": 12.87798075001092, + "pack_ms": 6.338750850022734, + "mpi_ms": 76.09802300000297, + "field_ms": 1.184742850011844, + "max_allocated_bytes_per_rank": 10739122192, + "speedup": 0.5694674211632178 + }, + { + "nodes": 2, + "ranks": 4, + "step_ms": 65.88151769999513, + "stream_ms": 12.31640194998249, + "spline_ms": 10.97326685001008, + "pack_ms": 3.172203949969798, + "mpi_ms": 38.54746239999827, + "field_ms": 0.9742971999912697, + "max_allocated_bytes_per_rank": 5370380304, + "speedup": 1.0459146518721145 + } + ], + "max_abs_mass_relative_change": 5.062453547606825e-12, + "max_abs_energy_relative_change": 3.391486345094563e-07, + "minimum_f": -2.32438533550678e-16, + "source_sha256": { + "CMakeLists.txt": "d5f11680925c336c63ea39e306230371f6dcb3e41ba423c8c09599c7c592664c", + "backend.cpp": "0921b3b030e6ff54cfa5a442ca660d9a51cc1ebb31d94205016c2e06786056de", + "backend.hpp": "d45dbddea158e2c8d78b3c803cd83490354211082b61e988cf07e55d67c0bc47", + "main.cpp": "5c13dff7de6e21aa536d90aa9e46ccbc68e8773d32fe84ea123c0390f80464d5", + "measure.py": "d896e2c6537f650280e3dc7795d255e6d7181241278360b58f098a3a6caef4fb", + "validate.py": "36fd3383185f1a3ddba754d587ca8d76f94c790597a86859200683330907955f" + }, + "completed_slurm_steps": "60204067.8 through .19 (single-node checks), .21 through .24 (two-node checks), .25 through .39 (measurements); COMPLETED with exit 0:0.", + "transport_setup": "MPICH_GPU_SUPPORT_ENABLED=1; --gpus-per-node=4 --gpu-bind=none; cudaSetDevice before MPI_Init." +} diff --git a/benchmarks/vlasov_cuda/results/2026-10-24/validation-cuda-two-nodes.json b/benchmarks/vlasov_cuda/results/2026-10-24/validation-cuda-two-nodes.json new file mode 100644 index 00000000..f8c758a8 --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24/validation-cuda-two-nodes.json @@ -0,0 +1,26 @@ +[ + { + "mode": "transpose", + "ranks": 4, + "max_abs_error": 2.7755575615628914e-16, + "passed": true + }, + { + "mode": "stream", + "ranks": 4, + "max_abs_error": 4.440892098500626e-16, + "passed": true + }, + { + "mode": "accelerate", + "ranks": 4, + "max_abs_error": 6.106226635438361e-16, + "passed": true + }, + { + "mode": "landau", + "ranks": 4, + "max_abs_error": 1.1102230246251565e-15, + "passed": true + } +] diff --git a/benchmarks/vlasov_cuda/results/2026-10-24/validation-cuda.json b/benchmarks/vlasov_cuda/results/2026-10-24/validation-cuda.json new file mode 100644 index 00000000..f2dde4c5 --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24/validation-cuda.json @@ -0,0 +1,74 @@ +[ + { + "mode": "transpose", + "ranks": 1, + "max_abs_error": 2.7755575615628914e-16, + "passed": true + }, + { + "mode": "transpose", + "ranks": 2, + "max_abs_error": 2.7755575615628914e-16, + "passed": true + }, + { + "mode": "transpose", + "ranks": 4, + "max_abs_error": 2.7755575615628914e-16, + "passed": true + }, + { + "mode": "stream", + "ranks": 1, + "max_abs_error": 4.440892098500626e-16, + "passed": true + }, + { + "mode": "stream", + "ranks": 2, + "max_abs_error": 4.440892098500626e-16, + "passed": true + }, + { + "mode": "stream", + "ranks": 4, + "max_abs_error": 4.440892098500626e-16, + "passed": true + }, + { + "mode": "accelerate", + "ranks": 1, + "max_abs_error": 6.106226635438361e-16, + "passed": true + }, + { + "mode": "accelerate", + "ranks": 2, + "max_abs_error": 6.106226635438361e-16, + "passed": true + }, + { + "mode": "accelerate", + "ranks": 4, + "max_abs_error": 6.106226635438361e-16, + "passed": true + }, + { + "mode": "landau", + "ranks": 1, + "max_abs_error": 1.0547118733938987e-15, + "passed": true + }, + { + "mode": "landau", + "ranks": 2, + "max_abs_error": 1.1102230246251565e-15, + "passed": true + }, + { + "mode": "landau", + "ranks": 4, + "max_abs_error": 1.1102230246251565e-15, + "passed": true + } +] diff --git a/benchmarks/vlasov_cuda/results/2026-10-24/validation-fftw.json b/benchmarks/vlasov_cuda/results/2026-10-24/validation-fftw.json new file mode 100644 index 00000000..7a9996cb --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24/validation-fftw.json @@ -0,0 +1,74 @@ +[ + { + "mode": "transpose", + "ranks": 1, + "max_abs_error": 2.220446049250313e-16, + "passed": true + }, + { + "mode": "transpose", + "ranks": 2, + "max_abs_error": 2.220446049250313e-16, + "passed": true + }, + { + "mode": "transpose", + "ranks": 4, + "max_abs_error": 2.220446049250313e-16, + "passed": true + }, + { + "mode": "stream", + "ranks": 1, + "max_abs_error": 3.3306690738754696e-16, + "passed": true + }, + { + "mode": "stream", + "ranks": 2, + "max_abs_error": 3.3306690738754696e-16, + "passed": true + }, + { + "mode": "stream", + "ranks": 4, + "max_abs_error": 3.3306690738754696e-16, + "passed": true + }, + { + "mode": "accelerate", + "ranks": 1, + "max_abs_error": 5.551115123125783e-16, + "passed": true + }, + { + "mode": "accelerate", + "ranks": 2, + "max_abs_error": 5.551115123125783e-16, + "passed": true + }, + { + "mode": "accelerate", + "ranks": 4, + "max_abs_error": 5.551115123125783e-16, + "passed": true + }, + { + "mode": "landau", + "ranks": 1, + "max_abs_error": 3.1839523243379595e-15, + "passed": true + }, + { + "mode": "landau", + "ranks": 2, + "max_abs_error": 3.1713215548549073e-15, + "passed": true + }, + { + "mode": "landau", + "ranks": 4, + "max_abs_error": 3.1710355546772805e-15, + "passed": true + } +] diff --git a/benchmarks/vlasov_cuda/results/2026-10-26-ions/.gitignore b/benchmarks/vlasov_cuda/results/2026-10-26-ions/.gitignore new file mode 100644 index 00000000..ee3acfa2 --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-26-ions/.gitignore @@ -0,0 +1,4 @@ +# Raw validation evidence remains local and in the isolated NERSC session. +* +!.gitignore +!README.md diff --git a/benchmarks/vlasov_cuda/results/2026-10-26-ions/README.md b/benchmarks/vlasov_cuda/results/2026-10-26-ions/README.md new file mode 100644 index 00000000..53dc7de4 --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-26-ions/README.md @@ -0,0 +1,85 @@ +# Native kinetic-species integration validation + +All five CUDA integration cases passed on allocation 60336491, nodes +nid003048/nid003744 (A100-SXM4-40GB, driver 580.178.04). The allocation was +released after preserving the outputs; all validation steps completed with +exit code 0. This is correctness evidence, not a production-scale ion timing. +Only this written report is published. Raw source archives, NetCDF outputs, +configs and MLflow metadata remain in the matching local result directory +and the isolated NERSC session recorded below. + +| Case | GPUs | Steps | Maximum absolute saved-value error vs ADEPT/JAX | +|---|---:|---:|---:| +| Kinetic electrons + protons | 1 | 100 | 1.12266e-12 | +| Kinetic electrons + protons | 4 | 100 | 9.30811e-13 | +| Kinetic electrons + protons | 8 / 2 nodes | 100 | 9.66338e-13 | +| Electron two-stream regression | 4 | 5 | 1.49214e-13 | +| Current turbulence custom module | 8 / 2 nodes | 5 | 1.01110e-11 | + +Each saved leaf and time was compared with rtol 1e-10, atol 1e-11, including +full distributions, fields, species moments and scalar diagnostics. The +proton case has nx=32, electron nv=64/vmax=8, ion nv=32, +ion vmax=6*sqrt(0.1/1836), qi=1, mi=1836, Ti=0.1 and dt=0.02. Only the +ions have the initial sine density perturbation. Both species change: max +|f(t=2)-f(t=0)| is approximately 0.02271 for electrons and 0.22539 for ions. +Constant-rate analytic-beta Chang-Cooper Dougherty collisions (nu=0.001) +act on electrons, matching the existing ADEPT reference-species convention. +No inter-species collision operator is claimed. + +The turbulence case uses VPTwoStream1D from vp-turbulence main commit +94df45d0172f861ae42bae5297bacc957294ba32, with no project edits. Its original +f0, velocity-spectrum and cascade callbacks and postprocessor ran through +ergoExo. `vp-artifacts/binary/` contains the actual postprocessed NetCDF +outputs; `vp-artifacts/native/` contains the request, statistics and library +hash logged by the run. `vp-source.tar.gz` preserves its source snapshot. + +ADEPT source: ddc4dd14082a60f9fe8a8f9dfdb80044f5ab91f0 plus the recorded +`run_integration.sh` launcher, archived in source.tar.gz. The compiled C++ +physics is unchanged in the final PR except for trailing blank-line cleanup. +Subsequent Python changes add float64/null-default guards and ensure every +registered Vlasov module honors backend selection. Runtime: JAX/JAXlib +0.10.2, interpax 0.3.14+10.g84aaf30, diffrax 0.7.2, cloudpickle 3.1.2, +CUDA 13.2.78, GNU 14 / Cray MPI 9.1. Existing environments were unchanged. + +Build/launch commands: `../../run_integration.sh` and +`../../integration_check.py`. The turbulence check additionally passes +`--module vp --ranks 8 --vp-source `. Machine paths in hashes, +requests and accounting refer to the isolated session at +/pscratch/sd/a/archis/adept-sessions/20261026T213621Z-5332-10833/. + +Local validation: 8 native/entropy tests, including 1/2-rank MPI and three +kinetic species with independent grids and charges/masses, plus 211 existing +lifecycle/core/builder/configuration regressions. The wheel contains the +CMake/C++ sources needed to build this backend from downstream projects. + +## Review corrections and masked-GPU regression + +ABI3 uses the CUDA ordinal selected at allocation time for both JAX and +DLPack, separately from the MPI local rank. Native synchronization restores +that allocation's device. Collision guards and the constant frequency use +the resolved ADEPT envelopes, so an omitted bump height defaults to one and +is rejected as a nonconstant profile; an omitted constant baseline retains +ADEPT's zero default. Eleven focused CPU tests pass, including both omitted +time/space pulse defaults, constant defaults and MPI/NetCDF parity. + +Supplemental ABI3 checks on shared allocation 60339880, nid200301: + +| Case | GPUs | Steps | Maximum absolute saved-value error | +|---|---:|---:|---:| +| Kinetic electrons + protons | 1 | 100 | 1.12266e-12 | +| Same, one GPU visible per worker | 2 | 100 | 8.31335e-13 | +| Current turbulence callbacks, one GPU visible per worker | 2 | 5 | 5.74119e-12 | + +Both MPI local ranks 0/1 use CUDA ordinal zero and report +CUDA_VISIBLE_DEVICES=0. A separate CUDA probe confirmed distinct physical PCI +devices 0000:41:00.0 and 0000:C1:00.0. Comparison tolerances remain unchanged. +The earlier 1/4/8-GPU tests above use the normal all-visible production path. + +Two launcher/transport failures were retained in the local evidence: shared +Slurm steps with gpu-bind=none exposed the same GPU to both workers, causing +NCCL to reject duplicate devices; explicit single-GPU binding exposed distinct +GPUs, but NCCL SHM imports failed with an invalid device ordinal. The masked +regression passed with host-staged MPI and socket NCCL, disabling P2P/SHM and +GPU-direct registration. These transport settings are for the regression, +not a new throughput benchmark. Reproduction: ../../run_visibility_check.sh. +The shared lease was released after preserving reports and source. diff --git a/benchmarks/vlasov_cuda/run_adept_comparison.sh b/benchmarks/vlasov_cuda/run_adept_comparison.sh new file mode 100644 index 00000000..2408c3e0 --- /dev/null +++ b/benchmarks/vlasov_cuda/run_adept_comparison.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# Run only inside an owned two-node allocation, after building both CUDA targets. +set -euo pipefail +if [ "${SLURM_PROCID:-0}" -ne 0 ]; then exit 0; fi +: "${EC_SESSION_ROOT:?}" "${SLURM_JOB_ID:?}" +source_root="$EC_SESSION_ROOT/src" +report_root="$EC_SESSION_OUTPUTS/${ADEPT_COMPARISON_OUTPUT:-adept-comparison}" +mkdir "$report_root" +prepared_root=/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs +vp_small=/pscratch/sd/a/archis/vp-turbulence-sessions/20261023T081121Z-nv4096-nx134400-5ce4fe14/src +vp_large=/pscratch/sd/a/archis/vp-turbulence-sessions/20261023T081121Z-nv4096-nx268800-93a418d7/src +export MPICH_GPU_SUPPORT_ENABLED=1 XLA_PYTHON_CLIENT_PREALLOCATE=false JAX_ENABLE_X64=true +export LD_LIBRARY_PATH="/global/common/software/nersc9/nccl/2.29.2-cu13/plugin/lib:/opt/nvidia/hpc_sdk/Linux_x86_64/26.5/cuda/13.2/lib64${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" +export NCCL_SOCKET_IFNAME=hsn NCCL_CROSS_NIC=2 NCCL_NET="AWS Libfabric" NCCL_NET_GDR_LEVEL=LOC FI_CXI_DISABLE_HOST_REGISTER=1 +export XLA_FLAGS="${XLA_FLAGS:+$XLA_FLAGS }--xla_gpu_autotune_level=0" +mapfile -t allocated_nodes < <(scontrol show hostnames "$SLURM_NODELIST") +for case in ${ADEPT_COMPARISON_CASES:-134400:1:4 134400:2:8 268800:2:8}; do + IFS=: read -r nx nodes ranks <<< "$case" + node_list=$(IFS=,; echo "${allocated_nodes[*]:0:nodes}") + prepared="$prepared_root/native-nx$nx" + vp_source="$vp_small" + if [ "$nx" = 268800 ]; then + prepared="$prepared_root/native-nx268800-distributed" + vp_source="$vp_large" + fi + # The pinned environment's own ADEPT must win over this development checkout. + # JAX CUDA wheels supply their runtime; unloading cudatoolkit matches the reference runs. + ( + module unload cudatoolkit 2>/dev/null || true + source /global/common/software/m4490/archis/venvs/vp-turbulence/bin/activate + export PYTHONPATH="$vp_source" + srun --jobid="$SLURM_JOB_ID" --overlap --nodes="$nodes" --nodelist="$node_list" --ntasks="$ranks" \ + --ntasks-per-node=$((ranks / nodes)) --gpus-per-node=4 --gpu-bind=none \ + --cpu-bind=none --mpi=none --kill-on-bad-exit=1 \ + python -u "$source_root/benchmarks/vlasov_cuda/adept_timing.py" \ + --prepared "$prepared" --config "$source_root/benchmarks/vlasov_cuda/reference/latest-nv4096/nx$nx.yaml" \ + --vp-source "$vp_source" --output "$report_root/jax-nx$nx-g$ranks.json" \ + > "$report_root/jax-nx$nx-g$ranks.log" 2>&1 + ) + python -u benchmarks/vlasov_cuda/measure.py "$EC_SESSION_WORKDIR/cuda-build/vlasov_bench" \ + --launcher "srun --jobid=$SLURM_JOB_ID --overlap --nodes={nodes} --nodelist=$node_list --ntasks={ranks} --ntasks-per-node={ranks_per_node} --gpus-per-node=4 --gpu-bind=none --cpu-bind=none" \ + --placements "$nodes:$ranks" --nx "$nx" --nv 4096 --steps 10 --warmup 3 --repeats 3 \ + --expect-cuda --velocity cubic --layout halos --integrator sixth --cells --nu 1e-12 \ + --length 12600 --vmax 15 --dt .02 --input "$prepared/initial.bin" \ + --v-grid "$prepared/v.bin" --background "$prepared/background.bin" --max-mass-change -1 \ + --source-state "$EC_SESSION_ROOT/source-state" --output "$report_root/cuda-nx$nx-g$ranks.jsonl" +done +sha256sum "$EC_SESSION_WORKDIR/cuda-build/vlasov_bench" "$EC_SESSION_WORKDIR/native-build/libadept_vlasov.so" > "$report_root/binaries.sha256" +sha256sum adept/_vlasov1d/native/api.cpp adept/_vlasov1d/native/core/* benchmarks/vlasov_cuda/adept_timing.py > "$report_root/source.sha256" +cp "$EC_SESSION_ROOT/source-state" "$report_root/source-state" +cp benchmarks/vlasov_cuda/run_adept_comparison.sh "$report_root/launch.sh" diff --git a/benchmarks/vlasov_cuda/run_integration.sh b/benchmarks/vlasov_cuda/run_integration.sh new file mode 100644 index 00000000..5e90bca9 --- /dev/null +++ b/benchmarks/vlasov_cuda/run_integration.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# Run inside an owned two-node A100 session; preserves reports beside source. +set -euo pipefail +if [ "${SLURM_PROCID:-0}" -ne 0 ]; then exit 0; fi +: "${EC_SESSION_ROOT:?}" "${EC_SESSION_OUTPUTS:?}" "${SLURM_JOB_ID:?}" +report_root="$EC_SESSION_OUTPUTS/native-ions" +mkdir "$report_root" +export MPICH_GPU_SUPPORT_ENABLED=1 XLA_PYTHON_CLIENT_PREALLOCATE=false JAX_ENABLE_X64=true +export LD_LIBRARY_PATH="/global/common/software/nersc9/nccl/2.29.2-cu13/plugin/lib:/opt/nvidia/hpc_sdk/Linux_x86_64/26.5/cuda/13.2/lib64${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" +export NCCL_SOCKET_IFNAME=hsn NCCL_CROSS_NIC=2 NCCL_NET="AWS Libfabric" NCCL_NET_GDR_LEVEL=LOC FI_CXI_DISABLE_HOST_REGISTER=1 +export XLA_FLAGS="${XLA_FLAGS:+$XLA_FLAGS }--xla_gpu_autotune_level=0" +cmake -S adept/_vlasov1d/native -B "$EC_SESSION_WORKDIR/native-build" \ + -DCMAKE_CXX_COMPILER=CC -DCMAKE_BUILD_TYPE=Release > "$report_root/build.log" 2>&1 +cmake --build "$EC_SESSION_WORKDIR/native-build" -j4 >> "$report_root/build.log" 2>&1 +library="$EC_SESSION_WORKDIR/native-build/libadept_vlasov.so" +sha256sum "$library" > "$report_root/library.sha256" +sha256sum adept/_vlasov1d/native/api.cpp adept/_vlasov1d/native/core/* > "$report_root/source.sha256" +nvidia-smi --query-gpu=name,driver_version,memory.total --format=csv > "$report_root/gpus.csv" +scontrol show job "$SLURM_JOB_ID" > "$report_root/allocation.txt" +python -u benchmarks/vlasov_cuda/integration_check.py --library "$library" \ + --output "$report_root/coupled" --ranks 1 4 8 --module base --ions --steps 100 \ + > "$report_root/coupled.log" 2>&1 +python -u benchmarks/vlasov_cuda/integration_check.py --library "$library" \ + --output "$report_root/electron" --ranks 4 --module base \ + > "$report_root/electron.log" 2>&1 +echo "Native electron/ion A100 integration checks passed: $report_root" diff --git a/benchmarks/vlasov_cuda/run_visibility_check.sh b/benchmarks/vlasov_cuda/run_visibility_check.sh new file mode 100644 index 00000000..b125997f --- /dev/null +++ b/benchmarks/vlasov_cuda/run_visibility_check.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# ABI3 regression on an owned two-GPU shared allocation. +set -euo pipefail +if [ "${SLURM_PROCID:-0}" -ne 0 ]; then exit 0; fi +: "${EC_SESSION_WORKDIR:?}" "${EC_SESSION_OUTPUTS:?}" "${SLURM_JOB_ID:?}" +report_root="$EC_SESSION_OUTPUTS/native-visibility" +mkdir "$report_root" +export MPICH_GPU_SUPPORT_ENABLED=0 XLA_PYTHON_CLIENT_PREALLOCATE=false JAX_ENABLE_X64=true +export NCCL_P2P_DISABLE=1 NCCL_SHM_DISABLE=1 +export LD_LIBRARY_PATH="/global/common/software/nersc9/nccl/2.29.2-cu13/plugin/lib:/opt/nvidia/hpc_sdk/Linux_x86_64/26.5/cuda/13.2/lib64${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" +export NCCL_SOCKET_IFNAME=hsn NCCL_CROSS_NIC=2 NCCL_NET=Socket NCCL_NET_GDR_LEVEL=0 FI_CXI_DISABLE_HOST_REGISTER=1 +export XLA_FLAGS="${XLA_FLAGS:+$XLA_FLAGS }--xla_gpu_autotune_level=0" +cmake -S adept/_vlasov1d/native -B "$EC_SESSION_WORKDIR/native-build" \ + -DCMAKE_CXX_COMPILER=CC -DCMAKE_BUILD_TYPE=Release > "$report_root/build.log" 2>&1 +cmake --build "$EC_SESSION_WORKDIR/native-build" -j4 >> "$report_root/build.log" 2>&1 +library="$EC_SESSION_WORKDIR/native-build/libadept_vlasov.so" +sha256sum "$library" > "$report_root/library.sha256" +sha256sum adept/_vlasov1d/native/api.cpp adept/_vlasov1d/native/core/* > "$report_root/source.sha256" +python -u benchmarks/vlasov_cuda/integration_check.py --library "$library" \ + --output "$report_root/shared" --ranks 1 --module base --ions --steps 100 \ + --gpus-per-node 2 --host-mpi > "$report_root/shared.log" 2>&1 +python -u benchmarks/vlasov_cuda/integration_check.py --library "$library" \ + --output "$report_root/bound" --ranks 2 --module base --ions --steps 100 \ + --gpus-per-node 2 --per-rank-gpu > "$report_root/bound.log" 2>&1 +python -u benchmarks/vlasov_cuda/integration_check.py --library "$library" \ + --output "$report_root/vp-main" --ranks 2 --module vp \ + --vp-source "$EC_SESSION_OUTPUTS/vp-source" --gpus-per-node 2 --per-rank-gpu \ + > "$report_root/vp-main.log" 2>&1 +echo "Single-worker and per-worker GPU visibility checks passed: $report_root" diff --git a/benchmarks/vlasov_cuda/validate.py b/benchmarks/vlasov_cuda/validate.py new file mode 100644 index 00000000..b72d3cff --- /dev/null +++ b/benchmarks/vlasov_cuda/validate.py @@ -0,0 +1,139 @@ +"""Numerical checks against NumPy FFTs and SciPy's natural CubicSpline. + +Use the CPU build locally or the CUDA build inside a GPU allocation. The launcher +is an argument template, e.g. 'mpiexec -n {ranks}' or 'srun -n {ranks} ...'. +""" + +import argparse +import json +import shlex +import subprocess +import tempfile +from pathlib import Path + +import numpy as np +from scipy.interpolate import CubicSpline + + +def initial(nx, nv): + x = np.arange(nx) * (4 * np.pi / nx) + v = np.linspace(-8, 8, nv) + m = np.exp(-(v**2) / 2) + m /= np.trapezoid(m, v) + return x, v, m[:, None] * (1 + 0.01 * np.cos(0.5 * x))[None, :] + + +def stream(f, v, dt): + k = 0.5 * np.arange(f.shape[1] // 2 + 1) + fk = np.fft.rfft(f, axis=1) * np.exp(-1j * v[:, None] * k[None, :] * dt) + return np.fft.irfft(fk, n=f.shape[1], axis=1) + + +def field(f, v): + rho = 1 - np.trapezoid(f, v, axis=0) + rk = np.fft.rfft(rho) + ek = np.zeros_like(rk) + k = 0.5 * np.arange(len(rk)) + ek[1:] = rk[1:] / (1j * k[1:]) + if f.shape[1] % 2 == 0: + ek[-1] = 0 + return np.fft.irfft(ek, n=f.shape[1]) + + +def accelerate(f, v, a, dt): + q = v[:, None] - np.broadcast_to(a, (f.shape[1],))[None, :] * dt + # Evaluate one spatial column per query column (SciPy otherwise broadcasts + # the whole output axis). Boundary values outside the velocity box are zero. + result = np.empty_like(f) + for x in range(f.shape[1]): + spline = CubicSpline(v, f[:, x], bc_type="natural", extrapolate=False) + result[:, x] = np.nan_to_num(spline(q[:, x]), nan=0.0) + return result + + +def reference(mode, nx, nv, steps, dt): + _, v, f = initial(nx, nv) + for _ in range(steps): + if mode == "stream": + f = stream(f, v, dt) + elif mode == "accelerate": + f = accelerate(f, v, 0.2, dt) + elif mode == "landau": + f = stream(f, v, dt / 2) + e = field(f, v) + f = accelerate(f, v, -e, dt) + f = stream(f, v, dt / 2) + return f + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("binary", type=Path) + parser.add_argument("--launcher", default="mpiexec -n {ranks}") + parser.add_argument("--ranks", nargs="+", type=int, default=[1, 2, 4]) + parser.add_argument("--host-mpi", action="store_true") + parser.add_argument("--output", type=Path) + args = parser.parse_args() + rows = [] + nx, nv, dt, steps = 128, 128, 0.1, 4 + with tempfile.TemporaryDirectory(prefix="vlasov-validation-") as directory: + for mode in ["transpose", "stream", "accelerate", "landau"]: + expected = reference(mode, nx, nv, steps, dt) + first = None + for ranks in args.ranks: + # MPI ranks on different nodes need a shared destination. + # Only rank 0 writes; MPI_Gather collects all velocity slabs. + dump = Path(directory) / f"{mode}-{ranks}.bin" + command = [ + *shlex.split(args.launcher.format(ranks=ranks)), + str(args.binary.resolve()), + "--nx", + str(nx), + "--nv", + str(nv), + "--steps", + str(steps), + "--dt", + str(dt), + "--warmup", + "1", + "--mode", + mode, + "--dump", + str(dump), + ] + if args.host_mpi: + command.append("--host-mpi") + completed = subprocess.run(command, capture_output=True, text=True, timeout=180) + if completed.returncode: + raise RuntimeError(f"{' '.join(command)}\n{completed.stdout}\n{completed.stderr}") + report = json.loads(completed.stdout.strip().splitlines()[-1]) + actual = np.fromfile(dump, dtype=np.float64).reshape(nv, nx) + error = float(np.max(np.abs(actual - expected))) + np.testing.assert_allclose(actual, expected, rtol=2e-11, atol=3e-13) + if first is None: + first = actual + else: + np.testing.assert_allclose(actual, first, rtol=2e-11, atol=3e-13) + if mode in ("transpose", "stream"): + assert abs(report["mass_relative_change"]) < 1e-12, report + if mode == "stream": + x, v, f0 = initial(nx, nv) + m = f0[:, 0] / 1.01 + analytic = m[:, None] * (1 + 0.01 * np.cos(0.5 * (x[None, :] - v[:, None] * steps * dt))) + np.testing.assert_allclose(actual, analytic, rtol=2e-12, atol=3e-14) + if mode == "landau": + assert report["final_field_energy"] < report["initial_field_energy"], report + assert abs(report["mass_relative_change"]) < 1e-9, report + initial_energy = report["initial_kinetic_energy"] + report["initial_field_energy"] + final_energy = report["final_kinetic_energy"] + report["final_field_energy"] + assert abs(final_energy / initial_energy - 1) < 1e-6, report + row = {"mode": mode, "ranks": ranks, "max_abs_error": error, "passed": True} + rows.append(row) + print(json.dumps(row), flush=True) + if args.output: + args.output.write_text(json.dumps(rows, indent=2) + "\n") + + +if __name__ == "__main__": + main() diff --git a/benchmarks/vlasov_cuda/validate_local.py b/benchmarks/vlasov_cuda/validate_local.py new file mode 100644 index 00000000..5f28e947 --- /dev/null +++ b/benchmarks/vlasov_cuda/validate_local.py @@ -0,0 +1,193 @@ +"""Compare C1 cubic halos and sixth-order splitting with interpax/NumPy. + +This is an operator test. Native-checkpoint and collision comparisons are +separate requirements before claiming turbulence-run reproduction. +""" + +import argparse +import json +import os +import shlex +import subprocess +import tempfile +from pathlib import Path + +os.environ.setdefault("JAX_PLATFORMS", "cpu") +import jax + +jax.config.update("jax_enable_x64", True) +import jax.numpy as jnp +import numpy as np +from interpax import interp1d + + +def cubic(f, v, acceleration, dt): + a = np.broadcast_to(acceleration, f.shape[1]) + q = v[:, None] - a[None, :] * dt + return np.asarray( + jax.vmap(lambda query, values: interp1d(query, jnp.asarray(v), values, extrap=1e-30))( + jnp.asarray(q.T), jnp.asarray(f.T) + ) + ).T + + +def stream(f, v, dt, length): + k = 2 * np.pi * np.fft.rfftfreq(f.shape[1], d=length / f.shape[1]) + return np.fft.irfft(np.fft.rfft(f, axis=1) * np.exp(-1j * v[:, None] * k * dt), n=f.shape[1], axis=1) + + +def field(f, dv, length, background): + rho = background - np.sum(f, axis=0) * dv + k = 2 * np.pi * np.fft.rfftfreq(f.shape[1], d=length / f.shape[1]) + ek = np.zeros(len(k), dtype=complex) + ek[1:] = np.fft.rfft(rho)[1:] / (1j * k[1:]) + if f.shape[1] % 2 == 0: + ek[-1] = 0 + return np.fft.irfft(ek, n=f.shape[1]) + + +def sixth(f, v, dt, length, background): + a = [ + 0.168735950563437422448196, + 0.377851589220928303880766, + -0.093175079568731452657924, + 0.377851589220928303880766, + 0.168735950563437422448196, + ] + d1 = 0.049086460976116245491441 - 2 * 0.000069728715055305084099 * dt**2 + d2 = 0.264177609888976700200146 - 2 * 0.000625704827430047189169 * dt**2 - 4 * 2.916600457689847816445691e-6 * dt**4 + d3 = ( + 0.186735929134907054308413 + - 2 * 0.002213085124045325561636 * dt**2 + + 4 * 3.048480261700038788680723e-5 * dt**4 + - 8 * 4.985549387875068121593988e-7 * dt**6 + ) + for i, d in enumerate([d1, d2, d3, d3, d2, d1]): + e = field(f, v[1] - v[0], length, background) + f = cubic(f, v, -e, d * dt) + if i < 5: + f = stream(f, v, a[i] * dt, length) + return f + + +def collide(f, v, dt, nu): + from adept._vlasov1d.solvers.pushers.fokker_planck import Collisions + + cfg = { + "grid": { + "parallel": False, + "species_grids": {"electron": {"v": jnp.asarray(v), "dv": v[1] - v[0]}}, + "species_params": {"electron": {"mass": 1.0}}, + }, + "terms": { + "species": [{"name": "electron"}], + "fokker_planck": {"is_on": True, "type": "chang_cooper_dougherty"}, + "krook": {"is_on": False}, + }, + } + result = Collisions(cfg)(jnp.full(f.shape[1], nu), jnp.zeros(f.shape[1]), jnp.asarray(f.T), dt) + return np.asarray(result).T + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("binary", type=Path) + parser.add_argument("--launcher", default="mpiexec -n {ranks}") + parser.add_argument("--ranks", nargs="+", type=int, default=[1, 2, 4]) + parser.add_argument("--output", type=Path) + args = parser.parse_args() + nx, nv, vmax, length = 128, 128, 8.0, 4 * np.pi + dv = 2 * vmax / nv + v = np.linspace(-vmax + dv / 2, vmax - dv / 2, nv) + x = (np.arange(nx) + 0.5) * length / nx + smooth = np.exp(-(v[:, None] ** 2) / 2) / np.sqrt(2 * np.pi) * (1 + 0.03 * np.cos(0.5 * x)) + signed = np.random.default_rng(4).normal(size=(nv, nx)) + # A shift longer than a rank's velocity block tests multi-neighbour halos. + tests = [ + ("signed-positive", signed, 1.2, 0.1, 0.0), + ("signed-negative", signed, -1.2, 0.1, 0.0), + ("signed-wide", signed, 45.0, 0.1, 0.0), + ("sixth", smooth, 0.0, 0.02, 0.0), + ("collide", smooth, 0.0, 0.1, 1e-3), + ("sixth-collide", smooth, 0.0, 0.02, 1e-12), + ] + records = [] + with tempfile.TemporaryDirectory(prefix="vlasov-local-") as directory: + tmp = Path(directory) + v.tofile(tmp / "v.bin") + background = np.ones(nx) + background.tofile(tmp / "bg.bin") + for name, initial, acceleration, dt, nu in tests: + initial.tofile(tmp / "initial.bin") + if name.startswith("sixth"): + expected = sixth(initial, v, dt, length, background) + elif name == "collide": + expected = initial + else: + expected = cubic(initial, v, acceleration, dt) + if nu: + expected = collide(expected, v, dt, nu) + for layout in ["transpose", "halos"]: + for ranks in args.ranks: + output = tmp / "out.bin" + cmd = [ + *shlex.split(args.launcher.format(ranks=ranks)), + str(args.binary.resolve()), + "--nx", + str(nx), + "--nv", + str(nv), + "--vmax", + str(vmax), + "--length", + str(length), + "--dt", + str(dt), + "--steps", + "1", + "--warmup", + "0", + "--cells", + "--velocity", + "cubic", + "--layout", + layout, + "--halo-capacity", + "64", + "--integrator", + "sixth", + "--mode", + "landau" if name.startswith("sixth") else ("collide" if name == "collide" else "accelerate"), + "--nu", + str(nu), + "--acceleration", + str(acceleration), + "--input", + str(tmp / "initial.bin"), + "--v-grid", + str(tmp / "v.bin"), + "--background", + str(tmp / "bg.bin"), + "--dump", + str(output), + ] + completed = subprocess.run(cmd, capture_output=True, text=True, timeout=90) + if completed.returncode: + raise RuntimeError(f"{' '.join(cmd)}\n{completed.stdout}\n{completed.stderr}") + actual = np.fromfile(output, dtype=np.float64).reshape(nv, nx) + np.testing.assert_allclose(actual, expected, rtol=2e-11, atol=2e-13) + row = dict( + test=name, + layout=layout, + ranks=ranks, + max_abs_error=float(np.max(np.abs(actual - expected))), + passed=True, + ) + records.append(row) + print(json.dumps(row), flush=True) + if args.output: + args.output.write_text(json.dumps(records, indent=2) + "\n") + + +if __name__ == "__main__": + main() diff --git a/docs/source/solvers/vlasov1d/config.md b/docs/source/solvers/vlasov1d/config.md index ae27e2a4..bebe3531 100644 --- a/docs/source/solvers/vlasov1d/config.md +++ b/docs/source/solvers/vlasov1d/config.md @@ -32,6 +32,111 @@ terms: # Solver configuration ``` +## Execution backend + +`backend: jax` is the default. The optional `backend: native` runs the +electrostatic solver in C++/CUDA/MPI while retaining Python initialization, +save callbacks, xarray/NetCDF postprocessing, and `ergoExo` tracking. It has no +automatic differentiation. Custom `BaseVlasov1D` subclasses passed to +`ergoExo.setup(..., adept_module=MyModule)` are wrapped automatically. + +Build the shared library on the target system (CUDA toolkit, MPI and CMake +are required). Install the Python extra with `pip install 'adept[native]'` +or your environment manager's equivalent: + +```sh +cmake -S adept/_vlasov1d/native -B build/native -DCMAKE_BUILD_TYPE=Release +cmake --build build/native -j4 +``` + +When ADEPT is installed in another project (such as turbulence), locate the +packaged sources in that project's Python environment: + +```sh +native_source=$(python -c 'from importlib.util import find_spec; print(find_spec("adept._vlasov1d.native").submodule_search_locations[0])') +cmake -S "$native_source" -B build/native -DCMAKE_BUILD_TYPE=Release +cmake --build build/native -j4 +``` + +On Perlmutter add `-DCMAKE_CXX_COMPILER=CC`, and use the existing allocated +job's `srun` launcher. A two-node example is: + +```yaml +backend: native +native: + library: /absolute/path/build/native/libadept_vlasov.so + ranks: 8 + workdir: /absolute/shared/scratch/path + scalar_stride: 1 + launcher: [srun, --overlap, --nodes=2, '--ntasks={ranks}', + --ntasks-per-node=4, --gpus-per-node=4, --gpu-bind=none, + --cpu-bind=none] +``` + +Run one outer Python driver inside the allocation; the launcher starts its +workers. Set `MPICH_GPU_SUPPORT_ENABLED=1` for CUDA-aware Cray MPI. The work +directory and library must be visible on every worker node. Native execution +uses its own velocity-slab decomposition; `native.ranks` controls it. +CUDA device ordinals follow each worker's visible device list, independently +of its MPI local rank. The example exposes all four GPUs per node for +CUDA-aware Cray MPI. A launcher that exposes one GPU per worker uses ordinal +zero within each worker. On Cray systems requiring all GPUs visible for GPU +transport, use `host_mpi: true` and `MPICH_GPU_SUPPORT_ENABLED=0` to test that +visibility mode with host-staged communication. +On the tested shared A100 node, masked-worker NCCL shared-memory imports also +reported an invalid device ordinal. The validated host-transport regression +uses `NCCL_P2P_DISABLE=1`, `NCCL_SHM_DISABLE=1`, `NCCL_NET=Socket` and +`NCCL_NET_GDR_LEVEL=0`. Keep the all-visible CUDA-aware example above for +the production throughput path. +If an outer Slurm step already restricted `CUDA_VISIBLE_DEVICES`, prefix the +launcher with `[env, -u, CUDA_VISIBLE_DEVICES, srun, ...]` so Slurm can apply +the new worker step's GPU binding. Separate GPU cgroups may each report +ordinal zero and `CUDA_VISIBLE_DEVICES=0`; physical GPUs still differ. + +| Native option | Default | Meaning | +|---|---|---| +| `library` | unset | Shared library path; alternatively set `ADEPT_VLASOV_LIBRARY` | +| `device` | `cuda` | `cpu` selects the FFTW validation build (`-DVLASOV_CUDA=OFF`) | +| `ranks` | `1` | One MPI worker per GPU; must divide `nx` and every species' `nv` | +| `launcher` | empty | Argument list, with `{ranks}` substitution; defaults to `mpiexec -n` for multiple ranks | +| `workdir` | `BASE_TEMPDIR` or system temporary directory | Shared staging/output directory, required for multiple ranks | +| `scalar_stride` | `1` | Save scalar diagnostics every this many actual timesteps, plus both endpoints | +| `custom_save_times` | `preserve` | Preserve subclass callback times; `steps` rounds those too | +| `halo_capacity` | `64` | Maximum velocity halo width; exceeding it fails explicitly | +| `host_mpi` | `false` | Stage MPI messages through host memory for MPI without CUDA support | +| `timeout` | unset | Worker timeout in seconds | + +Standard field/distribution save times round to the nearest timestep and +duplicate times are merged. Outputs record actual times in `result.ts` and +`cfg['save'][key]['t']['ax']`; the number of frames can therefore change. +The native backend streams saved outputs into host memory-mapped files instead +of retaining the full time history in GPU memory. Sparse custom callbacks +preserve their requested times by default for existing project postprocessors; +off-step callbacks require a previous-state GPU copy and interpolation. Use +`custom_save_times: steps` when the postprocessor reads actual result times. +Custom postprocessors that rebuild a uniform scalar axis must also use actual +times when the final scalar interval is shorter than `scalar_stride`. + +Kinetic electrons and ions use the existing `terms.species` and `density` +configuration below. Each species has its own symmetric cell-centred velocity +grid, charge and mass. All species contribute charge to the shared Poisson +solve at every splitting stage and advance with their own charge-to-mass ratio. +With no explicit species list, the usual electron species and static background +are retained. No additional ion parameters or preprocessing are required. + +Supported numerics are `time: sixth`, `edfdv: cubic-spline`, spectral spatial +streaming and `field: poisson`. The cubic operator reproduces ADEPT's local C1 +Hermite interpolation. Optional collisions are constant-rate +`chang_cooper_dougherty` with analytic beta, applied to the reference species +(electron if present, otherwise the first species), matching the JAX path. +Inter-species collisional exchange is not implemented. Drivers, Krook relaxation, +Hou–Li filtering, evolved df/dt diagnostics and trainable modules are rejected. + +This backend uses the `ergoExo`/`BaseVlasov1D` lifecycle. The explicit +`Vlasov1DBuilder` prepares JAX programs and rejects `backend: native`; it does +not silently fall back to JAX. Rebuild the shared library after updating ADEPT; +incompatible native library ABIs fail with an actionable error. + ## units Physical unit normalizations for the simulation. diff --git a/pyproject.toml b/pyproject.toml index 008555bc..763657bf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,6 +40,7 @@ Documentation = "https://ergodicio.github.io/adept/" [project.optional-dependencies] cpu = [] +native = ["cloudpickle"] gpu = [ "jax[cuda12]", "nvidia-ml-py" diff --git a/tests/test_vlasov1d/test_native.py b/tests/test_vlasov1d/test_native.py new file mode 100644 index 00000000..50a80d49 --- /dev/null +++ b/tests/test_vlasov1d/test_native.py @@ -0,0 +1,262 @@ +"""Native solves retain ADEPT's callbacks and Python outputs at actual save times.""" + +import os +import shutil +import subprocess +from copy import deepcopy +from pathlib import Path + +import jax +import jax.numpy as jnp +import numpy as np +import pytest +import xarray as xr +import yaml +from diffrax import SubSaveAt + +from adept import ergoExo +from adept._vlasov1d.storage import store_f, store_fields +from adept.vlasov1d import BaseVlasov1D, NativeVlasov1D, native_backend + + +@pytest.fixture(scope="module") +def native_library(tmp_path_factory): + configured = os.environ.get("ADEPT_NATIVE_TEST_LIBRARY") + if configured: + return Path(configured) + if not all(shutil.which(name) for name in ["cmake", "mpicxx", "pkg-config"]): + pytest.skip("native integration tests require CMake, MPI and FFTW or ADEPT_NATIVE_TEST_LIBRARY") + if subprocess.run(["pkg-config", "--exists", "fftw3"]).returncode: + pytest.skip("FFTW validation backend is not installed") + build = tmp_path_factory.mktemp("native-build") + source = Path(__file__).resolve().parents[2] / "adept/_vlasov1d/native" + subprocess.run( + ["cmake", "-S", str(source), "-B", str(build), "-DVLASOV_CUDA=OFF", "-DCMAKE_BUILD_TYPE=Release"], + check=True, + capture_output=True, + ) + subprocess.run(["cmake", "--build", str(build), "-j2"], check=True, capture_output=True) + return next(path for path in [build / "libadept_vlasov.so", build / "libadept_vlasov.dylib"] if path.exists()) + + +@pytest.fixture +def config(tmp_path): + cfg = yaml.safe_load(Path("tests/test_base/configs/example.yaml").read_text()) + cfg["grid"].update(nx=32, nv=64, dt=0.02, tmax=0.08, vmax=8.0) + cfg["drivers"] = {"ex": {}, "ey": {}} + cfg["terms"]["time"] = "sixth" + cfg["terms"]["fokker_planck"].update(is_on=True, type="chang_cooper_dougherty") + cfg["terms"]["fokker_planck"]["time"]["baseline"] = 0.001 + cfg["terms"]["fokker_planck"]["space"]["baseline"] = 1.0 + cfg["terms"]["krook"]["is_on"] = False + cfg["save"] = { + "fields": {"t": {"nt": 5}}, + "electron": { + "main": {"t": {"nt": 3}}, + "sampled": { + "t": {"nt": 4}, + "x": {"xmin": 0.0, "xmax": 20.94, "nx": 8}, + "v": {"vmin": -7.0, "vmax": 7.0, "nv": 16}, + }, + }, + } + beam = cfg["density"]["species-background"] + beam.update(v0=3.0, T0=0.2, noise_val=0.001) + cfg["density"]["species-left"] = deepcopy(beam) + cfg["density"]["species-left"].update(v0=-3.0, noise_seed=43) + cfg["native"] = { + "device": os.environ.get("ADEPT_NATIVE_TEST_DEVICE", "cpu"), + "workdir": str(tmp_path), + "timeout": 120, + } + return cfg + + +def setup(cls, config): + module = cls(deepcopy(config)) + module.write_units() + module.get_derived_quantities() + module.get_solver_quantities() + module.init_state_and_args() + module.init_diffeqsolve() + return module + + +def compare_results(original, native): + assert set(original.ts) == set(native.ts) + for key in original.ts: + np.testing.assert_allclose(original.ts[key], native.ts[key], atol=1e-14) + assert jax.tree.structure(original.ys[key]) == jax.tree.structure(native.ys[key]) + for a, b in zip(jax.tree.leaves(original.ys[key]), jax.tree.leaves(native.ys[key]), strict=True): + np.testing.assert_allclose(a, b, rtol=1e-10, atol=1e-11, equal_nan=True, err_msg=key) + + +def reference_at_native_times(original, native): + subs = original.diffeqsolve_quants["saveat"]["subs"] + for key, save in native.diffeqsolve_quants["saveat"]["subs"].items(): + subs[key] = SubSaveAt(ts=save.ts, fn=subs[key].fn) + return original({})["solver result"] + + +@pytest.mark.parametrize("ranks", [1, 2]) +def test_native_dense_saves_and_xarray(config, native_library, tmp_path, ranks): + config["native"]["library"] = str(native_library) + config["native"]["ranks"] = ranks + original, native = (setup(cls, config) for cls in [BaseVlasov1D, NativeVlasov1D]) + a, b = reference_at_native_times(original, native), native({})["solver result"] + compare_results(a, b) + for key, times in b.ts.items(): + np.testing.assert_allclose(np.asarray(times) / 0.02, np.rint(np.asarray(times) / 0.02), atol=1e-14) + assert len(b.ts["default"]) == 6 + output = tmp_path / "output" + (output / "binary").mkdir(parents=True) + fields = store_fields(native.cfg, str(output / "binary"), b.ys["fields"], b.ts["fields"], "fields") + dists = store_f(native.cfg, b.ts, str(output), b.ys) + assert isinstance(fields["electron"], xr.Dataset) + assert dists["electron.sampled"].sizes == {"t": 4, "x": 8, "v_electron": 16} + with xr.open_dataset(output / "binary/dist-electron.sampled.nc") as saved: + xr.testing.assert_allclose(saved, dists["electron.sampled"]) + + +def test_native_step_schedules(config): + config["native"].update(scalar_stride=3, custom_save_times="steps") + native = setup(native_backend(CustomTwoStream), config) + subs = native.diffeqsolve_quants["saveat"]["subs"] + np.testing.assert_allclose(subs["default"].ts, [0.0, 0.06, 0.1]) + np.testing.assert_allclose(subs["f0"].ts, [0.0, 0.04, 0.1]) + + +@pytest.mark.parametrize("axis", ["time", "space"]) +def test_native_rejects_omitted_collision_pulse_default(config, axis): + config["terms"]["fokker_planck"][axis].pop("bump_height") + with pytest.raises(NotImplementedError, match="constant frequency"): + NativeVlasov1D(config) + + +def test_native_constant_collision_defaults(config): + native = NativeVlasov1D(deepcopy(config)) + assert native.native_collision_frequency == 0.001 + config["terms"]["fokker_planck"]["time"].pop("baseline") + assert NativeVlasov1D(config).native_collision_frequency == 0.0 + + +class CustomTwoStream(BaseVlasov1D): + def init_diffeqsolve(self): + super().init_diffeqsolve() + self.diffeqsolve_quants["saveat"]["subs"]["f0"] = SubSaveAt( + ts=np.array([0.0, 0.037, self.time_quantities["t1"]]), fn=lambda t, y, args: jnp.mean(y["electron"], axis=0) + ) + k = jnp.abs(jnp.asarray(self.cfg["grid"]["kx"])) + self.diffeqsolve_quants["saveat"]["subs"]["peak"] = SubSaveAt( + ts=np.array([0.0, 0.04, self.time_quantities["t1"]]), + fn=lambda t, y, args: k[jnp.argmax(jnp.abs(jnp.fft.fft(y["e"])))], + ) + + def post_process(self, run_output, td): + sol = run_output["solver result"] + result = xr.Dataset( + {"f0": (("t", "v"), sol.ys["f0"])}, + coords={"t": sol.ts["f0"], "v": self.cfg["grid"]["species_grids"]["electron"]["v"]}, + ) + result.to_netcdf(Path(td) / "custom.nc") + return {"custom": result} + + +def test_ergoexo_custom_module_preserves_callbacks(config, native_library, tmp_path, monkeypatch): + from adept import patched_mlflow as mlflow + + mlflow.set_tracking_uri((tmp_path / "mlruns").as_uri()) + monkeypatch.setenv("MLFLOW_ENABLE_SYSTEM_METRICS_LOGGING", "false") + config["native"]["library"] = str(native_library) + original = setup(CustomTwoStream, config) + config["backend"] = "native" + config["mlflow"] = {"experiment": "native-integration-test", "run": "two-stream"} + exo = ergoExo() + modules = exo.setup(config, adept_module=CustomTwoStream) + result, post, run_id = exo(modules) + compare_results(reference_at_native_times(original, exo.adept_module), result["solver result"]) + np.testing.assert_allclose(result["solver result"].ts["f0"], [0.0, 0.037, 0.1]) + assert isinstance(exo.adept_module, CustomTwoStream) + assert not exo.adept_module.jit_compile + assert post["custom"].sizes == {"t": 3, "v": 64} + artifacts = Path(mlflow.get_run(run_id).info.artifact_uri.removeprefix("file://")) + assert (artifacts / "custom.nc").exists() + assert (artifacts / "native/request.json").exists() + assert (artifacts / "native/library.sha256").exists() + + +def test_backend_dispatch_and_guards(config): + assert type(ergoExo()._get_adept_module_(config)) is BaseVlasov1D + config["backend"] = "native" + assert isinstance(ergoExo()._get_adept_module_(config), NativeVlasov1D) + config["terms"]["fokker_planck"]["self_consistent_beta"] = {"enabled": True} + with pytest.raises(NotImplementedError, match="analytic beta"): + NativeVlasov1D(config) + with pytest.raises(TypeError, match="BaseVlasov1D"): + native_backend(object) + from adept import SimulationSpec, solver_registry + + with pytest.raises(NotImplementedError, match="ergoExo"): + solver_registry.prepare(SimulationSpec.from_legacy_config(config), key=0) + config["terms"]["fokker_planck"]["self_consistent_beta"] = None + config["solver"] = "vlasov-1d-iaw" + assert not ergoExo()._get_adept_module_(config).jit_compile + + +@pytest.mark.parametrize("ranks,collisions,mass,charge", [(1, False, 25.0, 1.0), (2, True, 1836.0, 2.0)]) +def test_native_kinetic_ions(config, native_library, tmp_path, ranks, collisions, mass, charge): + """Ion charge feeds back on electrons; all species move on independent grids.""" + config["native"].update(library=str(native_library), ranks=ranks) + config["terms"]["fokker_planck"]["is_on"] = collisions + electron = deepcopy(config["density"]["species-background"]) + electron.update(v0=0.0, T0=1.0, noise_val=0.0) + ion = deepcopy(electron) + ion.update(T0=0.1, basis="sine", baseline=0.8 / charge, amplitude=0.03, wavenumber=2 * np.pi / 20.94) + alpha = deepcopy(electron) + alpha.update(T0=0.2, **{"space-profile": {**electron["space-profile"], "baseline": 0.1}}) + config["density"] = { + "quasineutrality": True, + "species-electrons": electron, + "species-ions": ion, + "species-alpha": alpha, + } + # Put the electron last to exercise reference-species selection, rather than + # accidentally applying its collision operator to the first configured ion. + config["terms"]["species"] = [ + dict( + name="ion", + charge=charge, + mass=mass, + nv=32, + vmax=6 * np.sqrt(0.1 / mass), + density_components=["species-ions"], + ), + dict( + name="alpha", + charge=2.0, + mass=4 * mass, + nv=48, + vmax=6 * np.sqrt(0.2 / (4 * mass)), + density_components=["species-alpha"], + ), + dict(name="electron", charge=-1.0, mass=1.0, nv=64, vmax=8.0, density_components=["species-electrons"]), + ] + config["save"].update(ion={"main": {"t": {"nt": 3}}}, alpha={"main": {"t": {"nt": 3}}}) + original, native = (setup(cls, config) for cls in [BaseVlasov1D, NativeVlasov1D]) + a, output = reference_at_native_times(original, native), native({}) + b = output["solver result"] + compare_results(a, b) + for species in ["electron", "ion", "alpha"]: + values = b.ys[f"{species}.main"] + assert np.max(np.abs(values[-1] - values[0])) > 1e-9, species + assert np.max(np.abs(b.ys["fields"]["e"][-1])) > 1e-4 + assert {s["name"] for s in output["native stats"]["species"]} == {"electron", "ion", "alpha"} + directory = tmp_path / "multispecies" + (directory / "binary").mkdir(parents=True) + fields = store_fields(native.cfg, str(directory / "binary"), b.ys["fields"], b.ts["fields"], "fields") + dists = store_f(native.cfg, b.ts, str(directory), b.ys) + assert set(fields) >= {"electron", "ion", "alpha"} + for species, nv in [("ion", 32), ("alpha", 48)]: + with xr.open_dataset(directory / f"binary/dist-{species}.main.nc") as saved: + xr.testing.assert_allclose(saved, dists[f"{species}.main"]) + assert saved.sizes[f"v_{species}"] == nv diff --git a/tests/test_vlasov1d/test_observations.py b/tests/test_vlasov1d/test_observations.py new file mode 100644 index 00000000..4a778807 --- /dev/null +++ b/tests/test_vlasov1d/test_observations.py @@ -0,0 +1,17 @@ +"""Entropy diagnostics use the continuous value 0 log(0) = 0.""" + +import jax.numpy as jnp +import numpy as np + +from adept._vlasov1d.observations import FieldsObservation, ScalarsObservation + + +def test_entropy_with_empty_velocity_cells(): + grids = {"electron": {"v": jnp.array([-1.0, 0.0, 1.0]), "dv": 0.5}} + state = {"electron": jnp.tile(jnp.array([0.0, 1.0, np.exp(-1.0)]), (4, 1))} + state.update({name: jnp.zeros(6 if name in {"a", "prev_a"} else 4) for name in ["e", "de", "a", "prev_a"]}) + expected = 0.5 * np.exp(-1.0) + fields = FieldsObservation(grids, 1.0)(0.0, state, {}) + scalars = ScalarsObservation(grids, {"electron": {"mass": 1.0}}, 1.0)(0.0, state, {}) + np.testing.assert_allclose(fields["electron"]["-flogf"], expected) + np.testing.assert_allclose(scalars["mean_-flogf_electron"], expected) diff --git a/uv.lock b/uv.lock index a6cb1c3c..9c21b90a 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 3 +revision = 5 requires-python = ">=3.11" resolution-markers = [ "python_full_version >= '3.13'", @@ -70,12 +70,16 @@ gpu = [ { name = "jax", extra = ["cuda12"] }, { name = "nvidia-ml-py" }, ] +native = [ + { name = "cloudpickle" }, +] [package.metadata] requires-dist = [ { name = "adept", marker = "extra == 'docs'" }, { name = "adept", extras = ["docs"], marker = "extra == 'dev'" }, { name = "boto3" }, + { name = "cloudpickle", marker = "extra == 'native'" }, { name = "diffrax" }, { name = "equinox" }, { name = "h5netcdf" }, @@ -108,7 +112,7 @@ requires-dist = [ { name = "typeguard", marker = "extra == 'dev'" }, { name = "xarray" }, ] -provides-extras = ["cpu", "gpu", "docs", "dev"] +provides-extras = ["cpu", "native", "gpu", "docs", "dev"] [[package]] name = "alabaster" @@ -2291,20 +2295,20 @@ resolution-markers = [ "python_full_version < '3.12'", ] dependencies = [ - { name = "astropy", marker = "python_full_version < '3.12'" }, - { name = "h5py", marker = "python_full_version < '3.12'" }, - { name = "lmfit", marker = "python_full_version < '3.12'" }, - { name = "matplotlib", marker = "python_full_version < '3.12'" }, - { name = "mpmath", marker = "python_full_version < '3.12'" }, - { name = "numpy", marker = "python_full_version < '3.12'" }, - { name = "packaging", marker = "python_full_version < '3.12'" }, - { name = "pandas", marker = "python_full_version < '3.12'" }, - { name = "requests", marker = "python_full_version < '3.12'" }, - { name = "scipy", marker = "python_full_version < '3.12'" }, - { name = "setuptools", marker = "python_full_version < '3.12'" }, - { name = "tqdm", marker = "python_full_version < '3.12'" }, - { name = "wrapt", marker = "python_full_version < '3.12'" }, - { name = "xarray", marker = "python_full_version < '3.12'" }, + { name = "astropy" }, + { name = "h5py" }, + { name = "lmfit" }, + { name = "matplotlib" }, + { name = "mpmath" }, + { name = "numpy" }, + { name = "packaging" }, + { name = "pandas" }, + { name = "requests" }, + { name = "scipy" }, + { name = "setuptools" }, + { name = "tqdm" }, + { name = "wrapt" }, + { name = "xarray" }, ] sdist = { url = "https://files.pythonhosted.org/packages/2c/f8/941a2f3b1acda062c90ce9c47ad9cb8564df92f77e04db0a5663edc2af7e/plasmapy-2025.8.0.tar.gz", hash = "sha256:b26e6237000d8b69ffb98baa3f3d6043eb6c1828a3d0a1b44ff4053cf61afb90", size = 13148459, upload-time = "2025-08-07T21:16:01.725Z" } wheels = [ @@ -2320,19 +2324,19 @@ resolution-markers = [ "python_full_version == '3.12.*'", ] dependencies = [ - { name = "astropy", marker = "python_full_version >= '3.12'" }, - { name = "h5py", marker = "python_full_version >= '3.12'" }, - { name = "lmfit", marker = "python_full_version >= '3.12'" }, - { name = "matplotlib", marker = "python_full_version >= '3.12'" }, - { name = "mpmath", marker = "python_full_version >= '3.12'" }, - { name = "numpy", marker = "python_full_version >= '3.12'" }, - { name = "packaging", marker = "python_full_version >= '3.12'" }, - { name = "pandas", marker = "python_full_version >= '3.12'" }, - { name = "requests", marker = "python_full_version >= '3.12'" }, - { name = "scipy", marker = "python_full_version >= '3.12'" }, - { name = "tqdm", marker = "python_full_version >= '3.12'" }, - { name = "wrapt", marker = "python_full_version >= '3.12'" }, - { name = "xarray", marker = "python_full_version >= '3.12'" }, + { name = "astropy" }, + { name = "h5py" }, + { name = "lmfit" }, + { name = "matplotlib" }, + { name = "mpmath" }, + { name = "numpy" }, + { name = "packaging" }, + { name = "pandas" }, + { name = "requests" }, + { name = "scipy" }, + { name = "tqdm" }, + { name = "wrapt" }, + { name = "xarray" }, ] sdist = { url = "https://files.pythonhosted.org/packages/84/95/4c7d3f51e5af11286a04643be5596ef8ebcb3cac25386e8bcd5ff846d17a/plasmapy-2025.10.0.tar.gz", hash = "sha256:cfeb6f2785906e080977292e9b7f3bb6e71cdd8ef7273a877959a113d9e56295", size = 13144933, upload-time = "2025-10-25T01:59:00.912Z" } wheels = [ @@ -3057,23 +3061,23 @@ resolution-markers = [ "python_full_version < '3.12'", ] dependencies = [ - { name = "alabaster", marker = "python_full_version < '3.12'" }, - { name = "babel", marker = "python_full_version < '3.12'" }, - { name = "colorama", marker = "python_full_version < '3.12' and sys_platform == 'win32'" }, - { name = "docutils", marker = "python_full_version < '3.12'" }, - { name = "imagesize", marker = "python_full_version < '3.12'" }, - { name = "jinja2", marker = "python_full_version < '3.12'" }, - { name = "packaging", marker = "python_full_version < '3.12'" }, - { name = "pygments", marker = "python_full_version < '3.12'" }, - { name = "requests", marker = "python_full_version < '3.12'" }, - { name = "roman-numerals", marker = "python_full_version < '3.12'" }, - { name = "snowballstemmer", marker = "python_full_version < '3.12'" }, - { name = "sphinxcontrib-applehelp", marker = "python_full_version < '3.12'" }, - { name = "sphinxcontrib-devhelp", marker = "python_full_version < '3.12'" }, - { name = "sphinxcontrib-htmlhelp", marker = "python_full_version < '3.12'" }, - { name = "sphinxcontrib-jsmath", marker = "python_full_version < '3.12'" }, - { name = "sphinxcontrib-qthelp", marker = "python_full_version < '3.12'" }, - { name = "sphinxcontrib-serializinghtml", marker = "python_full_version < '3.12'" }, + { name = "alabaster" }, + { name = "babel" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "docutils" }, + { name = "imagesize" }, + { name = "jinja2" }, + { name = "packaging" }, + { name = "pygments" }, + { name = "requests" }, + { name = "roman-numerals" }, + { name = "snowballstemmer" }, + { name = "sphinxcontrib-applehelp" }, + { name = "sphinxcontrib-devhelp" }, + { name = "sphinxcontrib-htmlhelp" }, + { name = "sphinxcontrib-jsmath" }, + { name = "sphinxcontrib-qthelp" }, + { name = "sphinxcontrib-serializinghtml" }, ] sdist = { url = "https://files.pythonhosted.org/packages/42/50/a8c6ccc36d5eacdfd7913ddccd15a9cee03ecafc5ee2bc40e1f168d85022/sphinx-9.0.4.tar.gz", hash = "sha256:594ef59d042972abbc581d8baa577404abe4e6c3b04ef61bd7fc2acbd51f3fa3", size = 8710502, upload-time = "2025-12-04T07:45:27.343Z" } wheels = [ @@ -3089,23 +3093,23 @@ resolution-markers = [ "python_full_version == '3.12.*'", ] dependencies = [ - { name = "alabaster", marker = "python_full_version >= '3.12'" }, - { name = "babel", marker = "python_full_version >= '3.12'" }, - { name = "colorama", marker = "python_full_version >= '3.12' and sys_platform == 'win32'" }, - { name = "docutils", marker = "python_full_version >= '3.12'" }, - { name = "imagesize", marker = "python_full_version >= '3.12'" }, - { name = "jinja2", marker = "python_full_version >= '3.12'" }, - { name = "packaging", marker = "python_full_version >= '3.12'" }, - { name = "pygments", marker = "python_full_version >= '3.12'" }, - { name = "requests", marker = "python_full_version >= '3.12'" }, - { name = "roman-numerals", marker = "python_full_version >= '3.12'" }, - { name = "snowballstemmer", marker = "python_full_version >= '3.12'" }, - { name = "sphinxcontrib-applehelp", marker = "python_full_version >= '3.12'" }, - { name = "sphinxcontrib-devhelp", marker = "python_full_version >= '3.12'" }, - { name = "sphinxcontrib-htmlhelp", marker = "python_full_version >= '3.12'" }, - { name = "sphinxcontrib-jsmath", marker = "python_full_version >= '3.12'" }, - { name = "sphinxcontrib-qthelp", marker = "python_full_version >= '3.12'" }, - { name = "sphinxcontrib-serializinghtml", marker = "python_full_version >= '3.12'" }, + { name = "alabaster" }, + { name = "babel" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "docutils" }, + { name = "imagesize" }, + { name = "jinja2" }, + { name = "packaging" }, + { name = "pygments" }, + { name = "requests" }, + { name = "roman-numerals" }, + { name = "snowballstemmer" }, + { name = "sphinxcontrib-applehelp" }, + { name = "sphinxcontrib-devhelp" }, + { name = "sphinxcontrib-htmlhelp" }, + { name = "sphinxcontrib-jsmath" }, + { name = "sphinxcontrib-qthelp" }, + { name = "sphinxcontrib-serializinghtml" }, ] sdist = { url = "https://files.pythonhosted.org/packages/cd/bd/f08eb0f4eed5c83f1ba2a3bd18f7745a2b1525fad70660a1c00224ec468a/sphinx-9.1.0.tar.gz", hash = "sha256:7741722357dd75f8190766926071fed3bdc211c74dd2d7d4df5404da95930ddb", size = 8718324, upload-time = "2025-12-31T15:09:27.646Z" } wheels = [