From f9ff13c0b076f707c4b871a3fdb29316229993f4 Mon Sep 17 00:00:00 2001 From: archis Date: Wed, 5 Aug 2026 16:25:32 +0200 Subject: [PATCH 1/8] fix: vlasov1d2v gamma_func_for_sg.nc path (vlasov1d dir was renamed _vlasov1d) The module ships its own copy of the table; load it from the package dir. Fixes import-time crash of adept.vlasov1d2v.helpers. Co-Authored-By: Claude Fable 5 --- adept/vlasov1d2v/helpers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/adept/vlasov1d2v/helpers.py b/adept/vlasov1d2v/helpers.py index bf0e7f4f..2e8fc2f3 100644 --- a/adept/vlasov1d2v/helpers.py +++ b/adept/vlasov1d2v/helpers.py @@ -17,7 +17,8 @@ from .. import patched_mlflow as mlflow -gamma_da = xarray.open_dataarray(os.path.join(os.path.dirname(__file__), "..", "vlasov1d", "gamma_func_for_sg.nc")) +# vlasov1d was renamed _vlasov1d; this module carries its own copy of the table +gamma_da = xarray.open_dataarray(os.path.join(os.path.dirname(__file__), "gamma_func_for_sg.nc")) m_ax = gamma_da.coords["m"].data g_3_m = np.squeeze(gamma_da.loc[{"gamma": "3/m"}].data) g_5_m = np.squeeze(gamma_da.loc[{"gamma": "5/m"}].data) From 3e669b88350554dd7fddc05d3d68a527efac3f48 Mon Sep 17 00:00:00 2001 From: archis Date: Mon, 26 Oct 2026 21:21:34 +0000 Subject: [PATCH 2/8] feat(vlasov1d): add native CUDA MPI execution backend --- adept/_base_.py | 16 +- adept/_vlasov1d/datamodel.py | 21 + adept/_vlasov1d/native/CMakeLists.txt | 21 + adept/_vlasov1d/native/README.md | 43 + adept/_vlasov1d/native/__init__.py | 1 + adept/_vlasov1d/native/api.cpp | 157 +++ adept/_vlasov1d/native/bridge.py | 92 ++ adept/_vlasov1d/native/core/backend.cpp | 499 ++++++++++ adept/_vlasov1d/native/core/backend.hpp | 66 ++ adept/_vlasov1d/native/core/local_cubic.hpp | 30 + adept/_vlasov1d/native/core/solver.hpp | 326 +++++++ adept/_vlasov1d/native/module.py | 375 +++++++ adept/_vlasov1d/native/worker.py | 201 ++++ adept/vlasov1d.py | 1 + benchmarks/vlasov_cuda/CMakeLists.txt | 21 + benchmarks/vlasov_cuda/README.md | 308 ++++++ benchmarks/vlasov_cuda/adept_timing.py | 152 +++ benchmarks/vlasov_cuda/backend.cpp | 2 + benchmarks/vlasov_cuda/backend.hpp | 2 + benchmarks/vlasov_cuda/benchmark_latest.py | 93 ++ .../vlasov_cuda/checkpoint_reference.py | 178 ++++ benchmarks/vlasov_cuda/compare_checkpoint.py | 150 +++ .../vlasov_cuda/distributed_reference.py | 160 +++ benchmarks/vlasov_cuda/integration_check.py | 138 +++ benchmarks/vlasov_cuda/local_cubic.hpp | 2 + benchmarks/vlasov_cuda/main.cpp | 122 +++ benchmarks/vlasov_cuda/measure.py | 124 +++ .../reference/latest-nv4096/nx134400.yaml | 168 ++++ .../reference/latest-nv4096/nx268800.yaml | 166 ++++ .../reference/latest-nv4096/target.json | 39 + .../results/2026-10-24-adept/README.md | 48 + .../results/2026-10-24-adept/binaries.sha256 | 2 + .../2026-10-24-adept/cuda-nx134400-g4.jsonl | 3 + .../2026-10-24-adept/cuda-nx134400-g8.jsonl | 3 + .../2026-10-24-adept/cuda-nx268800-g8.jsonl | 3 + .../2026-10-24-adept/integration-checks.json | 210 ++++ .../2026-10-24-adept/jax-nx134400-g4.json | 120 +++ .../2026-10-24-adept/jax-nx134400-g8.json | 140 +++ .../2026-10-24-adept/jax-nx268800-g8.json | 142 +++ .../results/2026-10-24-adept/launch.sh | 49 + .../observer-native/library.sha256 | 1 + .../observer-native/request.json | 84 ++ .../observer-native/stats.json | 21 + .../observer-netcdf-check.json | 144 +++ .../observer-netcdf/dist-electron.f0.nc | Bin 0 -> 8687 bytes .../observer-netcdf/dist-electron.main.nc | Bin 0 -> 58136 bytes .../observer-netcdf/fields-electron-t=0.1.nc | Bin 0 -> 22528 bytes .../observer-netcdf/fields-shared-t=0.1.nc | Bin 0 -> 17408 bytes .../observer-netcdf/native-cascade.nc | Bin 0 -> 18659 bytes .../observer-netcdf/scalars-t=0.1.nc | Bin 0 -> 20290 bytes .../observer-netcdf/spectrum-electron.pkv.nc | Bin 0 -> 8843 bytes .../pinned-one-node/binaries.sha256 | 2 + .../pinned-one-node/cuda-nx134400-g4.jsonl | 3 + .../pinned-one-node/jax-nx134400-g4.json | 120 +++ .../pinned-one-node/launch.sh | 51 + .../pinned-one-node/source-state | 5 + .../pinned-one-node/source.sha256 | 6 + .../2026-10-24-adept/slurm-accounting.txt | 35 + .../results/2026-10-24-adept/source-state | 5 + .../results/2026-10-24-adept/source.sha256 | 6 + .../results/2026-10-24-adept/source.tar.gz | Bin 0 -> 58287 bytes .../results/2026-10-24-adept/summarize.py | 82 ++ .../results/2026-10-24-adept/summary.json | 306 ++++++ .../2026-10-24-adept/unmatched-node/README.md | 4 + .../unmatched-node/cuda-nx134400-g4.jsonl | 3 + .../unmatched-node/jax-nx134400-g4.json | 120 +++ .../comparison-nx134400-smoke.json | 446 +++++++++ .../comparison-nx134400-window.json | 914 +++++++++++++++++ .../comparison-nx268800-1node.json | 462 +++++++++ .../comparison-nx268800-window.json | 922 ++++++++++++++++++ .../2026-10-24-halos/compiled-source.sha256 | 5 + .../2026-10-24-halos/executable.sha256 | 1 + .../2026-10-24-halos/nx134400-halos.jsonl | 6 + .../2026-10-24-halos/nx134400-transpose.jsonl | 6 + .../2026-10-24-halos/nx268800-halos.jsonl | 6 + .../2026-10-24-halos/nx268800-transpose.jsonl | 6 + .../reference-library-versions.json | 9 + .../2026-10-24-halos/reference-nx134400.json | 35 + .../2026-10-24-halos/reference-nx268800.json | 37 + .../2026-10-24-halos/reference-source-state | 5 + .../2026-10-24-halos/reference8-launch.sh | 12 + .../2026-10-24-halos/slurm-accounting.txt | 31 + .../2026-10-24-halos/source-archive.sha256 | 1 + .../results/2026-10-24-halos/source-state | 5 + .../results/2026-10-24-halos/source.tar.gz | Bin 0 -> 33466 bytes .../results/2026-10-24-halos/summarize.py | 129 +++ .../results/2026-10-24-halos/summary.json | 302 ++++++ .../validation-local-cpu.json | 254 +++++ .../validation-local-cuda.json | 254 +++++ .../validation-natural-cpu.json | 74 ++ .../results/2026-10-24/baseline-source.tar.gz | Bin 0 -> 15627 bytes .../2026-10-24/scaling-32768x8192.jsonl | 15 + .../results/2026-10-24/summary.json | 101 ++ .../2026-10-24/validation-cuda-two-nodes.json | 26 + .../results/2026-10-24/validation-cuda.json | 74 ++ .../results/2026-10-24/validation-fftw.json | 74 ++ .../vlasov_cuda/run_adept_comparison.sh | 51 + benchmarks/vlasov_cuda/validate.py | 139 +++ benchmarks/vlasov_cuda/validate_local.py | 193 ++++ docs/source/solvers/vlasov1d/config.md | 67 ++ pyproject.toml | 1 + tests/test_vlasov1d/test_native.py | 182 ++++ uv.lock | 134 +-- 103 files changed, 10305 insertions(+), 66 deletions(-) create mode 100644 adept/_vlasov1d/native/CMakeLists.txt create mode 100644 adept/_vlasov1d/native/README.md create mode 100644 adept/_vlasov1d/native/__init__.py create mode 100644 adept/_vlasov1d/native/api.cpp create mode 100644 adept/_vlasov1d/native/bridge.py create mode 100644 adept/_vlasov1d/native/core/backend.cpp create mode 100644 adept/_vlasov1d/native/core/backend.hpp create mode 100644 adept/_vlasov1d/native/core/local_cubic.hpp create mode 100644 adept/_vlasov1d/native/core/solver.hpp create mode 100644 adept/_vlasov1d/native/module.py create mode 100644 adept/_vlasov1d/native/worker.py create mode 100644 benchmarks/vlasov_cuda/CMakeLists.txt create mode 100644 benchmarks/vlasov_cuda/README.md create mode 100644 benchmarks/vlasov_cuda/adept_timing.py create mode 100644 benchmarks/vlasov_cuda/backend.cpp create mode 100644 benchmarks/vlasov_cuda/backend.hpp create mode 100644 benchmarks/vlasov_cuda/benchmark_latest.py create mode 100644 benchmarks/vlasov_cuda/checkpoint_reference.py create mode 100644 benchmarks/vlasov_cuda/compare_checkpoint.py create mode 100644 benchmarks/vlasov_cuda/distributed_reference.py create mode 100644 benchmarks/vlasov_cuda/integration_check.py create mode 100644 benchmarks/vlasov_cuda/local_cubic.hpp create mode 100644 benchmarks/vlasov_cuda/main.cpp create mode 100644 benchmarks/vlasov_cuda/measure.py create mode 100644 benchmarks/vlasov_cuda/reference/latest-nv4096/nx134400.yaml create mode 100644 benchmarks/vlasov_cuda/reference/latest-nv4096/nx268800.yaml create mode 100644 benchmarks/vlasov_cuda/reference/latest-nv4096/target.json create mode 100644 benchmarks/vlasov_cuda/results/2026-10-24-adept/README.md create mode 100644 benchmarks/vlasov_cuda/results/2026-10-24-adept/binaries.sha256 create mode 100644 benchmarks/vlasov_cuda/results/2026-10-24-adept/cuda-nx134400-g4.jsonl create mode 100644 benchmarks/vlasov_cuda/results/2026-10-24-adept/cuda-nx134400-g8.jsonl create mode 100644 benchmarks/vlasov_cuda/results/2026-10-24-adept/cuda-nx268800-g8.jsonl create mode 100644 benchmarks/vlasov_cuda/results/2026-10-24-adept/integration-checks.json create mode 100644 benchmarks/vlasov_cuda/results/2026-10-24-adept/jax-nx134400-g4.json create mode 100644 benchmarks/vlasov_cuda/results/2026-10-24-adept/jax-nx134400-g8.json create mode 100644 benchmarks/vlasov_cuda/results/2026-10-24-adept/jax-nx268800-g8.json create mode 100644 benchmarks/vlasov_cuda/results/2026-10-24-adept/launch.sh create mode 100644 benchmarks/vlasov_cuda/results/2026-10-24-adept/observer-native/library.sha256 create mode 100644 benchmarks/vlasov_cuda/results/2026-10-24-adept/observer-native/request.json create mode 100644 benchmarks/vlasov_cuda/results/2026-10-24-adept/observer-native/stats.json create mode 100644 benchmarks/vlasov_cuda/results/2026-10-24-adept/observer-netcdf-check.json create mode 100644 benchmarks/vlasov_cuda/results/2026-10-24-adept/observer-netcdf/dist-electron.f0.nc create mode 100644 benchmarks/vlasov_cuda/results/2026-10-24-adept/observer-netcdf/dist-electron.main.nc create mode 100644 benchmarks/vlasov_cuda/results/2026-10-24-adept/observer-netcdf/fields-electron-t=0.1.nc create mode 100644 benchmarks/vlasov_cuda/results/2026-10-24-adept/observer-netcdf/fields-shared-t=0.1.nc create mode 100644 benchmarks/vlasov_cuda/results/2026-10-24-adept/observer-netcdf/native-cascade.nc create mode 100644 benchmarks/vlasov_cuda/results/2026-10-24-adept/observer-netcdf/scalars-t=0.1.nc create mode 100644 benchmarks/vlasov_cuda/results/2026-10-24-adept/observer-netcdf/spectrum-electron.pkv.nc create mode 100644 benchmarks/vlasov_cuda/results/2026-10-24-adept/pinned-one-node/binaries.sha256 create mode 100644 benchmarks/vlasov_cuda/results/2026-10-24-adept/pinned-one-node/cuda-nx134400-g4.jsonl create mode 100644 benchmarks/vlasov_cuda/results/2026-10-24-adept/pinned-one-node/jax-nx134400-g4.json create mode 100644 benchmarks/vlasov_cuda/results/2026-10-24-adept/pinned-one-node/launch.sh create mode 100644 benchmarks/vlasov_cuda/results/2026-10-24-adept/pinned-one-node/source-state create mode 100644 benchmarks/vlasov_cuda/results/2026-10-24-adept/pinned-one-node/source.sha256 create mode 100644 benchmarks/vlasov_cuda/results/2026-10-24-adept/slurm-accounting.txt create mode 100644 benchmarks/vlasov_cuda/results/2026-10-24-adept/source-state create mode 100644 benchmarks/vlasov_cuda/results/2026-10-24-adept/source.sha256 create mode 100644 benchmarks/vlasov_cuda/results/2026-10-24-adept/source.tar.gz create mode 100644 benchmarks/vlasov_cuda/results/2026-10-24-adept/summarize.py create mode 100644 benchmarks/vlasov_cuda/results/2026-10-24-adept/summary.json create mode 100644 benchmarks/vlasov_cuda/results/2026-10-24-adept/unmatched-node/README.md create mode 100644 benchmarks/vlasov_cuda/results/2026-10-24-adept/unmatched-node/cuda-nx134400-g4.jsonl create mode 100644 benchmarks/vlasov_cuda/results/2026-10-24-adept/unmatched-node/jax-nx134400-g4.json create mode 100644 benchmarks/vlasov_cuda/results/2026-10-24-halos/comparison-nx134400-smoke.json create mode 100644 benchmarks/vlasov_cuda/results/2026-10-24-halos/comparison-nx134400-window.json create mode 100644 benchmarks/vlasov_cuda/results/2026-10-24-halos/comparison-nx268800-1node.json create mode 100644 benchmarks/vlasov_cuda/results/2026-10-24-halos/comparison-nx268800-window.json create mode 100644 benchmarks/vlasov_cuda/results/2026-10-24-halos/compiled-source.sha256 create mode 100644 benchmarks/vlasov_cuda/results/2026-10-24-halos/executable.sha256 create mode 100644 benchmarks/vlasov_cuda/results/2026-10-24-halos/nx134400-halos.jsonl create mode 100644 benchmarks/vlasov_cuda/results/2026-10-24-halos/nx134400-transpose.jsonl create mode 100644 benchmarks/vlasov_cuda/results/2026-10-24-halos/nx268800-halos.jsonl create mode 100644 benchmarks/vlasov_cuda/results/2026-10-24-halos/nx268800-transpose.jsonl create mode 100644 benchmarks/vlasov_cuda/results/2026-10-24-halos/reference-library-versions.json create mode 100644 benchmarks/vlasov_cuda/results/2026-10-24-halos/reference-nx134400.json create mode 100644 benchmarks/vlasov_cuda/results/2026-10-24-halos/reference-nx268800.json create mode 100644 benchmarks/vlasov_cuda/results/2026-10-24-halos/reference-source-state create mode 100644 benchmarks/vlasov_cuda/results/2026-10-24-halos/reference8-launch.sh create mode 100644 benchmarks/vlasov_cuda/results/2026-10-24-halos/slurm-accounting.txt create mode 100644 benchmarks/vlasov_cuda/results/2026-10-24-halos/source-archive.sha256 create mode 100644 benchmarks/vlasov_cuda/results/2026-10-24-halos/source-state create mode 100644 benchmarks/vlasov_cuda/results/2026-10-24-halos/source.tar.gz create mode 100644 benchmarks/vlasov_cuda/results/2026-10-24-halos/summarize.py create mode 100644 benchmarks/vlasov_cuda/results/2026-10-24-halos/summary.json create mode 100644 benchmarks/vlasov_cuda/results/2026-10-24-halos/validation-local-cpu.json create mode 100644 benchmarks/vlasov_cuda/results/2026-10-24-halos/validation-local-cuda.json create mode 100644 benchmarks/vlasov_cuda/results/2026-10-24-halos/validation-natural-cpu.json create mode 100644 benchmarks/vlasov_cuda/results/2026-10-24/baseline-source.tar.gz create mode 100644 benchmarks/vlasov_cuda/results/2026-10-24/scaling-32768x8192.jsonl create mode 100644 benchmarks/vlasov_cuda/results/2026-10-24/summary.json create mode 100644 benchmarks/vlasov_cuda/results/2026-10-24/validation-cuda-two-nodes.json create mode 100644 benchmarks/vlasov_cuda/results/2026-10-24/validation-cuda.json create mode 100644 benchmarks/vlasov_cuda/results/2026-10-24/validation-fftw.json create mode 100644 benchmarks/vlasov_cuda/run_adept_comparison.sh create mode 100644 benchmarks/vlasov_cuda/validate.py create mode 100644 benchmarks/vlasov_cuda/validate_local.py create mode 100644 tests/test_vlasov1d/test_native.py diff --git a/adept/_base_.py b/adept/_base_.py index aa21bb83..a2d58801 100644 --- a/adept/_base_.py +++ b/adept/_base_.py @@ -52,6 +52,8 @@ class ADEPTModule: """ + jit_compile = True + def __init__(self, cfg) -> None: self.cfg = cfg self.state = None @@ -299,6 +301,11 @@ def _get_adept_module_(self, cfg: dict) -> ADEPTModule: elif cfg["solver"] == "vlasov-1d": from adept.vlasov1d import BaseVlasov1D as this_module + if cfg.get("backend", "jax") == "native": + from adept.vlasov1d import native_backend + + this_module = native_backend(this_module) + # config = ConfigModel(**cfg) elif cfg["solver"] == "vlasov-2d": @@ -340,6 +347,10 @@ def _setup_(self, cfg: dict, td: str, adept_module: ADEPTModule = None, log: boo if adept_module is None: self.adept_module = self._get_adept_module_(cfg) else: + if cfg.get("backend", "jax") == "native": + from adept.vlasov1d import native_backend + + adept_module = native_backend(adept_module) self.adept_module = adept_module(cfg) # dump raw config @@ -406,7 +417,10 @@ def __call__( run_id=self.mlflow_run_id, nested=self.mlflow_nested, log_system_metrics=True ) as mlflow_run: t0 = time.time() - run_output = filter_jit(self.adept_module.__call__)(modules, args) + solve = self.adept_module.__call__ + if self.adept_module.jit_compile: + solve = filter_jit(solve) + run_output = solve(modules, args) mlflow.log_metrics({"run_time": round(time.time() - t0, 4)}) # logs the run time to mlflow t0 = time.time() diff --git a/adept/_vlasov1d/datamodel.py b/adept/_vlasov1d/datamodel.py index 75d92293..56f4c2a3 100644 --- a/adept/_vlasov1d/datamodel.py +++ b/adept/_vlasov1d/datamodel.py @@ -231,7 +231,28 @@ class Vlasov1DConfig(BaseModel): grid: GridConfig save: SaveConfig solver: str + backend: Literal["jax", "native"] = "jax" + native: "NativeSolverConfig | None" = None mlflow: MLFlowConfig drivers: EMDriverSetConfig terms: TermsConfig diagnostics: DiagnosticsConfig = DiagnosticsConfig() + + +class NativeSolverConfig(BaseModel): + """Execution options for the optional native Vlasov backend.""" + + model_config = ConfigDict(extra="forbid") + library: str | None = None + device: Literal["cuda", "cpu"] = "cuda" + ranks: int = Field(default=1, ge=1) + launcher: list[str] = Field(default_factory=list) + workdir: str | None = None + halo_capacity: int = Field(default=64, ge=2) + scalar_stride: int = Field(default=1, ge=1) + custom_save_times: Literal["preserve", "steps"] = "preserve" + host_mpi: bool = False + timeout: float | None = Field(default=None, gt=0) + + +Vlasov1DConfig.model_rebuild() diff --git a/adept/_vlasov1d/native/CMakeLists.txt b/adept/_vlasov1d/native/CMakeLists.txt new file mode 100644 index 00000000..aa6e9108 --- /dev/null +++ b/adept/_vlasov1d/native/CMakeLists.txt @@ -0,0 +1,21 @@ +cmake_minimum_required(VERSION 3.24) +project(adept_vlasov_native LANGUAGES CXX) +option(VLASOV_CUDA "Use CUDA/cuFFT (OFF builds the FFTW validation backend)" ON) +find_package(MPI REQUIRED COMPONENTS CXX) +add_library(adept_vlasov SHARED api.cpp core/backend.cpp) +target_compile_features(adept_vlasov PRIVATE cxx_std_17) +target_link_libraries(adept_vlasov PRIVATE MPI::MPI_CXX) +set_target_properties(adept_vlasov PROPERTIES POSITION_INDEPENDENT_CODE ON) +if(VLASOV_CUDA) + enable_language(CUDA) + find_package(CUDAToolkit REQUIRED) + set_source_files_properties(core/backend.cpp PROPERTIES LANGUAGE CUDA) + set_target_properties(adept_vlasov PROPERTIES CUDA_STANDARD 17 CUDA_ARCHITECTURES 80) + target_compile_definitions(adept_vlasov PRIVATE VLASOV_CUDA) + target_compile_options(adept_vlasov PRIVATE $<$:--extended-lambda>) + target_link_libraries(adept_vlasov PRIVATE CUDA::cudart CUDA::cufft) +else() + find_package(PkgConfig REQUIRED) + pkg_check_modules(FFTW REQUIRED IMPORTED_TARGET fftw3) + target_link_libraries(adept_vlasov PRIVATE PkgConfig::FFTW) +endif() diff --git a/adept/_vlasov1d/native/README.md b/adept/_vlasov1d/native/README.md new file mode 100644 index 00000000..dba0cb25 --- /dev/null +++ b/adept/_vlasov1d/native/README.md @@ -0,0 +1,43 @@ +# Native Vlasov backend + +Select `backend: native` in the usual `solver: vlasov-1d` configuration and run +through `ergoExo`. See the [configuration and build instructions](../../../docs/source/solvers/vlasov1d/config.md#execution-backend). +An existing `BaseVlasov1D` subclass can still supply initialization, save +callbacks and postprocessing. C++/CUDA/MPI advances the distribution; JAX runs +the original save callbacks, without automatic differentiation of the solve. + +`core/` is the canonical numerical implementation. The standalone program in +`benchmarks/vlasov_cuda` includes these same sources. Build the shared library +on the target system with CMake; `VLASOV_CUDA=OFF` builds the FFTW validation +implementation. The library is loaded only by MPI workers, not the outer +Python driver. + +The outer driver performs initialization on the CPU and stages the initial +state in bounded blocks. One persistent worker owns the solver on each GPU. +At a save point, DLPack exposes synchronized native buffers as JAX arrays on +a velocity-slab mesh. CUDA views require zero-copy import. The CPU validation +build permits a copy when FFTW's host alignment is insufficient for JAX. +Callbacks complete and saved outputs reach the host before the next native +mutation. No borrowed state view may survive `av_close`; the library checks +this lifetime rule explicitly. + +Workers stream saved leaves into host `.npy` memory maps. The driver assembles +distributed saved outputs and returns the usual Diffrax `Solution` containing +NumPy arrays. Existing xarray/NetCDF writers consume it. The module retains +the backing workspace while its returned saves are in use. MLflow artifacts +include actual save schedules, worker statistics/logs and the library hash. + +Standard saves round to actual timestep boundaries and merge duplicates. +Scalars default to every step; `native.scalar_stride` can reduce the cadence. +Custom callbacks preserve their requested times by default because some +project postprocessors reconstruct those axes. Off-step custom saves require +a previous-state copy and interpolation. `custom_save_times: steps` avoids +that work when the postprocessor consumes actual result times. Postprocessors +that reconstruct uniform scalar axes also need actual result times if the +last scalar interval is shorter than the requested stride. + +The validated physics is spectral spatial streaming, ADEPT's local cubic C1 +velocity interpolation, sixth-order splitting, Poisson fields, and optional +constant-rate Chang-Cooper Dougherty collisions. Unsupported physics and +trainable modules fail explicitly. Performance and reproduction evidence is +under [the benchmark results](../../../benchmarks/vlasov_cuda/results/2026-10-24-adept). diff --git a/adept/_vlasov1d/native/__init__.py b/adept/_vlasov1d/native/__init__.py new file mode 100644 index 00000000..ca9c4720 --- /dev/null +++ b/adept/_vlasov1d/native/__init__.py @@ -0,0 +1 @@ +"""Optional native Vlasov timestepping with ADEPT preprocessing and saves.""" diff --git a/adept/_vlasov1d/native/api.cpp b/adept/_vlasov1d/native/api.cpp new file mode 100644 index 00000000..3db9a19a --- /dev/null +++ b/adept/_vlasov1d/native/api.cpp @@ -0,0 +1,157 @@ +// Narrow C ABI for Python orchestration. Physics is shared with the benchmark. +#include "core/solver.hpp" +#include +#include +#include +#include + +using namespace adept_native; +namespace { +thread_local std::string last_error; +bool own_mpi = false; +std::atomic live_engines{0}; +struct Engine { + Options options; + std::unique_ptr backend; + std::unique_ptr stepper; + int local_rank; + Engine(Options opts, int local) : options(std::move(opts)), local_rank(local) { + backend = std::make_unique(options.grid, local); + stepper = std::make_unique(*backend, options); + if (!options.coordinates.empty()) backend->set_coordinates(read_binary(options.coordinates, options.grid.nv)); + if (!options.background.empty()) backend->set_background(read_binary(options.background, options.grid.nx)); + initialize(*backend, options); + ++live_engines; + } + ~Engine() { --live_engines; } +}; +using Handle = std::shared_ptr; +template int guarded(F action) { + try { action(); return 0; } + catch (const std::exception& error) { last_error = error.what(); return -1; } +} +// Legacy DLPack 0.8 ABI, exported as a "dltensor" capsule by Python. +// https://dmlc.github.io/dlpack/latest/c_api.html +struct DLDevice { int32_t device_type, device_id; }; +struct DLDataType { uint8_t code, bits; uint16_t lanes; }; +struct DLTensor { + void* data; DLDevice device; int32_t ndim; DLDataType dtype; + int64_t* shape; int64_t* strides; uint64_t byte_offset; +}; +struct DLManagedTensor { + DLTensor dl_tensor; void* manager_ctx; void (*deleter)(DLManagedTensor*); +}; +struct Export { + DLManagedTensor managed; + Handle owner; + int64_t shape[2], strides[2]; +}; +void release_tensor(DLManagedTensor* tensor) { delete static_cast(tensor->manager_ctx); } +int device_type() { +#ifdef VLASOV_CUDA + return 2; // kDLCUDA +#else + return 1; // kDLCPU +#endif +} +} +extern "C" { +const char* av_error() { return last_error.c_str(); } +void* av_create(int argc, const char** argv) { + void* result = nullptr; + if (guarded([&] { + prepare_device_from_environment(); + int initialized; + mpi_check(MPI_Initialized(&initialized)); + if (!initialized) { mpi_check(MPI_Init(nullptr, nullptr)); own_mpi = true; } + MPI_Comm_set_errhandler(MPI_COMM_WORLD, MPI_ERRORS_RETURN); + int rank; + mpi_check(MPI_Comm_rank(MPI_COMM_WORLD, &rank)); + Options opts = parse(argc, const_cast(argv), rank); + Grid& g = opts.grid; + g.rank = rank; + mpi_check(MPI_Comm_size(MPI_COMM_WORLD, &g.ranks)); + g.collisions = opts.nu > 0; + if (g.nx < 8 || g.nv < 4 || g.nx % g.ranks || g.nv % g.ranks || + g.count() > std::size_t(std::numeric_limits::max())) + throw std::runtime_error("native grid must divide across ranks and fit MPI int counts"); + if (!g.local_cubic || !g.halos || !g.cell_centred || opts.dt <= 0 || opts.nu < 0 || + g.length <= 0 || g.vmax <= 0 || g.halo_capacity < 2 || + std::size_t(g.halo_capacity) * g.nx > std::size_t(std::numeric_limits::max()) || + (opts.integrator != "sixth" && opts.integrator != "strang")) + throw std::runtime_error("unsupported native solver options"); + MPI_Comm local; + mpi_check(MPI_Comm_split_type(MPI_COMM_WORLD, MPI_COMM_TYPE_SHARED, rank, MPI_INFO_NULL, &local)); + int local_rank; + mpi_check(MPI_Comm_rank(local, &local_rank)); + mpi_check(MPI_Comm_free(&local)); + result = new Handle(std::make_shared(std::move(opts), local_rank)); + })) return nullptr; + return result; +} +int av_info(void* handle, int* output) { + return guarded([&] { + auto& e = **static_cast(handle); + const Grid& g = e.options.grid; + const int values[] = {g.rank, g.ranks, e.local_rank, device_type(), g.nx, g.nv, g.local_v()}; + std::copy_n(values, 7, output); + }); +} +int av_broadcast(char* value, int capacity) { + return guarded([&] { mpi_check(MPI_Bcast(value, capacity, MPI_CHAR, 0, MPI_COMM_WORLD)); }); +} +int av_barrier() { return guarded([] { mpi_check(MPI_Barrier(MPI_COMM_WORLD)); }); } +int av_step(void* handle, int count) { + return guarded([&] { + if (count < 0) throw std::runtime_error("negative native step count"); + auto& e = **static_cast(handle); + for (int i = 0; i < count; ++i) e.stepper->step(); + e.backend->synchronize(); + }); +} +int av_set_field(void* handle, const char* path) { + return guarded([&] { + auto& e = **static_cast(handle); + e.backend->set_field(read_binary(path, e.options.grid.nx)); + }); +} +void* av_tensor(void* handle, int field) { + void* result = nullptr; + if (guarded([&] { + auto owner = *static_cast(handle); + auto& e = *owner; + e.backend->synchronize(); // Producer completion before a consumer stream reads. + auto* tensor = new Export{}; + tensor->owner = std::move(owner); + tensor->shape[0] = e.options.grid.nx; + tensor->shape[1] = e.options.grid.local_v(); + tensor->strides[0] = 1; + tensor->strides[1] = e.options.grid.nx; + tensor->managed = {{field ? e.backend->electric_buffer() : e.backend->velocity_buffer(), + {device_type(), device_type() == 2 ? e.local_rank : 0}, field ? 1 : 2, + {2, 64, 1}, tensor->shape, tensor->strides, 0}, tensor, release_tensor}; + result = &tensor->managed; + })) return nullptr; + return result; +} +int av_stats(void* handle, double* output) { + return guarded([&] { + auto& e = **static_cast(handle); + mpi_check(MPI_Allreduce(e.stepper->times.data(), output, StageCount, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD)); + }); +} +int av_close(void* handle) { + return guarded([&] { + auto* owner = static_cast(handle); + if (owner->use_count() != 1) throw std::runtime_error("native buffers still have live DLPack views"); + delete owner; + }); +} +int av_finalize() { + return guarded([] { + if (live_engines) throw std::runtime_error("native engines must close before MPI finalization"); + if (own_mpi) { mpi_check(MPI_Finalize()); own_mpi = false; } + }); +} +void av_abort(int code) { MPI_Abort(MPI_COMM_WORLD, code); } +} diff --git a/adept/_vlasov1d/native/bridge.py b/adept/_vlasov1d/native/bridge.py new file mode 100644 index 00000000..138b9cf4 --- /dev/null +++ b/adept/_vlasov1d/native/bridge.py @@ -0,0 +1,92 @@ +"""ctypes and DLPack boundary; importing this module does not initialize JAX.""" + +import ctypes as ct + + +class NativeCore: + """Own one persistent MPI/native solver inside a worker process.""" + + def __init__(self, library, options): + self.lib = ct.CDLL(str(library)) + signatures = { + "av_create": ([ct.c_int, ct.POINTER(ct.c_char_p)], ct.c_void_p), + "av_error": ([], ct.c_char_p), + "av_info": ([ct.c_void_p, ct.POINTER(ct.c_int)], ct.c_int), + "av_step": ([ct.c_void_p, ct.c_int], ct.c_int), + "av_tensor": ([ct.c_void_p, ct.c_int], ct.c_void_p), + "av_set_field": ([ct.c_void_p, ct.c_char_p], ct.c_int), + "av_broadcast": ([ct.c_char_p, ct.c_int], ct.c_int), + "av_barrier": ([], ct.c_int), + "av_stats": ([ct.c_void_p, ct.POINTER(ct.c_double)], ct.c_int), + "av_close": ([ct.c_void_p], ct.c_int), + "av_finalize": ([], ct.c_int), + "av_abort": ([ct.c_int], None), + } + for name, (arguments, result) in signatures.items(): + function = getattr(self.lib, name) + function.argtypes, function.restype = arguments, result + argv = [b"adept-native", *(str(value).encode() for value in options)] + self.handle = self.lib.av_create(len(argv), (ct.c_char_p * len(argv))(*argv)) + if not self.handle: + raise RuntimeError(self.lib.av_error().decode()) + values = (ct.c_int * 7)() + self.check(self.lib.av_info(self.handle, values)) + self.rank, self.size, self.local_rank, self.device_type, self.nx, self.nv, self.local_nv = values + + def check(self, result): + if result: + raise RuntimeError(self.lib.av_error().decode()) + + def step(self, count): + self.check(self.lib.av_step(self.handle, count)) + + def broadcast(self, value): + buffer = ct.create_string_buffer(value.encode(), 512) + self.check(self.lib.av_broadcast(buffer, len(buffer))) + return buffer.value.decode() + + def barrier(self): + self.check(self.lib.av_barrier()) + + def set_field(self, path): + self.check(self.lib.av_set_field(self.handle, str(path).encode())) + + def view(self, field=False): + return _View(self, field) + + def stats(self): + values = (ct.c_double * 7)() + self.check(self.lib.av_stats(self.handle, values)) + names = ["stream", "velocity", "pack", "mpi", "field", "collision", "total"] + return dict(zip(names, values, strict=True)) + + def close(self): + if self.handle: + self.check(self.lib.av_close(self.handle)) + self.handle = None + self.check(self.lib.av_finalize()) + + +class _View: + """Synchronized, read-only-by-convention view valid until the next native step.""" + + def __init__(self, owner, field): + self.owner, self.field = owner, field + + def __dlpack_device__(self): + return self.owner.device_type, self.owner.local_rank if self.owner.device_type == 2 else 0 + + def __dlpack__(self, stream=None, *, max_version=None, dl_device=None, copy=None): + if copy: + raise BufferError("the native exporter provides views; the consumer may copy") + if dl_device is not None and tuple(dl_device) != self.__dlpack_device__(): + raise BufferError("native DLPack export cannot change devices") + pointer = self.owner.lib.av_tensor(self.owner.handle, int(self.field)) + if not pointer: + raise RuntimeError(self.owner.lib.av_error().decode()) + # The consumer owns the tensor's C++ deleter after renaming this capsule + # to used_dltensor. Its manager keeps the native allocation alive. + capsule = ct.pythonapi.PyCapsule_New + capsule.restype = ct.py_object + capsule.argtypes = [ct.c_void_p, ct.c_char_p, ct.c_void_p] + return capsule(pointer, b"dltensor", None) diff --git a/adept/_vlasov1d/native/core/backend.cpp b/adept/_vlasov1d/native/core/backend.cpp new file mode 100644 index 00000000..94295779 --- /dev/null +++ b/adept/_vlasov1d/native/core/backend.cpp @@ -0,0 +1,499 @@ +#include "backend.hpp" +#include "local_cubic.hpp" + +#include +#include +#include +#include +#include +#include + +#ifdef VLASOV_CUDA +#include +#include +#define DEVICE __device__ +static void check_cuda(cudaError_t error) { + if (error != cudaSuccess) throw std::runtime_error(cudaGetErrorString(error)); +} +static void check_fft(cufftResult error) { + if (error != CUFFT_SUCCESS) throw std::runtime_error("cuFFT error " + std::to_string(error)); +} +template __global__ void kernel(std::size_t n, F fn) { + for (std::size_t i = blockIdx.x * std::size_t(blockDim.x) + threadIdx.x; + i < n; i += std::size_t(blockDim.x) * gridDim.x) fn(i); +} +template void parallel_for(std::size_t n, F fn) { + kernel<<((n + 255) / 256, 65535), 256>>>(n, fn); + check_cuda(cudaGetLastError()); +} +__global__ void maximum_absolute(const double* values, double* output, int n) { + __shared__ double partial[256]; + double maximum = 0; + for (int i = blockIdx.x * blockDim.x + threadIdx.x; i < n; i += blockDim.x * gridDim.x) + maximum = fmax(maximum, fabs(values[i])); + partial[threadIdx.x] = maximum; + __syncthreads(); + for (int stride = 128; stride; stride /= 2) { + if (threadIdx.x < stride) partial[threadIdx.x] = fmax(partial[threadIdx.x], partial[threadIdx.x + stride]); + __syncthreads(); + } + if (threadIdx.x == 0) + atomicMax(reinterpret_cast(output), + static_cast(__double_as_longlong(partial[0]))); +} +#else +#include +#define DEVICE +template void parallel_for(std::size_t n, F fn) { + for (std::size_t i = 0; i < n; ++i) fn(i); +} +#endif + +namespace { +constexpr double pi = 3.14159265358979323846; +class Buffer { +public: + explicit Buffer(std::size_t n) : size(n) { + if (!n) return; +#ifdef VLASOV_CUDA + check_cuda(cudaMalloc(reinterpret_cast(&ptr), n * sizeof(double))); +#else + ptr = static_cast(fftw_malloc(n * sizeof(double))); + if (!ptr) throw std::bad_alloc(); +#endif + } + ~Buffer() { +#ifdef VLASOV_CUDA + cudaFree(ptr); +#else + fftw_free(ptr); +#endif + } + Buffer(const Buffer&) = delete; + Buffer& operator=(const Buffer&) = delete; + std::vector download() const { + std::vector result(size); +#ifdef VLASOV_CUDA + check_cuda(cudaMemcpy(result.data(), ptr, size * sizeof(double), cudaMemcpyDeviceToHost)); +#else + std::copy_n(ptr, size, result.data()); +#endif + return result; + } + void upload(const std::vector& values) { + if (values.size() != size) throw std::runtime_error("buffer size mismatch"); +#ifdef VLASOV_CUDA + check_cuda(cudaMemcpy(ptr, values.data(), size * sizeof(double), cudaMemcpyHostToDevice)); +#else + std::copy(values.begin(), values.end(), ptr); +#endif + } + double* ptr = nullptr; + std::size_t size; +}; + +// Plans and workspaces persist for the full run. Complex storage is interleaved. +class FFT { +public: + FFT(int n, int batch, double* real, double* complex) { +#ifdef VLASOV_CUDA + check_fft(cufftPlan1d(&forward, n, CUFFT_D2Z, batch)); + check_fft(cufftPlan1d(&backward, n, CUFFT_Z2D, batch)); + std::size_t a = 0, b = 0; + check_fft(cufftGetSize(forward, &a)); + check_fft(cufftGetSize(backward, &b)); + workspace_bytes = a + b; +#else + int dims[] = {n}; + forward = fftw_plan_many_dft_r2c(1, dims, batch, real, nullptr, 1, n, + reinterpret_cast(complex), nullptr, 1, n / 2 + 1, FFTW_ESTIMATE); + backward = fftw_plan_many_dft_c2r(1, dims, batch, + reinterpret_cast(complex), nullptr, 1, n / 2 + 1, + real, nullptr, 1, n, FFTW_ESTIMATE); + if (!forward || !backward) throw std::runtime_error("FFTW plan creation failed"); +#endif + } + ~FFT() { +#ifdef VLASOV_CUDA + cufftDestroy(forward); + cufftDestroy(backward); +#else + fftw_destroy_plan(forward); + fftw_destroy_plan(backward); +#endif + } + void r2c(double* real, double* complex) { +#ifdef VLASOV_CUDA + check_fft(cufftExecD2Z(forward, real, reinterpret_cast(complex))); +#else + fftw_execute_dft_r2c(forward, real, reinterpret_cast(complex)); +#endif + } + void c2r(double* complex, double* real) { +#ifdef VLASOV_CUDA + check_fft(cufftExecZ2D(backward, reinterpret_cast(complex), real)); +#else + fftw_execute_dft_c2r(backward, reinterpret_cast(complex), real); +#endif + } + std::size_t workspace_bytes = 0; +private: +#ifdef VLASOV_CUDA + cufftHandle forward{}, backward{}; +#else + fftw_plan forward{}, backward{}; +#endif +}; + +std::string select_device(int local_rank) { +#ifdef VLASOV_CUDA + int count = 0; + check_cuda(cudaGetDeviceCount(&count)); + if (!count) throw std::runtime_error("no CUDA devices visible"); + // Slurm may expose one device per rank, or all devices on the node. + if (count != 1 && local_rank >= count) + throw std::runtime_error("more local ranks than visible GPUs"); + int device = count == 1 ? 0 : local_rank; + check_cuda(cudaSetDevice(device)); + cudaDeviceProp props{}; + check_cuda(cudaGetDeviceProperties(&props, device)); + return props.name; +#else + (void)local_rank; + return "CPU/FFTW (validation only)"; +#endif +} +} // namespace + +void prepare_device_from_environment() { + int local_rank = 0; + for (const char* name : {"SLURM_LOCALID", "OMPI_COMM_WORLD_LOCAL_RANK", "MPI_LOCALRANKID", + "PMI_LOCAL_RANK", "MV2_COMM_WORLD_LOCAL_RANK"}) { + if (const char* value = std::getenv(name)) { + local_rank = std::stoi(value); + break; + } + } + select_device(local_rank); +} + +struct Backend::Impl { + Grid grid; + std::string device; + Buffer fv, fx, send, recv, second, rhs, output, inverse; + Buffer spectrum, density_local, density_global, electric, field_spectrum; + Buffer coordinates, background, ghosts, field_maximum; + FFT streaming_fft, field_fft; + Impl(Grid g, int local_rank) + : grid(g), device(select_device(local_rank)), + fv(g.count()), fx(!g.halos || g.collisions ? g.count() : 0), + send((!g.halos || g.collisions) && g.ranks > 1 ? g.count() : 0), + recv((!g.halos || g.collisions) && g.ranks > 1 ? g.count() : 0), + second(!g.local_cubic || g.collisions ? g.count() : 0), + rhs(!g.local_cubic || g.collisions ? g.count() : 0), + output(g.count()), inverse(!g.local_cubic ? g.nv : 0), + spectrum(2 * std::size_t(g.nx / 2 + 1) * g.local_v()), + density_local(g.nx), density_global(g.nx), electric(g.nx), + field_spectrum(2 * (g.nx / 2 + 1)), + coordinates(g.nv), background(g.nx), + ghosts(g.halos ? 2 * std::size_t(g.halo_capacity) * g.nx : 0), + field_maximum(1), + streaming_fft(g.nx, g.local_v(), fv.ptr, spectrum.ptr), + field_fft(g.nx, 1, electric.ptr, field_spectrum.ptr) { + std::vector inv(g.nv, 0.0); + // LU factors for natural spline: m[i-1] + 4m[i] + m[i+1] = 6*d2f/dv^2. + for (int i = 1; i < g.nv - 1; ++i) inv[i] = 1.0 / (4.0 - inv[i - 1]); + if (!g.local_cubic) inverse.upload(inv); + std::vector v(g.nv), bg(g.nx, 1.0); + for (int j = 0; j < g.nv; ++j) v[j] = g.velocity(j); + coordinates.upload(v); + background.upload(bg); + electric.upload(std::vector(g.nx, 0.0)); + } +}; + +Backend::Backend(Grid grid, int local_rank) : p(std::make_unique(grid, local_rank)) {} +Backend::~Backend() = default; +void Backend::synchronize() { +#ifdef VLASOV_CUDA + check_cuda(cudaDeviceSynchronize()); +#endif +} +void Backend::collide(double dt, double nu) { + const Grid g = p->grid; + const int nx = g.local_x(), nv = g.nv; + const double dv = g.dv(), factor = dt * nu; + const double* f = p->fx.ptr; + const double* v = p->coordinates.ptr; + double* upper = p->second.ptr; + double* rhs = p->rhs.ptr; + double* out = p->output.ptr; + parallel_for(nx, [=] DEVICE (std::size_t x) { + double sum = 0, momentum = 0; + for (int j = 0; j < nv; ++j) { + double value = f[std::size_t(j) * nx + x]; + sum += value; momentum += value * v[j]; + } + double mean = momentum / sum, variance = 0; + for (int j = 0; j < nv; ++j) { + double delta = v[j] - mean; + variance += f[std::size_t(j) * nx + x] * delta * delta * dv; + } + double temperature = variance / (sum * dv); + // Production self_consistent_beta is disabled: beta=1/(2*T). + double beta = 1.0 / (2.0 * temperature), D = 1.0 / (2.0 * beta); + double safe_D = D > 1e-30 ? D : 1e-30; + auto edge = [=] DEVICE (int j, double& alpha, double& beta_edge) { + if (j < 0 || j >= nv - 1) { alpha = beta_edge = 0; return; } + double drift = 2.0 * beta * D * (0.5 * (v[j] + v[j + 1]) - mean); + double w = drift * dv / safe_D; + double delta = fabs(w) < 1e-8 ? 0.5 - w / 12.0 + w * w * w / 720.0 + : 1.0 / w - 1.0 / expm1(w); + alpha = -drift * delta + safe_D / dv; + beta_edge = -drift * (1 - delta) - safe_D / dv; + }; + double previous_upper = 0, previous_rhs = 0; + for (int j = 0; j < nv; ++j) { + double al, bl, ar, br; + edge(j - 1, al, bl); edge(j, ar, br); + double diag = 1.0 + factor * (ar - bl) / dv; + double lower = -factor * al / dv, higher = factor * br / dv; + std::size_t i = std::size_t(j) * nx + x; + double applied = diag * f[i]; + if (j > 0) applied += lower * f[i - nx]; + if (j < nv - 1) applied += higher * f[i + nx]; + // Match ADEPT's delta formulation rather than solving for f directly. + double residual = f[i] - applied; + double denominator = diag - lower * previous_upper; + previous_upper = higher / denominator; + previous_rhs = (residual - lower * previous_rhs) / denominator; + upper[i] = previous_upper; rhs[i] = previous_rhs; + } + double next = 0; + for (int j = nv - 1; j >= 0; --j) { + std::size_t i = std::size_t(j) * nx + x; + next = rhs[i] - upper[i] * next; + out[i] = f[i] + next; + } + }); + std::swap(p->fx.ptr, p->output.ptr); +} +void Backend::initialize() { + const Grid g = p->grid; + double norm = 0.0; + for (int j = 0; j < g.nv; ++j) { + double v = g.velocity(j); + norm += std::exp(-0.5 * v * v) * (!g.cell_centred && (j == 0 || j == g.nv - 1) ? 0.5 : 1.0) * g.dv(); + } + double* f = p->fv.ptr; + const int nx = g.nx, nv_local = g.local_v(), v0 = g.rank * nv_local; + const double* vgrid = p->coordinates.ptr; + const double amp = g.amplitude; + parallel_for(g.count(), [=] DEVICE (std::size_t i) { + int x = i % nx, vindex = i / nx + v0; + double v = vgrid[vindex]; + f[i] = exp(-0.5 * v * v) / norm * (1.0 + amp * cos(2.0 * pi * x / nx)); + }); +} +void Backend::stream(double dt) { + const Grid g = p->grid; + const int nk = g.nx / 2 + 1, nx = g.nx, v0 = g.rank * g.local_v(); + const double k0 = 2 * pi / g.length; + const double* vgrid = p->coordinates.ptr; + double* fk = p->spectrum.ptr; + double* f = p->fv.ptr; + p->streaming_fft.r2c(f, fk); + parallel_for(std::size_t(nk) * g.local_v(), [=] DEVICE (std::size_t i) { + int k = i % nk; + double v = vgrid[i / nk + v0]; + double angle = -k0 * k * v * dt; + double c = cos(angle), s = sin(angle), re = fk[2 * i], im = fk[2 * i + 1]; + fk[2 * i] = re * c - im * s; + fk[2 * i + 1] = (k == 0 || (nx % 2 == 0 && k == nx / 2)) ? 0.0 : re * s + im * c; + }); + p->streaming_fft.c2r(fk, f); + parallel_for(g.count(), [=] DEVICE (std::size_t i) { f[i] /= nx; }); +} +void Backend::density() { + const Grid g = p->grid; + const int nx = g.nx, nv = g.nv, nvl = g.local_v(), v0 = g.rank * nvl; + const double dv = g.dv(); + const bool cells = g.cell_centred; + const double* f = p->fv.ptr; + double* rho = p->density_local.ptr; + parallel_for(nx, [=] DEVICE (std::size_t x) { + double sum = 0.0; + for (int j = 0; j < nvl; ++j) + sum += f[std::size_t(j) * nx + x] * (!cells && (j + v0 == 0 || j + v0 == nv - 1) ? 0.5 : 1.0); + rho[x] = sum * dv; + }); +} +void Backend::field() { + const Grid g = p->grid; + const int nx = g.nx, nk = nx / 2 + 1; + const double k0 = 2 * pi / g.length; + const double* density = p->density_global.ptr; + const double* bg = p->background.ptr; + double* e = p->electric.ptr; + double* ek = p->field_spectrum.ptr; + parallel_for(nx, [=] DEVICE (std::size_t i) { e[i] = bg[i] - density[i]; }); + p->field_fft.r2c(e, ek); + parallel_for(nk, [=] DEVICE (std::size_t i) { + if (i == 0 || (nx % 2 == 0 && i == std::size_t(nx / 2))) { + ek[2 * i] = ek[2 * i + 1] = 0.0; + } else { + double re = ek[2 * i], im = ek[2 * i + 1], k = k0 * i; + ek[2 * i] = im / k; + ek[2 * i + 1] = -re / k; + } + }); + p->field_fft.c2r(ek, e); + parallel_for(nx, [=] DEVICE (std::size_t i) { e[i] /= nx; }); +} +void Backend::velocity(double dt, double acceleration, bool self_consistent) { + if (p->grid.local_cubic) { local_velocity(dt, acceleration, self_consistent); return; } + const Grid g = p->grid; + const int nxl = g.local_x(), nv = g.nv, x0 = g.rank * nxl; + const double dv = g.dv(); + const double* f = p->fx.ptr; + const double* inv = p->inverse.ptr; + double* m = p->second.ptr; + double* r = p->rhs.ptr; + // One thread per spatial line. Keeping x contiguous coalesces each sweep + // across a warp. This baseline has a serial recurrence along each v line. + parallel_for(nxl, [=] DEVICE (std::size_t x) { + m[x] = m[std::size_t(nv - 1) * nxl + x] = 0.0; + double previous = 0.0; + for (int j = 1; j < nv - 1; ++j) { + std::size_t i = std::size_t(j) * nxl + x; + double b = 6.0 * (f[i - nxl] - 2.0 * f[i] + f[i + nxl]) / (dv * dv); + previous = (b - previous) * inv[j]; + r[i] = previous; + } + double next = 0.0; + for (int j = nv - 2; j >= 1; --j) { + std::size_t i = std::size_t(j) * nxl + x; + next = r[i] - inv[j] * next; + m[i] = next; + } + }); + double* out = p->output.ptr; + const double* e = p->electric.ptr; + parallel_for(g.count(), [=] DEVICE (std::size_t i) { + int x = i % nxl, j = i / nxl; + double a = self_consistent ? -e[x0 + x] : acceleration; + double q = j - a * dt / dv; + if (q < 0.0 || q > nv - 1) { out[i] = 0.0; return; } + int lo = int(floor(q)); + if (lo == nv - 1) lo = nv - 2; + double b = q - lo, a0 = 1.0 - b; + std::size_t l = std::size_t(lo) * nxl + x, h = l + nxl; + out[i] = a0 * f[l] + b * f[h] + + ((a0 * a0 * a0 - a0) * m[l] + (b * b * b - b) * m[h]) * dv * dv / 6.0; + }); + std::swap(p->fx.ptr, p->output.ptr); +} +void Backend::local_velocity(double dt, double acceleration, bool self_consistent) { + const Grid g = p->grid; + const bool halo = g.halos; + const int width = halo ? g.nx : g.local_x(); + const int nv = g.nv, nvl = g.local_v(), v0 = halo ? g.rank * nvl : 0; + const int x0 = halo ? 0 : g.rank * g.local_x(), cap = g.halo_capacity; + const double dv = g.dv(); + const double* f = halo ? p->fv.ptr : p->fx.ptr; + const double* ghosts = p->ghosts.ptr; + const double* v = p->coordinates.ptr; + const double* e = p->electric.ptr; + double* out = p->output.ptr; + parallel_for(g.count(), [=] DEVICE (std::size_t i) { + int x = i % width, j = i / width + v0; + double a = self_consistent ? -e[x0 + x] : acceleration; + double query = v[j] - a * dt; + int guess = int(floor(j - a * dt / dv)); + auto sample = [=] DEVICE (int index) { + int local = index - v0; + if (!halo || (local >= 0 && local < nvl)) return f[std::size_t(local) * width + x]; + int row = local < 0 ? cap + local : cap + local - nvl; + return ghosts[std::size_t(row) * width + x]; + }; + out[i] = local_cubic_value(query, guess, v, nv, sample, 1e-30); + }); + if (halo) std::swap(p->fv.ptr, p->output.ptr); + else std::swap(p->fx.ptr, p->output.ptr); +} +double Backend::max_field() { +#ifdef VLASOV_CUDA + check_cuda(cudaMemset(p->field_maximum.ptr, 0, sizeof(double))); + maximum_absolute<<grid.nx + 255) / 256, 65535), 256>>>( + p->electric.ptr, p->field_maximum.ptr, p->grid.nx); + check_cuda(cudaGetLastError()); + return p->field_maximum.download()[0]; +#else + double maximum = 0; + for (int x = 0; x < p->grid.nx; ++x) maximum = std::max(maximum, std::abs(p->electric.ptr[x])); + return maximum; +#endif +} +double* Backend::velocity_buffer() { return p->fv.ptr; } +double* Backend::electric_buffer() { return p->electric.ptr; } +void Backend::set_field(const std::vector& values) { p->electric.upload(values); } +double* Backend::halo_buffer() { return p->ghosts.ptr; } +void Backend::upload_halos(const std::vector& values) { p->ghosts.upload(values); } +void Backend::upload_state(const std::vector& values) { p->fv.upload(values); } +void Backend::set_coordinates(const std::vector& values) { + if (values.size() != std::size_t(p->grid.nv)) throw std::runtime_error("incorrect velocity coordinate count"); + for (std::size_t i = 1; i < values.size(); ++i) { + double spacing = values[i] - values[i - 1]; + if (!std::isfinite(spacing) || spacing <= 0 || std::abs(spacing / p->grid.dv() - 1) > 1e-10) + throw std::runtime_error("velocity halos require uniform coordinates matching --nv/--vmax/--cells"); + } + p->coordinates.upload(values); +} +void Backend::set_background(const std::vector& values) { p->background.upload(values); } +void Backend::pack(bool to_x) { + const Grid g = p->grid; + const int nxl = g.local_x(), nvl = g.local_v(), nx = g.nx; + const std::size_t tile = std::size_t(nxl) * nvl; + const double* input = to_x ? p->fv.ptr : p->fx.ptr; + double* output = p->send.ptr; + parallel_for(g.count(), [=] DEVICE (std::size_t i) { + int dst = i / tile; + std::size_t in_tile = i % tile; + int v = in_tile / nxl, x = in_tile % nxl; + output[i] = to_x ? input[std::size_t(v) * nx + dst * nxl + x] + : input[std::size_t(dst * nvl + v) * nxl + x]; + }); +} +void Backend::unpack(bool to_x) { + const Grid g = p->grid; + const int nxl = g.local_x(), nvl = g.local_v(), nx = g.nx; + const std::size_t tile = std::size_t(nxl) * nvl; + const double* input = p->recv.ptr; + double* output = to_x ? p->fx.ptr : p->fv.ptr; + parallel_for(g.count(), [=] DEVICE (std::size_t i) { + int src = i / tile; + std::size_t in_tile = i % tile; + int v = in_tile / nxl, x = in_tile % nxl; + if (to_x) output[std::size_t(src * nvl + v) * nxl + x] = input[i]; + else output[std::size_t(v) * nx + src * nxl + x] = input[i]; + }); +} +double* Backend::send_buffer() { return p->send.ptr; } +void Backend::local_layout() { std::swap(p->fv.ptr, p->fx.ptr); } +double* Backend::recv_buffer() { return p->recv.ptr; } +double* Backend::density_buffer() { return p->density_local.ptr; } +double* Backend::reduced_density_buffer() { return p->density_global.ptr; } +std::vector Backend::download() { return p->fv.download(); } +std::vector Backend::download_field() { return p->electric.download(); } +std::vector Backend::download_send() { return p->send.download(); } +std::vector Backend::download_density() { return p->density_local.download(); } +void Backend::upload_received(const std::vector& values) { p->recv.upload(values); } +void Backend::upload_density(const std::vector& values) { p->density_global.upload(values); } +std::string Backend::device_name() const { return p->device; } +std::size_t Backend::allocated_bytes() const { + return (p->fv.size + p->fx.size + p->send.size + p->recv.size + p->second.size + p->rhs.size + + p->output.size + p->inverse.size + p->spectrum.size + + 3 * p->grid.nx + p->field_spectrum.size) * sizeof(double) + + p->streaming_fft.workspace_bytes + p->field_fft.workspace_bytes + + (p->coordinates.size + p->background.size + p->ghosts.size + p->field_maximum.size) * sizeof(double); +} diff --git a/adept/_vlasov1d/native/core/backend.hpp b/adept/_vlasov1d/native/core/backend.hpp new file mode 100644 index 00000000..933f9bd0 --- /dev/null +++ b/adept/_vlasov1d/native/core/backend.hpp @@ -0,0 +1,66 @@ +#pragma once + +#include +#include +#include +#include + +// Cray MPI initializes its GPU transport during MPI_Init, so bind first. +void prepare_device_from_environment(); + +struct Grid { + int nx = 4096, nv = 1024, ranks = 1, rank = 0; + double length = 12.566370614359172, vmax = 8.0, amplitude = 0.01; + bool cell_centred = false, local_cubic = false, halos = false, collisions = false; + int halo_capacity = 64; + int local_x() const { return nx / ranks; } + int local_v() const { return nv / ranks; } + std::size_t count() const { return std::size_t(nx) * local_v(); } + double dv() const { return 2.0 * vmax / (cell_centred ? nv : nv - 1); } + double velocity(int j) const { return -vmax + (j + (cell_centred ? 0.5 : 0.0)) * dv(); } +}; + +// Buffers are GPU-resident in the CUDA build; MPI must support device pointers. +// Both layouts keep the spatial index contiguous, including the spline batches. +class Backend { +public: + explicit Backend(Grid grid, int local_rank); + ~Backend(); + Backend(const Backend&) = delete; + Backend& operator=(const Backend&) = delete; + void initialize(); + void synchronize(); + void stream(double dt); + void density(); + void field(); + void velocity(double dt, double acceleration, bool self_consistent); + void local_velocity(double dt, double acceleration, bool self_consistent); + void collide(double dt, double nu); + double max_field(); + double* velocity_buffer(); + double* electric_buffer(); + void set_field(const std::vector& values); + double* halo_buffer(); + void upload_halos(const std::vector& values); + void upload_state(const std::vector& values); + void set_coordinates(const std::vector& values); + void set_background(const std::vector& values); + void pack(bool to_x); + void unpack(bool to_x); + void local_layout(); + double* send_buffer(); + double* recv_buffer(); + double* density_buffer(); + double* reduced_density_buffer(); + std::vector download(); + std::vector download_field(); + void upload_received(const std::vector& values); + std::vector download_send(); + std::vector download_density(); + void upload_density(const std::vector& values); + std::string device_name() const; + std::size_t allocated_bytes() const; +private: + struct Impl; + std::unique_ptr p; +}; diff --git a/adept/_vlasov1d/native/core/local_cubic.hpp b/adept/_vlasov1d/native/core/local_cubic.hpp new file mode 100644 index 00000000..b78570a5 --- /dev/null +++ b/adept/_vlasov1d/native/core/local_cubic.hpp @@ -0,0 +1,30 @@ +#pragma once +#include + +#ifdef __CUDACC__ +#define CUBIC_HD __host__ __device__ +#else +#define CUBIC_HD +#endif + +// interpax method="cubic": C1 Hermite interpolation, arithmetic mean of +// neighbouring secant slopes, one-sided slopes at physical endpoints. +template +CUBIC_HD double local_cubic_value(double query, int guess, const double* v, + int nv, Sample sample, double extrap) { + if (query < v[0] || query > v[nv - 1]) return extrap; + int lo = guess < 0 ? 0 : (guess >= nv - 1 ? nv - 2 : guess); + while (lo > 0 && query < v[lo]) --lo; + while (lo < nv - 2 && query >= v[lo + 1]) ++lo; + double f0 = sample(lo), f1 = sample(lo + 1), h = v[lo + 1] - v[lo]; + double secant = (f1 - f0) / h; + double d0 = lo == 0 ? secant : + 0.5 * ((f0 - sample(lo - 1)) / (v[lo] - v[lo - 1]) + secant); + double d1 = lo + 1 == nv - 1 ? secant : + 0.5 * (secant + (sample(lo + 2) - f1) / (v[lo + 2] - v[lo + 1])); + double t = (query - v[lo]) / h; + double s0 = d0 * h, s1 = d1 * h; + double c3 = 2.0 * f0 - 2.0 * f1 + s0 + s1; + double c2 = -3.0 * f0 + 3.0 * f1 - 2.0 * s0 - s1; + return ((c3 * t + c2) * t + s0) * t + f0; +} diff --git a/adept/_vlasov1d/native/core/solver.hpp b/adept/_vlasov1d/native/core/solver.hpp new file mode 100644 index 00000000..91b7055e --- /dev/null +++ b/adept/_vlasov1d/native/core/solver.hpp @@ -0,0 +1,326 @@ +#pragma once +#include "backend.hpp" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace adept_native { +void mpi_check(int error) { + if (error == MPI_SUCCESS) return; + char message[MPI_MAX_ERROR_STRING]; + int length = 0; + MPI_Error_string(error, message, &length); + throw std::runtime_error(std::string(message, length)); +} +struct Options { + Grid grid; + int steps = 20, warmup = 3; + double dt = 0.1, acceleration = 0.2, nu = 0; + bool host_mpi = false; + std::string mode = "landau", dump, integrator = "strang", input, coordinates, background; +}; +int integer(const std::string& text) { + std::size_t end; + int result = std::stoi(text, &end); + if (end != text.size()) throw std::runtime_error("invalid integer: " + text); + return result; +} +double real(const std::string& text) { + std::size_t end; + double result = std::stod(text, &end); + if (end != text.size() || !std::isfinite(result)) throw std::runtime_error("invalid real: " + text); + return result; +} +Options parse(int argc, char** argv, int rank) { + Options o; + for (int i = 1; i < argc; ++i) { + std::string key = argv[i]; + if (key == "--host-mpi") { o.host_mpi = true; continue; } + if (key == "--cells") { o.grid.cell_centred = true; continue; } + if (key == "--help") { + if (rank == 0) std::cout + << "vlasov_bench [--nx N] [--nv N] [--steps N] [--warmup N] [--dt D]\n" + << " [--mode landau|stream|accelerate|transpose] [--acceleration A]\n" + << " [--length L] [--vmax V] [--amplitude A] [--host-mpi] [--dump FILE]\n"; + if (rank == 0) std::cout << " [--velocity natural|cubic] [--layout transpose|halos] [--halo-capacity N]\n" + << " [--integrator strang|sixth] [--cells] [--input FILE] [--v-grid FILE] [--background FILE]\n"; + if (rank == 0) std::cout << " [--nu D] (Chang-Cooper Dougherty; requires --cells)\n"; + // All ranks take the same path; finalize normally in main. + o.mode = "help"; + return o; + } + if (i + 1 == argc) throw std::runtime_error("missing value for " + key); + std::string value = argv[++i]; + if (key == "--nx") o.grid.nx = integer(value); + else if (key == "--nv") o.grid.nv = integer(value); + else if (key == "--steps") o.steps = integer(value); + else if (key == "--warmup") o.warmup = integer(value); + else if (key == "--dt") o.dt = real(value); + else if (key == "--acceleration") o.acceleration = real(value); + else if (key == "--nu") o.nu = real(value); + else if (key == "--length") o.grid.length = real(value); + else if (key == "--vmax") o.grid.vmax = real(value); + else if (key == "--amplitude") o.grid.amplitude = real(value); + else if (key == "--mode") o.mode = value; + else if (key == "--integrator") o.integrator = value; + else if (key == "--input") o.input = value; + else if (key == "--v-grid") o.coordinates = value; + else if (key == "--background") o.background = value; + else if (key == "--halo-capacity") o.grid.halo_capacity = integer(value); + else if (key == "--velocity") { + if (value != "natural" && value != "cubic") throw std::runtime_error("unknown velocity method"); + o.grid.local_cubic = value == "cubic"; + } else if (key == "--layout") { + if (value != "transpose" && value != "halos") throw std::runtime_error("unknown layout"); + o.grid.halos = value == "halos"; + } + else if (key == "--dump") o.dump = value; + else throw std::runtime_error("unknown option: " + key); + } + return o; +} +// Times include explicit device completion, with no computation/communication +// overlap. Reduce each rank's accumulated component times with MPI_MAX later. +enum Stage { Stream, Spline, Pack, Exchange, Field, Collision, Total, StageCount }; +class Stepper { +public: + Stepper(Backend& backend, const Options& opts) : b(backend), o(opts) {} + void collision_step() { + if (o.nu == 0) return; + // One transpose pair remains for the implicit collision solve; the + // six Vlasov kicks use halos. Count its communication in MPI/pack times. + redistribute(true); + timed(Collision, [&] { b.collide(o.dt, o.nu); }); + redistribute(false); + } + void velocity_step(double dt, bool self_consistent) { + if (!o.grid.halos) { + redistribute(true); + timed(Spline, [&] { b.velocity(dt, o.acceleration, self_consistent); }); + redistribute(false); + return; + } + timed(Exchange, [&] { + double maximum = self_consistent ? b.max_field() : std::abs(o.acceleration); + double distance = maximum * std::abs(dt) / o.grid.dv(); + if (!std::isfinite(distance) || distance > o.grid.halo_capacity - 2) + throw std::runtime_error("departure point exceeds halo capacity; increase --halo-capacity"); + int width = int(std::ceil(distance)) + 2, global_width; + mpi_check(MPI_Allreduce(&width, &global_width, 1, MPI_INT, MPI_MAX, MPI_COMM_WORLD)); + width = global_width; + max_halo_width = std::max(max_halo_width, width); + exchange_halos(width); + }); + timed(Spline, [&] { b.local_velocity(dt, o.acceleration, self_consistent); }); + } + void exchange_halos(int width) { + const Grid& g = o.grid; + if (g.ranks == 1) return; + const int start = g.rank * g.local_v(), stop = start + g.local_v(); + std::vector requests; + std::vector host_send, host_recv; + double* send = b.velocity_buffer(); + double* recv = b.halo_buffer(); + if (o.host_mpi) { + host_send = b.download(); + host_recv.resize(2 * std::size_t(g.halo_capacity) * g.nx); + send = host_send.data(); recv = host_recv.data(); + } + b.synchronize(); + auto post = [&](bool receiving, int peer, int count, double* ptr) { + requests.push_back(MPI_REQUEST_NULL); + if (receiving) mpi_check(MPI_Irecv(ptr, count, MPI_DOUBLE, peer, 713, MPI_COMM_WORLD, &requests.back())); + else mpi_check(MPI_Isend(ptr, count, MPI_DOUBLE, peer, 713, MPI_COMM_WORLD, &requests.back())); + }; + for (int peer = 0; peer < g.ranks; ++peer) { + if (peer == g.rank) continue; + int ps = peer * g.local_v(), pe = ps + g.local_v(); + int lo = std::max(ps, peer < g.rank ? start - width : stop); + int hi = std::min(pe, peer < g.rank ? start : stop + width); + if (hi > lo) { + std::size_t offset = std::size_t(g.halo_capacity + lo - (peer < g.rank ? start : stop)) * g.nx; + post(true, peer, (hi - lo) * g.nx, recv + offset); + } + lo = std::max(start, g.rank < peer ? ps - width : pe); + hi = std::min(stop, g.rank < peer ? ps : pe + width); + if (hi > lo) { + int count = (hi - lo) * g.nx; + post(false, peer, count, send + std::size_t(lo - start) * g.nx); + halo_sent_bytes += std::uint64_t(count) * sizeof(double); + } + } + mpi_check(MPI_Waitall(int(requests.size()), requests.data(), MPI_STATUSES_IGNORE)); + if (o.host_mpi) b.upload_halos(host_recv); + } + template void timed(Stage stage, F fn) { + b.synchronize(); + double start = MPI_Wtime(); + fn(); + b.synchronize(); + times[stage] += MPI_Wtime() - start; + } + void redistribute(bool to_x) { + if (o.grid.ranks == 1) { b.local_layout(); return; } + timed(Pack, [&] { b.pack(to_x); }); + int count = o.grid.count() / o.grid.ranks; + timed(Exchange, [&] { + slab_sent_bytes += std::uint64_t(count) * (o.grid.ranks - 1) * sizeof(double); + if (o.host_mpi) { + auto send = b.download_send(); + std::vector recv(send.size()); + mpi_check(MPI_Alltoall(send.data(), count, MPI_DOUBLE, recv.data(), count, + MPI_DOUBLE, MPI_COMM_WORLD)); + b.upload_received(recv); + } else { + mpi_check(MPI_Alltoall(b.send_buffer(), count, MPI_DOUBLE, b.recv_buffer(), + count, MPI_DOUBLE, MPI_COMM_WORLD)); + } + }); + timed(Pack, [&] { b.unpack(to_x); }); + } + void solve_field() { + b.density(); + b.synchronize(); + if (o.host_mpi) { + auto density = b.download_density(); + std::vector global(density.size()); + mpi_check(MPI_Allreduce(density.data(), global.data(), o.grid.nx, + MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD)); + b.upload_density(global); + } else { + mpi_check(MPI_Allreduce(b.density_buffer(), b.reduced_density_buffer(), + o.grid.nx, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD)); + } + b.field(); + } + void step() { + b.synchronize(); + double start = MPI_Wtime(); + if (o.mode == "stream") timed(Stream, [&] { b.stream(o.dt); }); + else if (o.mode == "transpose") { redistribute(true); redistribute(false); } + else if (o.mode == "collide") collision_step(); + else { + if (o.mode == "landau" && o.integrator == "sixth") { + // Same force-gradient Hamiltonian coefficients and ordering as + // ADEPT SixthOrderHamIntegrator, including its dt corrections. + const double a[] = {0.168735950563437422448196, 0.377851589220928303880766, + -0.093175079568731452657924, 0.377851589220928303880766, 0.168735950563437422448196}; + double dt2 = o.dt * o.dt, dt4 = dt2 * dt2, dt6 = dt4 * dt2; + double d1 = 0.049086460976116245491441 - 2 * 0.000069728715055305084099 * dt2; + double d2 = 0.264177609888976700200146 - 2 * 0.000625704827430047189169 * dt2 + - 4 * 2.916600457689847816445691e-6 * dt4; + double d3 = 0.186735929134907054308413 - 2 * 0.002213085124045325561636 * dt2 + + 4 * 3.048480261700038788680723e-5 * dt4 - 8 * 4.985549387875068121593988e-7 * dt6; + double d[] = {d1, d2, d3, d3, d2, d1}; + for (int k = 0; k < 6; ++k) { + timed(Field, [&] { solve_field(); }); + velocity_step(d[k] * o.dt, true); + if (k < 5) timed(Stream, [&] { b.stream(a[k] * o.dt); }); + } + collision_step(); + b.synchronize(); + times[Total] += MPI_Wtime() - start; + return; + } + if (o.mode == "landau") { + timed(Stream, [&] { b.stream(0.5 * o.dt); }); + timed(Field, [&] { solve_field(); }); + } + velocity_step(o.dt, o.mode == "landau"); + if (o.mode == "landau") timed(Stream, [&] { b.stream(0.5 * o.dt); }); + collision_step(); + } + b.synchronize(); + times[Total] += MPI_Wtime() - start; + } + std::array times{}; + std::uint64_t halo_sent_bytes = 0; + std::uint64_t slab_sent_bytes = 0; + int max_halo_width = 0; +private: + Backend& b; + const Options& o; +}; +struct Diagnostics { double mass, kinetic, field, minimum; bool finite; }; +Diagnostics diagnostics(Backend& b, Stepper& stepper, const Grid& g) { + auto f = b.download(); + double mass = 0, kinetic = 0, minimum = std::numeric_limits::infinity(); + int finite = 1; + for (std::size_t i = 0; i < f.size(); ++i) { + int j = i / g.nx + g.rank * g.local_v(); + double v = g.velocity(j); + double weight = (!g.cell_centred && (j == 0 || j == g.nv - 1) ? 0.5 : 1.0) * g.dv() * g.length / g.nx; + mass += f[i] * weight; + kinetic += 0.5 * v * v * f[i] * weight; + minimum = std::min(minimum, f[i]); + finite &= std::isfinite(f[i]); + } + double local[] = {mass, kinetic}, global[2]; + mpi_check(MPI_Allreduce(local, global, 2, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD)); + double global_min; + int global_finite; + mpi_check(MPI_Allreduce(&minimum, &global_min, 1, MPI_DOUBLE, MPI_MIN, MPI_COMM_WORLD)); + mpi_check(MPI_Allreduce(&finite, &global_finite, 1, MPI_INT, MPI_MIN, MPI_COMM_WORLD)); + stepper.solve_field(); + auto field = b.download_field(); + double energy = 0; + for (double e : field) energy += 0.5 * e * e * g.length / g.nx; + return {global[0], global[1], energy, global_min, bool(global_finite)}; +} +void dump(Backend& b, const Grid& g, const std::string& path) { + auto local = b.download(); + std::vector full(g.rank == 0 ? std::size_t(g.nx) * g.nv : 0); + mpi_check(MPI_Gather(local.data(), int(local.size()), MPI_DOUBLE, full.data(), + int(local.size()), MPI_DOUBLE, 0, MPI_COMM_WORLD)); + if (g.rank == 0) { + std::ofstream out(path, std::ios::binary); + out.write(reinterpret_cast(full.data()), full.size() * sizeof(double)); + if (!out) throw std::runtime_error("cannot write dump: " + path); + } +} +std::string quoted(const std::string& text) { + std::string out = "\""; + for (char c : text) { + if (c == '"' || c == '\\') out += '\\'; + if (c == '\n') out += "\\n"; + else out += c; + } + return out + '"'; +} +std::vector read_binary(const std::string& path, std::size_t count) { + std::vector values(count); + std::ifstream in(path, std::ios::binary | std::ios::ate); + if (!in || in.tellg() != std::streamoff(count * sizeof(double))) + throw std::runtime_error("wrong binary size: " + path); + in.seekg(0); + in.read(reinterpret_cast(values.data()), count * sizeof(double)); + if (!in) throw std::runtime_error("cannot read " + path); + return values; +} +void initialize(Backend& backend, const Options& o) { + if (o.input.empty()) { backend.initialize(); return; } + MPI_File file; + mpi_check(MPI_File_open(MPI_COMM_WORLD, o.input.c_str(), MPI_MODE_RDONLY, MPI_INFO_NULL, &file)); + MPI_Offset bytes; + mpi_check(MPI_File_get_size(file, &bytes)); + if (bytes != MPI_Offset(o.grid.nx) * o.grid.nv * sizeof(double)) + throw std::runtime_error("input state must be raw float64 with shape (nv,nx)"); + std::vector values(o.grid.count()); + MPI_Offset offset = MPI_Offset(o.grid.rank) * values.size() * sizeof(double); + mpi_check(MPI_File_read_at_all(file, offset, values.data(), int(values.size()), MPI_DOUBLE, MPI_STATUS_IGNORE)); + mpi_check(MPI_File_close(&file)); + backend.upload_state(values); +} +} // namespace adept_native + diff --git a/adept/_vlasov1d/native/module.py b/adept/_vlasov1d/native/module.py new file mode 100644 index 00000000..df51550f --- /dev/null +++ b/adept/_vlasov1d/native/module.py @@ -0,0 +1,375 @@ +"""Native execution of existing ADEPT modules with their original save functions.""" + +import gc +import hashlib +import json +import os +import shutil +import signal +import subprocess +import sys +import tempfile +from contextlib import contextmanager +from functools import cache, lru_cache +from pathlib import Path + +import jax +import numpy as np +from diffrax import RESULTS, Solution, SubSaveAt + +from adept._vlasov1d.datamodel import NativeSolverConfig +from adept._vlasov1d.modules import BaseVlasov1D + + +@contextmanager +def _host_preprocessing(): + with jax.default_device(jax.devices("cpu")[0]): + yield + + +def _restore_array(host): + """Recreate callback constants as replicated JAX arrays on the worker mesh.""" + from jax.sharding import Mesh, NamedSharding, PartitionSpec + + sharding = NamedSharding(Mesh(np.array(jax.devices()), ("device",)), PartitionSpec()) + return jax.make_array_from_callback(host.shape, sharding, lambda index: host[index]) + + +def _host_pickle(payload, stream): + import cloudpickle + + class HostPickler(cloudpickle.CloudPickler): + def reducer_override(self, value): + if isinstance(value, jax.Array): + return _restore_array, (np.asarray(value),) + return super().reducer_override(value) + + HostPickler(stream).dump(payload) + + +def _write_vx(path, values): + """Export a host x-v state with bounded transpose temporaries.""" + output = np.memmap(path, mode="w+", dtype=np.float64, shape=values.shape[::-1]) + for start in range(0, values.shape[1], 64): + output[start : start + 64] = np.asarray(values[:, start : start + 64]).T + output.flush() + del output + + +def _collect(root, times, ranks): + import cloudpickle + + with (root / "layout.pkl").open("rb") as stream: + layout = cloudpickle.load(stream) + indices = [json.loads((root / f"rank-{rank:05d}/index.json").read_text()) for rank in range(ranks)] + result = {} + directory = root / "collected" + directory.mkdir() + for key_id, (key, definition) in enumerate(layout.items()): + arrays = [] + for leaf_id, leaf in enumerate(definition["leaves"]): + shape = (len(times[key]), *leaf["shape"]) + parts = [index[key][leaf_id] for index in indices if index[key][leaf_id] is not None] + if len(parts) == 1 and all( + start == 0 and stop == size and step == 1 + for (start, stop, step), size in zip(parts[0]["index"], leaf["shape"], strict=True) + ): + array = np.load(root / parts[0]["file"], mmap_mode="r") + else: + array = np.lib.format.open_memmap( + directory / f"{key_id:04d}-{leaf_id:03d}.npy", mode="w+", dtype=np.dtype(leaf["dtype"]), shape=shape + ) + for part in parts: + local = np.load(root / part["file"], mmap_mode="r") + slices = tuple(slice(*axis) for axis in part["index"]) + # Bound host copies for large saved distributions. + for frame in range(len(times[key])): + array[(frame, *slices)] = local[frame] + array.flush() + if array.shape != shape: + raise RuntimeError(f"native save shape mismatch for {key}") + arrays.append(array) + result[key] = jax.tree.unflatten(definition["tree"], arrays) + return result + + +class NativeVlasovMixin: + """Swap execution while retaining a Vlasov module's lifecycle and callbacks.""" + + jit_compile = False + + def __init__(self, cfg): + with _host_preprocessing(): + super().__init__(cfg) + self.native_options = NativeSolverConfig.model_validate(cfg.get("native") or {}) + self._validate_native_physics() + + def _validate_native_physics(self): + cfg, sim = self.cfg, self.simulation + if ( + len(sim.species) != 1 + or sim.species[0].name != "electron" + or sim.species[0].charge != -1 + or sim.species[0].mass != 1 + ): + raise NotImplementedError("native Vlasov currently supports one electron species with charge=-1, mass=1") + species = sim.species[0] + if not np.isclose(species.vmin, -species.vmax, rtol=0, atol=1e-12): + raise NotImplementedError("native Vlasov requires a symmetric velocity domain") + terms = cfg["terms"] + if terms["time"] != "sixth" or terms["edfdv"] != "cubic-spline" or terms["field"] != "poisson": + raise NotImplementedError( + "native Vlasov requires sixth-order time, cubic-spline velocity, and Poisson fields" + ) + if terms.get("vdfdx", "exponential") != "exponential": + raise NotImplementedError("native Vlasov requires exponential spatial streaming") + if ( + any(cfg["drivers"][key] for key in ["ex", "ey"]) + or terms["krook"]["is_on"] + or terms.get("hou_li_filter", {}).get("is_on", False) + ): + raise NotImplementedError("native Vlasov does not support external drivers, Krook relaxation, or filtering") + if any(cfg.get("diagnostics", {}).values()): + raise NotImplementedError("native Vlasov does not yet evolve df/dt diagnostic state") + fp = terms["fokker_planck"] + if fp["is_on"]: + if fp["type"] != "chang_cooper_dougherty" or fp.get("self_consistent_beta", {}).get("enabled", False): + raise NotImplementedError("native collisions require Chang-Cooper Dougherty with analytic beta") + for axis in ["time", "space"]: + envelope = fp[axis] + if envelope.get("bump_height", 0) != 0 or envelope.get("slope", 0) != 0: + raise NotImplementedError("native collisions currently require constant frequency profiles") + + def write_units(self): + with _host_preprocessing(): + return super().write_units() + + def get_derived_quantities(self): + with _host_preprocessing(): + return super().get_derived_quantities() + + def get_solver_quantities(self): + with _host_preprocessing(): + return super().get_solver_quantities() + + def init_state_and_args(self): + with _host_preprocessing(): + return super().init_state_and_args() + + def init_diffeqsolve(self): + with _host_preprocessing(): + super().init_diffeqsolve() + # Standard outputs use native steps, avoiding full-state copies for + # Diffrax's off-step interpolation. Keep sparse subclass callback times + # unless the caller opts into step rounding for those too; some existing + # project postprocessors reconstruct those axes from their own config. + t0, t1 = (float(self.time_quantities[key]) for key in ["t0", "t1"]) + dt = self.simulation.grid.dt + steps = round((t1 - t0) / dt) + subs = self.diffeqsolve_quants["saveat"]["subs"] + for key, save in list(subs.items()): + standard = key in self.cfg["save"] + if key == "default": + indices = np.arange(0, steps + 1, self.native_options.scalar_stride, dtype=int) + indices = np.union1d(indices, [steps]) + elif standard or self.native_options.custom_save_times == "steps": + requested = np.asarray(save.ts) + if ( + requested.size == 0 + or np.any(np.diff(requested) < 0) + or requested.min() < t0 - 1e-9 + or requested.max() > t1 + 1e-9 + ): + raise ValueError("native save times must be ordered and inside the solve interval") + indices = np.unique(np.clip(np.rint((requested - t0) / dt).astype(int), 0, steps)) + else: + continue + times = t0 + indices * dt + subs[key] = SubSaveAt(ts=times, fn=save.fn) + if standard: + self.cfg["save"][key]["t"].update(ax=times, nt=len(times), tmin=float(times[0]), tmax=float(times[-1])) + + def __call__(self, trainable_modules=None, args=None): + if trainable_modules: + raise NotImplementedError("native Vlasov does not execute trainable modules") + if args is not None and args is not self.args: + raise NotImplementedError("native Vlasov currently uses the configured static arguments") + options = self.native_options + library = options.library or os.environ.get("ADEPT_VLASOV_LIBRARY") + if not library or not Path(library).expanduser().is_file(): + raise FileNotFoundError( + "Build adept/_vlasov1d/native with CMake and set native.library or ADEPT_VLASOV_LIBRARY" + ) + library = str(Path(library).expanduser().resolve()) + grid = self.simulation.grid + t0, t1 = (float(self.time_quantities[key]) for key in ["t0", "t1"]) + steps = round((t1 - t0) / grid.dt) + if steps < 1 or not np.isclose(steps * grid.dt, t1 - t0, rtol=0, atol=1e-9): + raise NotImplementedError("native solve endpoints must be separated by an integer number of timesteps") + if grid.nx % options.ranks or self.simulation.species[0].nv % options.ranks: + raise ValueError("native ranks must divide both nx and nv") + known = {"electron", "e", "de", "a", "da", "prev_a"} + if set(self.state) - known or any(np.any(np.asarray(self.state[key])) for key in ["de", "a", "da", "prev_a"]): + raise NotImplementedError( + "native Vlasov requires electrostatic state without additional evolved diagnostics" + ) + subs = self.diffeqsolve_quants["saveat"]["subs"] + times = {key: np.asarray(save.ts, dtype=float).tolist() for key, save in subs.items()} + if any( + not values or min(values) < t0 - 1e-9 or max(values) > t1 + 1e-9 or np.any(np.diff(values) < 0) + for values in times.values() + ): + raise ValueError("native save times must be ordered and inside the solve interval") + base = options.workdir or os.environ.get("BASE_TEMPDIR") + if options.ranks > 1 and not base: + raise ValueError("set native.workdir to a shared directory for MPI workers") + if base: + base = os.path.expandvars(os.path.expanduser(base)) + if "$" in base: + raise ValueError("native.workdir contains an undefined environment variable") + Path(base).mkdir(parents=True, exist_ok=True) + workspace = tempfile.TemporaryDirectory(prefix="adept-native-", dir=base) + root = Path(workspace.name) + # Retain backing files for the NumPy memmaps returned in solver result. + self._native_workspaces = getattr(self, "_native_workspaces", []) + [workspace] + _write_vx(root / "initial.bin", self.state["electron"]) + species_grid = self.cfg["grid"]["species_grids"]["electron"] + for name, values in [ + ("v.bin", species_grid["v"]), + ("background.bin", self.cfg["grid"]["ion_charge"]), + ("field.bin", self.state["e"]), + ]: + np.asarray(values, dtype=np.float64).tofile(root / name) + # Initializer cache is not used by save callbacks. Avoid serializing a + # second full distribution through a closure over the grid config. + cache = self.cfg["grid"].pop("species_distributions", None) + try: + with (root / "callbacks.pkl").open("wb") as stream: + _host_pickle( + dict( + callbacks={key: save.fn for key, save in subs.items()}, + args=self.args, + state={key: value for key, value in self.state.items() if key not in ["electron", "e"]}, + ), + stream, + ) + finally: + if cache is not None: + self.cfg["grid"]["species_distributions"] = cache + fp = self.cfg["terms"]["fokker_planck"] + nu = float(fp["time"]["baseline"]) * float(fp["space"]["baseline"]) if fp["is_on"] else 0.0 + native_args = [ + "--nx", + str(grid.nx), + "--nv", + str(species_grid["nv"]), + "--length", + str(grid.xmax - grid.xmin), + "--vmax", + str(species_grid["vmax"]), + "--dt", + str(grid.dt), + "--nu", + str(nu), + "--cells", + "--integrator", + "sixth", + "--velocity", + "cubic", + "--layout", + "halos", + "--halo-capacity", + str(options.halo_capacity), + "--input", + str(root / "initial.bin"), + "--v-grid", + str(root / "v.bin"), + "--background", + str(root / "background.bin"), + ] + if options.host_mpi: + native_args.append("--host-mpi") + request = dict( + library=library, + options=native_args, + times=times, + t0=t0, + dt=grid.dt, + steps=steps, + device=options.device, + ranks=options.ranks, + pythonpath=[str(Path(entry or os.getcwd()).resolve()) for entry in sys.path], + ) + (root / "request.json").write_text(json.dumps(request, indent=2)) + launcher = [part.format(ranks=options.ranks) for part in options.launcher] + if options.ranks > 1 and not launcher: + launcher = ["mpiexec", "-n", str(options.ranks)] + command = [*launcher, sys.executable, str(Path(__file__).with_name("worker.py")), str(root / "request.json")] + environment = dict(os.environ, XLA_PYTHON_CLIENT_PREALLOCATE="false") + environment.setdefault("MPICH_GPU_SUPPORT_ENABLED", "1") + log = root / "worker.log" + with log.open("w") as stream: + process = subprocess.Popen( + command, stdout=stream, stderr=subprocess.STDOUT, env=environment, start_new_session=True + ) + try: + returncode = process.wait(timeout=options.timeout) + except BaseException as error: + os.killpg(process.pid, signal.SIGTERM) + try: + process.wait(timeout=10) + except subprocess.TimeoutExpired: + os.killpg(process.pid, signal.SIGKILL) + process.wait() + raise RuntimeError( + f"native Vlasov workers interrupted; log: {log}\n{log.read_text()[-12000:]}" + ) from error + if returncode: + raise RuntimeError(f"native Vlasov workers failed ({returncode}); log: {log}\n{log.read_text()[-12000:]}") + ys = _collect(root, times, options.ranks) + stats = json.loads((root / "stats.json").read_text()) + result = Solution( + t0=t0, + t1=t1, + ts={key: np.asarray(value) for key, value in times.items()}, + ys=ys, + stats={"num_steps": steps, "num_accepted_steps": steps, "num_rejected_steps": 0}, + result=RESULTS.successful, + interpolation=None, + solver_state=None, + controller_state=None, + made_jump=None, + event_mask=None, + ) + gc.collect() + return {"solver result": result, "native stats": stats, "native workdir": str(root)} + + def post_process(self, run_output, td): + source = Path(run_output["native workdir"]) + directory = Path(td) / "native" + directory.mkdir(exist_ok=True) + for name in ["request.json", "stats.json", "worker.log"]: + shutil.copyfile(source / name, directory / name) + request = json.loads((source / "request.json").read_text()) + library = Path(request["library"]) + (directory / "library.sha256").write_text( + f"{hashlib.sha256(library.read_bytes()).hexdigest()} {library.name}\n" + ) + with _host_preprocessing(): + return super().post_process(run_output, td) + + def vg(self, trainable_modules, args): + raise NotImplementedError("the native Vlasov backend does not provide automatic differentiation") + + +@cache +def native_backend(module_class): + """Adapt a BaseVlasov1D subclass without changing its pre/postprocessing.""" + if not isinstance(module_class, type) or not issubclass(module_class, BaseVlasov1D): + raise TypeError("the native backend requires a BaseVlasov1D subclass") + if issubclass(module_class, NativeVlasovMixin): + return module_class + return type(f"Native{module_class.__name__}", (NativeVlasovMixin, module_class), {}) + + +NativeVlasov1D = native_backend(BaseVlasov1D) diff --git a/adept/_vlasov1d/native/worker.py b/adept/_vlasov1d/native/worker.py new file mode 100644 index 00000000..d4c6263d --- /dev/null +++ b/adept/_vlasov1d/native/worker.py @@ -0,0 +1,201 @@ +"""Internal MPI worker. Run by filename so JAX initializes before importing ADEPT.""" + +import argparse +import gc +import importlib.metadata +import json +import os +import socket +import sys +import time +import traceback +from pathlib import Path + +from bridge import NativeCore + + +def run(request): + import numpy as np + + root = request.parent + spec = json.loads(request.read_text()) + os.environ["JAX_PLATFORMS"] = "cpu" if spec["device"] == "cpu" else "cuda,cpu" + os.environ.setdefault("XLA_PYTHON_CLIENT_PREALLOCATE", "false") + core = NativeCore(spec["library"], spec["options"]) + try: + if core.size != spec["ranks"]: + raise RuntimeError("MPI launcher rank count differs from native.ranks") + if core.device_type != (2 if spec["device"] == "cuda" else 1): + raise RuntimeError("native library was built for a different device") + core.set_field(root / "field.bin") + import jax + + jax.config.update("jax_enable_x64", True) + if core.size > 1: + coordinator = "" + if core.rank == 0: + with socket.socket() as connection: + connection.bind(("", 0)) + coordinator = f"{socket.gethostname()}:{connection.getsockname()[1]}" + coordinator = core.broadcast(coordinator) + jax.distributed.initialize( + coordinator_address=coordinator, + num_processes=core.size, + process_id=core.rank, + local_device_ids=[core.local_rank] if core.device_type == 2 else [0], + ) + elif core.device_type == 2: + jax.config.update("jax_cuda_visible_devices", str(core.local_rank)) + sys.path[:0] = spec.get("pythonpath", []) + import cloudpickle + import jax.numpy as jnp + from jax.sharding import Mesh, NamedSharding + from jax.sharding import PartitionSpec as P + + if len(jax.devices()) != core.size or len(jax.local_devices()) != 1: + raise RuntimeError("native workers require exactly one JAX device per MPI rank") + mesh = Mesh(np.array(jax.devices()), ("device",)) + slab = NamedSharding(mesh, P(None, "device")) + replicated = NamedSharding(mesh, P()) + with (root / "callbacks.pkl").open("rb") as stream: + payload = cloudpickle.load(stream) + + def replicate(value): + host = np.asarray(value) + return jax.make_array_from_callback(host.shape, replicated, lambda index: host[index]) + + static = jax.tree.map(replicate, payload["state"]) + callbacks = { + key: jax.jit(lambda t, y, fn=fn: fn(t, y, payload["args"])) for key, fn in payload["callbacks"].items() + } + + def snapshot(): + # FFTW may align host allocations to 32 bytes; JAX CPU requires 64. + # CUDA allocations satisfy JAX's alignment, so GPU views stay strict. + copy = False if core.device_type == 2 else None + f = jax.dlpack.from_dlpack(core.view(), copy=copy) + e = jax.dlpack.from_dlpack(core.view(field=True), copy=copy) + return dict( + static, + electron=jax.make_array_from_single_device_arrays((core.nx, core.nv), slab, [f]), + e=jax.make_array_from_single_device_arrays((core.nx,), replicated, [e]), + ) + + directory = root / f"rank-{core.rank:05d}" + directory.mkdir() + writers, metadata, layout = {}, {}, {} + + def save(key, frame, t, state): + value = callbacks[key](replicate(np.asarray(t)), state) + jax.block_until_ready(value) + leaves, tree = jax.tree.flatten(value) + if key not in layout: + layout[key] = {"tree": tree, "leaves": []} + metadata[key] = [] + for i, leaf in enumerate(leaves): + layout[key]["leaves"].append(dict(shape=leaf.shape, dtype=str(leaf.dtype))) + if leaf.is_fully_replicated and core.rank != 0: + metadata[key].append(None) + continue + shards = leaf.addressable_shards + if len(shards) != 1: + raise RuntimeError("unexpected native save sharding") + shard = shards[0] + index = [ + [part.start or 0, part.stop if part.stop is not None else leaf.shape[d], part.step or 1] + for d, part in enumerate(shard.index) + ] + filename = f"{len(layout) - 1:04d}-{i:03d}.npy" + shape = (len(spec["times"][key]), *shard.data.shape) + writers[key, i] = np.lib.format.open_memmap( + directory / filename, mode="w+", dtype=np.dtype(leaf.dtype), shape=shape + ) + metadata[key].append(dict(file=str((directory / filename).relative_to(root)), index=index)) + for i, leaf in enumerate(leaves): + if (key, i) in writers: + (shard,) = leaf.addressable_shards + expected = metadata[key][i]["index"] + index = [ + [part.start or 0, part.stop if part.stop is not None else leaf.shape[d], part.step or 1] + for d, part in enumerate(shard.index) + ] + if index != expected: + raise RuntimeError("save output sharding changed between frames") + writers[key, i][frame] = np.asarray(shard.data) + # No device save output survives a native mutation, including identity saves. + del leaves, value + + requests = {} + for key, times in spec["times"].items(): + for frame, t in enumerate(times): + position = (t - spec["t0"]) / spec["dt"] + nearest = round(position) + if abs(position - nearest) <= 1e-9: + position = float(nearest) + upper = int(np.ceil(position)) + requests.setdefault(upper, []).append((key, frame, t, position - (upper - 1))) + current = 0 + started = time.perf_counter() + for upper, saves in sorted(requests.items()): + off_grid = any(0 < alpha < 1 for _, _, _, alpha in saves) + previous = None + if off_grid: + core.step(upper - 1 - current) + source = snapshot() + previous = jax.tree.map(lambda value: jnp.array(value, copy=True), source) + jax.block_until_ready(previous) + del source + core.step(1) + else: + core.step(upper - current) + current = upper + state = snapshot() + for key, frame, t, alpha in saves: + if 0 < alpha < 1: + sample = jax.tree.map(lambda a, b, alpha=alpha: a + (b - a) * alpha, previous, state) + save(key, frame, t, sample) + del sample + else: + save(key, frame, t, state) + del state, previous + core.barrier() + core.step(spec["steps"] - current) + for writer in writers.values(): + writer.flush() + (directory / "index.json").write_text(json.dumps(metadata)) + if core.rank == 0: + with (root / "layout.pkl").open("wb") as stream: + cloudpickle.dump(layout, stream) + stats = core.stats() + if core.rank == 0: + (root / "stats.json").write_text( + json.dumps( + dict( + native_seconds=stats, + solve_and_save_seconds=time.perf_counter() - started, + steps=spec["steps"], + ranks=core.size, + device=spec["device"], + libraries={ + name: importlib.metadata.version(name) + for name in ["jax", "jaxlib", "cloudpickle", "diffrax"] + }, + ), + indent=2, + ) + ) + core.barrier() + gc.collect() + core.close() + if core.size > 1: + jax.distributed.shutdown() + except BaseException: + traceback.print_exc() + core.lib.av_abort(1) + raise + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("request", type=Path) + run(parser.parse_args().request) diff --git a/adept/vlasov1d.py b/adept/vlasov1d.py index 2a9c9c09..f72c252e 100644 --- a/adept/vlasov1d.py +++ b/adept/vlasov1d.py @@ -1,3 +1,4 @@ """Public entry point for the Vlasov-1D ADEPT module.""" from ._vlasov1d.modules import BaseVlasov1D +from ._vlasov1d.native.module import NativeVlasov1D, native_backend diff --git a/benchmarks/vlasov_cuda/CMakeLists.txt b/benchmarks/vlasov_cuda/CMakeLists.txt new file mode 100644 index 00000000..cebb6241 --- /dev/null +++ b/benchmarks/vlasov_cuda/CMakeLists.txt @@ -0,0 +1,21 @@ +cmake_minimum_required(VERSION 3.24) +project(vlasov_split_benchmark LANGUAGES CXX) + +option(VLASOV_CUDA "Build the CUDA/cuFFT backend (OFF uses FFTW for validation)" ON) +find_package(MPI REQUIRED COMPONENTS CXX) +add_executable(vlasov_bench main.cpp backend.cpp) +target_compile_features(vlasov_bench PRIVATE cxx_std_17) +target_link_libraries(vlasov_bench PRIVATE MPI::MPI_CXX) +if(VLASOV_CUDA) + enable_language(CUDA) + find_package(CUDAToolkit REQUIRED) + set_source_files_properties(backend.cpp PROPERTIES LANGUAGE CUDA) + set_target_properties(vlasov_bench PROPERTIES CUDA_STANDARD 17 CUDA_ARCHITECTURES 80) + target_compile_definitions(vlasov_bench PRIVATE VLASOV_CUDA) + target_compile_options(vlasov_bench PRIVATE $<$:--extended-lambda>) + target_link_libraries(vlasov_bench PRIVATE CUDA::cudart CUDA::cufft) +else() + find_package(PkgConfig REQUIRED) + pkg_check_modules(FFTW REQUIRED IMPORTED_TARGET fftw3) + target_link_libraries(vlasov_bench PRIVATE PkgConfig::FFTW) +endif() diff --git a/benchmarks/vlasov_cuda/README.md b/benchmarks/vlasov_cuda/README.md new file mode 100644 index 00000000..d95e7268 --- /dev/null +++ b/benchmarks/vlasov_cuda/README.md @@ -0,0 +1,308 @@ +# CUDA/MPI Vlasov split-step prototype + +A standalone, FP64, 1D1V Vlasov–Poisson benchmark targeting A100 GPUs. It does +not require AD or import ADEPT. The optional FFTW backend exists for numerical +validation; it is not a threaded CPU performance implementation. + +The numerical implementation now lives in +[`adept/_vlasov1d/native`](../../adept/_vlasov1d/native/README.md). Its optional +ADEPT backend retains Python initialization, custom diagnostics, MLflow and +xarray/NetCDF output. The standalone benchmark shares its C++ core. Controlled +ADEPT/JAX versus CUDA measurements are preserved in +[`results/2026-10-24-adept`](results/2026-10-24-adept/summary.json). + +## Latest turbulence target and halo mode + +The reproduction target is the **23 October Nv4096 campaign**, not the older +L8000 cases: Nx134400 and Nx268800, Nv4096, L12600, vmax15, dt0.02, sixth-order +Hamiltonian splitting, exponential/spectral x advection, `cubic-spline` velocity +advection, and Chang-Cooper Dougherty collisions at nu1e-12. The pinned runtime +is ADEPT `ddd70b957dc3df85f0b5ec84af927c25d09d24e0` and JAX 0.10.2. The exact +member configs and source identifiers are under +[reference/latest-nv4096](reference/latest-nv4096/target.json). + +**Correction to the original benchmark's method assumption:** ADEPT's +`VelocityCubicSpline` calls interpax's default `method="cubic"`, a **local C1 +Hermite interpolant**, with slopes averaged from adjacent secants and one-sided +endpoint slopes. It does not use the global natural C2 spline implemented in +the first benchmark. Consequently, preserving the actual velocity method does +not require full velocity lines to reside on one rank. + +`--velocity cubic --layout halos` keeps the velocity-slab layout for all six +Vlasov kicks. It communicates just the velocity rows required by the departure +points and the four-point local stencil. The active width is computed from the +maximum field, kick duration, and dv, with two extra rows for the stencil. +Requests may span several neighbouring ranks. If a departure exceeds the +allocated `--halo-capacity`, the program fails explicitly. Both signed values +and the original extrapolation value `1e-30` are retained; no positivity limiter +or mass normalization changes the interpolant. + +Each rank owns a velocity slab and the complete x axis, so the batched x FFTs +stay local. The Poisson solve reduces the density array across velocity ranks. + +`--integrator sixth` uses ADEPT's actual force-gradient coefficients and dt +corrections, with its six velocity/five streaming stages. `--cells` selects the +production cell-centred velocity grid and rectangular velocity quadrature. +`--nu` enables the production Dougherty moment closure, Chang-Cooper weights, +zero-flux endpoints, and delta-form implicit solve. Its self-consistent-beta +iteration is disabled, matching the target configs. **One transpose pair per +timestep remains for collisions**; the Vlasov halo path removes the other six +pairs. Spectral x streaming is retained. + +For an exact native-state comparison, import raw float64 arrays with +`--input` (v,x order), `--v-grid`, and `--background`. Import the original ion +background; replacing its seeded density profile with a constant would change +the simulation. `checkpoint_reference.py` verifies and exports a committed +native checkpoint and advances the pinned ADEPT operators. It writes only to +its own output directory. `compare_checkpoint.py` checks full-resolution f, +Poisson fields and the latest whole-box N observer (periodic Gaussian sigma3, +potential depths/vb², strict cuts0.02 and0.5). Neither script modifies an +existing production controller or MLflow parent. + +`distributed_reference.py` advances the same verified prepared state with +eight JAX processes on two nodes. This is needed for Nx268800 on 40 GB GPUs: +the pinned JAX collision solve could not allocate its gtsv2 workspace on four. +The eight-process reference uses one GPU per rank and the production NCCL +Libfabric plugin; its provenance and network log are preserved with the results. +This reference uses all numerical operators, including nu1e-12 collisions. + +Omitting `--checkpoint` from the reference exporter uses the pinned project's +original t0 initializer, including both seeded, band-limited beams. This provides +an exact initial-state input for a future full-duration reproduction; the C++ +benchmark's default Maxwellian initializer is not the turbulence initializer. + +`validate_local.py` checks interpax C1 parity, signed values, physical endpoints, +multi-neighbour shifts, the sixth-order step and collisions on both layouts: + +```sh +python benchmarks/vlasov_cuda/validate_local.py /path/to/vlasov_bench + +# Example latest-case invocation after checkpoint_reference.py exports input: +mpiexec -n 8 /path/to/vlasov_bench --nx 134400 --nv 4096 \ + --length 12600 --vmax 15 --dt .02 --cells --integrator sixth \ + --velocity cubic --layout halos --nu 1e-12 --steps 100 --warmup 0 \ + --input /path/to/reference/initial.bin --v-grid /path/to/reference/v.bin \ + --background /path/to/reference/background.bin +``` + +Short-window agreement is an implementation check. It does not establish the +full t0–2000 decay trajectory, resolution convergence, or equivalence of another +spatial/velocity discretization. The earlier natural-spline/Strang results below +are preserved as the initial experiment, not a matched turbulence performance +comparison. Their exact source is archived in +[baseline-source.tar.gz](results/2026-10-24/baseline-source.tar.gz). + +## Results using the latest turbulence operators + +On Perlmutter A100-SXM4-40GB GPUs, the halo path gives the following medians +over three repeats, each with 10 measured steps after 3 warmup steps. Both +layouts use the same native t600 input and all operators listed above. Timings +exclude initialization, diagnostics and I/O; communication/computation do not +overlap. These compare two CUDA layouts, not CUDA against JAX. + +| Nx × Nv | Nodes | GPUs | Full transpose (ms/step) | Halos (ms/step) | Speedup | +| --- | ---: | ---: | ---: | ---: | ---: | +| 134400 × 4096 | 1 | 4 | 323.5 | 174.3 | 1.86× | +| 134400 × 4096 | 2 | 8 | 490.5 | 140.0 | 3.50× | +| 268800 × 4096 | 1 | 4 | 649.6 | 342.5 | 1.90× | +| 268800 × 4096 | 2 | 8 | 965.0 | 275.0 | 3.51× | + +Maximum per-rank distribution bytes sent fall by 85.1–85.3%. One full transpose +pair remains for collisions. On eight GPUs, MPI still takes about 60 ms on the +smaller grid and 119 ms on the larger grid, so that remaining communication +limits further strong scaling. + +All 20 native-state comparisons pass across both layouts and GPU placements. +Through 100 steps (t600→602), maximum absolute f error is `6.73e-13`, relative +L2 f error is at most `9.52e-14`, and relative L2 E error is at most `5.64e-14`. +The exact latest whole-box hole counts match at both thresholds in every case: +at t602, 129/47 for Nx134400 and 129/50 for Nx268800. Another 84 CPU/CUDA/MPI +operator and initial natural-spline regression checks pass. Conservation diagnostics +are recorded without clipping f or renormalizing mass. + +Reports, commands, runtime versions, source/executable hashes and the pinned +reference provenance are in +[results/2026-10-24-halos](results/2026-10-24-halos/summary.json). +`results/2026-10-24-halos/summarize.py` rebuilds the compact report from the +adjacent raw JSONL and comparison files. Native distributions stay on scratch; +their exact locations and manifest hashes are in the reference provenance. +The owned development allocations were released after saving the results. +This is a short-window implementation check; the full t0–2000 decay trajectory +has not been reproduced yet. + +## Initial natural-C2/Strang benchmark: 24 October 2026 + +On Perlmutter A100-SXM4-40GB GPUs, a `32768 x 8192` FP64 distribution +(2 GiB of state), with 20 timesteps after 3 warmup steps, gave the following +medians over three runs. All runs use the same global problem size. + +| Nodes | GPUs | Step (ms) | Streaming (ms) | Spline (ms) | Pack/unpack (ms) | Slab MPI (ms) | Speedup | +| ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| 1 | 1 | 68.91 | 49.02 | 17.81 | 0.00 | 0.00 | 1.00x | +| 1 | 2 | 58.70 | 24.54 | 12.88 | 6.34 | 13.78 | 1.17x | +| 1 | 4 | 36.95 | 12.33 | 10.98 | 3.17 | 9.75 | 1.87x | +| 2 | 2 | 121.00 | 24.54 | 12.88 | 6.34 | 76.10 | 0.57x | +| 2 | 4 | 65.88 | 12.32 | 10.97 | 3.17 | 38.55 | 1.05x | + +The field stage adds about 0.8–2.1 ms. Owned device allocations, including +cuFFT workspaces, are about 20 GiB on one GPU, 10 GiB per GPU on two, and +5 GiB per GPU on four. GPU runtime/MPI allocations are additional. + +The full distribution agrees with NumPy FFTs and SciPy natural splines to +less than `4e-15` maximum absolute error in 28 checks: 12 CPU/MPI checks, +12 CUDA checks on one node, and 4 CUDA checks spanning two nodes. For the +larger timing runs, maximum relative mass change is `5.1e-12` and maximum +relative total-energy change is `3.4e-7` over physical time 2.0. The smallest +f is `-2.4e-16` to the stated precision. + +This establishes a working initial implementation. It also shows that the +two full-state redistributions limit multinode strong scaling: they consume +about 59% of the two-node/four-GPU timestep. Streaming is the largest component +on a single GPU; the spline recurrence also scales less than ideally. These +measurements do not establish a performance comparison with ADEPT/JAX. + +Raw measurements, source hashes, and correctness reports are in +[results/2026-10-24](results/2026-10-24/summary.json). All final validation and +measurement Slurm steps completed with exit status `0:0`. The development +allocation was released after downloading the results. + +## Original natural-spline baseline + +The normalized model has electrons with acceleration `-E`, an immobile uniform +ion background, and `dE/dx = 1 - integral(f dv)`. The initial condition is a +discretely normalized Maxwellian times `1 + amplitude*cos(2*pi*x/length)`. +Defaults are `length=4*pi`, `vmax=8`, `amplitude=0.01`, and `dt=0.1`. + +Each Strang step performs: + +1. Half a streaming step using batched real FFTs in x and spectral phase shifts. +2. A velocity-moment MPI Allreduce and a replicated 1D FFT Poisson solve. +3. An MPI Alltoall from velocity slabs to spatial slabs. +4. A full velocity step using natural cubic splines, including coefficient + construction and evaluation at the backward characteristic. +5. An MPI Alltoall back to velocity slabs, then half a streaming step. + +Velocity slabs have shape `(nv/ranks, nx)`; spatial slabs have shape +`(nv, nx/ranks)`. Both keep x contiguous. Spline construction uses a Thomas +solve per spatial line with precomputed LU factors. Threads run across x so +memory accesses in each recurrence sweep are coalesced. The recurrence along +v is serial within each thread; this is a baseline to profile before investing +in a parallel tridiagonal solver. One-rank runs swap layout pointers and skip +the redistribution copies and MPI Alltoall. + +Distribution, packing buffers, coefficients, FFT plans, and FFT workspaces +remain allocated throughout the run. GPU-aware MPI is the default. `--host-mpi` +stages MPI buffers through host memory for comparison or systems without device +pointer support. FFT streaming and the spline solve still execute on the GPU. + +## Build + +CUDA, cuFFT, MPI, and CMake >= 3.24 are required. The CUDA target defaults to +SM80. Use the system's MPI compiler wrapper when required by its toolchain: + +```sh +cmake -S benchmarks/vlasov_cuda -B /tmp/vlasov-cuda-build \ + -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=mpicxx +cmake --build /tmp/vlasov-cuda-build -j 4 +mpiexec -n 4 /tmp/vlasov-cuda-build/vlasov_bench --nx 32768 --nv 8192 +``` + +Each local MPI rank selects its GPU by local rank before MPI initialization, +using the launcher's local-rank environment variable. This is needed because +Cray MPI initializes its GPU transport in `MPI_Init`. If the launcher exposes +just one GPU per rank, each rank uses its visible device 0. Do not launch +multiple ranks bound to the same physical GPU. + +On Perlmutter, use `CC` instead of `mpicxx`, load an available CMake module, +and set `MPICH_GPU_SUPPORT_ENABLED=1`. Expose peer GPUs with +`--gpus-per-node=4 --gpu-bind=none`; the program assigns one rank to each GPU. +The tested CUDA 13.2 / Cray MPICH 9.1 setup reported invalid CUDA IPC handles +when Slurm restricted visibility to one GPU per rank. A development-session +build is: + +```sh +~/.ergodic-agent-workflows/ops/session.sh exec -- bash -lc ' + set -euo pipefail + if [ "${SLURM_PROCID:-0}" -ne 0 ]; then exit 0; fi + module load cmake/3.30.2 + cmake -S benchmarks/vlasov_cuda -B "$EC_SESSION_WORKDIR/cuda-build" \ + -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=CC + cmake --build "$EC_SESSION_WORKDIR/cuda-build" -j 4 +' +``` + +Use an isolated development session and verify the account, modules, resource +counts, and launcher on the actual system before allocating. The benchmark +does not allocate or cancel jobs. + +## Validate + +The reference checks require NumPy >= 2 and SciPy. They compare the entire +distribution against NumPy FFTs and SciPy's natural `CubicSpline`, verify +analytic streaming, test exact redistribution, and check rank independence. +They also check mass and short-time energy error for the coupled step. + +```sh +# CPU build on a machine with FFTW and MPI: +cmake -S benchmarks/vlasov_cuda -B /tmp/vlasov-cpu-build \ + -DVLASOV_CUDA=OFF -DCMAKE_BUILD_TYPE=Release +cmake --build /tmp/vlasov-cpu-build -j 4 +python benchmarks/vlasov_cuda/validate.py /tmp/vlasov-cpu-build/vlasov_bench + +# CUDA build on an allocated GPU node: +python benchmarks/vlasov_cuda/validate.py /path/to/cuda-build/vlasov_bench \ + --launcher 'srun --jobid=JOBID --overlap --nodes=1 --ntasks={ranks} --ntasks-per-node={ranks} --gpus-per-node=4 --gpu-bind=none' +``` + +For validation across nodes, put `TMPDIR` on shared scratch so the driver can +read rank 0's dump, and use an appropriate explicit node/task layout. + +## Measure + +`measure.py` runs serial repeats of the same fixed problem on different MPI +placements. It verifies the reported placement and saves JSONL with timings, +diagnostics, commands, rank mapping, and optional source-state provenance: + +```sh +python benchmarks/vlasov_cuda/measure.py /path/to/cuda-build/vlasov_bench \ + --launcher 'srun --jobid=JOBID --overlap --nodes={nodes} --ntasks={ranks} --ntasks-per-node={ranks_per_node} --gpus-per-node=4 --gpu-bind=none' \ + --placements 1:1 1:2 1:4 2:2 2:4 --expect-cuda \ + --nx 32768 --nv 8192 --steps 20 --warmup 3 --repeats 3 \ + --output /path/to/outputs/scaling.jsonl +``` + +Run the lightweight driver on a compute node, with overlapping inner Slurm +steps if using a development-session outer step. The source directory remains +the local worktree; synchronize edits before rebuilding. Keep builds under +the session `workdir` and results under `outputs` so source sync preserves them. + +Each executable prints one JSON object. `step_ms` excludes initialization, +planning, warmup, diagnostics, downloads, and optional dumps. Warmup is followed +by reinitialization so reported physical time is exactly `steps*dt`. Component +timings explicitly synchronize the device and do not overlap communication +with computation. Times are the maximum across ranks of each rank's mean; +the component maxima need not sum to `step_ms`. `mpi_ms` covers the slab/halo +exchanges and halo-width coordination; `field_ms` includes density integration, +its Allreduce, and Poisson. `pack_ms` includes packing and unpacking for all +full-state redistributions. + +## Scope + +This is an initial solver/communication benchmark, not a replacement for ADEPT. +It supports one species, a uniform 1D1V grid and periodic x. The original natural +spline uses zero extrapolation; local cubic mode uses ADEPT's `1e-30`. Remapping +does not guarantee positivity or exact discrete conservation; no clipping or +mass correction hides that error. Keep velocity tails negligible and inspect +the reported mass, energy, and minimum f. The validation is short-time and +does not establish nonlinear or long-time fidelity. + +Both dimensions must divide evenly by the MPI rank count. Local arrays must +fit MPI's int count limit. Memory reporting includes owned device buffers and +cuFFT workspaces, but excludes CUDA/MPI runtime overhead. The current storage +uses roughly eight copies of the distribution per rank plus FFT workspace when +full-line solves are enabled. Unused transpose/coefficient buffers are omitted +for collisionless local-cubic halo runs. +Packing copies, scratch reuse, spline batching, and communication overlap are +deliberately available optimization targets once measurements identify the +limiting operation. No speedup over ADEPT/JAX is claimed without an equivalent +comparison run. diff --git a/benchmarks/vlasov_cuda/adept_timing.py b/benchmarks/vlasov_cuda/adept_timing.py new file mode 100644 index 00000000..a90b3d5d --- /dev/null +++ b/benchmarks/vlasov_cuda/adept_timing.py @@ -0,0 +1,152 @@ +"""Time pinned ADEPT operators on prepared latest-run checkpoints, without saves. + +Launch with one Python process per GPU using the pinned vp-turbulence venv. +Compilation, initial input, warmup and correctness reductions are outside timing. +""" + +import argparse +import copy +import importlib.metadata +import json +import os +import socket +import sys +import time +from pathlib import Path + +os.environ.setdefault("XLA_PYTHON_CLIENT_PREALLOCATE", "false") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--prepared", type=Path, required=True) + parser.add_argument("--config", type=Path, required=True) + parser.add_argument("--vp-source", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--steps", type=int, default=10) + parser.add_argument("--warmup", type=int, default=3) + parser.add_argument("--repeats", type=int, default=3) + args = parser.parse_args() + sys.path.insert(0, str(args.vp_source)) + import jax + + jax.config.update("jax_enable_x64", True) + jax.distributed.initialize(local_device_ids=[int(os.environ["SLURM_LOCALID"])]) + import numpy as np + import yaml + from checkpoint_reference import sha + from jax.experimental import multihost_utils as mh + from jax.sharding import Mesh, NamedSharding + from jax.sharding import PartitionSpec as P + from vp_turbulence.adeptmodules import VPTwoStream1D + from vp_turbulence.multinode import initialize_module_sharded + + rank, ranks = jax.process_index(), jax.process_count() + dist = importlib.metadata.distribution("adept") + commit = json.loads(dist.read_text("direct_url.json")).get("vcs_info", {}).get("commit_id") + if commit != "ddd70b957dc3df85f0b5ec84af927c25d09d24e0" or jax.__version__ != "0.10.2": + raise RuntimeError("incorrect pinned numerical environment") + metadata = json.loads((args.prepared / "provenance.json").read_text()) + if sha(args.config) != metadata["config_sha256"]: + raise RuntimeError("config differs from verified checkpoint") + nx, nv = metadata["nx"], metadata["nv"] + cfg = yaml.safe_load(args.config.read_text()) + module = VPTwoStream1D(copy.deepcopy(cfg)) + module.write_units() + module.get_derived_quantities() + initialize_module_sharded(module) + np.testing.assert_array_equal( + np.fromfile(args.prepared / "v.bin", dtype=np.float64), + np.asarray(module.cfg["grid"]["species_grids"]["electron"]["v"]), + ) + np.testing.assert_array_equal( + np.fromfile(args.prepared / "background.bin", dtype=np.float64), np.asarray(module.cfg["grid"]["ion_charge"]) + ) + source = np.memmap(args.prepared / "initial.bin", mode="r", dtype=np.float64, shape=(nv, nx)) + mesh = Mesh(np.array(jax.devices()), ("device",)) + sharding = NamedSharding(mesh, P("device", None)) + initial = jax.make_array_from_callback((nx, nv), sharding, lambda index: source.T[index]) + module.state = {"electron": initial} + module.init_diffeqsolve() + operator = module.diffeqsolve_quants["terms"].vector_field.vpfp + if operator.fp._sc_max_steps != 0: + raise RuntimeError("unexpected self-consistent beta") + replicated = NamedSharding(mesh, P()) + + def replicate(value): + host = np.asarray(value) + return jax.make_array_from_callback(host.shape, replicated, lambda index: host[index]) + + a, prev = replicate(np.zeros(nx + 2)), replicate(np.zeros(nx)) + external = replicate(np.zeros((6, nx))) + nu_fp, nu_k = replicate(np.full(nx, metadata["nu"])), replicate(np.zeros(nx)) + + @jax.jit + def advance(values, iterations): + def body(_, current): + _, result, _ = operator({"electron": current}, a, prev, external, nu_fp, nu_k) + return result["electron"] + + return jax.lax.fori_loop(0, iterations, body, values) + + iterations = replicate(np.asarray(args.steps, dtype=np.int32)) + start = time.perf_counter() + compiled = advance.lower(initial, iterations).compile() + compile_seconds = time.perf_counter() - start + warm = compiled(initial, replicate(np.asarray(args.warmup, dtype=np.int32))) + warm.block_until_ready() + del warm + rows = [] + for repeat in range(args.repeats): + # Reset to the same checkpoint each repeat; execution uses immutable input. + mh.sync_global_devices(f"before-{repeat}") + start = time.perf_counter() + result = compiled(initial, iterations) + result.block_until_ready() + seconds = time.perf_counter() - start + all_seconds = np.asarray(mh.process_allgather(np.asarray(seconds))) + local = np.asarray(result.addressable_shards[0].data) + if not np.isfinite(local).all(): + raise RuntimeError("nonfinite ADEPT output") + local_sums = np.asarray([local.sum(), np.square(local).sum(), local.min(), local.max()]) + sums = np.asarray(mh.process_allgather(local_sums)).reshape(ranks, 4) + if rank == 0: + row = dict( + backend="adept-jax", + nx=nx, + nv=nv, + ranks=ranks, + repeat=repeat, + steps=args.steps, + warmup=args.warmup, + step_ms=1000 * float(all_seconds.max()) / args.steps, + rank_seconds=all_seconds.tolist(), + sum=float(sums[:, 0].sum()), + sum_squares=float(sums[:, 1].sum()), + minimum=float(sums[:, 2].min()), + maximum=float(sums[:, 3].max()), + ) + rows.append(row) + print(json.dumps(row), flush=True) + del result, local + hosts = np.asarray( + mh.process_allgather(np.asarray(list(socket.gethostname().encode().ljust(64, b"\0")), dtype=np.uint8)) + ) + if rank == 0: + report = dict( + rows=rows, + checkpoint=metadata, + adept_commit=commit, + libraries={name: importlib.metadata.version(name) for name in ["jax", "jaxlib", "interpax", "diffrax"]}, + hosts=[bytes(host).rstrip(b"\0").decode() for host in hosts.reshape(ranks, 64)], + compile_seconds=compile_seconds, + devices=[str(d) for d in jax.devices()], + scope="Warmed operator loop; no SaveAt, diagnostics, I/O, initialization or compilation in timings", + ) + with args.output.open("x") as stream: + json.dump(report, stream, indent=2) + jax.distributed.shutdown() + + +if __name__ == "__main__": + main() diff --git a/benchmarks/vlasov_cuda/backend.cpp b/benchmarks/vlasov_cuda/backend.cpp new file mode 100644 index 00000000..293d6790 --- /dev/null +++ b/benchmarks/vlasov_cuda/backend.cpp @@ -0,0 +1,2 @@ +// Compatibility include; the implementation ships with ADEPT. +#include "../../adept/_vlasov1d/native/core/backend.cpp" diff --git a/benchmarks/vlasov_cuda/backend.hpp b/benchmarks/vlasov_cuda/backend.hpp new file mode 100644 index 00000000..b6d96321 --- /dev/null +++ b/benchmarks/vlasov_cuda/backend.hpp @@ -0,0 +1,2 @@ +// Compatibility include; the implementation ships with ADEPT. +#include "../../adept/_vlasov1d/native/core/backend.hpp" diff --git a/benchmarks/vlasov_cuda/benchmark_latest.py b/benchmarks/vlasov_cuda/benchmark_latest.py new file mode 100644 index 00000000..53d77eef --- /dev/null +++ b/benchmarks/vlasov_cuda/benchmark_latest.py @@ -0,0 +1,93 @@ +"""Paired strong-scaling measurements after latest native-window checks pass.""" + +import argparse +import json +import subprocess +import sys +from pathlib import Path + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("binary", type=Path) + parser.add_argument("--data", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--launcher", required=True) + parser.add_argument("--source-state", type=Path) + parser.add_argument("--nx", nargs="+", type=int, default=[134400, 268800]) + parser.add_argument("--placements", nargs="+", default=["1:4", "2:8"]) + parser.add_argument("--steps", type=int, default=10) + parser.add_argument("--repeats", type=int, default=3) + args = parser.parse_args() + args.output.mkdir(parents=True, exist_ok=False) + for nx in args.nx: + reference = args.data / f"native-nx{nx}" + checks = json.loads((args.data / f"comparison-nx{nx}-window/comparison.json").read_text()) + if not checks or not all(row["passed"] for row in checks): + raise RuntimeError("native-state comparison must pass before timing") + for placement in args.placements: + nodes, ranks = map(int, placement.split(":")) + for layout in ["transpose", "halos"]: + if not any( + row["nodes"] == nodes and row["ranks"] == ranks and row["layout"] == layout and row["steps"] == 100 + for row in checks + ): + raise RuntimeError(f"missing 100-step native comparison: {nx}, {placement}, {layout}") + metadata = json.loads((reference / "provenance.json").read_text()) + if metadata["nx"] != nx or metadata["nv"] != 4096 or metadata["dt"] != 0.02 or metadata["nu"] != 1e-12: + raise RuntimeError("not the pinned latest turbulence target") + for layout in ["transpose", "halos"]: + command = [ + sys.executable, + str(Path(__file__).with_name("measure.py")), + str(args.binary), + "--launcher", + args.launcher, + "--placements", + *args.placements, + "--nx", + str(nx), + "--nv", + "4096", + "--length", + "12600", + "--vmax", + "15", + "--dt", + ".02", + "--integrator", + "sixth", + "--velocity", + "cubic", + "--layout", + layout, + "--cells", + "--nu", + "1e-12", + "--steps", + str(args.steps), + "--warmup", + "3", + "--repeats", + str(args.repeats), + "--expect-cuda", + "--input", + str(reference / "initial.bin"), + "--v-grid", + str(reference / "v.bin"), + "--background", + str(reference / "background.bin"), + "--max-mass-change=-1", + "--output", + str(args.output / f"nx{nx}-{layout}.jsonl"), + ] + if args.source_state: + command.extend(["--source-state", str(args.source_state)]) + print(json.dumps({"nx": nx, "layout": layout, "status": "measuring"}), flush=True) + # Conservation diagnostics are recorded without changing the target + # operator. Full-state/field/N parity is gated independently above. + subprocess.run(command, check=True) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/vlasov_cuda/checkpoint_reference.py b/benchmarks/vlasov_cuda/checkpoint_reference.py new file mode 100644 index 00000000..a819383f --- /dev/null +++ b/benchmarks/vlasov_cuda/checkpoint_reference.py @@ -0,0 +1,178 @@ +"""Read a committed native checkpoint and advance its pinned ADEPT operators. + +Run with the production vp-turbulence environment on an allocated GPU node. +Only --output is written; the source campaign and checkpoint are read-only. +This deliberately tests numerical operators directly, without production +tracking, save buffers, or changes to either existing MLflow parent. +""" + +import argparse +import copy +import hashlib +import importlib.metadata +import json +import os +import sys +import time +from pathlib import Path + +os.environ.setdefault("XLA_PYTHON_CLIENT_PREALLOCATE", "false") +os.environ.setdefault("JAX_ENABLE_X64", "true") + + +def sha(path): + with Path(path).open("rb") as stream: + return hashlib.file_digest(stream, "sha256").hexdigest() + + +def save_vx(path, array): + """Write x,v data as a contiguous v,x file with bounded copy temporaries.""" + import numpy as np + + out = np.memmap(path, mode="w+", dtype=np.float64, shape=array.shape[::-1]) + for v0 in range(0, array.shape[1], 64): + out[v0 : v0 + 64] = array[:, v0 : v0 + 64].T + out.flush() + del out + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--checkpoint", type=Path, help="committed native input; omit to export the original seeded t0 initializer" + ) + parser.add_argument("--config", type=Path, required=True) + parser.add_argument("--vp-source", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--steps", nargs="+", type=int, default=[1, 10, 100]) + parser.add_argument("--expected-manifest") + args = parser.parse_args() + args.output.mkdir(parents=True, exist_ok=False) + if args.checkpoint and ( + not args.expected_manifest or sha(args.checkpoint / "manifest.json") != args.expected_manifest + ): + raise RuntimeError("native checkpoint manifest checksum mismatch") + distribution = importlib.metadata.distribution("adept") + direct = json.loads(distribution.read_text("direct_url.json")) + commit = direct.get("vcs_info", {}).get("commit_id") + if commit != "ddd70b957dc3df85f0b5ec84af927c25d09d24e0": + raise RuntimeError(f"wrong ADEPT environment: {commit}") + sys.path.insert(0, str(args.vp_source)) + import adept + + if Path(adept.__file__).resolve().parent != Path(distribution.locate_file("adept")).resolve(): + raise RuntimeError("PYTHONPATH shadows the pinned ADEPT installation") + import jax + import jax.numpy as jnp + import numpy as np + import yaml + from jax.sharding import Mesh, NamedSharding + from jax.sharding import PartitionSpec as P + from vp_turbulence.adeptmodules import VPTwoStream1D + from vp_turbulence.multinode import initialize_module_sharded + + jax.config.update("jax_enable_x64", True) + cfg = yaml.safe_load(args.config.read_text()) + if cfg["terms"]["time"] != "sixth" or cfg["terms"]["edfdv"] != "cubic-spline": + raise RuntimeError("expected the latest sixth-order/cubic target") + if cfg["drivers"]["ex"] or cfg["drivers"]["ey"]: + raise RuntimeError("external forcing is outside this comparison") + manifest = ( + json.loads((args.checkpoint / "manifest.json").read_text()) + if args.checkpoint + else {"actual_time": 0.0, "ranks": []} + ) + nx, nv = int(cfg["grid"]["nx"]), int(cfg["grid"]["nv"]) + state = np.empty((nx, nv), dtype=np.float64) if args.checkpoint else None + covered = np.zeros(nx, dtype=bool) + for rank in manifest["ranks"]: + leaf = rank["leaves"]["electron"] + path = args.checkpoint / leaf["file"] + if sha(path) != leaf["sha256"]: + raise RuntimeError(f"native shard checksum mismatch: {path}") + (start, stop, stride), vrange = leaf["index"] + if stride != 1 or vrange != [0, nv, 1] or np.any(covered[start:stop]): + raise RuntimeError("unexpected native shard coverage") + state[start:stop] = np.load(path, mmap_mode="r", allow_pickle=False) + covered[start:stop] = True + if args.checkpoint and (not covered.all() or not np.isfinite(state).all()): + raise RuntimeError("incomplete or nonfinite native input") + v = ( + np.load(args.checkpoint / manifest["coordinates"]["v_electron"]["file"], allow_pickle=False) + if args.checkpoint + else None + ) + # Rebuild the original ion background/geometry from the frozen initializer; + # the native perturbed background is not replaced with uniform density. + module = VPTwoStream1D(copy.deepcopy(cfg)) + module.write_units() + module.get_derived_quantities() + initialize_module_sharded(module) + native_v = np.asarray(module.cfg["grid"]["species_grids"]["electron"]["v"]) + if args.checkpoint is None: + state = np.asarray(module.state["electron"]) + v = native_v + if not np.array_equal(v, native_v): + raise RuntimeError("native velocity coordinates differ from pinned setup") + v.tofile(args.output / "v.bin") + np.asarray(module.cfg["grid"]["ion_charge"]).tofile(args.output / "background.bin") + save_vx(args.output / "initial.bin", state) + module.init_diffeqsolve() + operator = module.diffeqsolve_quants["terms"].vector_field.vpfp + if type(operator.vlasov_poisson).__name__ != "SixthOrderHamIntegrator": + raise RuntimeError("incorrect time integrator") + if operator.fp._sc_max_steps != 0: + raise RuntimeError("this prototype expects production self-consistent beta disabled") + sharding = NamedSharding(Mesh(np.array(jax.devices()), ("device",)), P("device", None)) + f = jax.make_array_from_callback(state.shape, sharding, lambda index: state[index]) + module.state = {"electron": f} + nu = float(cfg["terms"]["fokker_planck"]["time"]["baseline"]) + a, prev, external = jnp.zeros(nx + 2), jnp.zeros(nx), jnp.zeros((6, nx)) + nu_fp, nu_k = jnp.full(nx, nu), jnp.zeros(nx) + + @jax.jit + def advance(values, count): + def body(_, current): + _, result, _ = operator({"electron": current}, a, prev, external, nu_fp, nu_k) + return result["electron"] + + return jax.lax.fori_loop(0, count, body, values) + + compiled = advance.lower(f, jnp.asarray(1)).compile() + record = dict( + checkpoint=str(args.checkpoint) if args.checkpoint else None, + manifest_sha256=args.expected_manifest, + actual_time=manifest["actual_time"], + config_sha256=sha(args.config), + adept_commit=commit, + jax=jax.__version__, + nx=nx, + nv=nv, + dt=cfg["grid"]["dt"], + nu=nu, + devices=[str(d) for d in jax.devices()], + milestones=[], + ) + (args.output / "provenance.json").write_text(json.dumps(record, indent=2) + "\n") + print(json.dumps({"prepared": str(args.output), "nx": nx, "nv": nv, "time": manifest["actual_time"]}), flush=True) + previous = 0 + for stop in sorted(set(args.steps)): + if stop < 1: + raise ValueError("milestones must be positive") + start = time.perf_counter() + f = compiled(f, jnp.asarray(stop - previous)) + f.block_until_ready() + elapsed = time.perf_counter() - start + result = np.asarray(f) + if not np.isfinite(result).all(): + raise RuntimeError("nonfinite reference result") + save_vx(args.output / f"reference-{stop:04d}.bin", result) + row = dict(steps=stop, advanced_steps=stop - previous, seconds=elapsed) + record["milestones"].append(row) + (args.output / "provenance.json").write_text(json.dumps(record, indent=2) + "\n") + print(json.dumps(row), flush=True) + previous = stop + + +if __name__ == "__main__": + main() diff --git a/benchmarks/vlasov_cuda/compare_checkpoint.py b/benchmarks/vlasov_cuda/compare_checkpoint.py new file mode 100644 index 00000000..b2e387af --- /dev/null +++ b/benchmarks/vlasov_cuda/compare_checkpoint.py @@ -0,0 +1,150 @@ +"""Compare CUDA layouts at full native resolution with a pinned ADEPT window.""" + +import argparse +import json +import shlex +import subprocess +from pathlib import Path + +import numpy as np + + +def hole_counts(electric, k): + # Same whole-box observer as the latest Nv4096 comparison: Gaussian + # sigma_x=3, adjacent-minimum potential depths / vb^2, strict cuts. + kernel = np.zeros(len(k), dtype=complex) + kernel[1:] = 1j / k[1:] * np.exp(-0.5 * (3 * k[1:]) ** 2) + phi = np.fft.irfft(np.fft.rfft(electric) * kernel, n=len(electric)) + peaks = np.flatnonzero((phi > np.roll(phi, 1)) & (phi > np.roll(phi, -1))) + minima = np.flatnonzero((phi < np.roll(phi, 1)) & (phi < np.roll(phi, -1))) + if not len(minima): + return [0, 0] + i = np.searchsorted(minima, peaks) + depth = phi[peaks] - np.maximum(phi[minima[(i - 1) % len(minima)]], phi[minima[i % len(minima)]]) + return [int(np.count_nonzero(depth / 9 > cut)) for cut in [0.02, 0.5]] + + +def compare(candidate, reference, nx, nv, dv, length, background): + actual = np.memmap(candidate, mode="r", dtype=np.float64, shape=(nv, nx)) + expected = np.memmap(reference, mode="r", dtype=np.float64, shape=(nv, nx)) + error2, reference2, maximum = 0.0, 0.0, 0.0 + density = np.zeros(nx) + reference_density = np.zeros(nx) + for start in range(0, nv, 64): + a, b = actual[start : start + 64], expected[start : start + 64] + difference = a - b + if not np.isfinite(difference).all(): + raise RuntimeError("nonfinite checkpoint comparison") + maximum = max(maximum, float(np.max(np.abs(difference)))) + error2 += float(np.sum(difference * difference)) + reference2 += float(np.sum(b * b)) + density += np.sum(a, axis=0) * dv + reference_density += np.sum(b, axis=0) * dv + k = 2 * np.pi * np.fft.rfftfreq(nx, d=length / nx) + ef, rf = np.zeros(len(k), dtype=complex), np.zeros(len(k), dtype=complex) + ef[1:] = np.fft.rfft(background - density)[1:] / (1j * k[1:]) + rf[1:] = np.fft.rfft(background - reference_density)[1:] / (1j * k[1:]) + ef[-1] = rf[-1] = 0 + electric, reference_electric = np.fft.irfft(ef, n=nx), np.fft.irfft(rf, n=nx) + counts, reference_counts = hole_counts(electric, k), hole_counts(reference_electric, k) + return dict( + max_abs_f_error=maximum, + relative_l2_f_error=float(np.sqrt(error2 / reference2)), + relative_l2_e_error=float(np.linalg.norm(electric - reference_electric) / np.linalg.norm(reference_electric)), + max_abs_e_error=float(np.max(np.abs(electric - reference_electric))), + hole_counts=counts, + reference_hole_counts=reference_counts, + hole_counts_match=counts == reference_counts, + ) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("binary", type=Path) + parser.add_argument("--reference", required=True, type=Path) + parser.add_argument("--launcher", required=True) + parser.add_argument("--placements", nargs="+", default=["1:4", "2:8"]) + parser.add_argument("--steps", nargs="+", type=int, default=[1, 10, 100]) + parser.add_argument("--output", required=True, type=Path) + args = parser.parse_args() + source = json.loads((args.reference / "provenance.json").read_text()) + nx, nv, dt, nu = (source[key] for key in ["nx", "nv", "dt", "nu"]) + # Thresholds are implementation-matching checks for a short mature window, + # not criteria for long-time turbulent convergence or positivity. + limits = dict(max_abs_f_error=1e-7, relative_l2_f_error=1e-8, relative_l2_e_error=1e-7) + background = np.fromfile(args.reference / "background.bin", dtype=np.float64) + records = [] + args.output.mkdir(parents=True, exist_ok=False) + for placement in args.placements: + nodes, ranks = map(int, placement.split(":")) + for layout in ["transpose", "halos"]: + for steps in args.steps: + candidate = args.output / f"{nodes}n-{ranks}g-{layout}-{steps:04d}.bin" + command = [ + *shlex.split(args.launcher.format(nodes=nodes, ranks=ranks, ranks_per_node=ranks // nodes)), + str(args.binary.resolve()), + "--nx", + str(nx), + "--nv", + str(nv), + "--length", + "12600", + "--vmax", + "15", + "--dt", + str(dt), + "--nu", + str(nu), + "--cells", + "--velocity", + "cubic", + "--integrator", + "sixth", + "--layout", + layout, + "--steps", + str(steps), + "--warmup", + "0", + "--input", + str(args.reference / "initial.bin"), + "--v-grid", + str(args.reference / "v.bin"), + "--background", + str(args.reference / "background.bin"), + "--dump", + str(candidate), + ] + result = subprocess.run(command, text=True, capture_output=True, timeout=300) + if result.returncode: + raise RuntimeError(f"{' '.join(command)}\n{result.stdout}\n{result.stderr}") + metrics = compare( + candidate, args.reference / f"reference-{steps:04d}.bin", nx, nv, 30.0 / nv, 12600.0, background + ) + passed = ( + all(metrics[key] <= threshold for key, threshold in limits.items()) and metrics["hole_counts_match"] + ) + row = dict( + nodes=nodes, + ranks=ranks, + layout=layout, + steps=steps, + **metrics, + passed=passed, + limits=limits, + benchmark=json.loads(result.stdout.strip().splitlines()[-1]), + command=command, + rank_log=result.stderr, + ) + records.append(row) + (args.output / "comparison.json").write_text(json.dumps(records, indent=2) + "\n") + print( + json.dumps({key: row[key] for key in ["nodes", "ranks", "layout", "steps", *metrics, "passed"]}), + flush=True, + ) + if not passed: + raise RuntimeError("native checkpoint comparison did not meet predeclared matching limits") + + +if __name__ == "__main__": + main() diff --git a/benchmarks/vlasov_cuda/distributed_reference.py b/benchmarks/vlasov_cuda/distributed_reference.py new file mode 100644 index 00000000..62930ebc --- /dev/null +++ b/benchmarks/vlasov_cuda/distributed_reference.py @@ -0,0 +1,160 @@ +"""Advance a verified prepared native state with pinned ADEPT over GPU ranks. + +Used when the production collision workspace does not fit four 40GB GPUs. +All writes go under a new --output; the prepared input is read-only. +""" + +import argparse +import copy +import importlib.metadata +import json +import os +import socket +import sys +import time +from pathlib import Path + +os.environ.setdefault("XLA_PYTHON_CLIENT_PREALLOCATE", "false") +os.environ.setdefault("JAX_ENABLE_X64", "true") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--prepared", type=Path, required=True) + parser.add_argument("--config", type=Path, required=True) + parser.add_argument("--vp-source", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--steps", type=int, nargs="+", default=[1, 100]) + args = parser.parse_args() + sys.path.insert(0, str(args.vp_source)) + import jax + + jax.config.update("jax_enable_x64", True) + jax.distributed.initialize(local_device_ids=[int(os.environ["SLURM_LOCALID"])]) + import jax.numpy as jnp + import numpy as np + import yaml + from checkpoint_reference import save_vx, sha + from jax.experimental import multihost_utils + from jax.sharding import Mesh, NamedSharding + from jax.sharding import PartitionSpec as P + from vp_turbulence.adeptmodules import VPTwoStream1D + from vp_turbulence.multinode import initialize_module_sharded + + rank, count = jax.process_index(), jax.process_count() + if count != 8 or len(jax.local_devices()) != 1: + raise RuntimeError("expected eight ranks with one GPU each") + d = importlib.metadata.distribution("adept") + commit = json.loads(d.read_text("direct_url.json")).get("vcs_info", {}).get("commit_id") + if commit != "ddd70b957dc3df85f0b5ec84af927c25d09d24e0" or jax.__version__ != "0.10.2": + raise RuntimeError("incorrect pinned numerical environment") + metadata = json.loads((args.prepared / "provenance.json").read_text()) + if sha(args.config) != metadata["config_sha256"]: + raise RuntimeError("configuration differs from verified prepared input") + nx, nv = metadata["nx"], metadata["nv"] + if (args.prepared / "initial.bin").stat().st_size != nx * nv * 8: + raise RuntimeError("incorrect prepared state size") + if rank == 0: + args.output.mkdir(parents=True, exist_ok=False) + for name in ["initial.bin", "v.bin", "background.bin"]: + os.link(args.prepared / name, args.output / name) + metadata.update( + process_count=count, + devices=[str(d) for d in jax.devices()], + distributed=True, + milestones=[], + prepared_input=str(args.prepared), + ) + (args.output / "provenance.json").write_text(json.dumps(metadata, indent=2) + "\n") + multihost_utils.sync_global_devices("output-created") + (args.output / f"rank-{rank:02d}.json").write_text( + json.dumps( + dict( + rank=rank, + host=socket.gethostname(), + pid=os.getpid(), + local_devices=[str(d) for d in jax.local_devices()], + ) + ) + + "\n" + ) + cfg = yaml.safe_load(args.config.read_text()) + module = VPTwoStream1D(copy.deepcopy(cfg)) + module.write_units() + module.get_derived_quantities() + initialize_module_sharded(module) + v = np.fromfile(args.prepared / "v.bin", dtype=np.float64) + np.testing.assert_array_equal(v, np.asarray(module.cfg["grid"]["species_grids"]["electron"]["v"])) + np.testing.assert_array_equal( + np.fromfile(args.prepared / "background.bin", dtype=np.float64), np.asarray(module.cfg["grid"]["ion_charge"]) + ) + source = np.memmap(args.prepared / "initial.bin", mode="r", dtype=np.float64, shape=(nv, nx)) + mesh = Mesh(np.array(jax.devices()), ("device",)) + sharding = NamedSharding(mesh, P("device", None)) + f = jax.make_array_from_callback((nx, nv), sharding, lambda index: source.T[index]) + module.state = {"electron": f} + module.init_diffeqsolve() + operator = module.diffeqsolve_quants["terms"].vector_field.vpfp + if operator.fp._sc_max_steps != 0: + raise RuntimeError("unexpected self-consistent-beta iterations") + replicated = NamedSharding(mesh, P()) + + def replicate(value): + host = np.asarray(value) + return jax.make_array_from_callback(host.shape, replicated, lambda index: host[index]) + + a = replicate(np.zeros(nx + 2)) + prev = replicate(np.zeros(nx)) + external = replicate(np.zeros((6, nx))) + nu_fp = replicate(np.full(nx, metadata["nu"])) + nu_k = replicate(np.zeros(nx)) + + @jax.jit + def advance(values, iterations): + def body(_, current): + _, result, _ = operator({"electron": current}, a, prev, external, nu_fp, nu_k) + return result["electron"] + + return jax.lax.fori_loop(0, iterations, body, values) + + compiled = advance.lower(f, replicate(np.asarray(1, dtype=np.int32))).compile() + if rank == 0: + print(json.dumps(dict(status="compiled", nx=nx, nv=nv, ranks=count)), flush=True) + previous = 0 + for stop in sorted(set(args.steps)): + start = time.perf_counter() + f = compiled(f, replicate(np.asarray(stop - previous, dtype=np.int32))) + f.block_until_ready() + elapsed = time.perf_counter() - start + shards = f.addressable_shards + if len(shards) != 1: + raise RuntimeError("unexpected endpoint layout") + shard = shards[0] + values = np.asarray(shard.data) + if not np.isfinite(values).all(): + raise RuntimeError("nonfinite reference result") + np.save(args.output / f"step-{stop:04d}-rank-{rank:02d}.npy", values, allow_pickle=False) + (args.output / f"step-{stop:04d}-rank-{rank:02d}.json").write_text( + json.dumps(dict(start=shard.index[0].start or 0, stop=shard.index[0].stop or nx, seconds=elapsed)) + "\n" + ) + multihost_utils.sync_global_devices(f"shards-saved-{stop}") + if rank == 0: + whole = np.empty((nx, nv), dtype=np.float64) + for r in range(count): + part = json.loads((args.output / f"step-{stop:04d}-rank-{r:02d}.json").read_text()) + whole[part["start"] : part["stop"]] = np.load( + args.output / f"step-{stop:04d}-rank-{r:02d}.npy", mmap_mode="r" + ) + save_vx(args.output / f"reference-{stop:04d}.bin", whole) + del whole + row = dict(steps=stop, advanced_steps=stop - previous, rank0_seconds=elapsed) + metadata["milestones"].append(row) + (args.output / "provenance.json").write_text(json.dumps(metadata, indent=2) + "\n") + print(json.dumps(row), flush=True) + multihost_utils.sync_global_devices(f"reference-saved-{stop}") + previous = stop + jax.distributed.shutdown() + + +if __name__ == "__main__": + main() diff --git a/benchmarks/vlasov_cuda/integration_check.py b/benchmarks/vlasov_cuda/integration_check.py new file mode 100644 index 00000000..5edba331 --- /dev/null +++ b/benchmarks/vlasov_cuda/integration_check.py @@ -0,0 +1,138 @@ +"""Exercise the native ADEPT/ergoExo path, original callbacks and NetCDF writers.""" + +import argparse +import copy +import json +import os +import sys +from pathlib import Path + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--library", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--ranks", nargs="+", type=int, default=[1, 4, 8]) + parser.add_argument("--vp-source", type=Path) + parser.add_argument("--module", choices=["base", "vp", "both"], default="both") + args = parser.parse_args() + os.environ["JAX_PLATFORMS"] = "cpu" + os.environ["JAX_ENABLE_X64"] = "true" + os.environ["MLFLOW_ALLOW_FILE_STORE"] = "true" + import numpy as np + import yaml + from diffrax import SubSaveAt + + from adept import ergoExo + from adept import patched_mlflow as mlflow + from adept._vlasov1d.storage import store_f, store_fields + from adept.vlasov1d import BaseVlasov1D + + args.output.mkdir(parents=True, exist_ok=False) + mlflow.set_tracking_uri((args.output / "mlruns").as_uri()) + classes = [] if args.module == "vp" else [BaseVlasov1D] + if args.vp_source: + sys.path.insert(0, str(args.vp_source)) + from vp_turbulence.adeptmodules import VPTwoStream1D + + if args.module != "base": + classes.append(VPTwoStream1D) + checks = [] + for cls in classes: + for ranks in args.ranks: + cfg = yaml.safe_load(Path("tests/test_base/configs/example.yaml").read_text()) + cfg["grid"].update(nx=32, nv=64, dt=0.02, tmax=0.08, vmax=8.0) + cfg["drivers"] = {"ex": {}, "ey": {}} + cfg["terms"]["time"] = "sixth" + cfg["terms"]["fokker_planck"].update(is_on=True, type="chang_cooper_dougherty") + cfg["terms"]["fokker_planck"]["time"]["baseline"] = 0.001 + cfg["terms"]["fokker_planck"]["space"]["baseline"] = 1.0 + cfg["terms"]["krook"]["is_on"] = False + cfg["save"] = {"fields": {"t": {"nt": 5}}, "electron": {"main": {"t": {"nt": 3}}}} + beam = cfg["density"]["species-background"] + beam.update(v0=3.0, T0=0.2, noise_val=0.001) + cfg["density"]["species-left"] = copy.deepcopy(beam) + cfg["density"]["species-left"].update(v0=-3.0, noise_seed=43) + cfg["native"] = dict(workdir=str(args.output)) + if cls.__name__ == "VPTwoStream1D": + cfg["two_stream"] = dict( + v_b=3.0, + delta_v_b=0.5, + native_cascade_budget=True, + native_kv_spectrum=dict(is_on=True, t=dict(tmin=0.0, tmax=0.1, nt=3)), + ) + cfg["postprocess"] = { + key: False + for key in [ + "plot_scalars", + "plot_fields", + "plot_fullres_dists", + "plot_lowres_dists", + "plot_transformed_dists", + "plot_field_spectrum", + ] + } + original = cls(copy.deepcopy(cfg)) + original.write_units() + original.get_derived_quantities() + original.get_solver_quantities() + original.init_state_and_args() + original.init_diffeqsolve() + nodes = 2 if ranks > 4 else 1 + cfg["backend"] = "native" + cfg["native"].update( + device="cuda", + library=str(args.library), + ranks=ranks, + timeout=180, + launcher=[ + "srun", + f"--jobid={os.environ['SLURM_JOB_ID']}", + "--overlap", + f"--nodes={nodes}", + "--ntasks={ranks}", + f"--ntasks-per-node={ranks // nodes}", + "--gpus-per-node=4", + "--gpu-bind=none", + "--cpu-bind=none", + ], + ) + cfg["mlflow"] = dict(experiment="native-adept-integration", run=f"{cls.__name__}-{ranks}") + exo = ergoExo() + modules = exo.setup(copy.deepcopy(cfg), adept_module=cls) + output, post, run_id = exo(modules) + for key, save in exo.adept_module.diffeqsolve_quants["saveat"]["subs"].items(): + subs = original.diffeqsolve_quants["saveat"]["subs"] + subs[key] = SubSaveAt(ts=save.ts, fn=subs[key].fn) + reference = original({})["solver result"] + root = args.output / f"{cls.__name__}-{ranks}" + (root / "binary").mkdir(parents=True) + sol = output["solver result"] + store_fields(exo.adept_module.cfg, str(root / "binary"), sol.ys["fields"], sol.ts["fields"], "fields") + store_f(exo.adept_module.cfg, sol.ts, str(root), sol.ys) + errors = [] + import jax + + for key in sol.ys: + np.testing.assert_allclose(reference.ts[key], sol.ts[key], atol=1e-14) + for a, b in zip(jax.tree.leaves(reference.ys[key]), jax.tree.leaves(sol.ys[key]), strict=True): + np.testing.assert_allclose(a, b, rtol=1e-10, atol=1e-11, equal_nan=True, err_msg=key) + delta = np.not_equal(a, b) if np.asarray(a).dtype.kind == "b" else np.asarray(a) - np.asarray(b) + errors.append(float(np.max(np.abs(delta))) if delta.size else 0.0) + row = dict( + module=cls.__name__, + ranks=ranks, + passed=True, + max_abs_error=max(errors), + mlflow_run_id=run_id, + stats=output["native stats"], + keys=list(sol.ys), + ) + checks.append(row) + (args.output / "checks.json").write_text(json.dumps(checks, indent=2)) + print(json.dumps(row), flush=True) + (args.output / "checks.json").write_text(json.dumps(checks, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/vlasov_cuda/local_cubic.hpp b/benchmarks/vlasov_cuda/local_cubic.hpp new file mode 100644 index 00000000..0dbdc4e9 --- /dev/null +++ b/benchmarks/vlasov_cuda/local_cubic.hpp @@ -0,0 +1,2 @@ +// Compatibility include; the implementation ships with ADEPT. +#include "../../adept/_vlasov1d/native/core/local_cubic.hpp" diff --git a/benchmarks/vlasov_cuda/main.cpp b/benchmarks/vlasov_cuda/main.cpp new file mode 100644 index 00000000..c4b45a40 --- /dev/null +++ b/benchmarks/vlasov_cuda/main.cpp @@ -0,0 +1,122 @@ +#include "../../adept/_vlasov1d/native/core/solver.hpp" + +using namespace adept_native; + +int main(int argc, char** argv) { + try { + prepare_device_from_environment(); + } catch (const std::exception& error) { + std::cerr << "device binding before MPI_Init: " << error.what() << '\n'; + return 1; + } + MPI_Init(&argc, &argv); + MPI_Comm_set_errhandler(MPI_COMM_WORLD, MPI_ERRORS_RETURN); + int rank = 0; + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + try { + Options o = parse(argc, argv, rank); + if (o.mode == "help") { MPI_Finalize(); return 0; } + Grid& g = o.grid; + g.collisions = o.nu > 0; + g.rank = rank; + mpi_check(MPI_Comm_size(MPI_COMM_WORLD, &g.ranks)); + if (g.nx < 8 || g.nv < 4 || g.nx % g.ranks || g.nv % g.ranks) + throw std::runtime_error("nx >= 8 and nv >= 4 must both be divisible by MPI ranks"); + if (g.count() > std::size_t(std::numeric_limits::max())) + throw std::runtime_error("local state exceeds this prototype's MPI int count limit"); + if (o.steps < 1 || o.warmup < 0 || o.dt <= 0 || g.vmax <= 0 || g.length <= 0 || std::abs(g.amplitude) >= 1) + throw std::runtime_error("require steps>0, warmup>=0, dt/vmax/length>0 and |amplitude|<1"); + if (o.mode != "landau" && o.mode != "stream" && o.mode != "accelerate" && o.mode != "transpose" && o.mode != "collide") + throw std::runtime_error("unknown mode: " + o.mode); + if (o.integrator != "strang" && o.integrator != "sixth") throw std::runtime_error("unknown integrator"); + if (g.halos && (!g.local_cubic || g.halo_capacity < 2)) + throw std::runtime_error("halos require --velocity cubic and capacity >= 2"); + if (g.halos && o.mode == "transpose") + throw std::runtime_error("transpose-only mode requires --layout transpose"); + if (o.nu < 0 || (o.nu > 0 && !g.cell_centred)) + throw std::runtime_error("collisions require nonnegative --nu and --cells"); + if (std::size_t(g.halo_capacity) * g.nx > std::size_t(std::numeric_limits::max())) + throw std::runtime_error("halo exceeds MPI int count limit"); + MPI_Comm local; + mpi_check(MPI_Comm_split_type(MPI_COMM_WORLD, MPI_COMM_TYPE_SHARED, rank, MPI_INFO_NULL, &local)); + int local_rank; + mpi_check(MPI_Comm_rank(local, &local_rank)); + int is_head = local_rank == 0, nodes; + mpi_check(MPI_Allreduce(&is_head, &nodes, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD)); + char hostname[MPI_MAX_PROCESSOR_NAME]; + int host_length; + mpi_check(MPI_Get_processor_name(hostname, &host_length)); + { + Backend backend(g, local_rank); + std::cerr << "rank=" << rank << " host=" << std::string(hostname, host_length) + << " local_rank=" << local_rank << " device=" << backend.device_name() << '\n'; + Stepper stepper(backend, o); + if (!o.coordinates.empty()) backend.set_coordinates(read_binary(o.coordinates, g.nv)); + if (!o.background.empty()) backend.set_background(read_binary(o.background, g.nx)); + initialize(backend, o); + for (int i = 0; i < o.warmup; ++i) stepper.step(); + // Warmup is excluded from both timings and reported physical time. + initialize(backend, o); + auto initial = diagnostics(backend, stepper, g); + stepper.times.fill(0.0); + stepper.halo_sent_bytes = 0; + stepper.slab_sent_bytes = 0; + stepper.max_halo_width = 0; + backend.synchronize(); + mpi_check(MPI_Barrier(MPI_COMM_WORLD)); + for (int i = 0; i < o.steps; ++i) stepper.step(); + std::array times; + mpi_check(MPI_Reduce(stepper.times.data(), times.data(), StageCount, + MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD)); + auto final = diagnostics(backend, stepper, g); + if (!initial.finite || !final.finite) throw std::runtime_error("nonfinite distribution"); + if (!o.dump.empty()) dump(backend, g, o.dump); + std::uint64_t allocated = backend.allocated_bytes(), max_allocated; + std::uint64_t max_halo_bytes; + mpi_check(MPI_Reduce(&stepper.halo_sent_bytes, &max_halo_bytes, 1, MPI_UINT64_T, MPI_MAX, 0, MPI_COMM_WORLD)); + mpi_check(MPI_Reduce(&allocated, &max_allocated, 1, MPI_UINT64_T, MPI_MAX, 0, MPI_COMM_WORLD)); + if (rank == 0) { + const char* names[] = {"stream_ms", "spline_ms", "pack_ms", "mpi_ms", "field_ms", "collision_ms", "step_ms"}; + std::cout << std::setprecision(16) << "{\"backend\":" +#ifdef VLASOV_CUDA + << "\"cuda\"" +#else + << "\"fftw\"" +#endif + << ",\"device\":" << quoted(backend.device_name()) + << ",\"precision\":\"float64\",\"mode\":" << quoted(o.mode) + << ",\"integrator\":" << quoted(o.integrator) + << ",\"nu\":" << o.nu + << ",\"velocity\":" << quoted(g.local_cubic ? "cubic" : "natural") + << ",\"layout\":" << quoted(g.halos ? "halos" : "transpose") + << ",\"cell_centred\":" << (g.cell_centred ? "true" : "false") + << ",\"max_halo_width\":" << stepper.max_halo_width + << ",\"rank0_halo_sent_bytes\":" << stepper.halo_sent_bytes + << ",\"halo_sent_bytes_per_rank_max\":" << max_halo_bytes + << ",\"slab_sent_bytes_per_rank\":" << stepper.slab_sent_bytes + << ",\"input\":" << quoted(o.input) + << ",\"nx\":" << g.nx << ",\"nv\":" << g.nv << ",\"ranks\":" << g.ranks + << ",\"nodes\":" << nodes << ",\"steps\":" << o.steps << ",\"warmup\":" << o.warmup + << ",\"dt\":" << o.dt << ",\"length\":" << g.length << ",\"vmax\":" << g.vmax + << ",\"amplitude\":" << g.amplitude << ",\"acceleration\":" << o.acceleration + << ",\"host_mpi\":" << (o.host_mpi ? "true" : "false") + << ",\"state_bytes\":" << std::uint64_t(g.nx) * g.nv * sizeof(double) + << ",\"max_allocated_bytes_per_rank\":" << max_allocated; + for (int i = 0; i < StageCount; ++i) + std::cout << ",\"" << names[i] << "\":" << times[i] * 1000.0 / o.steps; + std::cout << ",\"initial_mass\":" << initial.mass << ",\"final_mass\":" << final.mass + << ",\"mass_relative_change\":" << (final.mass - initial.mass) / initial.mass + << ",\"initial_kinetic_energy\":" << initial.kinetic << ",\"final_kinetic_energy\":" << final.kinetic + << ",\"initial_field_energy\":" << initial.field << ",\"final_field_energy\":" << final.field + << ",\"minimum_f\":" << final.minimum << ",\"finite\":true}\n"; + } + } // Destroy CUDA resources before MPI finalization. + mpi_check(MPI_Comm_free(&local)); + MPI_Finalize(); + return 0; + } catch (const std::exception& error) { + std::cerr << "rank " << rank << ": " << error.what() << '\n'; + MPI_Abort(MPI_COMM_WORLD, 1); + return 1; + } +} diff --git a/benchmarks/vlasov_cuda/measure.py b/benchmarks/vlasov_cuda/measure.py new file mode 100644 index 00000000..ed7d6c85 --- /dev/null +++ b/benchmarks/vlasov_cuda/measure.py @@ -0,0 +1,124 @@ +"""Run repeated strong-scaling measurements with an MPI launcher template. + +Placements have the form nodes:ranks. For Slurm, pass an explicit job ID, +--overlap, --nodes={nodes}, --ntasks={ranks}, and +--ntasks-per-node={ranks_per_node}. This script never allocates resources. +""" + +import argparse +import json +import shlex +import subprocess +from pathlib import Path + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("binary", type=Path) + parser.add_argument("--launcher", default="mpiexec -n {ranks}") + parser.add_argument("--placements", nargs="+", default=["1:1", "1:2", "1:4"]) + parser.add_argument("--nx", type=int, default=32768) + parser.add_argument("--nv", type=int, default=8192) + parser.add_argument("--steps", type=int, default=20) + parser.add_argument("--warmup", type=int, default=3) + parser.add_argument("--repeats", type=int, default=3) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--source-state", type=Path) + parser.add_argument("--host-mpi", action="store_true") + parser.add_argument("--expect-cuda", action="store_true") + parser.add_argument("--velocity", choices=["natural", "cubic"], default="natural") + parser.add_argument("--layout", choices=["transpose", "halos"], default="transpose") + parser.add_argument("--integrator", choices=["strang", "sixth"], default="strang") + parser.add_argument("--cells", action="store_true") + parser.add_argument("--input", type=Path) + parser.add_argument("--v-grid", type=Path) + parser.add_argument("--background", type=Path) + parser.add_argument("--nu", type=float, default=0.0) + parser.add_argument("--length", type=float, default=12.566370614359172) + parser.add_argument("--vmax", type=float, default=8.0) + parser.add_argument("--dt", type=float, default=0.1) + parser.add_argument( + "--max-mass-change", + type=float, + default=1e-8, + help="negative records conservation diagnostically without an absolute cutoff", + ) + args = parser.parse_args() + placements = [] + for placement in args.placements: + nodes, ranks = map(int, placement.split(":")) + if nodes < 1 or ranks < 1 or ranks % nodes: + parser.error("placements must have positive nodes/ranks and equal ranks per node") + placements.append((nodes, ranks)) + if args.repeats < 1: + parser.error("repeats must be positive") + provenance = args.source_state.read_text() if args.source_state else None + # Exclusive create prevents accidentally replacing an earlier measurement. + with args.output.open("x") as output: + for nodes, ranks in placements: + for repeat in range(args.repeats): + command = [ + *shlex.split(args.launcher.format(nodes=nodes, ranks=ranks, ranks_per_node=ranks // nodes)), + str(args.binary.resolve()), + "--nx", + str(args.nx), + "--nv", + str(args.nv), + "--steps", + str(args.steps), + "--warmup", + str(args.warmup), + "--mode", + "landau", + ] + if args.host_mpi: + command.append("--host-mpi") + command.extend( + [ + "--velocity", + args.velocity, + "--layout", + args.layout, + "--integrator", + args.integrator, + "--nu", + str(args.nu), + "--length", + str(args.length), + "--vmax", + str(args.vmax), + "--dt", + str(args.dt), + ] + ) + if args.cells: + command.append("--cells") + for key in ["input", "v_grid", "background"]: + value = getattr(args, key) + if value: + command.extend(["--" + key.replace("_", "-"), str(value)]) + completed = subprocess.run(command, capture_output=True, text=True, timeout=300) + if completed.returncode: + raise RuntimeError(f"{' '.join(command)}\n{completed.stdout}\n{completed.stderr}") + row = json.loads(completed.stdout.strip().splitlines()[-1]) + if row["nodes"] != nodes or row["ranks"] != ranks: + raise RuntimeError(f"placement mismatch: {row}") + if args.expect_cuda and row["backend"] != "cuda": + raise RuntimeError("expected the CUDA backend") + if (args.max_mass_change >= 0 and abs(row["mass_relative_change"]) > args.max_mass_change) or not row[ + "finite" + ]: + raise RuntimeError(f"invalid physical output: {row}") + row.update(repeat=repeat, command=command, rank_log=completed.stderr, source_state=provenance) + output.write(json.dumps(row) + "\n") + output.flush() + print( + json.dumps( + {key: row[key] for key in ["nodes", "ranks", "repeat", "step_ms", "spline_ms", "mpi_ms"]} + ), + flush=True, + ) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/vlasov_cuda/reference/latest-nv4096/nx134400.yaml b/benchmarks/vlasov_cuda/reference/latest-nv4096/nx134400.yaml new file mode 100644 index 00000000..16753567 --- /dev/null +++ b/benchmarks/vlasov_cuda/reference/latest-nv4096/nx134400.yaml @@ -0,0 +1,168 @@ +solver: vp-two-stream-1d +mlflow: + experiment: vp-turbulence + run: nv4096-fine-20261023-nx134400-nv4096-t600-seed42 +units: + normalizing_density: 1.0e20/cc + normalizing_temperature: 1000eV +grid: + nx: 134400 + nv: 4096 + xmin: 0.0 + xmax: 12600.0 + tmin: 0.0 + tmax: 600.0 + vmax: 15.0 + dt: 0.02 + parallel: + - x + - v +two_stream: + v_b: 3.0 + delta_v_b: 0.03 + noise_val: 1.0e-06 + save_e_only: true + native_cascade_budget: true + native_kv_spectrum: + is_on: true + t: + tmin: 0.0 + tmax: 600.0 + nt: 61 +density: + quasineutrality: true + beam-right: + basis: uniform + noise_seed: 42 + noise_type: band_limited_uniform + noise_reference_nx: 51610 + noise_val: 1.0e-06 + v0: 3.0 + T0: 0.00405 + m: 100.0 + baseline: 0.5 + bump_or_trough: bump + center: 0.0 + rise: 25.0 + bump_height: 0.0 + width: 100000.0 + noise_max_mode: 25804 + beam-left: + basis: uniform + noise_seed: 43 + noise_type: band_limited_uniform + noise_reference_nx: 51610 + noise_val: 1.0e-06 + v0: -3.0 + T0: 0.00405 + m: 100.0 + baseline: 0.5 + bump_or_trough: bump + center: 0.0 + rise: 25.0 + bump_height: 0.0 + width: 100000.0 + noise_max_mode: 25804 +terms: + field: poisson + edfdv: cubic-spline + time: sixth + hou_li_filter: + is_on: false + alpha: 36.0 + order: 36 + dimensions: + - x + species: + - name: electron + charge: -1.0 + mass: 1.0 + vmax: 15.0 + nv: 4096 + density_components: + - beam-right + - beam-left + fokker_planck: + is_on: true + type: chang_cooper_dougherty + time: + baseline: 1.0e-12 + bump_or_trough: bump + center: 0.0 + rise: 25.0 + slope: 0.0 + bump_height: 0.0 + width: 100000.0 + space: + baseline: 1.0 + bump_or_trough: bump + center: 0.0 + rise: 25.0 + slope: 0.0 + bump_height: 0.0 + width: 100000.0 + krook: + is_on: false + time: + baseline: 0.0 + bump_or_trough: bump + center: 0.0 + rise: 25.0 + slope: 0.0 + bump_height: 0.0 + width: 100000.0 + space: + baseline: 1.0 + bump_or_trough: bump + center: 0.0 + rise: 25.0 + slope: 0.0 + bump_height: 0.0 + width: 100000.0 + vdfdx: exponential +drivers: + ex: {} + ey: {} +diagnostics: + diag-vlasov-dfdt: false + diag-fp-dfdt: false +postprocess: + plot_scalars: false + plot_fields: false + plot_field_spectrum: false + plot_lowres_dists: false + plot_fullres_dists: false + plot_transformed_dists: false +save: + fields: + t: + tmin: 0.0 + tmax: 600.0 + nt: 301 + electron: + lowres: + t: + tmin: 0.0 + tmax: 600.0 + nt: 61 + x: + xmin: 0.046875 + xmax: 12599.953125 + nx: 2048 + v: + vmin: -14.996337890625 + vmax: 14.996337890625 + nv: 256 + full: + t: + tmin: 0.0 + tmax: 600.0 + nt: 7 + x: + xmin: 0.046875 + xmax: 12599.953125 + nx: 8192 + v: + vmin: -14.996337890625 + vmax: 14.996337890625 + nv: 1024 diff --git a/benchmarks/vlasov_cuda/reference/latest-nv4096/nx268800.yaml b/benchmarks/vlasov_cuda/reference/latest-nv4096/nx268800.yaml new file mode 100644 index 00000000..5c956dd6 --- /dev/null +++ b/benchmarks/vlasov_cuda/reference/latest-nv4096/nx268800.yaml @@ -0,0 +1,166 @@ +solver: vp-two-stream-1d +mlflow: + experiment: vp-turbulence + run: nv4096-fine-20261023-nx268800-nv4096-t600-seed42 +units: + normalizing_density: 1.0e20/cc + normalizing_temperature: 1000eV +grid: + nx: 268800 + nv: 4096 + xmin: 0.0 + xmax: 12600.0 + tmin: 0.0 + tmax: 600.0 + vmax: 15.0 + dt: 0.02 + parallel: + - x + - v +two_stream: + v_b: 3.0 + delta_v_b: 0.03 + noise_val: 1.0e-06 + save_e_only: true + native_cascade_budget: true + native_kv_spectrum: + is_on: true + t: + tmin: 0.0 + tmax: 600.0 + nt: 61 +density: + quasineutrality: true + beam-right: + basis: uniform + noise_seed: 42 + noise_type: band_limited_uniform + noise_reference_nx: 51610 + noise_val: 1.0e-06 + v0: 3.0 + T0: 0.00405 + m: 100.0 + baseline: 0.5 + bump_or_trough: bump + center: 0.0 + rise: 25.0 + bump_height: 0.0 + width: 100000.0 + beam-left: + basis: uniform + noise_seed: 43 + noise_type: band_limited_uniform + noise_reference_nx: 51610 + noise_val: 1.0e-06 + v0: -3.0 + T0: 0.00405 + m: 100.0 + baseline: 0.5 + bump_or_trough: bump + center: 0.0 + rise: 25.0 + bump_height: 0.0 + width: 100000.0 +terms: + field: poisson + edfdv: cubic-spline + time: sixth + hou_li_filter: + is_on: false + alpha: 36.0 + order: 36 + dimensions: + - x + species: + - name: electron + charge: -1.0 + mass: 1.0 + vmax: 15.0 + nv: 4096 + density_components: + - beam-right + - beam-left + fokker_planck: + is_on: true + type: chang_cooper_dougherty + time: + baseline: 1.0e-12 + bump_or_trough: bump + center: 0.0 + rise: 25.0 + slope: 0.0 + bump_height: 0.0 + width: 100000.0 + space: + baseline: 1.0 + bump_or_trough: bump + center: 0.0 + rise: 25.0 + slope: 0.0 + bump_height: 0.0 + width: 100000.0 + krook: + is_on: false + time: + baseline: 0.0 + bump_or_trough: bump + center: 0.0 + rise: 25.0 + slope: 0.0 + bump_height: 0.0 + width: 100000.0 + space: + baseline: 1.0 + bump_or_trough: bump + center: 0.0 + rise: 25.0 + slope: 0.0 + bump_height: 0.0 + width: 100000.0 + vdfdx: exponential +drivers: + ex: {} + ey: {} +diagnostics: + diag-vlasov-dfdt: false + diag-fp-dfdt: false +postprocess: + plot_scalars: false + plot_fields: false + plot_field_spectrum: false + plot_lowres_dists: false + plot_fullres_dists: false + plot_transformed_dists: false +save: + fields: + t: + tmin: 0.0 + tmax: 600.0 + nt: 301 + electron: + lowres: + t: + tmin: 0.0 + tmax: 600.0 + nt: 61 + x: + xmin: 0.0234375 + xmax: 12599.9765625 + nx: 2048 + v: + vmin: -14.996337890625 + vmax: 14.996337890625 + nv: 256 + full: + t: + tmin: 0.0 + tmax: 600.0 + nt: 7 + x: + xmin: 0.0234375 + xmax: 12599.9765625 + nx: 8192 + v: + vmin: -14.996337890625 + vmax: 14.996337890625 + nv: 1024 diff --git a/benchmarks/vlasov_cuda/reference/latest-nv4096/target.json b/benchmarks/vlasov_cuda/reference/latest-nv4096/target.json new file mode 100644 index 00000000..92f78b07 --- /dev/null +++ b/benchmarks/vlasov_cuda/reference/latest-nv4096/target.json @@ -0,0 +1,39 @@ +{ + "campaign": "nv4096-t2000-20261023-nx134400", + "git_commit": "2c4904968009d695e2625fd264f057dd9c846039", + "config_sha256": "5a9f5347e6253a3d4201745318843857c568b3787cf91e9fe81a623d0cdbb214", + "pinned_adept_commit": "ddd70b957dc3df85f0b5ec84af927c25d09d24e0", + "pinned_jax": "0.10.2", + "matched_ladder": { + "nx": 134400, + "L": 12600.0, + "nv": 4096, + "dt": 0.02, + "beam_seeds": [ + 42, + 43 + ], + "noise_reference_nx": 51610, + "noise_max_mode_both_beams": 25804, + "noise_value": 1e-06, + "noise_rms_renormalized": false, + "nyquist_mode_dropped": true, + "projection": "Same seeded reference Fourier coefficients; truncate to representable conjugate pairs, retain cell-centre phase and amplitudes." + }, + "members": [ + { + "nx": 134400, + "nv": 4096, + "parent_run_id": "aa042900211640d9a90f20001772f957", + "config": "nx134400.yaml", + "config_sha256": "5a9f5347e6253a3d4201745318843857c568b3787cf91e9fe81a623d0cdbb214" + }, + { + "nx": 268800, + "nv": 4096, + "parent_run_id": "916dc390224d4c14a417d772991a2812", + "config": "nx268800.yaml", + "config_sha256": "6adb42855432b79a74f63290678103c17c53842074d77bf092df38483f634a83" + } + ] +} diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-adept/README.md b/benchmarks/vlasov_cuda/results/2026-10-24-adept/README.md new file mode 100644 index 00000000..357ca7ef --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-adept/README.md @@ -0,0 +1,48 @@ +# ADEPT/JAX versus native CUDA on A100s + +The native backend is integrated into ADEPT (`backend: native`). It retains +`ergoExo`, ordinary/custom module initialization, original save callbacks and +xarray/NetCDF postprocessing. Standard saves use actual timestep boundaries; +outputs are streamed to host memory maps. The C++ physics core is shared with +the standalone timing executable. + +| Grid | Nodes / A100s | ADEPT/JAX ms/step | Native ms/step | Speedup | +|---|---:|---:|---:|---:| +| 134400 × 4096 | 1 / 4 | 223.54 | 174.24 | 1.283× | +| 134400 × 4096 | 2 / 8 | 383.94 | 140.07 | 2.741× | +| 268800 × 4096 | 2 / 8 | 756.70 | 275.99 | 2.742× | + +FP64, spectral x, local cubic C1 v, sixth-order splitting, dt=0.02, +Chang-Cooper Dougherty nu=1e-12. Both paths reset to the same mature t600 +checkpoint before each repeat. Median of three repeats, ten timed steps, +three warmup steps. Timings exclude compilation, preprocessing, save callbacks, +I/O and postprocessing; these are physics-step speedups, not complete-run +wall-clock speedups. No concurrent benchmark workloads were run. + +Allocation 60221398 used nid001600 and nid003745, four A100-SXM4-40GB cards +per node. ADEPT was pinned to ddd70b957dc3df85f0b5ec84af927c25d09d24e0, +JAX/JAXlib 0.10.2, interpax 0.3.14+10.g84aaf30, Diffrax 0.7.2. +The four-GPU pair was rerun with both solvers pinned to nid001600 after an +audit found the initial pair used different nodes. Excluded measurements are +retained under `unmatched-node/`; the final pair's provenance is under +`pinned-one-node/`. Both eight-GPU pairs used the same physical nodes/GPUs. + +Five native CPU integration tests and 33 existing configuration/save/logging +regressions passed. Seven GPU integration cases passed: ordinary ADEPT and +VPTwoStream1D at 1/4/8 ranks, plus the latest native velocity-spectrum and +cascade callbacks at eight ranks. Each compares original save callbacks at +the actual native save times (rtol=1e-10, atol=1e-11). The largest absolute +error was 1.16e-11 across all saved diagnostics; ordinary ADEPT saves were +within 1.32e-13. `observer-netcdf/` contains outputs from the unchanged +VPTwoStream1D postprocessor; `observer-native/` contains its MLflow provenance. +The initial `integration-v3.log` also records a subsequently fixed callback +constant serialization failure; its three ordinary ADEPT cases passed. + +`summary.json` is rebuilt by `python summarize.py`, which verifies source +archive hashes, matched hosts, finite outputs, endpoint aggregate checks and +all seven integration results. Raw repeated measurements, software versions, +checkpoint provenance, command lines and executable hashes are adjacent. +Pointwise full-grid t600-to-t602 validation of the shared numerical core is +in `../2026-10-24-halos/`; a full t0-to-2000 trajectory has not been tested. +The NERSC session's files remain under +`/pscratch/sd/a/archis/adept-sessions/20261024T190159Z-99199-22637/`. diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-adept/binaries.sha256 b/benchmarks/vlasov_cuda/results/2026-10-24-adept/binaries.sha256 new file mode 100644 index 00000000..9bfe9c31 --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-adept/binaries.sha256 @@ -0,0 +1,2 @@ +13429e0e2873ab774bdb695874836b4133750a4f36bc96d20bd22ab73aa104a4 /pscratch/sd/a/archis/adept-sessions/20261024T190159Z-99199-22637/workdir/cuda-build/vlasov_bench +8ea49ba7a81a1e0074e35e9f63ae73680f38005adc7b49acd12343b7e7951dd7 /pscratch/sd/a/archis/adept-sessions/20261024T190159Z-99199-22637/workdir/native-build/libadept_vlasov.so diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-adept/cuda-nx134400-g4.jsonl b/benchmarks/vlasov_cuda/results/2026-10-24-adept/cuda-nx134400-g4.jsonl new file mode 100644 index 00000000..8dfcec5a --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-adept/cuda-nx134400-g4.jsonl @@ -0,0 +1,3 @@ +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "integrator": "sixth", "nu": 1e-12, "velocity": "cubic", "layout": "halos", "cell_centred": true, "max_halo_width": 4, "rank0_halo_sent_bytes": 215040000, "halo_sent_bytes_per_rank_max": 430080000, "slab_sent_bytes_per_rank": 16515072000, "input": "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/initial.bin", "nx": 134400, "nv": 4096, "ranks": 4, "nodes": 1, "steps": 10, "warmup": 3, "dt": 0.02, "length": 12600, "vmax": 15, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 4404019200, "max_allocated_bytes_per_rank": 11155281976, "stream_ms": 96.76214629996593, "spline_ms": 29.39172449987382, "pack_ms": 6.529895999938162, "mpi_ms": 21.04299519983215, "field_ms": 6.631393699944965, "collision_ms": 14.49398829993243, "step_ms": 174.2776944999605, "initial_mass": 12597.6196507818, "final_mass": 12597.62016522545, "mass_relative_change": 4.083657516783715e-08, "initial_kinetic_energy": 55713.43281125277, "final_kinetic_energy": 55714.15660222915, "initial_field_energy": 787.6538181371966, "final_field_energy": 786.9554651665927, "minimum_f": -0.1916308738890627, "finite": true, "repeat": 0, "command": ["srun", "--jobid=60221398", "--overlap", "--nodes=1", "--nodelist=nid001600", "--ntasks=4", "--ntasks-per-node=4", "--gpus-per-node=4", "--gpu-bind=none", "--cpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T190159Z-99199-22637/workdir/cuda-build/vlasov_bench", "--nx", "134400", "--nv", "4096", "--steps", "10", "--warmup", "3", "--mode", "landau", "--velocity", "cubic", "--layout", "halos", "--integrator", "sixth", "--nu", "1e-12", "--length", "12600.0", "--vmax", "15.0", "--dt", "0.02", "--cells", "--input", "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/initial.bin", "--v-grid", "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/v.bin", "--background", "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/background.bin"], "rank_log": "rank=0 host=nid001600 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001600 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001600 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001600 local_rank=1 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=c0a2abe12f3d288f697c8b183b00313aa4c6ba063d5122a570dfae4a37751380\nsynced_at=2026-10-24T19:31:46Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "integrator": "sixth", "nu": 1e-12, "velocity": "cubic", "layout": "halos", "cell_centred": true, "max_halo_width": 4, "rank0_halo_sent_bytes": 215040000, "halo_sent_bytes_per_rank_max": 430080000, "slab_sent_bytes_per_rank": 16515072000, "input": "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/initial.bin", "nx": 134400, "nv": 4096, "ranks": 4, "nodes": 1, "steps": 10, "warmup": 3, "dt": 0.02, "length": 12600, "vmax": 15, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 4404019200, "max_allocated_bytes_per_rank": 11155281976, "stream_ms": 96.7626505999533, "spline_ms": 29.38959880000311, "pack_ms": 6.514031399979103, "mpi_ms": 21.03211520004606, "field_ms": 6.66575620000458, "collision_ms": 14.49633700001414, "step_ms": 174.2364576000227, "initial_mass": 12597.6196507818, "final_mass": 12597.62016522545, "mass_relative_change": 4.083657516783715e-08, "initial_kinetic_energy": 55713.43281125277, "final_kinetic_energy": 55714.15660222915, "initial_field_energy": 787.6538181371966, "final_field_energy": 786.9554651665927, "minimum_f": -0.1916308738890627, "finite": true, "repeat": 1, "command": ["srun", "--jobid=60221398", "--overlap", "--nodes=1", "--nodelist=nid001600", "--ntasks=4", "--ntasks-per-node=4", "--gpus-per-node=4", "--gpu-bind=none", "--cpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T190159Z-99199-22637/workdir/cuda-build/vlasov_bench", "--nx", "134400", "--nv", "4096", "--steps", "10", "--warmup", "3", "--mode", "landau", "--velocity", "cubic", "--layout", "halos", "--integrator", "sixth", "--nu", "1e-12", "--length", "12600.0", "--vmax", "15.0", "--dt", "0.02", "--cells", "--input", "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/initial.bin", "--v-grid", "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/v.bin", "--background", "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/background.bin"], "rank_log": "rank=1 host=nid001600 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001600 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001600 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001600 local_rank=0 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=c0a2abe12f3d288f697c8b183b00313aa4c6ba063d5122a570dfae4a37751380\nsynced_at=2026-10-24T19:31:46Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "integrator": "sixth", "nu": 1e-12, "velocity": "cubic", "layout": "halos", "cell_centred": true, "max_halo_width": 4, "rank0_halo_sent_bytes": 215040000, "halo_sent_bytes_per_rank_max": 430080000, "slab_sent_bytes_per_rank": 16515072000, "input": "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/initial.bin", "nx": 134400, "nv": 4096, "ranks": 4, "nodes": 1, "steps": 10, "warmup": 3, "dt": 0.02, "length": 12600, "vmax": 15, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 4404019200, "max_allocated_bytes_per_rank": 11155281976, "stream_ms": 96.76016480002545, "spline_ms": 29.38630270000431, "pack_ms": 6.511509500023749, "mpi_ms": 21.00970499996038, "field_ms": 6.653158599988274, "collision_ms": 14.5029157999943, "step_ms": 174.2416196000249, "initial_mass": 12597.6196507818, "final_mass": 12597.62016522545, "mass_relative_change": 4.083657516783715e-08, "initial_kinetic_energy": 55713.43281125277, "final_kinetic_energy": 55714.15660222915, "initial_field_energy": 787.6538181371966, "final_field_energy": 786.9554651665927, "minimum_f": -0.1916308738890627, "finite": true, "repeat": 2, "command": ["srun", "--jobid=60221398", "--overlap", "--nodes=1", "--nodelist=nid001600", "--ntasks=4", "--ntasks-per-node=4", "--gpus-per-node=4", "--gpu-bind=none", "--cpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T190159Z-99199-22637/workdir/cuda-build/vlasov_bench", "--nx", "134400", "--nv", "4096", "--steps", "10", "--warmup", "3", "--mode", "landau", "--velocity", "cubic", "--layout", "halos", "--integrator", "sixth", "--nu", "1e-12", "--length", "12600.0", "--vmax", "15.0", "--dt", "0.02", "--cells", "--input", "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/initial.bin", "--v-grid", "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/v.bin", "--background", "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/background.bin"], "rank_log": "rank=1 host=nid001600 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001600 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001600 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001600 local_rank=2 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=c0a2abe12f3d288f697c8b183b00313aa4c6ba063d5122a570dfae4a37751380\nsynced_at=2026-10-24T19:31:46Z\n"} diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-adept/cuda-nx134400-g8.jsonl b/benchmarks/vlasov_cuda/results/2026-10-24-adept/cuda-nx134400-g8.jsonl new file mode 100644 index 00000000..a86d7ad8 --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-adept/cuda-nx134400-g8.jsonl @@ -0,0 +1,3 @@ +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "integrator": "sixth", "nu": 1e-12, "velocity": "cubic", "layout": "halos", "cell_centred": true, "max_halo_width": 4, "rank0_halo_sent_bytes": 215040000, "halo_sent_bytes_per_rank_max": 430080000, "slab_sent_bytes_per_rank": 9633792000, "input": "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/initial.bin", "nx": 134400, "nv": 4096, "ranks": 8, "nodes": 2, "steps": 10, "warmup": 3, "dt": 0.02, "length": 12600, "vmax": 15, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 4404019200, "max_allocated_bytes_per_rank": 5650233400, "stream_ms": 48.50647130010657, "spline_ms": 12.24466279991248, "pack_ms": 3.281769600107509, "mpi_ms": 60.21116610008903, "field_ms": 6.163560300092286, "collision_ms": 11.23915750001743, "step_ms": 140.0714161999531, "initial_mass": 12597.61965078436, "final_mass": 12597.62016522508, "mass_relative_change": 4.083634327505363e-08, "initial_kinetic_energy": 55713.43281126581, "final_kinetic_energy": 55714.15660220217, "initial_field_energy": 787.6538181371961, "final_field_energy": 786.9554651665925, "minimum_f": -0.1916308738890627, "finite": true, "repeat": 0, "command": ["srun", "--jobid=60221398", "--overlap", "--nodes=2", "--ntasks=8", "--ntasks-per-node=4", "--gpus-per-node=4", "--gpu-bind=none", "--cpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T190159Z-99199-22637/workdir/cuda-build/vlasov_bench", "--nx", "134400", "--nv", "4096", "--steps", "10", "--warmup", "3", "--mode", "landau", "--velocity", "cubic", "--layout", "halos", "--integrator", "sixth", "--nu", "1e-12", "--length", "12600.0", "--vmax", "15.0", "--dt", "0.02", "--cells", "--input", "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/initial.bin", "--v-grid", "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/v.bin", "--background", "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/background.bin"], "rank_log": "rank=3 host=nid003745 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=5 host=nid003745 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid003745 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=7 host=nid003745 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001600 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001600 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=6 host=nid001600 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=4 host=nid001600 local_rank=2 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=afdc248c4553148cd3ca5a63cb0431b8e474acd835d818af386e898beb75eb2d\nsynced_at=2026-10-24T19:13:57Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "integrator": "sixth", "nu": 1e-12, "velocity": "cubic", "layout": "halos", "cell_centred": true, "max_halo_width": 4, "rank0_halo_sent_bytes": 215040000, "halo_sent_bytes_per_rank_max": 430080000, "slab_sent_bytes_per_rank": 9633792000, "input": "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/initial.bin", "nx": 134400, "nv": 4096, "ranks": 8, "nodes": 2, "steps": 10, "warmup": 3, "dt": 0.02, "length": 12600, "vmax": 15, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 4404019200, "max_allocated_bytes_per_rank": 5650233400, "stream_ms": 48.50949399988167, "spline_ms": 12.24919390003834, "pack_ms": 3.278222700009792, "mpi_ms": 60.14856769998005, "field_ms": 5.799373300033039, "collision_ms": 11.23889549996875, "step_ms": 140.015343999994, "initial_mass": 12597.61965078436, "final_mass": 12597.62016522508, "mass_relative_change": 4.083634327505363e-08, "initial_kinetic_energy": 55713.43281126581, "final_kinetic_energy": 55714.15660220217, "initial_field_energy": 787.6538181371961, "final_field_energy": 786.9554651665925, "minimum_f": -0.1916308738890627, "finite": true, "repeat": 1, "command": ["srun", "--jobid=60221398", "--overlap", "--nodes=2", "--ntasks=8", "--ntasks-per-node=4", "--gpus-per-node=4", "--gpu-bind=none", "--cpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T190159Z-99199-22637/workdir/cuda-build/vlasov_bench", "--nx", "134400", "--nv", "4096", "--steps", "10", "--warmup", "3", "--mode", "landau", "--velocity", "cubic", "--layout", "halos", "--integrator", "sixth", "--nu", "1e-12", "--length", "12600.0", "--vmax", "15.0", "--dt", "0.02", "--cells", "--input", "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/initial.bin", "--v-grid", "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/v.bin", "--background", "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/background.bin"], "rank_log": "rank=2 host=nid001600 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001600 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid003745 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=6 host=nid001600 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=4 host=nid001600 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid003745 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=5 host=nid003745 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=7 host=nid003745 local_rank=3 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=afdc248c4553148cd3ca5a63cb0431b8e474acd835d818af386e898beb75eb2d\nsynced_at=2026-10-24T19:13:57Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "integrator": "sixth", "nu": 1e-12, "velocity": "cubic", "layout": "halos", "cell_centred": true, "max_halo_width": 4, "rank0_halo_sent_bytes": 215040000, "halo_sent_bytes_per_rank_max": 430080000, "slab_sent_bytes_per_rank": 9633792000, "input": "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/initial.bin", "nx": 134400, "nv": 4096, "ranks": 8, "nodes": 2, "steps": 10, "warmup": 3, "dt": 0.02, "length": 12600, "vmax": 15, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 4404019200, "max_allocated_bytes_per_rank": 5650233400, "stream_ms": 48.49759509997966, "spline_ms": 12.24273310001536, "pack_ms": 3.282368400095947, "mpi_ms": 60.46683179993124, "field_ms": 6.560649900029603, "collision_ms": 11.23777200000404, "step_ms": 140.3418015999932, "initial_mass": 12597.61965078436, "final_mass": 12597.62016522508, "mass_relative_change": 4.083634327505363e-08, "initial_kinetic_energy": 55713.43281126581, "final_kinetic_energy": 55714.15660220217, "initial_field_energy": 787.6538181371961, "final_field_energy": 786.9554651665925, "minimum_f": -0.1916308738890627, "finite": true, "repeat": 2, "command": ["srun", "--jobid=60221398", "--overlap", "--nodes=2", "--ntasks=8", "--ntasks-per-node=4", "--gpus-per-node=4", "--gpu-bind=none", "--cpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T190159Z-99199-22637/workdir/cuda-build/vlasov_bench", "--nx", "134400", "--nv", "4096", "--steps", "10", "--warmup", "3", "--mode", "landau", "--velocity", "cubic", "--layout", "halos", "--integrator", "sixth", "--nu", "1e-12", "--length", "12600.0", "--vmax", "15.0", "--dt", "0.02", "--cells", "--input", "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/initial.bin", "--v-grid", "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/v.bin", "--background", "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/background.bin"], "rank_log": "rank=0 host=nid001600 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=6 host=nid001600 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001600 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid003745 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=5 host=nid003745 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=4 host=nid001600 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid003745 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=7 host=nid003745 local_rank=3 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=afdc248c4553148cd3ca5a63cb0431b8e474acd835d818af386e898beb75eb2d\nsynced_at=2026-10-24T19:13:57Z\n"} diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-adept/cuda-nx268800-g8.jsonl b/benchmarks/vlasov_cuda/results/2026-10-24-adept/cuda-nx268800-g8.jsonl new file mode 100644 index 00000000..705db02d --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-adept/cuda-nx268800-g8.jsonl @@ -0,0 +1,3 @@ +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "integrator": "sixth", "nu": 1e-12, "velocity": "cubic", "layout": "halos", "cell_centred": true, "max_halo_width": 4, "rank0_halo_sent_bytes": 438681600, "halo_sent_bytes_per_rank_max": 877363200, "slab_sent_bytes_per_rank": 19267584000, "input": "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/initial.bin", "nx": 268800, "nv": 4096, "ranks": 8, "nodes": 2, "steps": 10, "warmup": 3, "dt": 0.02, "length": 12600, "vmax": 15, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 8808038400, "max_allocated_bytes_per_rank": 11300409400, "stream_ms": 97.32425760009846, "spline_ms": 30.11911129995042, "pack_ms": 6.521739400091064, "mpi_ms": 119.9787687000708, "field_ms": 10.82064670004002, "collision_ms": 14.49070120004308, "step_ms": 276.0648734999677, "initial_mass": 12597.62207265221, "final_mass": 12597.62136207489, "mass_relative_change": -5.640567068345301e-08, "initial_kinetic_energy": 55701.51067942529, "final_kinetic_energy": 55701.00688716807, "initial_field_energy": 791.8606180599771, "final_field_energy": 792.3077830250799, "minimum_f": -0.09580882788649095, "finite": true, "repeat": 0, "command": ["srun", "--jobid=60221398", "--overlap", "--nodes=2", "--ntasks=8", "--ntasks-per-node=4", "--gpus-per-node=4", "--gpu-bind=none", "--cpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T190159Z-99199-22637/workdir/cuda-build/vlasov_bench", "--nx", "268800", "--nv", "4096", "--steps", "10", "--warmup", "3", "--mode", "landau", "--velocity", "cubic", "--layout", "halos", "--integrator", "sixth", "--nu", "1e-12", "--length", "12600.0", "--vmax", "15.0", "--dt", "0.02", "--cells", "--input", "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/initial.bin", "--v-grid", "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/v.bin", "--background", "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/background.bin"], "rank_log": "rank=7 host=nid003745 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=6 host=nid001600 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001600 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=4 host=nid001600 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001600 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid003745 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=5 host=nid003745 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid003745 local_rank=1 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=37396dd0852f9e6ff790f730c645eaa6681006a0a135e7039737d9472112a7d3\nsynced_at=2026-10-24T19:19:12Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "integrator": "sixth", "nu": 1e-12, "velocity": "cubic", "layout": "halos", "cell_centred": true, "max_halo_width": 4, "rank0_halo_sent_bytes": 438681600, "halo_sent_bytes_per_rank_max": 877363200, "slab_sent_bytes_per_rank": 19267584000, "input": "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/initial.bin", "nx": 268800, "nv": 4096, "ranks": 8, "nodes": 2, "steps": 10, "warmup": 3, "dt": 0.02, "length": 12600, "vmax": 15, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 8808038400, "max_allocated_bytes_per_rank": 11300409400, "stream_ms": 97.33070519996545, "spline_ms": 30.12465930000872, "pack_ms": 6.524045600053796, "mpi_ms": 119.7810640999251, "field_ms": 10.77050900000813, "collision_ms": 14.49306849997356, "step_ms": 275.9853768999847, "initial_mass": 12597.62207265221, "final_mass": 12597.62136207489, "mass_relative_change": -5.640567068345301e-08, "initial_kinetic_energy": 55701.51067942529, "final_kinetic_energy": 55701.00688716807, "initial_field_energy": 791.8606180599771, "final_field_energy": 792.3077830250799, "minimum_f": -0.09580882788649095, "finite": true, "repeat": 1, "command": ["srun", "--jobid=60221398", "--overlap", "--nodes=2", "--ntasks=8", "--ntasks-per-node=4", "--gpus-per-node=4", "--gpu-bind=none", "--cpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T190159Z-99199-22637/workdir/cuda-build/vlasov_bench", "--nx", "268800", "--nv", "4096", "--steps", "10", "--warmup", "3", "--mode", "landau", "--velocity", "cubic", "--layout", "halos", "--integrator", "sixth", "--nu", "1e-12", "--length", "12600.0", "--vmax", "15.0", "--dt", "0.02", "--cells", "--input", "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/initial.bin", "--v-grid", "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/v.bin", "--background", "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/background.bin"], "rank_log": "rank=0 host=nid001600 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid003745 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid003745 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=7 host=nid003745 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=5 host=nid003745 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001600 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=6 host=nid001600 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=4 host=nid001600 local_rank=2 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=37396dd0852f9e6ff790f730c645eaa6681006a0a135e7039737d9472112a7d3\nsynced_at=2026-10-24T19:19:12Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "integrator": "sixth", "nu": 1e-12, "velocity": "cubic", "layout": "halos", "cell_centred": true, "max_halo_width": 4, "rank0_halo_sent_bytes": 438681600, "halo_sent_bytes_per_rank_max": 877363200, "slab_sent_bytes_per_rank": 19267584000, "input": "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/initial.bin", "nx": 268800, "nv": 4096, "ranks": 8, "nodes": 2, "steps": 10, "warmup": 3, "dt": 0.02, "length": 12600, "vmax": 15, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 8808038400, "max_allocated_bytes_per_rank": 11300409400, "stream_ms": 97.31154520000018, "spline_ms": 30.11697609997555, "pack_ms": 6.524896600012653, "mpi_ms": 119.2875023999704, "field_ms": 10.904615400068, "collision_ms": 14.49597959997391, "step_ms": 275.6066252999744, "initial_mass": 12597.62207265221, "final_mass": 12597.62136207489, "mass_relative_change": -5.640567068345301e-08, "initial_kinetic_energy": 55701.51067942529, "final_kinetic_energy": 55701.00688716807, "initial_field_energy": 791.8606180599771, "final_field_energy": 792.3077830250799, "minimum_f": -0.09580882788649095, "finite": true, "repeat": 2, "command": ["srun", "--jobid=60221398", "--overlap", "--nodes=2", "--ntasks=8", "--ntasks-per-node=4", "--gpus-per-node=4", "--gpu-bind=none", "--cpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T190159Z-99199-22637/workdir/cuda-build/vlasov_bench", "--nx", "268800", "--nv", "4096", "--steps", "10", "--warmup", "3", "--mode", "landau", "--velocity", "cubic", "--layout", "halos", "--integrator", "sixth", "--nu", "1e-12", "--length", "12600.0", "--vmax", "15.0", "--dt", "0.02", "--cells", "--input", "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/initial.bin", "--v-grid", "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/v.bin", "--background", "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/background.bin"], "rank_log": "rank=3 host=nid003745 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid003745 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=7 host=nid003745 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=5 host=nid003745 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001600 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001600 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=4 host=nid001600 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=6 host=nid001600 local_rank=3 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=37396dd0852f9e6ff790f730c645eaa6681006a0a135e7039737d9472112a7d3\nsynced_at=2026-10-24T19:19:12Z\n"} diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-adept/integration-checks.json b/benchmarks/vlasov_cuda/results/2026-10-24-adept/integration-checks.json new file mode 100644 index 00000000..6c396f06 --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-adept/integration-checks.json @@ -0,0 +1,210 @@ +[ + { + "module": "BaseVlasov1D", + "ranks": 1, + "passed": true, + "max_abs_error": 1.0302869668521453e-13, + "mlflow_run_id": "10db23e28ed8428d9849bce2dea93d3f", + "stats": { + "native_seconds": { + "stream": 0.000619517000018277, + "velocity": 0.0003302759998859983, + "pack": 0.0, + "mpi": 0.0008770139997977822, + "field": 0.0017125090001854915, + "collision": 0.0005039860001261331, + "total": 0.004283324999960314 + }, + "solve_and_save_seconds": 0.7859312820073683, + "steps": 5, + "ranks": 1, + "device": "cuda" + }, + "keys": [ + "fields", + "electron.main", + "default" + ], + "log": "integration-v3.log" + }, + { + "module": "BaseVlasov1D", + "ranks": 4, + "passed": true, + "max_abs_error": 1.2434497875801753e-13, + "mlflow_run_id": "c15bc81503a44c02bb7f0d22043aa3df", + "stats": { + "native_seconds": { + "stream": 0.0005902340002421624, + "velocity": 0.0003807449994610579, + "pack": 0.00023884999984602473, + "mpi": 0.005609786999912103, + "field": 0.0067621719998669505, + "collision": 0.000502029000017501, + "total": 0.013536288000068453 + }, + "solve_and_save_seconds": 1.881096900004195, + "steps": 5, + "ranks": 4, + "device": "cuda" + }, + "keys": [ + "fields", + "electron.main", + "default" + ], + "log": "integration-v3.log" + }, + { + "module": "BaseVlasov1D", + "ranks": 8, + "passed": true, + "max_abs_error": 1.3145040611561853e-13, + "mlflow_run_id": "fb178ca1d165436c98457fd4ba0db80f", + "stats": { + "native_seconds": { + "stream": 0.0006939400003602714, + "velocity": 0.0005676090001998091, + "pack": 0.00025299700007508363, + "mpi": 0.01576393800007736, + "field": 0.013597956999888083, + "collision": 0.0005164489999742727, + "total": 0.02774611500001356 + }, + "solve_and_save_seconds": 3.1865445940056816, + "steps": 5, + "ranks": 8, + "device": "cuda" + }, + "keys": [ + "fields", + "electron.main", + "default" + ], + "log": "integration-v3.log" + }, + { + "module": "VPTwoStream1D", + "ranks": 1, + "passed": true, + "max_abs_error": 1.0457412713549274e-11, + "mlflow_run_id": "ac6e44af25584f74add8c11f040061c7", + "stats": { + "native_seconds": { + "stream": 0.0005986490004943334, + "velocity": 0.0003280549998407878, + "pack": 0.0, + "mpi": 0.0008833609997509484, + "field": 0.0016982180000013614, + "collision": 0.000500530999829607, + "total": 0.004253136000102131 + }, + "solve_and_save_seconds": 0.8431226150132716, + "steps": 5, + "ranks": 1, + "device": "cuda" + }, + "keys": [ + "fields", + "electron.main", + "default", + "electron.f0" + ], + "log": "integration-vp.log" + }, + { + "module": "VPTwoStream1D", + "ranks": 4, + "passed": true, + "max_abs_error": 6.3788974102862994e-12, + "mlflow_run_id": "d16748ce2f364431bb4fa17896661f44", + "stats": { + "native_seconds": { + "stream": 0.0006106250001494118, + "velocity": 0.000375073000213888, + "pack": 0.00023151799985043908, + "mpi": 0.005738596999663059, + "field": 0.006173489000275367, + "collision": 0.0005022090000466051, + "total": 0.012953435000099489 + }, + "solve_and_save_seconds": 1.350150696001947, + "steps": 5, + "ranks": 4, + "device": "cuda" + }, + "keys": [ + "fields", + "electron.main", + "default", + "electron.f0" + ], + "log": "integration-vp.log" + }, + { + "module": "VPTwoStream1D", + "ranks": 8, + "passed": true, + "max_abs_error": 1.1592504733926035e-11, + "mlflow_run_id": "e3e58e103a6747489eadadb244806865", + "stats": { + "native_seconds": { + "stream": 0.0006200880002324993, + "velocity": 0.0004060329994217682, + "pack": 0.0002525769999692784, + "mpi": 0.008024029000125665, + "field": 0.007057047000103012, + "collision": 0.000508114000012938, + "total": 0.01669021899999734 + }, + "solve_and_save_seconds": 7.212047922017518, + "steps": 5, + "ranks": 8, + "device": "cuda" + }, + "keys": [ + "fields", + "electron.main", + "default", + "electron.f0" + ], + "log": "integration-vp.log" + }, + { + "module": "VPTwoStream1D", + "ranks": 8, + "passed": true, + "max_abs_error": 1.1592504733926035e-11, + "mlflow_run_id": "a31a0009b9a440c79965f3e70332ccde", + "stats": { + "native_seconds": { + "stream": 0.000619704999962778, + "velocity": 0.0004217190003146243, + "pack": 0.00027711899952009844, + "mpi": 0.008916388000216102, + "field": 0.010889343000826557, + "collision": 0.0005323509999470843, + "total": 0.02131122699961452 + }, + "solve_and_save_seconds": 3.074192091007717, + "steps": 5, + "ranks": 8, + "device": "cuda", + "libraries": { + "jax": "0.10.2", + "jaxlib": "0.10.2", + "cloudpickle": "3.1.2", + "diffrax": "0.7.2" + } + }, + "keys": [ + "fields", + "electron.main", + "default", + "electron.f0", + "electron.pkv", + "electron.cascade" + ], + "log": "integration-observers-v2.log" + } +] diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-adept/jax-nx134400-g4.json b/benchmarks/vlasov_cuda/results/2026-10-24-adept/jax-nx134400-g4.json new file mode 100644 index 00000000..7dc3bfba --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-adept/jax-nx134400-g4.json @@ -0,0 +1,120 @@ +{ + "rows": [ + { + "backend": "adept-jax", + "nx": 134400, + "nv": 4096, + "ranks": 4, + "repeat": 0, + "steps": 10, + "warmup": 3, + "step_ms": 223.41581310029142, + "rank_seconds": [ + 2.233701307006413, + 2.234158131002914, + 2.233267256000545, + 2.23381844398682 + ], + "sum": 18346614.114405707, + "sum_squares": 2101439.806997545, + "minimum": -0.19163087388906355, + "maximum": 0.3687389216845132 + }, + { + "backend": "adept-jax", + "nx": 134400, + "nv": 4096, + "ranks": 4, + "repeat": 1, + "steps": 10, + "warmup": 3, + "step_ms": 223.62807510071434, + "rank_seconds": [ + 2.2362807510071434, + 2.2362757319933735, + 2.235863984999014, + 2.2360537410131656 + ], + "sum": 18346614.114405707, + "sum_squares": 2101439.806997545, + "minimum": -0.19163087388906355, + "maximum": 0.3687389216845132 + }, + { + "backend": "adept-jax", + "nx": 134400, + "nv": 4096, + "ranks": 4, + "repeat": 2, + "steps": 10, + "warmup": 3, + "step_ms": 223.53556799935177, + "rank_seconds": [ + 2.235278229985852, + 2.2352399160154164, + 2.2353556799935177, + 2.235204117023386 + ], + "sum": 18346614.114405707, + "sum_squares": 2101439.806997545, + "minimum": -0.19163087388906355, + "maximum": 0.3687389216845132 + } + ], + "checkpoint": { + "checkpoint": "/pscratch/sd/a/archis/vp-turbulence-sessions/20261023T081121Z-nv4096-nx134400-5ce4fe14/workdir/campaigns/production-nx134400/checkpoint-state/step-030000-1a19a5ae15f8", + "manifest_sha256": "9ab7e963d2699777bd935ca84fad446b89846e64c416d10db6d3c14829af8627", + "actual_time": 600.0, + "config_sha256": "5a9f5347e6253a3d4201745318843857c568b3787cf91e9fe81a623d0cdbb214", + "adept_commit": "ddd70b957dc3df85f0b5ec84af927c25d09d24e0", + "jax": "0.10.2", + "nx": 134400, + "nv": 4096, + "dt": 0.02, + "nu": 1e-12, + "devices": [ + "cuda:0", + "cuda:1", + "cuda:2", + "cuda:3" + ], + "milestones": [ + { + "steps": 1, + "advanced_steps": 1, + "seconds": 3.020330819010269 + }, + { + "steps": 10, + "advanced_steps": 9, + "seconds": 2.0284883639833424 + }, + { + "steps": 100, + "advanced_steps": 90, + "seconds": 20.115412370010745 + } + ] + }, + "adept_commit": "ddd70b957dc3df85f0b5ec84af927c25d09d24e0", + "libraries": { + "jax": "0.10.2", + "jaxlib": "0.10.2", + "interpax": "0.3.14+10.g84aaf30", + "diffrax": "0.7.2" + }, + "hosts": [ + "nid001600", + "nid001600", + "nid001600", + "nid001600" + ], + "compile_seconds": 0.9581184719863813, + "devices": [ + "cuda:0", + "cuda:1", + "cuda:2", + "cuda:3" + ], + "scope": "Warmed operator loop; no SaveAt, diagnostics, I/O, initialization or compilation in timings" +} \ No newline at end of file diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-adept/jax-nx134400-g8.json b/benchmarks/vlasov_cuda/results/2026-10-24-adept/jax-nx134400-g8.json new file mode 100644 index 00000000..fd9145ce --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-adept/jax-nx134400-g8.json @@ -0,0 +1,140 @@ +{ + "rows": [ + { + "backend": "adept-jax", + "nx": 134400, + "nv": 4096, + "ranks": 8, + "repeat": 0, + "steps": 10, + "warmup": 3, + "step_ms": 383.0053849000251, + "rank_seconds": [ + 3.8297905579966027, + 3.830011726997327, + 3.82965898400289, + 3.8299861470004544, + 3.8296211309789214, + 3.830053849000251, + 3.829693990992382, + 3.8298543929995503 + ], + "sum": 18346614.114405707, + "sum_squares": 2101439.806997545, + "minimum": -0.19163087388906322, + "maximum": 0.36873892168451344 + }, + { + "backend": "adept-jax", + "nx": 134400, + "nv": 4096, + "ranks": 8, + "repeat": 1, + "steps": 10, + "warmup": 3, + "step_ms": 383.94048050104175, + "rank_seconds": [ + 3.839310933020897, + 3.8392813240061514, + 3.8394048050104175, + 3.8391784759878647, + 3.8392548949923366, + 3.8392177119967528, + 3.839308759983396, + 3.8392889490060043 + ], + "sum": 18346614.114405707, + "sum_squares": 2101439.806997545, + "minimum": -0.19163087388906322, + "maximum": 0.36873892168451344 + }, + { + "backend": "adept-jax", + "nx": 134400, + "nv": 4096, + "ranks": 8, + "repeat": 2, + "steps": 10, + "warmup": 3, + "step_ms": 384.9415123986546, + "rank_seconds": [ + 3.848791770025855, + 3.8486333749897312, + 3.848707717988873, + 3.8486705959949177, + 3.8486820879916195, + 3.848695304011926, + 3.8488800520135555, + 3.849415123986546 + ], + "sum": 18346614.114405707, + "sum_squares": 2101439.806997545, + "minimum": -0.19163087388906322, + "maximum": 0.36873892168451344 + } + ], + "checkpoint": { + "checkpoint": "/pscratch/sd/a/archis/vp-turbulence-sessions/20261023T081121Z-nv4096-nx134400-5ce4fe14/workdir/campaigns/production-nx134400/checkpoint-state/step-030000-1a19a5ae15f8", + "manifest_sha256": "9ab7e963d2699777bd935ca84fad446b89846e64c416d10db6d3c14829af8627", + "actual_time": 600.0, + "config_sha256": "5a9f5347e6253a3d4201745318843857c568b3787cf91e9fe81a623d0cdbb214", + "adept_commit": "ddd70b957dc3df85f0b5ec84af927c25d09d24e0", + "jax": "0.10.2", + "nx": 134400, + "nv": 4096, + "dt": 0.02, + "nu": 1e-12, + "devices": [ + "cuda:0", + "cuda:1", + "cuda:2", + "cuda:3" + ], + "milestones": [ + { + "steps": 1, + "advanced_steps": 1, + "seconds": 3.020330819010269 + }, + { + "steps": 10, + "advanced_steps": 9, + "seconds": 2.0284883639833424 + }, + { + "steps": 100, + "advanced_steps": 90, + "seconds": 20.115412370010745 + } + ] + }, + "adept_commit": "ddd70b957dc3df85f0b5ec84af927c25d09d24e0", + "libraries": { + "jax": "0.10.2", + "jaxlib": "0.10.2", + "interpax": "0.3.14+10.g84aaf30", + "diffrax": "0.7.2" + }, + "hosts": [ + "nid001600", + "nid003745", + "nid001600", + "nid003745", + "nid001600", + "nid003745", + "nid001600", + "nid003745" + ], + "compile_seconds": 0.9945056700089481, + "devices": [ + "cuda:0", + "cuda:4", + "cuda:1", + "cuda:5", + "cuda:2", + "cuda:6", + "cuda:3", + "cuda:7" + ], + "scope": "Warmed operator loop; no SaveAt, diagnostics, I/O, initialization or compilation in timings" +} \ No newline at end of file diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-adept/jax-nx268800-g8.json b/benchmarks/vlasov_cuda/results/2026-10-24-adept/jax-nx268800-g8.json new file mode 100644 index 00000000..01a8b9b6 --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-adept/jax-nx268800-g8.json @@ -0,0 +1,142 @@ +{ + "rows": [ + { + "backend": "adept-jax", + "nx": 268800, + "nv": 4096, + "ranks": 8, + "repeat": 0, + "steps": 10, + "warmup": 3, + "step_ms": 757.4678515986307, + "rank_seconds": [ + 7.574635903991293, + 7.574306378985057, + 7.574678515986307, + 7.574352947995067, + 7.574508777994197, + 7.574119797995081, + 7.574576738988981, + 7.574465805984801 + ], + "sum": 36693231.71487449, + "sum_squares": 4200670.248861345, + "minimum": -0.09580882788649053, + "maximum": 0.4397238821796596 + }, + { + "backend": "adept-jax", + "nx": 268800, + "nv": 4096, + "ranks": 8, + "repeat": 1, + "steps": 10, + "warmup": 3, + "step_ms": 754.9265112000285, + "rank_seconds": [ + 7.54897340000025, + 7.549149327998748, + 7.548913473001448, + 7.549265112000285, + 7.548920306988293, + 7.5492265569919255, + 7.548932781006442, + 7.549169297009939 + ], + "sum": 36693231.71487449, + "sum_squares": 4200670.248861345, + "minimum": -0.09580882788649053, + "maximum": 0.4397238821796596 + }, + { + "backend": "adept-jax", + "nx": 268800, + "nv": 4096, + "ranks": 8, + "repeat": 2, + "steps": 10, + "warmup": 3, + "step_ms": 756.6989740997087, + "rank_seconds": [ + 7.566910344001371, + 7.566823539993493, + 7.566708603000734, + 7.566989740997087, + 7.566779902001144, + 7.566962459008209, + 7.566738772002282, + 7.566934483998921 + ], + "sum": 36693231.71487449, + "sum_squares": 4200670.248861345, + "minimum": -0.09580882788649053, + "maximum": 0.4397238821796596 + } + ], + "checkpoint": { + "checkpoint": "/pscratch/sd/a/archis/vp-turbulence-sessions/20261023T081121Z-nv4096-nx268800-93a418d7/workdir/campaigns/production-nx268800/checkpoint-state/step-030000-167819aeccca", + "manifest_sha256": "c9b9083f55cfcee8595556685d9e22eb9d87829fa6748726f7f44d5749fbec1f", + "actual_time": 600.0, + "config_sha256": "6adb42855432b79a74f63290678103c17c53842074d77bf092df38483f634a83", + "adept_commit": "ddd70b957dc3df85f0b5ec84af927c25d09d24e0", + "jax": "0.10.2", + "nx": 268800, + "nv": 4096, + "dt": 0.02, + "nu": 1e-12, + "devices": [ + "cuda:0", + "cuda:1", + "cuda:2", + "cuda:3", + "cuda:4", + "cuda:5", + "cuda:6", + "cuda:7" + ], + "milestones": [ + { + "steps": 1, + "advanced_steps": 1, + "rank0_seconds": 60.40102391698747 + }, + { + "steps": 100, + "advanced_steps": 99, + "rank0_seconds": 74.4436078449944 + } + ], + "process_count": 8, + "distributed": true, + "prepared_input": "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800" + }, + "adept_commit": "ddd70b957dc3df85f0b5ec84af927c25d09d24e0", + "libraries": { + "jax": "0.10.2", + "jaxlib": "0.10.2", + "interpax": "0.3.14+10.g84aaf30", + "diffrax": "0.7.2" + }, + "hosts": [ + "nid001600", + "nid003745", + "nid001600", + "nid003745", + "nid001600", + "nid003745", + "nid001600", + "nid003745" + ], + "compile_seconds": 1.1942315599881113, + "devices": [ + "cuda:0", + "cuda:4", + "cuda:1", + "cuda:5", + "cuda:2", + "cuda:6", + "cuda:3", + "cuda:7" + ], + "scope": "Warmed operator loop; no SaveAt, diagnostics, I/O, initialization or compilation in timings" +} \ No newline at end of file diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-adept/launch.sh b/benchmarks/vlasov_cuda/results/2026-10-24-adept/launch.sh new file mode 100644 index 00000000..5c4131d4 --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-adept/launch.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# Run only inside an owned two-node allocation, after building both CUDA targets. +set -euo pipefail +if [ "${SLURM_PROCID:-0}" -ne 0 ]; then exit 0; fi +: "${EC_SESSION_ROOT:?}" "${SLURM_JOB_ID:?}" +source_root="$EC_SESSION_ROOT/src" +report_root="$EC_SESSION_OUTPUTS/adept-comparison" +mkdir "$report_root" +prepared_root=/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs +vp_small=/pscratch/sd/a/archis/vp-turbulence-sessions/20261023T081121Z-nv4096-nx134400-5ce4fe14/src +vp_large=/pscratch/sd/a/archis/vp-turbulence-sessions/20261023T081121Z-nv4096-nx268800-93a418d7/src +export MPICH_GPU_SUPPORT_ENABLED=1 XLA_PYTHON_CLIENT_PREALLOCATE=false JAX_ENABLE_X64=true +export LD_LIBRARY_PATH="/global/common/software/nersc9/nccl/2.29.2-cu13/plugin/lib:/opt/nvidia/hpc_sdk/Linux_x86_64/26.5/cuda/13.2/lib64${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" +export NCCL_SOCKET_IFNAME=hsn NCCL_CROSS_NIC=2 NCCL_NET="AWS Libfabric" NCCL_NET_GDR_LEVEL=LOC FI_CXI_DISABLE_HOST_REGISTER=1 +export XLA_FLAGS="${XLA_FLAGS:+$XLA_FLAGS }--xla_gpu_autotune_level=0" +for case in 134400:1:4 134400:2:8 268800:2:8; do + IFS=: read -r nx nodes ranks <<< "$case" + prepared="$prepared_root/native-nx$nx" + vp_source="$vp_small" + if [ "$nx" = 268800 ]; then + prepared="$prepared_root/native-nx268800-distributed" + vp_source="$vp_large" + fi + # The pinned environment's own ADEPT must win over this development checkout. + # JAX CUDA wheels supply their runtime; unloading cudatoolkit matches the reference runs. + ( + module unload cudatoolkit 2>/dev/null || true + source /global/common/software/m4490/archis/venvs/vp-turbulence/bin/activate + export PYTHONPATH="$vp_source" + srun --jobid="$SLURM_JOB_ID" --overlap --nodes="$nodes" --ntasks="$ranks" \ + --ntasks-per-node=$((ranks / nodes)) --gpus-per-node=4 --gpu-bind=none \ + --cpu-bind=none --mpi=none --kill-on-bad-exit=1 \ + python -u "$source_root/benchmarks/vlasov_cuda/adept_timing.py" \ + --prepared "$prepared" --config "$source_root/benchmarks/vlasov_cuda/reference/latest-nv4096/nx$nx.yaml" \ + --vp-source "$vp_source" --output "$report_root/jax-nx$nx-g$ranks.json" \ + > "$report_root/jax-nx$nx-g$ranks.log" 2>&1 + ) + python -u benchmarks/vlasov_cuda/measure.py "$EC_SESSION_WORKDIR/cuda-build/vlasov_bench" \ + --launcher "srun --jobid=$SLURM_JOB_ID --overlap --nodes={nodes} --ntasks={ranks} --ntasks-per-node={ranks_per_node} --gpus-per-node=4 --gpu-bind=none --cpu-bind=none" \ + --placements "$nodes:$ranks" --nx "$nx" --nv 4096 --steps 10 --warmup 3 --repeats 3 \ + --expect-cuda --velocity cubic --layout halos --integrator sixth --cells --nu 1e-12 \ + --length 12600 --vmax 15 --dt .02 --input "$prepared/initial.bin" \ + --v-grid "$prepared/v.bin" --background "$prepared/background.bin" --max-mass-change -1 \ + --source-state "$EC_SESSION_ROOT/source-state" --output "$report_root/cuda-nx$nx-g$ranks.jsonl" +done +sha256sum "$EC_SESSION_WORKDIR/cuda-build/vlasov_bench" "$EC_SESSION_WORKDIR/native-build/libadept_vlasov.so" > "$report_root/binaries.sha256" +sha256sum adept/_vlasov1d/native/api.cpp adept/_vlasov1d/native/core/* benchmarks/vlasov_cuda/adept_timing.py > "$report_root/source.sha256" +cp "$EC_SESSION_ROOT/source-state" "$report_root/source-state" +cp benchmarks/vlasov_cuda/run_adept_comparison.sh "$report_root/launch.sh" diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-adept/observer-native/library.sha256 b/benchmarks/vlasov_cuda/results/2026-10-24-adept/observer-native/library.sha256 new file mode 100644 index 00000000..e871ac63 --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-adept/observer-native/library.sha256 @@ -0,0 +1 @@ +8ea49ba7a81a1e0074e35e9f63ae73680f38005adc7b49acd12343b7e7951dd7 libadept_vlasov.so diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-adept/observer-native/request.json b/benchmarks/vlasov_cuda/results/2026-10-24-adept/observer-native/request.json new file mode 100644 index 00000000..0cb6206b --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-adept/observer-native/request.json @@ -0,0 +1,84 @@ +{ + "library": "/pscratch/sd/a/archis/adept-sessions/20261024T190159Z-99199-22637/workdir/native-build/libadept_vlasov.so", + "options": [ + "--nx", + "32", + "--nv", + "64", + "--length", + "20.94", + "--vmax", + "8.0", + "--dt", + "0.02", + "--nu", + "0.001", + "--cells", + "--integrator", + "sixth", + "--velocity", + "cubic", + "--layout", + "halos", + "--halo-capacity", + "64", + "--input", + "/pscratch/sd/a/archis/adept-sessions/20261024T190159Z-99199-22637/outputs/integration-observers-v2/adept-native-gz4p09y5/initial.bin", + "--v-grid", + "/pscratch/sd/a/archis/adept-sessions/20261024T190159Z-99199-22637/outputs/integration-observers-v2/adept-native-gz4p09y5/v.bin", + "--background", + "/pscratch/sd/a/archis/adept-sessions/20261024T190159Z-99199-22637/outputs/integration-observers-v2/adept-native-gz4p09y5/background.bin" + ], + "times": { + "fields": [ + 0.0, + 0.02, + 0.04, + 0.08, + 0.1 + ], + "electron.main": [ + 0.0, + 0.04, + 0.1 + ], + "default": [ + 0.0, + 0.02, + 0.04, + 0.06, + 0.08, + 0.1 + ], + "electron.f0": [ + 0.0 + ], + "electron.pkv": [ + 0.0, + 0.05, + 0.1 + ], + "electron.cascade": [ + 0.0, + 0.05, + 0.1 + ] + }, + "t0": 0.0, + "dt": 0.02, + "steps": 5, + "device": "cuda", + "ranks": 8, + "pythonpath": [ + "/pscratch/sd/a/archis/vp-turbulence-sessions/20261023T081121Z-nv4096-nx134400-5ce4fe14/src", + "/pscratch/sd/a/archis/adept-sessions/20261024T190159Z-99199-22637/src/benchmarks/vlasov_cuda", + "/pscratch/sd/a/archis/adept-sessions/20261024T190159Z-99199-22637/src", + "/pscratch/sd/a/archis/adept-sessions/20261024T190159Z-99199-22637/src/src", + "/opt/nersc/pymon", + "/global/common/software/m4490/archis/uv-python/cpython-3.13.7-linux-x86_64-gnu/lib/python313.zip", + "/global/common/software/m4490/archis/uv-python/cpython-3.13.7-linux-x86_64-gnu/lib/python3.13", + "/global/common/software/m4490/archis/uv-python/cpython-3.13.7-linux-x86_64-gnu/lib/python3.13/lib-dynload", + "/global/common/software/m4490/archis/venvs/adept/lib/python3.13/site-packages", + "/pscratch/sd/a/archis/adept" + ] +} \ No newline at end of file diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-adept/observer-native/stats.json b/benchmarks/vlasov_cuda/results/2026-10-24-adept/observer-native/stats.json new file mode 100644 index 00000000..7c169385 --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-adept/observer-native/stats.json @@ -0,0 +1,21 @@ +{ + "native_seconds": { + "stream": 0.000619704999962778, + "velocity": 0.0004217190003146243, + "pack": 0.00027711899952009844, + "mpi": 0.008916388000216102, + "field": 0.010889343000826557, + "collision": 0.0005323509999470843, + "total": 0.02131122699961452 + }, + "solve_and_save_seconds": 3.074192091007717, + "steps": 5, + "ranks": 8, + "device": "cuda", + "libraries": { + "jax": "0.10.2", + "jaxlib": "0.10.2", + "cloudpickle": "3.1.2", + "diffrax": "0.7.2" + } +} \ No newline at end of file diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-adept/observer-netcdf-check.json b/benchmarks/vlasov_cuda/results/2026-10-24-adept/observer-netcdf-check.json new file mode 100644 index 00000000..053112b6 --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-adept/observer-netcdf-check.json @@ -0,0 +1,144 @@ +{ + "dist-electron.f0.nc": { + "sizes": { + "t": 1, + "v_electron": 64 + }, + "variables": [ + "electron.f0" + ], + "times": [ + 0.0 + ] + }, + "dist-electron.main.nc": { + "sizes": { + "t": 3, + "x": 32, + "v_electron": 64 + }, + "variables": [ + "electron.main" + ], + "times": [ + 0.0, + 0.04, + 0.1 + ] + }, + "fields-electron-t=0.1.nc": { + "sizes": { + "t": 5, + "x": 32 + }, + "variables": [ + "fields--flogf", + "fields-f^2", + "fields-j", + "fields-n", + "fields-p", + "fields-q", + "fields-v" + ], + "times": [ + 0.0, + 0.02, + 0.04, + 0.08, + 0.1 + ] + }, + "fields-shared-t=0.1.nc": { + "sizes": { + "t": 5, + "x": 32 + }, + "variables": [ + "fields-e", + "fields-de", + "fields-a", + "fields-prev_a", + "fields-pond" + ], + "times": [ + 0.0, + 0.02, + 0.04, + 0.08, + 0.1 + ] + }, + "native-cascade.nc": { + "sizes": { + "t": 3, + "kx": 17 + }, + "variables": [ + "c2", + "c2_background", + "c2_fluctuation", + "collision_loss_rate_estimate", + "collision_moments_valid", + "density_min", + "edge_absolute_mass", + "f_min", + "invalid_moment_cell_fraction", + "mass", + "negative_mass", + "pkx", + "temperature_min" + ], + "times": [ + 0.0, + 0.05, + 0.1 + ] + }, + "scalars-t=0.1.nc": { + "sizes": { + "t": 6 + }, + "variables": [ + "c2_0", + "delta_c2", + "gamma2", + "mean_-flogf_electron", + "mean_P_electron", + "mean_de2", + "mean_e2", + "mean_f2_electron", + "mean_field_energy", + "mean_j_electron", + "mean_kinetic_energy", + "mean_n_electron", + "mean_pond", + "mean_q_electron", + "mean_total_energy", + "n_holes_kc", + "n_holes_kpeak", + "v_the" + ], + "times": [ + 0.0, + 0.02, + 0.04, + 0.06, + 0.08, + 0.1 + ] + }, + "spectrum-electron.pkv.nc": { + "sizes": { + "t": 3, + "kv_electron": 33 + }, + "variables": [ + "electron.pkv" + ], + "times": [ + 0.0, + 0.05, + 0.1 + ] + } +} diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-adept/observer-netcdf/dist-electron.f0.nc b/benchmarks/vlasov_cuda/results/2026-10-24-adept/observer-netcdf/dist-electron.f0.nc new file mode 100644 index 0000000000000000000000000000000000000000..85854528c1d4f011788c2d6d74bce43aff408eeb GIT binary patch literal 8687 zcmeI1du&rx9LLY?p0pk9)~$5!q2ZCRlmXi^S$prW>)hCAmsTcOFx^5sXxM{n6~RE5 z5CTjMQ3*JC1vXIw5-~w1Au>cD8e@V2Lq#2?u#ka9WPl`Wp5HycL#0v>!XFy%S$aR` zp7T5B{C?l_XtT4fytFJqou(#`;^Rp)(aTx>7{T=GSXRQ{d3_zvM4+?0)TPq)s~(f< zqhipEcN0#viU?p5n+#7OS_Pm<2vo}Dv?f!2kB*Yd2~jWh_!@ol0Q9rPT1Yn2>oLb)KAph^&!!{{~( ziF~I1Gpg)FOJ@`&D@JbR zm@&%^-9amu1>Pz^wD~j!`aYUysinu^3(WI2;Ew4QzS$oap%gYCrfnxA2YWAsq1QmF*K zJz@T}J@nf^_8~xhaM{c3F1y2K_b?xfv;*+LTOX)z^KlJmL4?G8_4=E9&29dcX0!{9 zc=v`d#vLbEgnhujcz7@4Q};=s)q6IFmG8xB*fDS>KinP+*BLqmZM`3i;7N$XgrPmV zS1xJKLS`Z}km<-YWGd2x9D_6>Q;^9>12PGjh}1I@9Wnu_L5@PkA!CqHNC7!~h{)+C z@*46A@)9zLJcI1zlvw*S$RP3(@(S`A@+NW^DU`7GQOFo%9C8#=gG@l`7~A#8L}U`u zfJ{cFAdSc|NE0#@nTAY9W*{?>S;*{?r)GeEIXAUwmota{ikOnnLQ; zF)B|*0Yw2t0Yw2t0Yw2t0Yw2t0Yw2t0Yw2t0Y!m-p8|82P0A{&dv!Yb+OH}6IJI~{ zY+SS8?B?RRd*3yDZZKL7Ts7>@>Re}Os@=4E=))n4`dCey=4!3=3E{w2^$R`LIfH#| z={CLi#p9W)=Wp|f!8O-wvzG4?f3IsXb<7_SU(4N7F??dIbYtms*X!T8rEUG<(G%u& zX?S6eBWYHbbn?i1lUMfjO8br`RbOmBB~7)g?Xnqz(u8HBo!x=K+vMfkj<@6e@P7F? zTX0<0apuPk``%~4ua^tXJ-?FrS+pkokKt~;ICn#lNu<1V@y$lnR*$&lhuXq~WxK?a zO~+3(H4KQ|zim4`j`~0ImM(Yaw{EH5wV@z^`v2wH%AY1wbxFN?!}k5Bd!@r2nK{7~ zr=%a}dERlR1f@?dzxYi}+u&{Ta&E`l@qT!}e4ICM+(mKBj|;^XdS_LF+?*7Xb3=a%$XGmdlOZg>%wt6JMIx% zKD$JHuuEH59B?rMm!psN5-|4dnSStA`wWZ$m&tGy?{V0wTU!?SS_6Lh($zqNn=IFZ z6}{y3wZcV}b!zT{f@b+%-8#iwWS)|{z}rw@oodcA=h2!)ORdw)Qwq&_o6nCO{1Z4~ Bo7DgS literal 0 HcmV?d00001 diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-adept/observer-netcdf/dist-electron.main.nc b/benchmarks/vlasov_cuda/results/2026-10-24-adept/observer-netcdf/dist-electron.main.nc new file mode 100644 index 0000000000000000000000000000000000000000..dcd89f3fc8fea386d880470e9d6131ad4d377e5f GIT binary patch literal 58136 zcmeFZ2{_f?+c#>;5TVFeiV`Jdtk`>hzhsEYtV~ISOi7{)VH+b-6lEy0OvzYMrc4n@ zMTV3yGAEjG?w!5*Kj(Sh=XuY2uID+|b$Ii4^F1oE^&4OdXlUqZY5qMl^gRs+mBGIyCqI8p`14WMXwcH~QGP;8 z`{#quP^JHia#KFx+RZo(^B)FEG;*voa?3DnYj{FS+sq3?do>nIcu zQZ@BQ9E>37O#dPD6$ziH%-&Jhz!#eopoRY!K?*aC%3);#0~qCk|0x1iJ_QYps-~8j zwt=ROwuOO;@?kX^Hqac58QsAwEwI}E!H z(zwAU`2O02%9%5+*7nYpZZ_xsi25IFgX&QbN<=A>*>q{Z5@~5zfhTZOw|8vNHbHZ~T{j3W{vXr=a+X|JqL>C>QYK zU!NjnCDhUU_k4;!C-cAMQ%JJ;@~_yl6fNyOiY90w1W>+EgxUP(yMJTX)bIa?@&ARl z^r4snd@)mx!++u}x2(*hsa~PT-WmR5Z|OprNeeNQ6VB z;b%#!-=@vJ!M7i~`gFl=25)VFubn^frwii8uIke%Z3>>yD~n`O`m?T?O$oje3Z`GH z^k;nvmy*VS7k56+qx2P~KkegFN_PQ<-hxW{Fl~ZgNa@c`AKR?-HVuA|E38D3&izwF ziNaP$bE^_XtoSWaC0WYX=r$!l<3idau0&ybAb7hHg^k{u9ZHmG$F}TLvZZ{vN-9xc zyVUGjna?b>BLBYB$~q%yr9YozNqS{{rpyX1E!h=53$iPpCsA$%A9+*tADm<%D&0t> z8>m!}N(HEtpGx_tl$T0*sB}G*a#JZ6m9C@GwN%QvLTNatbPbiVQYj0SGEwO&Dy64V zS}I+tqWSanlS;o*=^T|#Qt2?2c2Q|HqWH@CE-D?S(n%_vqtdTb`jbkRsFYS|Wj#HW zuA)*VDrKQkRw`XXr5r1i%t@tdsdOEca#1Nam9D2!9xCOfQa&o>r&0kb6{OM)RJxH$ zsppcsN$G%!&f!1thmx1U7Ya)#jYGj{C1v|pg9bIEC}}6<3(ZGesFhI~nymj>7YY6F zANtP@{Idi9?7%-e@XrqXvjhL^z&|_i&kp>v1OM#6KRfWx4*atN|BviIq+2(e*2ySE zXTG_``U-wRh(=$C$v^@@am|uPF)t6%w;^mY>vkCNs2g`nxGK@ANfs$KHUO4*V5rE=%0x#Y?!Gka2&<#8AWZyI)rh9s9m*u?+x%P zd!oY%q{%p+SZAiMZW`QU>TPVDRA)YgTY7T9s` zBr|u9OspU{GN8a}6f6DlW#;e;VZ7(=dcO0W1~}^*LyZd(WIQ#Icxs(y8m<~OU%~LE z8ZQ{NjB>l!fj6}ptz=y$g=k~AlKy!*5%Np1T6Bln^OqSB9^(wEo6h|Gb`Bk~+P z#0$6GE~R!R5ZWvn&I}au6RbXR8s9kJTjMqf3|7P&o0O$7+zVJz$jQb5>35@D#_{RnZ ztz+%g_;7@36NhI9{zAdYcba_|7ZK8myyGzOm!fer53NId5MLx`j4C&~Y6Ztg-%vdT zmuZVf);GZEU}=XlC2?YLKYXs&r*dNHVl%Cd@fMht1n<1OS0=VASj-~OdKBYLziC&{ zCXC)vlGX+<8VEK?P;GDD$+(xSmxF?ov`p=h< zHqr|zU!652`9@j{Y%5tHMqf@eY>Npa_KDPKOmp%OOPb;}Uvwo9_GRZq4wmv0{DP#u zd^e9$JVx#y(H)OcOjT#sorQ5D#ap}kVBY;=THc?qj-R80p%3^}_x|va0lq&?dU9@q zoM>X{#VL@RX;#$50ysoBY-zCsT(=I+eNzUUMcwO$uEPG+T<4m71pBAmHdV<6`xi#j zR{0e6k5RDZ@om^YyC2cT=CFTiR%vvSuz!IEj(L>A{@o9a-mwk#4^cFZ=Am_n58{jD zv{2;=D{=k7;d3tgCk0oFJV)(Cz)ACr{=#q2`^N*nZWm{AVq3n(-5E@=z&e;jmxKH= zu`!w2Y8Ho4Y+vbJgOS(5xceif*}``Q__5`A_6R95E_F?$U05d#zk9vbTf4g&PwcwJ zARpL)_a$W~+wl$KcY%)$2%>4 zTYg9&B%j!_e&I1cp))v*j>{@a(c9@WQ?YfF;tYTNm$y~?g#FfHAvcE-2*JuSx?cP| zMECFVT4hOL#8(PyendW5ATHd@2wnPVN{VrvQZyx(k(x$5dk!Dq#BMwKT#3xIz$)oQ zs>cH|F$LcC>FCp=n1T>7EVfe^7bvoOUiiTP|NX6Ic2S0mS0ArG#HOEyFItv%clTD~ z)f*Y|-9kF>yL-aOX9b7x1?FoJW!@8iDH=!f{;s3)L41)MRjS-@pA{Ugwp~vtxa55v zUCaiYJid$JTe3N^g7xdh<>4HZ3~c$)cgq6vx3K&o;F*d2c6I4Gc5)PZXAvd#68wLv z)rqQlu~@c%^Hts%+a|HD3t*O0;gPZzU}t@G)?TXl55+ynp5!m0eT;>^Tf zipJ4Av<~q>e36`Xs$3to6&$Ip*|QW}>?Wd1PXXsgi*5dLN}Sk^>mw)f+{;MEV{fh{ z&X|&-dJXQZFJ2%%FYcAOawClRgeAtk3;e&O_4j*c-X;)kj&gP1DFHn{n$Q#i``2(E z*UAU`zdb@`nR&oZFlk8S`rMsBnErII^vzlx;+tGQa!+g+k$V$oFiYtI@yv-&QCXi& zNfnU^eIG8Ak@{=B2Xesvw+9eD#wS={^hFxko4qr!w55uS3pS(J)|%(G4ljjq+$hJm z9qeE0T+x!1BpFYs+yCy^p)}k;c-WJnvl{n488k=h*MZAbi(6jg9>!N6S;M-=b>c5Y z<7ggQhxj1ANX~AmTmhTPKR6~hX!0nyPBZTLPzpH1qZYrVr*UFs0&nRTAP*2HOo|_d zyrAT0@?Ik33A7XGy{{mDVS62PTp9A0+sXTi@xFzlEo2~c)^0u0j*D_v19!D6I z6Apm9ZsTN!*97Ew?w1@aY$5OS6?k0#0rbC`QC@A|Lw*9+NiiFxo&>^dbL>^Fbv#6u z4R23uxfw?EUFI=50Qw&foYGnV{crNW8zJgZMv7;Om+S`pkIV;+&%?e(Y;?bW0`^%+ zn(_y<(r{bza^r%x)%ehF2Boe39XM_9r9v;BVf>N0>eU&yiN6$$qj_i@;)D1i zIoqjn?_68K(Vn$aE7^bSqtLC`G@UE+z>aqB2vd2A&&HSnMUn}xN;>uzqSkF%#SVdt70MU)bH{o zwLlz_&5>O$fVh+?d!ok;;?&yBVPC)7k77^RzFzKv__R#U-FuCTcNjk$ zIUZ2$KJk~LaWoIDLwpclB&Uih_YrXghn`pORtheGU99c-fU}|WMpG^5zbY>C7n64x zX`zSJO=Q88#O|u*+6nsauFgmoiw`5Jzu%K^5A^>c=G0VZUjkvcPC`$qoS(p4E%R#v z^zY%kcl0*s-|u{C@zW>#gki(p0;T>0g1c(S%p1^ulX=NVW?~p|%TjPAtLEr;Joq7c*i4^`8Kbuv1;C1?C=lr)nQL6B89~?5KnMX>jIV{kb>7c!}WQ zG~GS}oOkM|!y53POfHA{pMd@oW3C8xfc}5eSC96C{`VWONwa|ev=Cb@=Ik-?m!fer z53NId5MLx`iYnLn(h82we8Z;{Tmf4cO+Ellnk&-0;^6ng^A+lQ=-2dNW-76l67DG zx*C^Sn@lV3-GO^vbq$^29L6s_;%hNGJ@J>KaWoIDLwpclBqxz7*I0Q4hiaX$COTS7YpT;w|IPMT95sXbZz|EA z{T0-IVp*F#2BH4bnuy-A8S1~-+e7x(>c5e`;{hb7|2~#ft1LrZc2&ojU~=d{}J=w z9T0>1Pj~Zk7Y2@D{26b_l_yaDA&SP)JhTq+L41*%E~?ypT`M@$f>@0yxEi+NiUQX` z{|AktcR~F(cCszT6#PJp@`f*#;1_r*1O2vxpLh{=>{1T+uRQjvuM)w3-6{PbZVCSD zP?N#lgW$jF7fUo>Yp3C5X)OyZaDMm9a9!#P=)mX0?tbI}|7ANt_e%u)7oun!%|q)D zAH)~QNukP(+_HjWT~F&p3a;24&CCY?rytY1iWu;_?->P`RKO3P$^LN07yR;CrM~CW zWeddoPIqBf@Y@H5u(am&Jj4WRsg&9G2?S{;EvE?Z^DTw0x-FJbivAoSZmf1uihG~e zy?kEBPdKJRu9zQ7AY=rZo-g0XL$rT!@pndg7}4PV5?)cgK;(M2mwq>m8EGyyEuAyC zj5Pbs&dT{9;0Sy7D?QHw+u;3862Fv*O_;p)<#rv#z8Z!0G4u%IC$8hBK|=<(Vt2(g z!98SLCUZ+-?U6Lxm2KTT?}uu9p^M?lt;-#F$anoKI-7=ZbK!GMwEh!+DH=!f&^p8i z@kMfIjaTFzu3o|6sFW{7!9}i}n63hxjY)am_47EfE8K~pH`6(>ypPkVrnfDy{ZdCJ z4qeQ|E}TBaecyHzQ_tIa$*5fz=cd~p+70=OiQ2& z?d|%_xrx6NjiY&J9pZ!dA~^)A++oHQ9Pxa&$0)emhN@C= zz$qQVq8_-P6KgA-y(a2eMtW|Ts=)CD>OW%4g&hwTh_iLwn|9v{BZgMfc>jbv?x*g- zwv)YJ|B3?!YGwQc>(}~0F&0sZwpXK*%E13`4U*kI4*uWh=1{03_-^tKSBQAOPd(}8S?+^fVV_Ts9z(c zrEK-|((v5wBXxVAetj^L>Xshdfj5NJEaeLfPA^UylP2k}L6lBsf& z7Z3lz5lJ8Yn}SQ&@wDYJz**EFXmua*f0hiRy{G-kNbcOdVLyIAU9P56^5pRXvGQBP z>FPUS#HNtA&~!c?;%t;*Mc7~h;Yn(2-}_2_LTo@?!)mDeKj>fDFAn;@_}lq1H|RgB zzK1;s^1n11p7S{9KTo(Z<_Gu@$?$+;ONjr$rr$Z%fc~)}gTQ+5J4Y+==M&(ER12nz zr@=1`m#^|k20zsv&3s}T=->KVzR)l5pDd9@dz-<3-mE&-odo`qpIq&z0Q#@re1ZEU z=)Y9xk**HJe;r)|X@BsaowczlCE!01MdN55T8H=`zDUj)s$9((c%RQOSFwal^U zv%onOcUL&|5&XVFP-eXl=-<1$O5+05Ra-4PKA1wCCFbc+I|ucj5??^J3Fu!krvIrf z=)d1RW!HN+zg;f&yroe8oe-cukq`CXLP?g+S2({Lex>QNf&PoPG#b5x`VUbwj^?3t zh!5h60` z2Kpy|`Mv2g)PE($l9ivK{;OU`*pLPF-;(^NaCXrDQD1EiU#S0{80|?~3w?n1?}Paw zpfA97Gw`?>^a)0_FPcAtzQNM1qA)M$BLq2qY+40<1(7!)j^fZ~U^-7977u*~;wsHd zap*%dojmn97y1$%8I9qqp-&+*k^Srt^iKj0Pi(J+{)vKjQ`-;dpO4!bWeh_9?4a ze{eKy_lcz7@`+e}+WXU8dHZp!u%KfH`N*c~Ly+2K$tr|aMvq5p!`3{`t zMprP2Z5X$?C(t!(JMovIaWoIDLwpclBxf^K?jG$G9F{-N+f#5=tuNk_3^+YUA1^E_ za$=JmyGmc5E+ct8D;B@?(Ug>ZZZ2_C-U9Jjh78#_B8+JB!?&o8nTHrMrl#TlB7yLG zr1zag9zWr`$dqw}5ybtMeNt%<|2ay7UGwVs33sOMvy6R8Akb)hQt}q!A!6rfLV`2G zh#%I@eekMTAi7;tw-TT=Bi%fwaPlL>|ImUZ>)#OnTm7E!J-TOs-IXxCMjs0K|8DL6 zeu)3FA`afk5dRx6(Q@fw13V;j`(PHte-T+@ai^ncxT4Yx#$1U14W||j4a1@TH{cEFLjKrox{$D5)d2GRH;x9$xXdYUJ_#nPWjucgH ztiuY9b-!as6kJa?TIoju&UT+^he*i(2Xj-nr6KW~@a|AF+nCyl}Xx2)D_HUs}(kbYdT z9sGa#k+8&z;Qw=1*IiVC_pYN{{LBNOnDUe z|L>2?`Y(h37y9J$rVRXlkW+yU4a9#$(Kwoi)*(KKFOpM5mD^<){s)Jf&1@3|SN!+o zo)ducb$B%M1o;1MD~r?i;0Ju>_HTE9d$ncJnt*k1&n9?AH%$}n|EgF;6PG~${lGp<@h!yWGb)b@Y78Ugoz?+#xRUY;@~ z3HH4>zytSxqQ^)wC*c0?S4Md68+IPzNN37gak&30>=CeVf&0G)8b;ss;Qp`WcFt)= zhy!x3@w4X87ku6zesB!(zkw;PN)w11dQEDBN)Sh0xF#E4fc($!gi3)8^cfYeD0nV` z{!d(+E_)36Z(}`{8~}M$)YlhG)sSahC>2T8gZSC{>hkyZp#MS7%C*lx|CwXywaySf z`$~i5q#%C&Cf;^bg!ozYpkcoi#LxXak&FS5|IOG7>vBQ-L==sqd1xKtgZLsjT~xVK zf?NLJVATv+O~IwfK62|e;H>(&S+)f7zdNS3bWK3%u54y*gz;tbUjcnE&uG>2LoR^d zzoS$CE8suUP2w2@{I`o!z6S#Sh3XhTO~C)rEZavH@K3ETcku=M4%uoZiGcsBdEddk zfFDsbj^?3th!5h602M0HYDw%?->RmZU9^kZh^Ji{{eeb*(A0Y(sUvu7* zE)4WSY&g|=9rQ%c(0ken^mbzxIqy8^v5uy{g$?5Ws`O{>2X_uYUFd2=vmq1e#Ak1{W3NKp_-wEwTN3iWlr;H=B~Vw2c}W}ZfjToMOD=>C z^1r8L4>1SG|H4m)o}Wv$z-Cp1k4{0K>Jt6EDMw2EtP&u88uCBpQhWBDkpCG9otqW{ z`^U@5vy-5H9(?}j(>v&&4Hf*RONaVd{nzE41JFPFUT{3T`~1XTipJ4Av<~q>e36`o zRJoiED>(cGyNxNh(&-3pYamZu{lm-C9qhmKRL`0BIhHk^yfQpugcnOH7M z>br?Equ4UH63>Y)VO&M3E%N?=0shEG_}pzdGJa#nY~L?~G~7Wz@4)50YP{_i)|wdF zfk!>?RdU}rjQ5aITA~ zv6K~E|dh-%&IA~jRt-mc*CjzU2m;)5Gc`mH`D z5XA4rJ2^dt`rq?`#93SD`{&6yL_z=O8rk@$IrM+r=*7${+aXVkQ_npD{U6W7hnPO} zf7;p3JkQ8qAbKs$DVBeN{C}~ZJI@9BKYAsD14K@YEx=1i3;I7^#^uQYaL+#7_b~Ma z-2aR3Dq-FU{hu5s;g%n-3~-uX;@^{?|8u75^IX1K8lHE_@FfTIe}pm8=6ny(zwAlX z>u~??lP%OI4E-NO(Kwoi)*(KKFOnlmm3u<9><

>u1v_xE_b|^Vk8-XZ5tKZ|`to zB*{~ej9~vtKYPg~5Er&|1{@LqJ6ozeC7c2A%WmE46f>~@vrVK*d9Z&ID_!4jaDF3Z zr@o{>{0gq1KU4|vOZD}QhFXYU?e|4~O+oy6v%YRY1L7B=XdKN$>kuEr7s)ZF%B?Y8 z!QnvLszJe}RV92g4sdE@bG=ug*#GxqTnE7q_g3VyBtcxR;^WbN3vt@Y{pP+Ph}#ig zt(hJ|96w~p@I?Up`tJK|y-5(~eHkvAhd|u7%gR`#2z>xH5|Zg*x-@4H4g4o zWX5ZJI^do~Q*q~8S?Hgze6T3|0rx*cO)`rv&_B_3cp@|c_dh#Dez1Rs`=7@G#vj9> zf3iG6-lE0z<|N4tl?=k5AKRT{F?g#yU zku&m@!r-SbF%O&~L;s&Sh-v#z=>NAV8i##?{{MN!{J=Qq|MzdXMGyzS?KXdU8%_#!#$ zsd9OA%>Lk*36!v*;9{s5NzwwGbEPxqRN>x#Pp$qD2B0)iXXjg?F0|HqAIkyrXuQa) z^Z@@5)7x#0P`}En$}2BH{+Cwaruq@`zhjD{^|63IRPrb00r=^6%&%q!{9nSIThjpl zk^bX?1yH{tipJ4Av<~q>e36_ks@y!T6&(BnVmBzbxHLK$iUB7>#;oQ}*!SV41N6yI zhwoZ{@7ylX3(Jn_jMbp0*jRa0GU&~d@t}4q)bR^r+3}X3*O%sdd!j(kx<7LR)`Q+@ z-%C^u0=@X`Vtx(0|G8nsm7fFef5`X4*1iJ$zfg0zoe%GS?lv@TW`*}ZM@6o_aDw+g zQpJxp)`0#w&kmSR0*=+J0f&vnlosd1qH={((zp#O;tS(RS!{->X{LIQ{PKX+0cnHb^y54%D*dlbC?A&-dL%!B?BMdN55T8H=` zzJKLVqd9W%iHOZvwNmP1~pvd_cl4fxq_ zr;Dm>A^$UM%4DAb|9{}fv&C-k|8v3S7Cw;wm8!f>R)+j9`cz_)Cggv2QrPp{A^)rP z@)W-T`5#;N0N+l?{}4svXdYUJ_#nPWjwV%ZGSdo<4-aY#DY)(w?UYEO;0*j`+YJ6+ zumxey`k)b4975dKn3dvbF!T;;H8mpaxzO+Z3F~1%3sq>5$9a_NuZ;PwyI}QEw z);UARIq?70!LE0-p?~fdd1ufa`sc!qb%j5`|L4{_ttLSKeBaPWS2FnjoW7Wrdhq{< zqH#12twVeeUnEDGDwno$1qbI|jtvxCpS5J<0{~~)L#>-G;Q#k)-LLn6bFgdd80$^& z3!h9wkQbC2RyveI{VyRs5N!*2L*=WJGut4K znC0)02!*^NOIr3N5%P?(0^|N!$p4@1z8B97=kUvbfcZAaOE_1bHhczkc13zcrw-&# z`wN~G2!Q>_cC59Yg8X0VS69#z$e+@_I^Q;f{K@KC>o^(mC;D$X_U@2BkzLO=rbGTz z^(cLG82Z16qH#12twVeeUnED1Dt8lW>TX?yI0X~_SzEuPmr zgmWOVJaeWC&c(#o_(O?p3W%yx{zL*2($j z!1?Vlrb`rq_aAbvC3D>2{5ISDPIwFF*HYLnlM~*5Ad1G(JhTq+L41)M2dZ4-MEgHD z`nTrWQE+|VRqRm#IKzKyF4cnF?e(_%83J}#lY7Y53+xhCT{T1oJ3U$Rdie&}E#pL= zkv`aQxpZm4E3oSa3DL#5VCTjn`*VB1?w#0!TMxnepUK9$J?8NK%QjLMX3!EA_;QdeSO}Pz6;QiN+S(8F-c>goQzAkGN{Oa1S zk2_Pq&wkzH{(TPoe|7a~%TQO{(mC}X-E?8e-K6E zXdYUJ_#nPWP6Ji$4aXH6IET?i3NC3=W4332(^l*??@8$Yw`t!#`4jyAvsGQZ-q8P7 zxK`A06#D-eVwy%8aQ_p%Ip`}9?th{Zi7ZTT|D&%MA!Q5w|LwD%_df#vzoa*|@)BGRvB2Kq*X<0e*q&_{Y9m-9pl`bs6!rr$TgJM)Q2ZL1tg z->JrsFAVxnT#^G1-Jvg4?`7~&68cne)AC}r&_6S~V5rXx`P1!lR~Ys}{zSa!@?Hh< zr^RH|^&QYZ+xkeD*arPGju-2cq@aJ+QPQN)1O2ma=2v18p?`)b8b|ZcI>ZO@MRIJZ zavKcy{K4^akogk@*PT~3x32-t@MsHPBJ}-2oHMO=K>uG}bHnEgSD=4={dM@V3*e7h zJu_(v?^mUpVop92#znk?*!tTHa3#;&UwqL27dJ^foVz~_ulrr%?fjw|FVubJsPEo^ zuQ}r)C6G=E18Xt zjiCSkMxrF3+o_CnVB&H{(1wMv!n{x>qJcpXzR z)c<@3p0~sM-%;99PrYBJB)&4bD;e6-XRa~Jnu~D%N2O>S%|q)DAH)~Qc}11`rC>E4 z;)~>bp~^jSX$8lSXsQhb*VvAN%uc|WAx@iU2=$+#$Ht5k@cu8rbKPzSc>kB$^lVWX z-ha*dTuAeP_kT<8HstX@{kPe^`&=`;|6AH{%k3Jx|JxAiDCG|CziN1o&_TlFkuEr7s*kj$~8Ks{s#x? zvg=z4E`Qg0?gqfwlbU*~8uI?E%FXO2ArA=J?kK z{f{UbNAu7+#0T+3a$Ko$qfAzCcuUc~q~J=P9~Axq^*@79cPypvDwx!>njZ47%p)q} zl)mh2&d|gZE#Bn6BM4^#6pi$wu_o!iXb$YK%m9|MiY*u-FaWe|?aN zbUXs@zd{+pt@YvkS8|~Jdog%dRk`19j}W}G+QD-uau2-!GLe$hzYG1pY3H3{f$;u| z^=sM`$96Ih)Y@&r)`-hTzPJ#PB|{Xay}IGTsn zAwGyNl4DJkd-UQ8j`DOl1qv>=69u*V0p}x0@o@%t|E1F3+w?7g6YIS?>$)5KswCg? z(sS^$@{PS355fPBCHg&41pmKo|3p|3`2W~Fq@gdOWPIC+6YPHA|6do~>OTPf|2@Vn zEDZi%x|Bg|KluOj>tFWAf&WJojiY&J9pZ!dA~}6jxeF&&aO6p8{-)si(d!+`4LCjS zO1jB_|37^A;Hebw|2_|2zf%YQf3_gv0|)s3D{~Fo`N035^t#|!0RI1t?e*lX;Qt#p zcg=4E|355#ZnOye{}aAtZ%xpDPrP{NY0$rENrwLj=)X2@`ZCVOLp1yt?O+M|Pf4$O z9S-_`ym7MTE$H7uEY9LS=s!YMN<0$u|L9@U+C+z>b%+n*i{yk*X=?{)3|LOM>Tu1!QFFpXAuT{M!j6nZkQ`$})KuM)70|hrM zux$Ob1QEDrjVt|fSq<)gd)p>6-CLl3Wk1Jc|Hc3pt)9=3NF1b z^Iu~C=Va*1e0I41eWe%l%Eh&elyp3?F>k_@bkxGobuZli-fg!mmw@|U&4g`C)1b#z z{CRQB>jXkmw1p4^(X6kL2Rm*4dP&X-O}x?eCR}zC)ctO$mqI(?9_Y7PvNv5VKVj}%_o-E* z34}}SHuKUOc!)(Wh6dilJ(6MDaVbL80`ZZ?gO@r>rlg|9IZyMzGSYbka}_O3PVABy z*IU;d3#=_f*X%yrQ%O~yvvzYC#Wp6?uYLyiXV-$apV%@8@4wSd_WXkTGyK&ncPGO% z{AO!%$RymKxi)EI0%0Baq5`Y)INYDfVCh->eiMHw8b|ZcI>ZO@MRLTca*emS{lQ_@ zwCXqo*K2l>$5#NSsxgPzFSz%s_sWV61pmLOGtaOl9L}$})#b)Bnb<8MHH9pbQ7mZi zh>24py#IBHiEDWQ=l6Ky`F-I3ml+3xl9b{7nyo7|=&SLAmmSxOyLI4vr>z4`RuAJE zA2~G4tSA0bG>+z>b%+n*i{yAx<*q4H{e$DVQqCm`F8fvHM>7HEi~Q;h=M~`Ha`C0f zV8=4j>y3wt*@sO@zXm2PxpNkXGy>8^Ghtyw7IJ-g0V5AlIeYZW!1Dyc`i(bP=Ck4c z_v`oUPe-B@_oxRFG@%c~=Opg*9{R#JSWkzQK%cmCXT>TG=o>v(NMs!4vKIPMdqYHn^`TFtbVQ>d7W!ugI89`J zL;YvLs%xqM{j;X~3NAd*KO1ytQFsacb6wy56FJa7JA74sjRy43Dl_G8eSrS?GehML zTIio4ipJ4Av<~q>e32Y|s@$H(D>zJE_g$jkI#yvVx)X3398UM4^!;e-njAy!!S4YY zEPvQ_AL{>}ah0F&PAHx~BXQu|D3=j+K{VLboojX>5B16-nF!q#*z8J7)@7{HCv zaKnX$`jh=o|MlT}l_EOu8>*Swy_<(|j%UVYLV**1DH=!f&^p8i@kMfCsd5h#r~ko` z?jom5!8MUip5+3Z;~TY}qD^GCL@LqQ_Ao|1;-thpgRbv|r7*_DjnHas(* ze=!Yr6m{6xIa!VSdHBXo!u#*{@>QbK&_BD&Zs24J^*`bA1!*I=M`_-^R$&MD)ugw4 zwA-(C;E(d$T0VWQ#vOlLu;30(!wHX!twS!7@wL~^2emsJ;;C`DMu+1>aN~{N40At@ zVf$TeqY|%WVg0=veajy$v4tJtZEMOqmug>_lJwbixNqEM(-|q}-(}jA_tM z?`7W@mJ<9VL+rW;ej!9I`l_8F?)GGG!@CP)e80!%EE;keKB~qmIXngZi%-Ktp@83F zK9f-o`d8Dc$M%*({;Eiq`_vlf)hq7HLC_a=Z+#=(5AOsRWgaVMPgLUqaa#+!ywh;! z)mvUzz`nRRH%Q4JH-tR1@-%<22>yU2?$h3mO2xDtuUrR~Y8BJT<27ZUiG+Q0b=z7%|Id%Mv6O{~6Ggv0u^H@& zB@RB}XOIo1Azi7iSU93&MG{f%zf|U0Nm4Yhcq~oNg^kpGKK(7*64MZnX?&rbh3$Io zr;ZhlVMpj!dlz|#;8TIGUd`$n;#JAdw=SI`dfDR{!i2vgxP}rW#2h^u8$VM)i==;#@HI- zgWLzMrMr{y&Z%Q-vf=znw)P#j9IwXjiVuirL;tV;?WLg#c)ws&`uVGb)daq0plx{c z!~}l4MeWK6;4koLs`>@@+*RRERD_1%-wk+9yZ@4F8ovD1N%EIH89%f`YnOxp*vZDj zdEQHSnQ(8F8T>o#X_E0WN? z?PJM-G^9-4cRuGjVu@i=WEU5IaUwCkB(bi$gpku2wMa8cB#hEtI2-6+thi&*ee33j zj}#mHcpp0dCK9A_UwE^8Dj}?|*Gu8OE>4ttkrQ)uB$ntT7j}3#l7{4!c&+;GAuE!& zi}CucA(f=9CN#hNIk>R;a;w`OrIwggQvJ5X<5`%3b=G5vnlbEUcA(#Ruz%*+duwvR z{uSpX7dSyrT6*V?rog$cxFQ@D2X@dpZeunNcH#CaYtIv~lakU9^M0@!bE_n4Yp4TG zhRk?RgIzJOM_o6dyc3vuL?4Q97Q}BM33J-W!?03YZ+TIYqrun8{|Aq+8 za8Uo9_lGe|PGg{RE;I{Mc^Q0j7WC{}=@6CG%7wM&ITcyF=EB}(oSB>*vBX5YKC|dw z&B8k0=U=)sGKSH-QOvo1O9Z#C%hDR*uQAxCYL*~&kM@C zKZ(G;GtX4I1_f1;zRNnCnbx!-r38yaB}UMY8g0vkhWo*9kE;8bMu-z9anlFs10{qv z=b9agXNZK$M&;8nmx>kdW@OX*0ae`bJ}?W$#Z%K3+xtrh4Bf|PZ(SBA`h6UFdJp{n z;88uc2jKr5f30%74gUXT&Erq!$d#m?j#Ac0HZCmRD)S@>{QskT(iWLxSy)U}-6oT& zG0gpQWKBe%2;P5r)BUZW|0@hbhgX6AOVdBZGl2ax)<*5t1OHza`0RxV_)lrKRm?u% z{|m_RG{f-wb7$X%YB9q;?j>&S=YW09zjbk)H0-lY)yz}&Irw*1z6j@Aga0=*$v-p+ zda>p{%nem59#5uI>BKW;SMtJ?ZxIG|Mv3zcixZ6@LTut8J%H`0~jK{Py_Zw{<{+e(cPBvSt|PS6^R9 zfIiWlo9*ueKUU*z#8w*~_cZ)r{h7rRPGsEIA+q<}QA6Cf#h|3oM+CPS>P-+X8^b;u z)}Oz3Bnx}YQEIyCz9p8Y-I`j&%!NgiX{085SCR;XSnfw^R-_k&ksCIEo-)&Z*_C{V zB@$YKU$91s69c5z(9wS^A!s`+tigX02}6Qqeh!5-ikmu*t$zEgM)A7`&s{wj-~9FN z^#YjZp`0ida#fr-5X8itIuJ`V>)X-Y5l%xgTx+jA3Hqn-w&<%0j*r%n?%0`)VWGK+xTsR?{>jg7Im9n?`zHXM5ad7;^e*02uL z2|;nL4J5Fi`!D? zwA|Dyt^rG|{MD_+_2Bmv3a?%qhCJYimfFKDqn22V-(u@S$P+vWV}_-WH-rg}9nyq6 z;_MHv_jO?Zc6;50&%-(6)rva)3+#Vd+s9^P{-XeX#S=%C?7Ou>0d5e?=x0S1H1+ zpZCN9k)V7c^U%mp2_d^V;&KM~4gVa?6Pmy$zgxJ@2+q~X1NV#F)vZWBW7~Utd@4x{ zXEtu%xf=Fw*(CgSfhERevdU z>*uB%`2ukw((>~n1L$8dKZV;F^iQaN#99gZf3uZ#;|I`x(btEHy`X=SO+JkJ6#u`9 zztA&bZ|A}`1~p`44_RU<#5^gsh%9WAUymO9;274T z*~8TrBZ50zUQ^29Xo#ynWnB*Nggmn-E9A02{QmNrl%F3U4oRyB>^1?vo}e0eGY{fa zj_0E4J&0fVw<>P6*-zm9!(Ke4fZyuT2aQ{oI&c;Bin=4Rd)gj|$ZScD)7w<1b zgCEw3<1AkV`WJND>uU%4uTD*l69)f}OLmGp06jj5+Iv+1^g8~i%G~n{k#N&ttc(6~ zv7&)p^Yz`4#ftCFWE)=pK_rB>E|sN@mJp7vlCjzmElza#e1J|H;y+*BWmcK1G^CBO z4r#>0R-|!eo9{`NDoIfrI9W>}53RYdKfV?6(kSIa!X+oNFgK&mSZM7S=3D#{LVTsFDAu_7`Hi{c37cYruwty{%i`-%$_xp#B;80h~+^zOGT!Jnq12m5EX3!cve`}YlOug?ViZ|m}X{R8yh8RYeD2K28=tfx!90{)-q<^37-Z#yaf zx^bRJ&|i=l-}vZ>qLJtuZJoek#ixo@{Z~H`3C`QUtW|hdLQoU>{A4LeoOq0p!B?+4 zmYAzvdbuuyhBR0dqQ6hYiey;JG$`m51?e`1EXRLU=M&iiNF_H+7pOT8P|RghP}f(x7z#yIXgS=jeVx3j|XpofBzFdIG4 z%Z)vB505z+;_kXn7xiz7;5yJVIOP^g{eFFW9Zhjk826>;mX2V=| zGZ(gRHu=eIs~ z!*(I?+i#onb|`=yJKr+5_J#Om5itBf4dQ>_vmZSc5ch={J{^jII>1_}x5WeE{{gY! z4pxZ&v27xsnIZny-HYhwhWNkdq-)d}i2o56J zjIggKs@K0i2>UE2O>3|O@qg#s`{U~_R+3)yeLgV<@jo)Y)))srowKw)rxyJ7q|Tmu zlc4_(-5wSXM@k4qbF^(&z|JGAJ*Tm)wTk+mRXN$!sucOG#`;;7pe~f~F`OPNAw-OG zw`ER z{G`iv&C6{Yx~Y zr}@TZAz-%xkJxnGz>XPdudDy+DIxftAFNCMNF-cX)Hvb{^`G3qfdIwuV#WFMl0TRs z{vX<&_Syr^30;zwVgEI8B0QL*+YbJ}B=j@WQ}F-k)3WS7&}V4sY`kLbV zeOF-rV(Eh)jDjE6xSK;`6xD%CPF?!l5BL8CXFa-|;r{e2@e`9J^wG6svgTF^4dcgC zdNUkiI&i_udVy_U;ok7QYDMwoH2k`6$Zi#Xi2oJoT=(4!an^%RvGhA4c>Cz|aMu*% z&*wI#RNTzMj(VM0qdaMevD#Hgx%O~jOq-VZ<{=N*oh0$9AMyfQ!R1o!%URg|AU)G_ z{bQI_+`8crs3X6KR*%a&7~%^0`oZ6z&a`zCWLpjKU$tLTB?97q!cS*IcF@1Llv<}h z_^C`h^lS{oe-E4b;V#hsgYwd{*Pwr|%GB{!p#P7?6c~ z9sjo5BH;g->nBEN!T+;56+e;$|L>=FaX16~|8Wje5)b5m+afZb3bAuxFR#2O$wK`n zs4Z}#>LA#E<8D!ZsMBZFhm0zZ}l{i#%RJ_)&E9Y#sD}UUKfZ94A$! z*f-;|J7;gLVk1+5O)%8|8yNGjC#}ZdP^}R|P(uZLG`t8#$azgx^x!&-b0qVfE z$Y(xokQWGWyF4|x(t(fuJ}l`yTaBMqBGI@8rs2v>wr|BD{vX*J?z9!+|DN)lIgMbq zfnJ+xlfjO+?Xqyc3U+An*{m4 zmKu|{ALReu_ol*`ApgHx<8jj*^8fVvTlX_V{%_YEzhDe`M`l;f+%`DB+7b+{ERh}f zr`+cqzW`@kn6+>u;QlRp>60CNj@tPc-Ba+no(DTLOTp(%ugbb@51+f4S+{i&^iOzN zUAhzY#dOzQWs3gK*niB+hx{+bBsEl{atvz-4C_~i{BQq`7R6%7|LB!(j)+74S5SV4 zuov>bvVHebpFv%wywAT=0pfptM_K~|#Q#B#f!#fj|HUr6yM7ts|0RXUb*T{l$6eNm z7DE1)>hf+C4e5#EPPGW9onMIr!#go%p2H=C+%1n5svvGG%8T~o0-vJ0)^X9m_n;%Y z!D{gT4hhzn8zt{djCvyrI10}`&&~u~NxEqq{D6~*QIcmF^43~|u}o|D9GkhXhdhD2 zR`iIec{7}I)s`{F^YFRz?#bi~f*sWPL_Bncyr@d!B>W3>!}$5>cn^8-pN707c}3X2 z^46fTPvBRDlhuASK%V6ibNDAG=%4rMpx;mMBTh{IN_F6uO~3FuXMmsHyM60!9nilv zdq~mz&={r}u}ZP?N*2~Juw2^ju2%; zkx@!TAxR+;$|##MOUqXF&dkaxMD|D;l>FZQ>Unzjy7zO=dC&X4=iK{Uc@$~@{&)IV z+mSZtzhf%iQilbO@rIZci(34LO!soX)AD;`3e8`7~nGI?uT{Z|DGIY?Tr5tK`Xk> zdO;iYe`;W$$Ov^ESr9-GgU%1O%n^+M@9%!AB*1t3HARk5tJYc~dhzkYs0jL)x5o344bcCe_iR|X2>y2}(UjvV_#c1rK=u~!zm>FD zg;eOjz)+*^kLdHd!aIVRu|LuGoMZYfbl~6k%%)KALUoVG6Enci@4K_1}|ILsv2^}khK2N>G2r8ZRnNN)j9BSjVi1)O5p2io{l#Pfd9w3>kQQk z{$J>Tz3kp<=%}tv&ad58MAe2TRvg#Ui1`QiePuep8!hE~oq~kP%NF1M^jjK|J{E-r zcBs?x(HHt$@GoYpIAkioUz#Qx$zAZJ=WW&-YQX2c62n`%`s5^8C#i5z2mZCl9xaB< zwZr7yn`?tuvFBR35_^1lsFZwP6Q~*Nl1v)cos{sl$9r*Z?zzDL|5wVm@k1f}-=(Vc zM?|y72#c@XtquB!e^$P+{8KC9Zu>s-IA$)w;nuNo-^<1Hqz6SRR~4=4{;rN}Q{d?) zJa4YF^hD6Y1X;p`5XUvQ^&Y6g*UOZ$Nc#^yzg+1=;3LISinEBgrK3OQKe@d(U(SIS zx*eQ80-R{~GdW=j|7Xof+KGF>e?F-&;b!!I=X}S#KapoL`!);C1{TwsFaPY_%FRXW z&(e$bDYPP#HXU)dJDo;6iEuS<%nn<$`6eb@_%;%PaKl6H}TTY>WrWLO~zz_Un zx;6Wlg8Dhip;WXS)eR9@s{s>=guO36L#W} zGefdq)dAdUFQYm!up_?I7KV?D;gMI88)X~EzI9TpNZ zlMP+wKo!1s-JgNZv;KI@P(S=1pG&&w!2hNS_RD9%>)yodRo6lNGas5e^}N57(z@~S z%^B3clA6GkFEcdij}5#5Q>cHpcw0UZ%>QOQBX|Eq{eQSnp+%tnN9-5_s!;!Le^^#+ zK>er73!eU0|D3~irx#KGllnUS6R7{N^!tMYsDH+ma{qqR|7&;eNCEI#7f~+5o8YNd3 z`4T+qH_!gP`oPbV-_MyI2L9(S+Hq(A|Bbw+XXwEH7R%(vJAwaSuSUR593yJFYc3d} z|96J-c!!|>w`=_OLmT~{{j=#n_&Y9Qupm@E_apS*+wdXJTWQ3(D)HFxUigNq+&P8d z|K4=7F`q>K4q2&{*#f7Z@9xxj3H^5tOoYP+co{pWqf zU@Qvy@5%Ks7?NYeA@kj-hyVG%iFJG9vaE>Dg^{Tu(0?0F7@YOIT1+pj63*!d|BtAd z4B3e|g{bTg<%DnFBJt_`4E(?RmZ5EfsO!>@pbhWQ=a*7AkF?5{QFe+gR&j^Eri96# z{yw%0{&&!Al?HesW9;1C`Um2)+Qvx{OUw~-)?Y}thQ6|=U^dO;-<+{8$LvM`_&L&GXxOj`|6cU&7ZuUjlyG$-Ybcr=Meg zize{Gw_E75TP_5Dz9$}gkPp0$q8=;w7WEC@A{qq@3zwre2LiaL> z7SKaF$m12QBM;&KvquRQK8OE5wq}4f2wqkowo8lyeY(=}+Z99bwig<*5f{MY@*j-M z3WC>B?dlTjG5@u3B(+8n|Dy27ug~%Ct7VZ-^T-R&=#NAEnE$-03A?ifzR{Iomd+*Q zQHatN&BNf8CEDDH|9B>$Thw$6yfY(Da^?tlh`qrw`yzPBlx|KFBlzD>E#HO1;4Smc z_uYiSW1?-tcY1=?d|H>2<#KFK>2VzceHc9;8Z+@_B+6nT}{rX@e(qEUeL+ zzk&G6U-)GJ9;H%Sv(4;y8nOQ<*YbMsEN2U^JU8fWa~G}D9nj&j)voR$;D4)|WKJeS z|LY4KrOHD8PX~EwjU)bB9@DCx4P{e(Rt5N)!T;m-*T1=Cs)Qn=^*~XbqnyH1b$_4l zFB*0Daoo%>_}`PDS5h;;4Ijzxw}+5NesMp}y+U63{pN1IhWaW@DSVf(s{y)&{=g|kRF~!+-z~z(w!V54z@6gl*9;= z+`R>_A6+mc16e*sMqJ19OGU)?LjPn^twa9vmhY(J4J<55_dCeB-5ySB1o8elncZS9 zvA?!kPVZDLo?~MjUsU*o{N+uF9W{i1X-U@oHAmjO-v5T74cA|3L7tHVf61;;h;p?y z#PhNH7`(y%`g=r-wpNc3x4%2M{NsNc!hZf$1pjN!UJ+dj{`brD!zKgN;Sp^n?H}-E zL+|?u2cu87Z94SF82T^x#_)UwbR5rIO=OKd(rWCR)Z(3_r zN;zXOfH$Or7f7qHQt@+Ng&!5|!Q6;%^~)oL$fLMLw$XLStFu!FS0l!Lfm%cr~ z+$Vi_-?e<`z@Wg7hr^&>k}hwGNJswO*gK&92K?2&rH|ni`pBrw(G#T?yh$eg)32!5 z|9H^x^z2K_f8u6GQ`s;_m+VogS~-zMEZ1qSc?6x!)L1>c8u~vYHS&6@PceP5tT3_^ zx-3k8!sjw{+86Q5%3_GyefF0cs+i+mmSpHjM_&)V>E-$adpZ@xan-p{r*R~Pv# z$|&!@ZWVHe{}WY2;mk*$xb;mUB?k4+`lhq^G(M-idFN_;CVRsenbH5x zotM3(iT-ceDBv}RIPN?AX>t#HmUt?}C_$#VT7WMPSr0A#lf z{=b*(sXi0%if)$Kbq~NZK3YFF?*Q-6JlE;XggVZg$h!I-_^+BrKg9w3zh$wKbRYP? zw8`yq4)EXIC_0}9{N=N?xGD#(|3E^J*%j!2&oRctL*S7eGp1X7peqD_?|S_j`hQh5 zb;*i_H)fti#8v^7x>27OXKI(tlVyu!S7XM#YrpSbCSLYjEmrN zkG;{-v%ud;IaPJ58-JH6sP@APT!;D^RWk!zm(%c&`&V4&1@n8eF}Ut6TgMO8;666w zOg~r;UbyC>*NK<7&zZFMS2c0p>8?`A?l^}jmhBIMaV|fM(-((vPTH>zN3Mr&KFDjW zm4kDfHcz?6W}HSGzWsvZJPsiY(f=ixW;c)G9Bv++ zn!Aqvf2`uMOdHPW&t=o@0i54zl~B`S=zrf;^k;Rzg<;2%TpIXOPW_frJK*Q&CbiCV z=>Of9n0nX1SANRJ@^%by3VtlfG>o|E3t#**gg9P~;$nP`xORTKw2_WD`)@2~kVM@7 zh&Cs<0slAM`F86m@ZUNspy(g|JG~NRZ2r9%#|E9NXk9h?A7o_oB z{s{h8QMPcs0`<>4m$B;s_-msbfh}|>W*Gq z|MF#R-Cf9={<#3o7wBJU)prk%g8%K7s#dN5|C{2jCXT>2JN9KFGadYoSEhphHuzsh z?Scy<_#gYKP0jM)e~)_KRMQZLV_XeMbl`GR;*5|z;=eG!SzC>|br9ViXb=5=_F}SA z^T%xJ_IO#Rc;G*;OoFv7@IO>!kESc zf2-MM&qKx46yW|BWa=V`{*4mHawtOgBp-$Pd=iYvEgt zus!*ykNYqw+oF6M_vLpdI??;zeLkGpxrn*P=A)1A8iEHe=04E@*UcS$TB{O{LhPs(2S@R?(8cSwN$rAMw9 z+ynnR&olYa2>dVH(6W;b{^z;(ah*_L8WFg(%J=h-74huMV^NM4E<%O~-u=%9GF<(m zGah+SXp%Wk!=8vqUFe$IAIFHC|J(yEVvnHXLvh?=>=j&M{89V{d6wv~+?x#ja=}=L zVuL=EZZr8H1Gw&C+}EIqK9!NR!D%h7zjf!iv-ap;Uo4{f{yopth-@TN~%Hp`!Vw1J3D_5zWpF=VmmJm8Xnz+;aC&Y8uY||L2{>O;#bY{i=7mrrzldG81oeX*W`2+TL=vs0QTT#ctql}-nL;u?> zE=wgr|0O4FSe4iOhB6@Tbi5MtpYPXC74zW#rwgv|#)AJ>fBRar9luxBdi0w-KIf0F zgyf#Xb$M%iS@R$Il*Z0E$~N>Z?P4jp zL+E2nyR*(FqOTpQkgoGYpOf!-I-rQYcR@6O>n`d*>?G3}I?k_XS~c+~;;iGLkhuf> zGnH}qKYrAamAzBnP1F^Sv6Rgz)ET$o-I-gcJJp1n93H4cVodG0Eb8*9OHa83>U3n< z%b^jvE%MuY;iu65!h1{pBt!puq$vvpL+4pMtZLg1-8b}mrS~;>fF$RUOUlR#y)D&C zESUck9O$;|`7}nvoTyai0*{D})p<36_^&zNq#2L+AKK`2Z42VRc($_WK5%Fzhoesx zb^Y!ctB*J0{P0Zg`nA9p)aKHz`5;x7GO^tBs-+vybG{|%*=zw@F0*Jj?}ih}-k z<(Qx*LjT`7C}8Xl{jZU~WnCile@ZBQPc(S+wx~0on!&4;YjlJK!Ly^Ex71r;?*8`N zOy2M!AnO0Zuqv$@_}`Xk?NW_C&qr4m z*DNciTs>jSxa)E$C8^EF*$?r*$~4sc1^ln&&!AN(02~jhf2OI|7zvqs%}{dhGOj``0zTI7Lo`2PzG@tWV@|7WTeM(rFM zBaWv&GWLQ0-|PInQ|&x-#gJHeEBt@c7n4e5@c*e%?8{#8FTc3%JoRsWDHD~XW(NQN z!}RULcj5nQH=iu2g@19WwIt;c{C}~NLmWSmzbt#`R_3#%BzLVybTRViGRpgj({$N`{=0xYVx0Wwp$h(YSHvW;3j9xi zw_Niy;=g?F{$FYE5cMw?h6fSy1@IJk#ZURm;4O1$I)BvB|BJW9J(WQJ4>XDu zu0s4h_|NXlLHt+0dzOC%@z1&OY2e@YKXCs1bx0ch|B=^b#}UNeh3#cx2I5~D{73rV z_bTkE^(a+D{4ebK6aNUfaqQ}Zb`E%$@A#RigNXm&cjb;L>}l4^ZDMCf{L5;uX?(#R zr{wcz<5RcOi1lvS3X|CLraFPmREb5z6Yyo@j<2&}f=YjtprnIZ|;rV2;jX?AaVE8GXlu`Gp$4 z3FT5TUBk1@E>FRlt}+_yOo#tJBDqfH6XyR3g)*u9h+}>0#{uy{ygn zu$WRrktrCsLB||u&S$nZ0{l-xw}0&%_P-2wsF;4l{@1tlS4EXEM|%6Z<32C^zm2bX ziyYzqt@LQ{j=*^=z}qdui! z@gRJShOz;MIplfTRc4NJ&~4Xj1-^Vk9avZd{mO%WroK}%(}PcP&Xwifg3sefa-Qe63a3|DA}ITMWhiuUe^=883Li11FE+7r_7N+S!x} z;Q#M}h{OWuMy~xAKQ#dVt$U6QS3_4edF+-qgU;O1@%#WIuK%sRK%F)EzooO5H4XjC z`ML9LPw)albDF4-LLzlI;;DI1{w%h?}>j${ARHNjam4E4`u>&+an zy@FElb-U#ygEtiUU5Ebgf&Y=MF=sOIb5FgLHRJGmmrR;JjX_69$e9g12mUkji6(J@ zXUgq*`u8OK|5_8-4jb_Q$Z}a1F6e(Y6WdFirfCFAsz)L>_J4gsBx*Lo$Gq8G;cNr{ ze_M@qas%}LRif?aY4DywgTrDQ!T*BuI*M+huF2H}OoKSTi}PI-mf(GU;nxm?fj^t6 z9oGnff3jbsohJx7JNXw+F$4O)F4ZJyXlS`LB5A zh92~Pu^@WPcl7^Vm$L#sqW=?;9|ywG|M#uSJYWp{|F!K*>jdZ zSJhhwz%i1*1fLQm<|w*RpIU#h{>w+j3Jp|k5ESKY$= zXSKR`4fZm8Bzikpf&XWd%=R!~Zs(wAPnU=PXQ_QAMh$f>w;-C`iaKwhyx(^ibzkp$ zY-+;$H6?qWbLO`Z8a3Tz3u|6Skul>t8B zc?M&c>1Fj>G?3SXDeR0RL}` zcFfA#519X?Ih`5y!T!t@1${>B`IV3L7o}kCHuvp62=hO?1OA=6 zu3-QFe)Y*5%zvNN+j@Ql{{Qm5!Egif-_X$2Lq@>=wzsQ~i2?uBf9q9!#hlMl#pjvR zl@iKS=t^7%_Q2=n+VAMX7nuK`cCHIQXI0ZL+Kb=2Z>Kffh|ihaeY}mOR1}s5zSO0PZ*{BQBHl^nyKc6rXD z3~Ttd`gYZRHrV5Su+yjdH}t>Fb)&VKnE&|bZP;XvzRyYtW0pZ5X!yvQSA)JVl@T=~ z3;*Z0L+XJ`@PFKVB4vZ&{|IcEjg*A{KOs^Qv;%WSmiPB>m&5-r{~1IthyO3ht{A=* zbDi0hpgBeGd`3b?CJlS*@`W;$-RJ|37IJNlxc-x(>qjEtU&vG^8oPlvY; zM^4!Pdt>Flkrnu#Gb8->Gjw`*Gpn%Jud5N{s+Xry$&t*Ro>+5BoW^iqN7!@lf8ISsKQ5yG@77^j zSPTDe$)A_^KKvhBp?4Fzq5oeUBR%$FF5z|S?C8{qG~#~grI$K&$PMaO`N02DO4rOjT>T9tUnq!qEIv0QSvn~Rvgb8Dox9CM(}JxT4I*k4uQqp@8} zBd&HVPN=qy5qM5A>H+pwjX%w8t-^CTqB+^x+PQDEzs?iF{e)N27YR+&~RCk*j^3GrD#`p{D|7^S%yr={HFJiOV8_$2H3z!FM zpzn?hvvgks|Mz>)D)AJ!{E-a%wGsSJQ(?8mzxn@$dW-mXh=1x%uY+{-e{<;{N}Sm9 z?hap_o>GnHzM5m4?%0=7nvN(1H8atv!ATA|xl`Fx53k+iDCYmy(shdzQ2$2FHyUHG z|9RDV`?l@yne>hd%xC=D|1%Gcx&TUY&#fJN_qE`V)BW%k+o2Z@_zl8}-XwfS;>g&jz&s*MI+sdA$WZIoeK9qyhSG z`0+=M2he|&r03!!%+c{A%Hg8Q|9@4#n{}Vba~uiLO57hc@bFG_lrF9d*>?skS){btNpcD*7vYw!cS9 z!Uo~HmZ;q}-itUy5*;2>|NLK`A*O#m&Hi^kN~FO5n=Qq)`tTh0ufQX&nRu?lys3uK zXN*SmrvEB>;q#ipe2VA5bw^P7cB}DAw}I6i%)JbBGVqO;ry=!JZH-;3ys2Lvip$StpuhrGEm*|v=H_> zGNLsl`@!?vIdkmhP(Suoas$igpWQ#-tGf`<^=WTXLphz@_lOZW%+JvmQ7c0Jt!%$1L7gPNwx8C@8OkJ7_rA5hcfyvqx4wIy z{c(PxbD;6cgcd(>=3cS;m7})A?|c!z$*xR7TlXNXMQ)NXRu+h1trQ{ka?F*a)s4vT z^YXFXlX(6=u%l=BeKP43?2$UmT}FD|vP!Os?;x*jpf6{D{~zm&<~)e}4YlezwF-Mc zwVUoox5L+LuHJ68=@IVp(3sLK_A)YMxMR4u3jU=-LegLl=06Fe^mcJ0^8RU!vOf<+ z$SbFHU%L-X5Kp%I=X+XZ67hdNzHsQUAwJx!xz*OiOC&rzYtoWkNnfLR{s4Eh1AQWP z&^9cMjUFZyPH6~BrR}3nACNOAX_jvbSRE8vsZ^!4y5hW+)Byv-H)Gj@l&T!DgQJFD zDV}G_zDcQBQtjS9Yj}0Gl{(M5V_!r#NwZ^*cPo0CO6zNE)!tCXMn7Xewe3@?13gRZ zXYQ-KN;++WdCTW^ULt7ST%}Kw4Z)eW?(F>0Ok!^DA;;$C3F7|b--AntKfh3!7nRS5 z+#C|;+|l4ork-e@P0R*=Hcr#I^tY7c@(Ql#hcBLom&tOk#&cfXAF`#7Op$f*!nDg_ ze}M?;r?<)I{LV?@{kfUno$oS<{!_vqZXU5EY;~m098}>a6zWt|_a5aZUfeGEiFdye z?;0$HcOd>7q9e;vWhRLzJ{{tHu?V@1^SaK3!$zcAL06B&Fy-kPrO7zA$QgTnF=uwIRhHaUbDaF>3^HUrS@9Vb-`$`9o?lPnSzc z!G-!_+H35QCoRhEdxyDL`<=g$n~g~A#;r4B{vu>YV}4Og{RFYa>v84?yl`t8c|$r5&wmKhc^6RsH)V3V zNYtV0+ge)>)aBsClp;pd=}4~+A1~@wZGRcT_unK@SI2f*<3lE~dy$xZfjWOv%P#W+ zbx)sG`qTg%$QX*&838WvDr{M&{5g~OLw@d`0d9;57|~DGiIBN*-a3!9j7a7XhLUqL zz<(%QVq%JiV#C)9GONTlYlB8BXrE zFIy%64;)pqx|u}MbtkpSY8&GA2dpbp@e-e;Ubb!k4!cSWhl$;Epj$8`b@2xSr|S>e z7GF!HF{U}%{-I(WN~^{5k4!6db0GJ)GqWZ2qh4m$N_+=J?WZN9Pkjf4(dW5tm#8H* z+;#GW-jP=7d7s-ca;Nb%_rAKT)*=2fE%W5)6E?aiCbR|*9OzS?Ir0Y2D(S?6vXyWn zFClQ;U!=9(hB(q=aof`%llXqKPEWRVf?zkk#+@51Lf$T~SzRk=M1Ji1G#K3KO&;92 zPEn@_^V2a`W^U#(vMg7DhX(w=Qxg9&7WkjOu7vdv@c&QM8!iUmKbxPT*=P7d9$)KY zxMDg;$NgHV{~`XLxWY|?5&yE1*lUAR-lRRFLx(l+Kfd@G?+@VrrY8QUQM5^-gX+`d zIFd<(Cx=IlS=thX79O)4mi$Dxb5Uct8b49Wv0Lt#jxDk4KM&HY1$h$5<5(;)N!V-4 zeZ2Wrgp6|&-syXXWRYRZg6~m$k6H2E7G>;zUoH1<4@3Xo|HjF-Vs#lgO^;G2M<0|kHj6HsJe)=GW;SBv`Hi=;iFP3QzXPNiiQ+huFyu+fFH z466<$IM5e7AKkM--w`p%V)JR^B^*CSDsFGKAxs$GcQ+r)Bvzm2;x}ubAl|uM5O&odPu62APiTVl6;0g|jf_`dwSJ%!_Mn2yaHrCykOxjo64&oZa_j0Qa3P>s& zk!DWbN?%Jw$eQaPT1;doiRgi~UhEy2grlBTz;k_DqC#Qw)niBa2?6&n3`HmSiNYt= zejXOK#0C!uHl5*2LSI`?a0~LrT(*(pC-Ugprq6eBkyoQDKNL*CGtQ5(D;xpuXpmrT z`iDc`zwwaU!ArhG+}ZpEJmpE;0X{S6f7zYvJ3WqLk6kyf%K>?@>Y$wa1oGUDcO#n< z@_ya^FsAc3huk}yQnrY{%`llDg*uUDp9#;uD?+Zlo7g0dK6-F|HCGzypZqT)dZosO zaNd>N%3jAy?mB(ADe`g&ZiCiaX z0rPU>)=yGt8fK0c06+N6x zGKrteQq^%C6U2o}E6nSW#|mYV#~qQ^LZ*v;RK#Chhdsp}_bu#tJ1-9L|Mj}<*mgxlNzi3pz82svAp1kAeT}zGB`-b@&OM ztd3ysWBi1V^9$1wQ(HoBgY|FU{!HS*+3zKM`zMKar_~f#t3=2xW!~e{YDVN_Sie~A z1m0h}Avz?zC7Dz*k>4%ISw?UWE8dWfb33 z!(8*>nizv9=-2DQ3{fXo%g8>a2q}Buf8e9R7eVdbB)e+SzR~SQzlnc=Ry2;auZt#KNhlvnZVy*}7I<;3w{ zoQVEkZ%?^+1^quuwK=^X{r}0ixjAo~-y;{5@KX4Hf?|>U6WE_uxV^T|5xjb&!cIgh z0sC_duCZUycN>cgvworv-??t}_doRI$ja?67}2L0Z!pPUMBR3~Fsn*YCW-YTJ@F$0 znMAp`M*PJtfBDvAaw$-^$s6&{Hy{R{BmR2NS|`VG{X4HS?7oKh zOGHlZ;70socgY*LqyAr*(z#E7C;r}j<>D3=^u1xf*|;~!Bx}n(wo256^_uUqR;ZJN zEbrdKz|G@(7&jenn;;g1I8pQltpatA2Kn6K6D zYU`kU2)s<#NLo^Pw2yzfsMktWl)IHH??}>oGD_~!BEchS!Y^-1VWWraKgfJG%z+*# zwS4_iMkPJ%HTV9OT3&)fu(0tEc!>tf4u;3jeYZfywu ze?W-M6X(~h_SoDVdGea8xo>Vm8Cia(VWl4RztL#J_66X7dKt@(2vs9eB-5NP0{pL{ zYE$h))UjK6OUIs1nS{pisf;WC+JM{lYKK;xe(EmxbSGCsAajr@qPrk-;$+v%o z^mgJp|1+y;RsnuZ9?tbm!1eE-2QF^}u02|vP+|d`i;J;R(F5+;$*g&y036IY@U=z} z@pr0B(n&-7+vd-(UIcC?&vi0h?wlYzx;IqEp{}%q6=$o#vwt$JJQRWc8%w*EvMIlk z?kCIb@&1VeJtFYJnWiE(`f|r=ch`(mnw{Gt+D$)_R<_imWoFq*?OAyt#Z0xN8u~Oe z@$H$TIQ_UkcHrhPMRT*a%6mCW>fg(Q8cM+bo&4e7Ak%5)^7)U)iYn>b?KJlO?d2uTFCVy<(P=}l{x10zW{LV~D~-F`KS9v9NHIQ* z5Fv*+gUu=6|2xhVy$MDA7Y}~l%v*}*XXfIc5a7Wv{i~H%!9N*^@M&fQPO`G3N4cZ# z3*6q~7>xSq?fO{t@3{bf>40S+^#5*&n_6EG{}}z1S6dsB$-=w+dpN#eFR!P}>kr}- zvSDkJJ>qtL^(Nz@-IGL;R>rTOrc9!KkL)@TEn6bwUh+GdGC$F0>d(<_#!oni-lHUe z{|oV7;I|&kBzhORyy_255+{u34k}cOkVKA5-*wc#cCuM*IPkwM@%P`ZX6&7=s-AA! zSVoTfom{=@A@&zP-M{q>eTs36=WQ?ei^SAA`w7HL~neNS&w<`r%eYVw!A2)5FBRtAPNfAgL`I+0BWW9J z{{|J&Td8SA(g`U%meld9r`uKYIw%1%>bY(?9h6Evbnm&A(glb9&`mr*-jl z4MQ?57whk&(j@FdwBCSMtQ-7rlqJN0o>-YsZt}E}Ui~a0btiDhdhX-ygN-(XX3x>R zipH75(Rq%`BIsN9cO3fGa9@Ocb!|)7l&}$L{xhzmAMp?Qz>_oy{FmT3^J4w_GLn7x z;%+v?^_&UMT{ie6m-uS)ec+oID(8lt!2JAztbC;x{9j|Ea|#C@V@}Y@CmzgSMmC*f zUTEk}CNIYB*fBMa_wtx#{jM}HA}fwBhDJ0)w^y+jnyE|@*BEb4{+Z4ss9k&9x1F&i zxE=+ce}9^vShJmG->k_`j3sRU@dCVXbaR^4JMhF@u{g^N@P?0pR}7TPMaaPwDX}>f zBQlCv-m`EVI-PfX9XtAe%;&4`<~Eg)m6R9*FW~>%`0`J3;C}<<(OW&i|8ia^8N9;& z$LiRE-haO4((+1v6!LsW-U>k??~_!cLW4_^$!Hr6?~D%k7hH>}K{%(I%=Exy^i9bN z0=suZM}OTf8TH>W(6II{d>E5k^A}B%9q4)0p{h)IZ1lu= z&l~38?H`JrZG=2XTA-2O7ar*Uv-K?P#9m8kLd4%|ZUrNh7uyTOo@mZeauZx9Mh;j~ z)dXnrb|-)nOfSQhd`Mc|_tmQlQ{n$f>FSvlu+fX!WFpibJJ5fMoBi^|-wk@B+boIe zpeuVN9e-~_u<=KvZ-7p@%3S#TI{JUnRD$41#D5`~=$G7KM5>*;*8UJY=T2M)i#2%9 ziQ9*S48emgSspiz058fh%6@5#`j=h4?feyWzn(Wvi4*52$tS)+2IpFIrD0bG&UsMm z&c-|FyKMu`LUQQCSuH&5yU~}2f^Nk*qE2@!?K*FYzP+otxa|P&zqV*Ysu+0Pe(lUF zv*`0o8GJU|jQEMvd#9pX&*lY;CB4$nloinbv;1#daFm}U#3T5ot9mkt z=@qr%OUDpr7w_7``us%GmV9@2=s$TS%IOK{KgW3uiFWt|_Y3|Ubt5K;o%-den(su& zuGK+ZT{=dj>AtIa`g7i7Uro1H!> zb{dgXmM%74^i8~C>1}ir2jdmzL*DJ zM(XJft)e70`k@N>n0fH_!L{qznoUSrJNdY>9K4=+zUx6Jc>d$f%16F~_h)*?H5onY zpmauSYbR+J5MK8WfMEswM zCsfy<&z1;1T%C%(d*{OM=jBbwWNf;#{8#Y5%Dle}VZiBK@?QFt!0n5NcGdlYkI!>; zqsSS=zlS~kp0}|rvFgo6jwbN_S0^JC_bT!eaYxMqR%zN2!J91oH^L`&2)LS(1>g8b z|2ok%g(9TAAKTm)nh`l8P``Y22sm@jvn04CndBPjXL|7nXn6Zr9v(>-$L;>D|F3BHOhPtRY zxy;;;I(h2P!uuZe@3~D?O#(j3*2mVG8_`!){mOUv;GBo_S-WT9yR>dpzi1%e6ZpRwQC~Y5kF`=WF8yNEm$jrGzIo_l^np=I zh-sc*Pecc0ulw@Jg+CV5wEMR#Id`{GFXi?(@XC`k-&fu$Jy%j`+71%?;{(~~ZhFIZ zDLxML+svz@KY@32YR&vusNp3vi(fpxS!YAszb=^~4V;>eSGDqi{N#tCM|MUkq z>(hL^#1-+++BX#UI~P7&+awz+csX}j_3Zm{HoACT`J2IP`2Pdt?(Q)pZ9&6HZ`b)& z>Qw!a&K_+`>glp89gOQHDY0v@vi`1vqHI~Vx?m@G!Sk`%B`x#`j=WtmXGvPy$s_SM zqf%*i?u8FKX0Xv8`24NRje`FZOR=~H{GVGGv%ZS@cks(qK83zCrEvALff4*Oie$Sb zbn7`QCyPGhv0JP1T0i)Idc9V+%hBha z7_g0h3H`KF;-M)6aNu;W?O6-(f77PlE*!{{^R9j4QK0PS3`ywwLe2xXlQj5=C9S;- z*}AsGjo$S1zSd0QSqlHxYUsbn)z;0tnEx0Y@4PVtT`?}6FJ}n-7s%xE)EoLQD>Njb z2l;!VTa)uO{NJG|g}MjuuXnFL%(fZ4f3M~BKeMR+^pi3@xv2j`r#V=hQ2(J@$0Jk0 z|1Aq9v)|?6J?OUIzc4g=lMh!_L><7K)bDW72M+jua|#^6QtxAjO}CY5K6pcL(21n+esy}Y z@FmlFZg($39HQ{KTs*^WX^jr`*GdayiV;pWovzlTF0@SF+~y3!XCay55NOPzT=sS6^Pl zI_ph7)~)#?sBJ{vj(_zauug8FQRBst*FnG5`HS-urYAI?rZy4+{h4K1uqk zE>9cs6XWar&v$_rcKRuJeTDy1^ybHZt?++{2SU6(;1LX!`7CkBE1sL1WL3d4rsfwa zKOpZ)zyFow0}n~dz9N@|yi8}lrC5hN_6_V#_&vo~ib9~1lb<{Yu4{__8&z39|R6}9u_GM^s-$KM79EVMrk%Iz3+L60E)%aTIA)o zw1LHC;8apcCa+D?1aVy=B*;DxI7TZ>+z(yL)e*vRtP#(DpF3&V5C12*WaKh0Lm4SQ zyt4k^^HWFk6YiM8{~4Yuvh~x0@0UG!cMs+#16{mx$52bcZp5b zVhwtS?~!uM+k6-L|LECT^J+yS@-zEMJs$Y~v8ST#o?(s~olop3YtJOyrWWH8pz~ff zvgriC{}=8NA!V)`(7}5}$Li9ci%kR0M)pDfjd?ss{))c2 zYn5hj3Uu_5^Ojb-;s2KY2!EmopH=%++qX{eZdRjXYG&YJ+myr{BjC%C%H?ebC9Q?~gYlP9%KI4Q_bj9( z^+I#)O5>?kY67E44kd`BxgH;Tzwk7b#eUXURxvnr!W8I zk^}B4SQJ@M>+t@(fr~$?K4UI7Uh!IqFd|_=>CA(-dCNwNtBFk#(`rJhmG#(5pt!Mr zRJSFAAr*;B?}ptHEMUq{j>iI|@X*}cFKwe#&l ziSU&fG}v^+;4^M49MQEJCJdBor7{zSqp#6SBf2fYaK zFLPPuQA7NDx})!%0{=VxaOENg`fPIBdv+bnUB62-oKArcOVPNI*%RnMSKoE##4Gqe z78e3TM14|euH(EL=g*`sI~2U{*%08+gN~$#+OrH=>jJk_ok3}n4kV=s+{7%{HJI_;$0T}lMk9|iSp1- zo34belwkgoeS%Sp0sOyf-L_{A;Qt;PV`o2O{xiA2Y&C-U&xLwtL7u&ngsx2WYJ;v! z!Y<>`-4B@mTs3m&I)wSp-+-u1K_>k8zOT;NK=1;Qtnicl;0gN{7L&}N|6N9Z1SCNJ zk3GoUcpv(|VZA(k7W)6)tKSMg;XAbdS{uuU`JdutwfkoX;n6P0%lY1MmE%!`C?Y#!7$&oxr{2|B9|Vp2|OrCMCsBgvu!6nu%mI?s{z%**i1J zDvFF!MkyhLP%4rYQb>d%BP+5KNoGXZsigdl|NC@b@AG}mbIy6r_g<@OZSeo!msR_F zzrsBNU7=AQ?El-oFloyP7!zq}-o}II8^;vkgbDDd#HwYMPVnlkAtt}gz**A8lD`?K z|IRy`60+dK%@6X2oQZa&)xRhi*z{5bx-#uxf;>FUAJ zI_N*o?QfWxq5m!@MbVuN#>wKf*C}_Q|Jl43|DDC0m}Eb4mIHG`-&5538Rp2T(1ms# z%$1`Haf%zE|Cy#;sAHHr#dN;6lBmC7XF<>)>L1JTu9XY*U;l7nj2-%)azy!uBf!o!X>_`kTc3xDc<)L)Ule=2JB~L4UM(I)4QJ z&Nwh_ni>AzCV_u_AMyXlOP&i%!~gr7e`%dNKIh_=9W8VCT++~@`x=}>E!n`jYru71 zy?sjNIH%K$uU~NE+}e3EmdfJ!yW%ri)~)&)@yFtKt@=MjJ}jl+y)vl_qpSPhBpq1> z{?+}@=nLWxaPrQ^mEEiRpP2O>UM;BqmC13HJK)tt*}iv9q5of28*8Y6cZ=j)xvbNTkjc(IM`5_V2Aa)i0kG5cn7yN(eYoudR;FZf6vlm3+ zGxEq^U1G%iA7p%{+5rB?(9ovb0A1P_WjQAfoOQ643p#Mp@7~udPXtQ_B z_#Eo5u~<1SgZIqP#Y3$|{lmoKO4Ct?3$+fUt*DEnji2%o=Kq5u9|A@yNW_-xJn6sE zlk^oW`;qF79J<&8UC+0AC+ODSURj@p?@MRln8~!{p--p$bTF|#K@XvQUULOJQP=U3 z6({zFZ=Mz%I$8%GZi+p>6#cIuxpFK8{h#Ubo+^y~Z<_GQTn7J3rR-q8248Bs)h}l! z;QT%V0q}qG3ikrx8ziFiifou6^q&X65+SLRL*J2@yG|VY|Jg0a`kb+MqXb9yrepuF zAf;F)jQ#)1yPNd)!Kb(Wq-il@`HNT<`%t zlhk@&mqG9jQ|b3*Bj|r8oms0!oJ%>gUANwwV9&}YvL*xb|IBnt$1m)$OjXZF$O;-0 zrm|+AdEx(R__=>u5C5n3SJfq9_&-u2bCoW@#T0qUS_RZO@!o~368L`2ib13G*rS%V z5!XhsS7muxaMBI_uWHVQWnuWgskfisNQVC>FM&iI%^fBW9i`Z4^U+YU3LlJI}ZMN9-Aq5uDJ%+Q#S|8qOs6xjFy{!jd^ zp)~kLq+N|9N;Q1=o9j5zu~)1+DF2@@K8G*lm9!f^cT@3CV;Y{LYIrk!5YOeiem@Zp z{12{T{=<&B|23}aKRrBu)p_syo4|#S40CPl;4ewRt|^i5|EZk!TI=Eef6@CCkPBbz zT$9VY1DOB+T)K7)Vg5Jx<-b*fk9PRO>~A0VY9$MXo@MAW?@-Umwcy>kJ<=@+=>Km| zj+u;O|9gG%>u+)FY1L&aRmzb6RC{*6mB)slW)^-ReL`J#w<;WyL!Fz?elh-n`qSnF z`((Sj$h?m6?1wS`*($c^b{V!&CSJ`4eS-hnkX3Lp75;A&BG-_<xVLDwQS=6KFe%~n#ihu_xSJYJFM{VWAJU>+kw;x82Ttv21~bI_J@Q7%;9t!9dRzdK`nF~c}{ zSvvGQ3Hh(k`sc}B$bZFN3DjDP{Ez;TVL>hIkwy~rwq7q}rA_jMegA_!0|Tw^1{?H0 zPdiWUW%v*+3Da8{(ElMvgr{RM{{yG??`sDC`>#t$DiS`%FB@O;a^TwA9@e3gz`3(} zb2`))`2OryO5}C$K?lsO4F}+h%o%h`iy%KMGfD}H!24^IIZ#po9-u^sPw#*)OmbP2 zHUfV(osna(hyKIQrFn@!|A{CZOzT5kcCu^?IQPhmJ|l;JoJ-Tu+_mM!)t=)-yy37^ODo@Ydy6O}MGij$hn znn=W(wIu&gU1Q=!bd*)}4D#QN6h`gtRN}mjTkbvh|3|kZFRg|C+xfYGBO5-2NR+Rh z5cFT{q=J_}bimk0&ENSrzw2XqnVy2b_~vc6&j9`RBR{vN2>f51m%`VC{;zg_lyCt) z`N!yi9zEoLh!(;APR!BL+R?U1^c7dJk#sk5mh62hzm^*@|4VhmrLt;hubKF4N#Oq# zV>OfC5?E=T8w9egu4YnYOVckZAjdBpz{S&l0KER^>tp-bZ7F^ICZCF*bdi<)o~!2- zc9H+SRKDWDWlL#0Sm~L8{jW+d^Y}3KzcFS_I=92I|1D5UeFy(vg*(0dJ@&t}FD6QN zz&qnb446NFho%%`6`*(3L&@rU>vgZ}>!z2Ab4{ulclsIvO~ZyR~fa|U8=Qj^to9|I0JKT1wlhOZK5 zR{QfJ_`guF*t2Tzu9izRY@d+-nA6}0R|o&!Jd<^%9sGY$^ynTE{QpE(8IN7Saittf z(|bKRbiUDwyf4`Q*qJXTu>=2kLbV3JDe=%5SbZD5Ko^Q#|2L7?kVBX2TpUq?|J%E{ z(Qys@-=--A@m7j4Veg&Sn>vX5pOt6D@7JahH!~dejYG$A*t4Ix3jg=uU;8p?%=tWv z@3D#S{pR0YIH!gF7ua5EB?AAi#f@_)0Ovf+jX^~T=l+SSOguN{|I;-WU)=;RR#%n~ z&gRFSF|R~39k{uxteiXq{`bZ_MKTL>HG3m7_a@-~wTFIlyzqbPCfIVE(1+&S=8Zo0 zk;}N1*`|=kN~4&!SC2xsal3WfYygg*pA>o$rQb%W8DujT+GR^wXL>+Zp=5x(#5=m| z2p>SQRBxFVykHmO-W4bK1eXWy7gKR>VA$8zz#t6Lbmbe3c{c|03GyDr)fmCivHm zy@3B`^?kd4HvGSD@>PE4u>ZfBvt3acb!~d&vVI2h^V{`=&TsHdE(~sPF~T0LO|Dq+ z#On6~DEI&cgbYFte=ZM>^ZC|--^DR+`T!Y0)1M4$*A4~eOtv@eE2r{ z_)YEunWyM$&a3x>tB#+bGdt=I-n8SPOK?X1UW5;HYa!OWAHGnKf}Tqke4=%?OeYh7 z|Gy;;*Utg}<@`-A#sL4@c6TvZ!Dphyrtj7P?xcO)?Y;{*G%M31eH-|1DJGoGi1T|; zL|ZEt_*LEz6+VeQz|oG!tS#&D{U2oxqN~4ORQTF@-T}EM=a9tozQRUtXY&)2A=A5z)}2b(*)g+^d?^$bFBB_ku{^3 zYirgqmslg`lG(1vbF~=vr@Uh4Ti}ELXZ}i52)P&^)oq<0fd72${YB!y{kOkp(kswU zcFa9t1o)Fz*v=dYoD;6ERgxU=|4B0EwckPiQw4v;2EqTYzG8o*5j<@ZbMRhy@IOI` zMRs=hxE@jg3P<7VeiaB2ISZe+zIB4*8hqc_!V@lx(Eqp3tsK*W{=eF$*|rS*|7%a> z(@)?Hv8IvHH1PkJ+bkJgN8tZj@9b_s&RkWwQ+~4CaRB+>{kD-J zjM!i092(#{jr$8(R~#N}hYyg!IZl+~`N!hwAG9L>5fn!9VgdiN>ytg#2OeDTA$=wT zyjXdvWZiS{c|C8Zt@l8bEHIqXl59Qa; z-f-G_G=Z00FMrH7lEX^lH_F?66u!M4<3L^oeEcT~yG>i*>yNwdi?NWjr7($eT|M90 zMeg}yH`}nWfm}IrC@X=-mZJOYX?Ht(g0U^*gAa@es%qxnMFs4Chu^h*Gf!ltmG3F@ zbPRK)g)kpe3(u&b(X9F%dTzPeB$A`JX z*@AGczw=}}9mhFWsx#tW#<^Epqx#VXec;0$ot1&UurFuxlt-U1D!q9agZneORW!j` z;O9T5I=q5jRVjLHz=LV%b1^42D4;8+`~B=)tFod|4-=ii_=uXn26Z#S^fPb z@_&`_g&n&l>3M3JZ^|(L*)F;cIYR$+QH86&L;pD_ib`4I9?+-#_KJGQg;}Jm6#6vd z`6HgTr;AV08EhI=waZ9^q}1gcY50owJQx;y@j3JndQAyFx2o0K>>2ROQI|4bfx3np zEa~n={~MVyPg8Mz>m}aYt_2Qs+jb|l;#_b2u06&BpCkA12a+N5Uv}fUCU)rHT_uV} zBdkf;y0MO-~H>I=F7S|F{kP&y~`&@fY;J z>rRK?skjI9dQOe`H}Zd$*S)udASZO`=UtY_3k207Y{&3J?2##(3~BrYth8E^LA4p? zj0T@nsS0?f!JJCOZ#+leIzNvZ%>RmJUoj8xzav}y?!N;6n{f~;@CX01UMh4*1pgy^ z3^VtF|HbJ041YlXk4g(a4n_YnrX9bx1#_{v$1h9)b29Duq}O`P&*3AQQcHM$ac-}! zW`S38_>^>{;61xux7odn_dcQHQE~lwkgUy{^uYihE|Q+Dr%RYw)`bLX8Ie(64Lm(pA3zzksx7(Rj@Yz$3XNRHh>@IBm z76$$Q6QsBaye!)9+M9IT({YXb@MHn?FMTHH8RbS$-z5fkdIQ%B8AW_%vH#Jw574Q+ zF-CT!)r1^(Y&3%4&L zIA`{rBL~b-|2ywP@2=ke+hb7qvmW`c4g7bTe_G|xC)KwfT7iQD4Dg#_IH{E?d> zD~9`1N4tjmk#iY0W4{!N{r{I^2PErJN4p#TQ#)62U1UOA8}nb!Jm-iRcysgR#SUid z{~}(p4ND{c9Ufky_%s>c6K8)cUKYOl*se17I^b|dcgypyz~%YC6s2{*=?69cuJ42X zd(r%4&xL)H^xUk;Tyx<6bkpOl$D#lGpL$3BLH@_5K9;*0_@9uN2|AuXfL{wtL3vB?f4d_V3LG*v47Fs)}r{?jY&Tf*p3g8JxjsDlj!`(IDStLHh9{~X+mr5WV?qAsUQ%=U@M``?-=@uVUmHcFmET8N9rj7o2d-If5S%^Up$ci{>zpW*8|@2NKaqlE#|+* z0b}bB^#8NX^SAB5b2mo>2}J?-7T+(Qn1m0?nQO2&9{is>=We7kbe_x3x1(R+|7e$% zdDGydP>zokeMf%Ed)(kx0p9bLVs6!Bymy_1BL8OWf1Rmk4|0P4sYDojtJ+gB`C#G}gSz|FQLM|uh8&t|;4 zzes@jlni`5a>KV%hQO~`wx;Awm|sWto5*iL{-gMiuf%_tYc<`sv=X5In__G2IDmUW z7tiWTLkDZs7PlV3d$EYKOO57)tEpzDk>>|foaR0YO z)mcWke;YlPElb_u7@Be8(POjtdAZ)S!y(Gq0#D<*6Xdx*h@Bu;X73W;Y zEX|~{u8R@6(}ev0>$pc(p1IO~uq0Dn;d51mPFUXhjQrQJqTx*+9O$p_82`@3@{QBe zEX}?kP`-;HB&HeW>jzRw` zKNax)1^j1U6FU$9{O2qDJ@^UvZ`%QbsMY*;pw7>wA>9A9mpJ4(r^-Voe0^+MU3lnB z55LIqIG>;odbS%DOy$sP&gn3p!ns+|eO%aybClI#RMP}qQQ@w5`t>yG|D70;#<>gZ zG&s@@{^#Y?&94Lg7s2-8ffDemmNkUa6?>ogqNR{<{2ghj$@gQxUn({c|4Cr}drk3k zETI1VpArOz;Qs_j9J1R9|0hUuM%Dp5DbVte3CDF3;n3;4O|));uI$7ec2YBk9=Y^cITXX?>XPS_kYek_xpX#ei>_wIp%oHG1p#e%(d2jqvZy?T%ufv zY@D2k4e^g3GXIQCG-Su1e8(pkoSBTj0aqCJ?pSFsEmztkPkej=w_Ch5MrG*M!?w)GeDoV;q%FyG#ReO=r!o^C;Xg*`uKmC`r z$@kf~tGq@OiZD5hLK$}iu}{cgLRKNtLjg$)R+Ej2RJoDr82HaW9{DMx-_VK}0I~6q z{0v@_G_;FzDhVMp6GS-@g~p5-?{CJ8)WD6RH5mL_32VsVp=!B z|JyN@V+MM{ZDve)elw<|f27_2nt!DDKQr#32&*iyTV`dn9Htc}WD4kWmvb~BZAEA@ z{A09&-52&u9FBukdPXaGfQuD85`q~M{yt-Rn>Gcyd;7TrjjykNFmn_6&de7kB*pTW zBghj9(XB$U_B8 zVjrsjb$t9OL8{TyFrflGo%#vbt2>3MS^gk!n;3O9u)P(LppM(&AW3b{gpPBwsiZYR zU!|y|SP5fkDydfPxD0g>*>|dbC{z*^(0%MB9%1T z|GE-`pHrEkzfhSm&UPVn+>gaFsBJd*uKO*oW z0zV?~BLY7n@FN00BJd*uKO*oW0zV?~AB@23=yJPVXJ}YSeGz3{9u24D?C9_ZJaXpj zEv-2;e06HWa0S3HE-|SJpy&HXI{5-_H0!mNNG=T@vVCvz6(ADGe_xeF!@F_}uk&Zq zaMSX7tq_3etxpGcpQGXEsN_m67?)&~81Vt*c!mVz8bg0^OoK}bjFaRst$he`9zUzh zv4(zK%jBB@XKA>wYw6utfF~d1I1mc`X%}C{mx3H2zwQ220444PEgk{+p*Hdl4uQPG zQv}%>0ZKlSc=$A*hCi9lO}B=A{!dEES-|gaAwo?9Im=%k**gRHYS-q(3l-3CN3<+Y z_5~W|doe2d<~&JXhBAR}e<`P*3t;Xk#qA1(H0*Ab_1GHv<0G3XuK|C&LNG?Ph=%t| z_EOqG4?@NIuR-sDd2X@$0iJN*elHE=mJBz~G=*_O1tzlPFwSg;V0k9=U){|d2HH|;%T1vyCaZ^3rFVS#?!~2jJz=wxl3YbCvix(dsoV`rL z0(zqqXMpb|`q5YLQ?tP}{Q~q;%IA0%gWjbjmnnzJAug;ROmcxfO~rb*5Adh)%BUg0 z1L`?z!~t^1hdkp2dOBgaC9KL z9r!NvZ^)%qL)`LGZ)|R)Vb-mF-@aa_;h83)DMxS6@PLM_Fh7Y4rX~R0KviMZMi}?yW^u|?;9t`2*sceB&7uW2M}TiDT{7lu4XiWG z!|!$jp0uc0r5XBD*lze;1A2Skq%Ut_T!MAGPXa)>f_yU{;QN#l-RlbSll*Nrw*g-x z_o7v=09UuN}U0E1kx}*5o)dm`#du!nx9q`MizWGW6 z*k`rZoK^*LZ=_A8r2yZHo}soSH(|Z3sg-zlmxg^~ZfPOFjp|ra$C_wZ_4DpqH*Uea z_;Z)e1UxreVQMM#mu{2(x}}AN_nOS%IstV5OiJoC(3dPwB$oyJX6>!V&jVjYzvzkZ zW?27I+yqU4K5*-(?MuM98dglz1G`H1*%ZeCUHS`mKpnuzr5~Su1K66OXch*1Hu)Eu zzJmPhTQSeh0)M9YBlkq0#}>@KqYix6{cn#h0y{eORQp5Ke=q{%76dy#S%&IU)cX@33B!TTNG{_RbkKPM_2XI6OsIh%l|I2iF@ZY=%w zCLofVh}^`M#9y%qNbZu58!609LDH1GMSIMCXD0vDEM+{Bks`y03^Rs*PVNpqaq(p6 z`SoAYcZMBFvW%Yp)(y~IpHJzJKXSlM?(mX3Pw>dW#9rvnj~twQG*Zd*@XyKl({mW| zne+8KIj=Zj$zPcB(su!j z^ZjY14)7_=U<=s2h>m&qUsE`g=(zTt?K+wHbUdB+)hn6?9fy1gnf^hYj>A?_GqM-c zG3VBWyE>)l`1qRiwkxW1JS@J&I2y*yyQ^ue4txvgpQmirq+R__^#JJ#dtOyca)ZGw*Y-w z_Ye0SQKMr=OS?=52|7-;SemqH4%qv^Vz3eTd$eLjEkTa!o`dr3pg;4DkN*^~=c#pB zn8Ylw_f5L`3BX-##By|iPs%LRVFBQ~G7R7Ekf&n@mZhV+z&>4tE#A}R=$Kj%nk6=u zju+H9d%1$%`uNnb*I@5r&i+&}h|^Xf3o}iSdu6xxf+QH{d@OZpF^r226+B)K_9UjY zTh#zx{`<`IT9EU=_vNKspqFob&3jQ9I##@7tf>vShH)f&H^f6ocf+w5!08{GXCDQ- zdOeORpAx6zGgHqi`h(tvY8TfNphtCeqfM149pBK-(pm@oTvHB@TmX7pi?hK!nCFKd z2K}xAKbqOn@($w8ap8uFh%ne)=j~Jq^8K%u5AFayHZR(ji=cM_->LVX!5*;_Y!`JE zAwC;AWmm$y?u~C0T?+9Pez(aW736)qomapP%$P~XC6|(IrUJdK zcUF!#@W+iE*4796*F@$HJ_C7u3zD>rrRkWTW}B%2@!`5F;NK-l$EBN8^8_I-2Akyh z1pwBjoNkB)yIP90%EDnj7V$SV`+%Q3lL#(5kpGGOq`wdN^Ze^gCI1<8tg`g@v8@25 zSLd}Z1AVV|Hi@M`z8nom-@h05SnAVlw}U*1Tw0|F55%*wfX^EA>@&XE{y~h6H3I@} z3xXY|J-FDeK^&93^n33@91`SP4BS9&k4vt1JMg{A+9hKRaWcB?#ZKBSR5TV52Xg%j zx&4*{Urr5W10D1l#Xh;!4RQWr5h(cx{56+#vh4tRg{zy%7m%+O5WA8O@H zO91%K4%8%rU;0)>b2uTcLdm#yGU$)}tZl##dRE#y%o&1tbd?)rv6)222aA+s@?d`6 z4!q7;26k&rT3`Me^xYbKrosnM#ASFdsW+`@8@E-V+p;a4eD0r<0{gy(49XM(ziZxy_^)6$x*u$98WKpL zsxo!;)1DZ0w-($}yCSZ_P1^*SodC!-zo9&V_f@ZYEIc z+*sd&V2l>W_fnj<5a^D`r<6eU*r4a&9zXBqWM%4t*tQ%cXI4lB1a(q8$08#`eM|e+;7)pOQ6?o zd+*h+ArODAZ}iAUj3l#OMCXG14w*QsV_u;DLP>3eIe`XJMOy=13FQ2^IHFV?9cH`2-FkST3#A}(RA&3Cyl{>|1*a#E9+rol;yKJ z-w~sr?wW=#8wsSq@h);;56IiyBT=&f`1YSaR0wiR@FlSd7$14P`QZ%kzr)l<(!>tx z4H{ir1tbV$bvn%|4D?IVwJ4{+uc3Vd><`=sB>Ey+p1K($>HE`54uKt!y7$wi0|*qJ z_vFedDU6EV&wEe-@siAKlg%(D5LHQ@Umff(xW0~kAd*0Fod$Aqcfx!=zj-p(0p@?5 zSJ+%dq@A-=z8$GaNo9lHW(dmyzX?yovyD<5hzQ!OJ2tt z@-Mow;CKkc*K5$z7yRc`U#E31m_VT{oN;VcFy1ZL{nm2uEB(}KYe%p%CO!YQF-A5u zNl`n%jvkYddODeJd;C@V)`7hzwSOvve3d+acfZmCJ8r4eWx_o9*80nexWIgd_B)>h zKK^TYEHdtpzuP(=WpD#rw`YJ`2;!M9U9kk>Pjxm399>DEy!lB+nt>SQ^Uc@vg8USV zyMxpr?);%ow(N0%aV_l~(XbAZ`iE$4+rhs%LWQOfhq65Prdgm5DLgte69eRoTfE5@ zql^vlr=u2ORBOIXG9LVm$hq4$EezI43fX41AV1>mgxl$!1Zr4N5;P0qtjr=^SP%K*&=TXh9PAJt+?X`g zhCmNJi@y;;a6ZAxE^k2p(My^flfeH5e?duA@c&^!sqbybFX0N8gZ_|*InM?pGab%=vFu{gt-)+U4s9dJ20-f zjH}Ck;tp)d#4W>Lcn3y4JNe5yF!Ii|Bk<$azv3=rA{zhe02wPrmBM`D)t>*>yC>^~ zeVhOP!u#i}8PSQV1v75HQ?-Dz;&}TD*9`gm@GsX4AK=mFMymgcHDj>a$Q+Sn6c*;w z*<^ISzc5B3(3nr~6Xr}VFJr1EQARqfZfullcXWc+J9#UT5@uPOf; z_VrN%w+=#7$xyX5ykEvJ$Gle-7J$3Y{B}#6f~W>upKrq}Rn(H~OY>ihjp2hM+MiDY z?kp%b`fhX#D;0BBXMG#PcFXiz8d(wbab8ZnJ3uPEHR34?qQ29T3K9G?hA%br+e-lM ze~2}}3gqPKK2;L~zQsJNw~hcb{cv<(5abwVy{dy*q*A7ERr?Q*;Vb&dvnl`=FEv^B zd1MUly%Kq68R+kx8+TF-;N$r#pSRM-un@O}Ko!WJ;jvLP7+{a})WqqA3dW z*A>6bSp)ofho!j=gPh~k8*%oaw{uv@4+EWrJ#{b>_>O2V&fNmIRPI{wD9{^el_mu* zeo(Il&HtPScFWjQsi=ee# zenzYuN8fdAfa#7y4Y1azkF9baB!gX|3xcsf#P{Pl%5-ax zGuke%^BL$I-kaQPYOE#Hz_!R%fX)4eWsSfpwFAjQxD#|rGDs`^hFo?PYbK%&H(#-VtQOp0N>u6 zB>i0Q|2=Q#8FSD-=iA-&2Z2w*rNd$L>lj`b@e?WodP}!~vmxkDC=2O%3i?g6<@Aby z-r2Z*-&~O2w&u*@M1X31OO*YAPv=Q%^K`AL) z*TTA8>cdxZ1aOL1zuO#ur@ezLRzX}$w;l|M0Ke!*9?hK%_BnY~ze)zXU2kuxI0kx} z1hx!50y{HWpFMr~VGMiJ=!B7c6|$T|QQ((OhtU@JWub~`qTY1?(--ULngjp)^0WI+ zgJ09j?9vUvzH8U7O+E?ZR1V%+2=Ba6V-=&q$@4(l>)p!yU~ka~?{fp7KU;O=j41fU zTY6vXEcjpJc=sJI(3?a`9+!jsA^rYgI44uP@=BukVBDl*5;e|XM~GN)tr+k}I|`Qe z!FUe(%mh~$$Hz*)q6+@>Ze~9q0=Of)GtPwZk;^7Oi-x>wb>W!10Q?X7oO0R{>}!cW zMm-1oHjiVhyTR^q51Z-dJyoc@9xfZ#yQ)z6I^LYC0o7=gsAns{i2Vx& zY=GX=X}HkHONE-y)W5&pLxrl`f8S?|y9)KEPkGhXeN?ELM`A)v0oK_+h$!_2IW^~` zik(%ci!xic-h*-1k~c191vz)#T0I|v{`<@K6W@UUQ(E@jY>;#9h(+c*=tqOowCded zs1oh(Cu@QJ-hS!NOF*C8-sIQ^fUgo1p413%?ZXqzLryAGfmWx6Lm;pBO!rTk0Dsb| zSeNOjLM?c}wv7(@S5&u;(t*F}GJD@J=xMVJ^wt5rRr&e%o&w#xV@UWD@JSDURFnog z`8l(izknT8hq(ftL;uH$#4W0T=QPLd`~v+O57RFTfjx(EXT3QG@-BCOBL$iwadnP4(0ej8l;w4sw8TS(F21-+-^IdfxDQ;EOiV@B9LC zg3fgd%m#f_&bG5v;D1KN>l-7WcXYu<_dysppl`}64*f;_vRDY@#R`59SPOQ~x1EvU z0P=dKj8aB`{&kOf4Hv{^UE=f;3!wk(rgJ?)(68%rBW5S)*~Cdbw*~Mc&G&*xf8zE# zrCkU9rsBeE8=yCIrP?}z-tzanhEG7B@0ms!CBS{Z?X^4r{VIjJ(wjj}xGnbv8o;K* z=RfQM{|5{WpF9J3R_3B;BlxxI(dGOK@IUTlta=~Nx67Fa-v@pnxpIR+kfZuo&#(fZ z*!HPQ1Hr#HT}VR}^tpIZ?mPzn`3rV0%mKY6yX9z=0MoqjLP27Xv zU)JFEJv<=iTd7G#2GHkR6|qSIf6^X!TkR(O>W|Ar1&}Di(MZ)I}P@zJby=ofjuYg4IYRBy|Nd+u9AoTKnr7@D6p^R zz15~X;Nx_+H>m>ui?ZE2$o#ZB#XmC`_@V=>OG<#xJYmuya^2+c4k&1YezqOg)X#&x z5%Zf8G=M(C)|0IOhbPR#dSvw3(0bEPJ;-Wm@ zZIxG^UxRh4F}JHs0`Sqq(AnuQo~5yx`vb&9ro3<<8T^XqoocHB_Qe)2THguwaJC4# z(Lo=Nab->?*cqqR`P>@#+@BqjmjgcJI@c#2^jveI>=FTaXOga0ZcW8PbUF7(lQ`Z|E-FPGeV2=dr2xcJ-zK9=JrF1djnpI#WX^n-t^xeb>b z1iI9L%SYak(WS8-HafV%gmp3q_o0_9A8FnbK-KMT>lasLYFNjYPHjzhifs`sL0 z8!(!d-ODduL?H93kI!zTKpi&lcC%zUMqBS#rgZ`Rib;Hv4#=;5eoiOv97a~Vmx&)q zfckBjg2MGIs4IqWSe_39I~Eo0ympKTzoz;u(zFqWmxWjAG*v?#^4J5*zyhdKs$Tr) zkpuPSn`dU*WfREW{_Rl~@c&vbP94pMx_eD#d-hSNzp6W*+LMUUx-)ye#erU*-mm9) zN(j`OsNSyu{tG8qvb}=(YM@W>NB&a;nyi;u(^Z5~I-zT|z78Y#El=Vd3*ipl$=f0p zlD!H+ebM2V&c#WoP`ACAWYq`rB|H@3S#gvI zzqM-IAo#&b&f7|}Yp+g6nmNS30ypcMR9x_9foJ?L+^ z;kl*wD1nq!#E;cPW3)umuP&SPFTb_p<3Xr<``VV9HehtSbje!Zs}T3-Ly4iFucU9+ zK<8oD2Z_JarLqU)a?Mx&0Q2M}aesA3CV|2ZC)=h#Tmru|NxMM&Gy2xv;DP+4zM_}j zT}vSEqp5S7A^!fXVx}h#Vw4;F>bWuGMQrLP!g zUBGBp=b71U1ngh1SP+^Jhvz5q%rrrtoULp|bvV>>?+4zw2Kitv?dh{&Gl2}3z7(hc ze+A8Y?KV^r$gyvsyaeRe{yAj^WmOo79-5yXOU_r_^387X5J%(f(Z`_Q%YkPdOB4=2 zCcbBXLN!LARxE|=Fh9ZyJ3iA2VSj-2`ke~U*F8w7Td@i1&S@4shhctsQVp)WIRX1u zxkjE#AwRzH3)Q8bg?UsB?y80J!BEkP&7P1CVml^N!$6PFirY=K$1vI~O4V6-nLydK zDGzply^f7eE*#+h(6cF>`@sJXx5Q_CD}i+wVj?mJ@^E_NiDXaUmz%PGkRR6H9lmQS z?+(Cx#O}b^;KxAw0_SJ2uC`eRZIFSuiGOHmI1m0yTW}2ep2y*5?~lFT27Yz$i<=A; z!FoMv*&qhDadr0m5BjjmpE3r z0NsLaF;r4Gb0vcN2qca+O*&+UipHt6W9c3!Z2YijVA)P|bo{V~-zycp96ubM^5P_d z+doh{V|1__kp(s(bY@@vS2S<@P;&1)UpV;D{<|VDKK~rw%cYEIgR%mnF{X}DRRaoR ze!thx7+J!>WM-ttM6RKU6!~7YW8~4qr)1DXvN6AjF_AW}~VvL)*3s{C*9Yd)$WqO*LfZnU`5ZX5!2mG81RkkeN8MhRnp7 zHDo5vtRXY;q9rd*KnVX&wsQrM*Vsc`2^HC`mQrVIu$$@rD3=f$%$Dr6)Mdb($QIIm zzf@oies5>L@cLUSww!IJ-Q=o4csYm*ZgFrUi1|K)7m7Tv!>W{l&^7CckKA3I#C&Z$ z%OSO4$qsEiz1Xod`G6zgQTh0?tW6+s)gXJ?rEF(nYgXmh3coR(z)XKp-OM#Ou2Q&8 z@46Ru2y2TNveL(OFHFvx=m-;U$E5X_U2(vDp8_=>HyaZBSo|C=-&jGoc23t`KD-Ve znEkd<_o5?_a^b1G*bzITBtqx)(r_vv%pbe!MA&NLh_SZf{>r(yRAQ3)Y7u=r|52)m zoZA-SYOaPc{{??Mka0uCy=e^*m!L3}`p^=yib;)J?plpkxy^|Q_jM-<)mO9dD=I=WrqjwWm)!%3Lr<*3E56dg}=ndVsGqDz zju}Q5%|C8qaU_VIy_K(mkb-{=HWp%EKcwr@u za46{;;R~~xj3eTeEYkJ$@M#p5k|kN@%Yt*4OB;B%W7j-1gCN!_U(`Ul31m zBJOQxYkDx-g)mw!c~!=AD%Kj2O>(X^ByPOjWnY)8NeEVHEncOnhlO8we_Qi0gm_?P zEhix5hqtUDW)oQ^#J$J9C$$yLv0$(zJoadUDYLa-z8`fZ*riXr9Qoy(ReyQ;`Z-@lrOlq%PZy6=HGU0meiG}hye7T!r#F)Q)D z#j%(As(W?z#csAe<7tiMN8+y^kWwLNZ$BQD(6J|2Y^KZjCVAr4+E+f!!A``nClU^P zcRYxP8EeagS(f8h8yeTh-BTrIo@*CZb1@^n7*f)T8Z~i3lCqg(cra1?!1TbG4_^3P zYvuIJm!`OMXT!8hSGBQmPsEh?ZUuaV^-^ee)>`6*>-jzrVGCTod;cta(iEP5{JiUp zq!mF`3(yqF9=_)IZu`X+9yTKC!K2HU)m#XRh>HO--4qm*~eW^i$6Y*L!%hMU2%ZohiuB>k9Kv=4Zy;Xa+k%&1X zc5rCK4$IzXj=C3xF&9lz-%G@iFjdQyz2c}&Y%Z?v4xZ^x(0%S0js{s_4qd1Dk<%S9 zZ}?{4%QKB7y` zOYl{@{3k3is`#k;D_qRuOhi%nmy}8R;QDsysL%|1;_GKTGJmH(VPd56({P$2)~Y@? zlTXzMH_ysr1Eb~35c6xSroam;oF)!YDJ-t1A-8U;L!n0`WJ(-)J6gKA79M?p@r=!aE zUo$r5wH@a~r_c8vHa6yU9<$5u`)?c@^O}#@<^TQXj*WT!$LtFD{>#V4ycT43g)n~v M|4;tv{p|Gr0}JL4Pyhe` literal 0 HcmV?d00001 diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-adept/observer-netcdf/fields-shared-t=0.1.nc b/benchmarks/vlasov_cuda/results/2026-10-24-adept/observer-netcdf/fields-shared-t=0.1.nc new file mode 100644 index 0000000000000000000000000000000000000000..cbc273c90faf503d83406184440560c96a1af891 GIT binary patch literal 17408 zcmeHO30M?I7Vd$ObHL$HMEF5TwY05mSAR(QG{_Ej2ad1c%Ver zMP)Goul3015sw93r3Ric#BtvX9O{cx>em>pu{7(vbb&+-^}ax`qiuV{(eTaQD87Iu#HfJ85ne*n>oEkCH zklL700<0otyeRcz)uXsxtkm9Ryj-{+Q z1&wxw8>hjUuxMZ+ZCpE6!j7C^$#Mal8bXU}JvU9zGojkW2m}%=6bP6=C^9Ia=FT`m z2N|HOn7%X>WJ?7-F()xM=WMRyEDJ?gpVh^p+FUC~Ow9#E=#5#_TDUU)nA=hfQ6)T% z!kS79Vf9^E2Pu&pd5iumkU=XdGOkou^-^Q2xR=>ey^NG7h(c}&D0@nTG*jcX#!ofrK`Px)od)TNQDAJqGQ`!7 zhL4XHtow+?RM7xa89t4gjQK`byaH3mVY-{se=(-s1ZXa7f~n05F~#v=zh5stY`!sZ zA4xR}@!P_M`x+!6GNz{}4k3ynaz`c5F-n)2*)J_KN6R|)yt#nSkQqcQ5z6+e&8=P-tHu7}thS+_AAIq&)My~%$vcC1RLy)W& zzBubY!94_2bF@ZCo?08?Vte(-m)De-|&Y#EP_=QCc{UT;3*-o~8+r($m{WSA2OIru3%?&n5t z(2ht=qZIfoMcwVI8&e?JiyYW9Nh>C z7vfe1hK1XlSaPQV_zcJ|ug@#W&_3c=3ot)UWZy+c3IWdK1Jl}W( zcm#L^cm#L^cm#L^cm#L^cm#L^cm#L^cm&>41j_vfX;eNAaP3%S-ssk1aB%c~IBc01 z{HC6rQ6ZJUG3EEFR94R5*qA_1-{BO9sq&1vyXk}}ugkzCi>jSq%4d=v?rw1fgUWoq zESTm3y_A!jZUjGyNOk-=x9w-<;OMr+eRo$ihyAW@xH&Vy3l7c77bd!kA+^Qr@^NXI zY4VC2zD}xEfSO-?aO`;*s69U_v$A(HaMIKHUKltX;qLcWW9%+xFmC@~>31L8F^!0O z+`Y2vBU7A*vUvFj39!9mIsf>4G5p~BSwpHn6+`Eiy9+}rk3>wg9`?;{>XGT=N5?kr ztGyU8RWz~Wnw=SF87_1)I{3kTqx4l%*0cv}E|#ud;@|?)2If5cvs4Q1UovSRYf7;k$i^=gUQTKWsIiNa+L>-e=%TjgO)c;MV=OPLq?>C@hlX^E z9;$bQt-I@T2VeIE(xus9_x~Y-5qSrTq75FfP3SJ`qy7|F*uVSZ#Bo9>`}olMOFfR6 zgb~`}QBVx*idC|kIbGmVm!Fr6Ob!C$QWUG2f9MQnOZprvm-&FNLT{xgmwAI*6%mO_ zuV7fd@3>)>QUPic{uR_>vIqDw>uRsCI5zjz69s5eu*P7zD2D@q#PX{nmEDF*n$`z?716 zJpvSM6m>|qqrsS*jiW>v-atNa{>j{23veb^KK$AtIoN(VIeJCzIg{T0$Uj51-tb6b z^-#&g2w+p;@t2u>bvK9Ea{bcGFB-wVqwiPU0Z_wO4 z-4bLiQ`cT_jD_Y~J39JLqX9)7zWI%=(qw$ps;Jb)54fpiE~YWQVCTuo{gHz^!{~We zs2W``xb1x^U%k}_6ntAbGOWE6>Y=Ad+{y#~SUjt2TbKfLu_;6$Uj3sf z^r}UNm2mr=5}7!u8;o)Zylijq2P@onZ9ZpR8!=$cq+QpqwS)43`(g{L9YOce{dLZC zFF1EYVp86LZYX}fy^jdsEr@1f{5adp%uGO@660-VyuE_A0pIR+SI<5Aiu!Gj!uBkt zbz3@by!q13K4w?L>+_P>-gunGLB-w;30T6FG?0y@7jFVm-W-11mPAu*0^(f~ zypcjS1+goul#}9LawY!@S!$FaH&FYKC5^N>S8;SgC^x_}q<*#(aD^{*v*jJz0L^_+ zqJGBpS2B&0^NT&$f&W0x`*SWfVb0^7zNd2@ufgA~RZMECoVResmynWeH^=#Rmc*`7O64 zlhI^x4au-sIU!=h0&qh@KoUD#gC#&Nl)v0Xxb4yh7DMmIlX~wkS zT5=b`#rYgU%A_^Aq1vqBy6h}Yhx`ZsoGy$AkHOZJ${3!MS#wa<6^xVjU~r7M&2VEW z;mJ!bgeN7+geP|mguh_O{0d`3?i5MQtH4OANJahe2)KwEzfzJKADL*#)@uz$9rN;l z6Ef71InVkrLTkWLhb#Otg0q=nJmnGK5#SNv5#SNv5qKvM7|(gZQx*sw2ObBE0G|i%c^>>5 D-n>QR literal 0 HcmV?d00001 diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-adept/observer-netcdf/native-cascade.nc b/benchmarks/vlasov_cuda/results/2026-10-24-adept/observer-netcdf/native-cascade.nc new file mode 100644 index 0000000000000000000000000000000000000000..9399ca423b5991f40649b79d65795de915fb824d GIT binary patch literal 18659 zcmeGk3v^ROc3zs6e$o%Lw1|8jlyB(|XiHn{OX!b+rBqs24pq{$d1*t_q$DqGQEC0L zh=_>biYQPNP!tgmR@CDcaKQtsi>QmRu1E#t2+BuRR?c?syqQT$N{FS&vYwOaY3|H@ zGk5OXd*^rG%v)5Dn;#n7HyGg%7>JZ8LXNVZHX2S85DZJR=jANo0!}T+ou*J-QVf=# z`}L&Rni5Imiyj1C4j?dCfjAtCkh!Q;5Ns|1NL7p@9O{XpP#~(bm`p~S(PGwC*!5MS zL^GjVI8JT^CLt4pyf~0JLI#{5C>f`Mrg88WE>o?mY1HaPgVAi1fHZPTP2Yi<%r0#S zMi2bxtq@rNp@`%9U3g@Sd z09wB^2lkTba!(OSBe^<2_3j5cg^D5(Q=K#PkHEAP3I*4Z9?^icFQ$r+KmgUtzBH0d zrdTw9MO?%49|Z_ga|&)}{P?-#ht%AZPiTo9NXDn^SdD35J5b6Ri-30KPMVTeR5EF5 zk+yJBNhxRyN?_f`z&~yglR1%myB)}B+T%7IO&pmo|JQL5A%QgV&8Mr?@@&>pOAgn0S4W1Cf= zE^OxkP%aSV6iz59fjp7$kEO+Owo9`lXF_2fQpJL8z*B#@RMCVfd6+ORg>9IXs%b>3 zLZXx?V62aWu^fxVsyCW-l4!%_Ix|Q+1S8ZDVm13i6(P_ufnvVVWSXHf*{MZbtFz!5 zT&WvB!yRRJ!j;=dy#3b`ofM%2I+(pg3A)zBALZ8h9xvJ?V=eS(-flCRt9a<# z#(I&j6wQ)V)bsMxDe`)w!5~^ibERk-&zmh0FI8itt&!C0!-1OKPz227tvnt4*xenQ%R9BBt{I(oq4BL9~5}Wq5yF2!eBd zHtRd%>5$MVKg-AJsjygNeu4=3PWj0Kt#iFty*My>j87FocfyCdz=yfO%d3iO(2sJF zzqbo~v}s#XS{7w>6Y8Qx2+FP*kF4~=4Aa}d)xj@H>u3U_QvrACp!t~1`*zLO^wkI zrn>K%T0KlxAaj~bT!dY7MMo9~Jx4hC{aWol>cDVswaGkb>&>>>`P3!CtoG37<{HA8 z2YzpgBJmrfLhtyfAJ-6}u&(mp*eF$gNl6}xpbl6L#(YG= z7xO|npqn>br2|Pj-J5X%PjcRDGj66HKW4_Be}{-&nRU5O+=gVedfdR!WFkmU{SNqI z({O|iEn5}wd{V1g5zz2(^N?2c^M^NywYRjYn&GcR??mOS6 zu3x@#`??El>Xt(%j-Nc&rq-{DOndI+i&;-Ra;D#phOJpUQeRy#li!&&=F%^lzMsA< z^U?AvJJ0NYGxNx=x^w%3_S1K&r1={!FARj9q+xIaO}i3KPR6L)znnZIV?;&aOAE)B z35}VD6QsR5;k&C&g`>t`*KTn5IwcQ6{9%DK!mWST7_zHt?zQgEYdS z?RkwG{-qbz4_mafach#Wf2n=Camifa@|5%sqwc5|`t*Ng*5=F_;oa|+U3o8}TqwQx z(?g0jtI+@b0iT5B8-%!XmkuO-o*~p`6~)XcwPlpP7Qg$%s50T#|Afwe>rI{TLet8y zs+NhuuKok3Z^_UKy{dyxpE^BDNUORc1sUswCtv?|O>B@xxO>~8rC)!n7amRwo1a~s zB&4v$trvz9ZxDcEDCEr z$oX>hg>gd5(Zj;t+@Tq#klzDK`j-hqg|hI)z=yeCP5Qy-Cklt&X?P%EoK{#Ht6YEi zr&&VlyzJQ5YU+iozntWoqBKJCfFW!1ztIavS6oEBmn8}1pR8H@WaC`n&L#Ihl{uwe zNHl1_|8{JR@P5g4G&696YlNXz*iUsYsUSf zzI-acw6+C`2>_WMS3QlxCN+U?fccRS6gP1VpYo}iz!U1wcjuf*{_77keL=nW;@|g< z;C`X_E9!l@O^-Z#5NSw$Kh5Oui0$iF`D;kNK+S<8>yB)$3f2%hDoxS)vE0s-a1ALJ zp)tN*vEnP#ir>BXa;~Th2Q9zkPqMCqgLdwKp^rD#8G^N4p+yH;+6BHa<-^; z?{}c5az>DHUlAamtn%!E8bm!2w1Q6Hi@6I6y|aqNy7s)U$@3P3x9Ojq9mtMnbdu0- zv=lensJ?N8Z~!r+>l_rU8|CF7UsqiYFlW%H+}-~p|jr}&YxCLjCuem zMMvvm2b9?$#e66DIGY?8q)B$9jV4_J=**6iq{HzrY@f(y1xk7zJCR=k#{wP0Bkh)S zoM-v)IoD#;h<0t3(k8(PEBNQm}E*(!4! z+5$A9(XID2j7Ip3>>$*82$ z3H3O5@uYmdyEodH7PJQn`GO}UAnQNCVR+y%0nM-_0o9jdnjr?j4jED-U}p=d_Hloa z12HGMkb;w*pw6L)9U!DS4`!K(ve|J#Y6gzOX2itU;XrDV>`WlFFGn{ch84jm6H|`# z3!1|)KG;D5zS$9>?^|L9cMwSzqMn&8&(m4U{cO3MC(Lfc%9w* z$#ZgjU}M+5s-gh?VRV#(gjzD!!j89VmepCV{_-vm@A=5+{a&+D-UYH0;dOH=sdY20 z-TfYr?H*s|Nf91wpD#%o6iY$V%v_c%A_M?7PBJE#S0CDmWYOYt#JvUJz-ZhTZDqhwts;X|Svzc~Kq zVr+RXte_an^hPhN;kg9|25}r~=*HXN9UocJ?n%St-dEV2)-u~;5Vw}dJl)Cn@lMj} zyYN#Ih$TjFob21=h@;f84RTzprMVqxH~zEJ^d_g0vg6tda*8LUSJIqp9#*=%aj7cD zCGfTIu3R!^R=}WK9UTB-WqR;K(?cLSYv!p?FOG@c@}EJE@$3ccXpqI3sEtdQEIdmc z`(uu-uX_!E+u)fo^K2HWsE*d>+A^C5lPujGGf6Leh`u84{%rm8+Xu+1+T*&${)72T zpY@Wiv14@M;umz@u_Wy1Ybm?5$2-&q0rgc4J5B>|6TDMDzAgEq?U@FddLthD@c;kg z+K9QT7z}Xu%!uG5)}5#vo)`G6!yGMzeFrq1ifs{{kA5`j`L! literal 0 HcmV?d00001 diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-adept/observer-netcdf/scalars-t=0.1.nc b/benchmarks/vlasov_cuda/results/2026-10-24-adept/observer-netcdf/scalars-t=0.1.nc new file mode 100644 index 0000000000000000000000000000000000000000..5cbbb671220fc84899bc1f24afaabe02c6bd072b GIT binary patch literal 20290 zcmeHP2~@4n1%TScnDn4pEe^gtFpqNt2CDC zs!+KPk`hJgF&DrhU1e2iMR{CIc*(?aow2Z}I4(MJN@R3+Nm22{xR^*~q!QH_mc&hn zjGh#!gfM#A9sh>@)^iKFRL?d+@WZDG0=W=yVKO*%^2Jx^EtiniIH@7l9b#`e0eB(H zkAS>XdcwE#bPzo-Kjp^MC%@cIymd}-rAI90#Xj~xZF7l9rDdf>sP;BVL>~R| z$O5FV0PaW|q>H9hug@#cFVxZ6pRzu#v8D^Q{KsKVC#aVZ>pgfGz~7l_f>b2QvX_;l z9RPDLiEU@bl5m zoZsC7zlVi#PYdM_StuW7p`4lfHhX$o;3sngRA;U~V+S=?fEPHZ!vYQqI4t0>fWra~ z3pgy`uzm-s_9A&*86uzR+iEFW$9|>2}5i5vg8w^sOo#z71(}pXF@~W7u-CySsIRG zOCdX$Sj$Uh^>WHGkV|G{J%=xrFpX#bKH)nnen*S?cejw+KZ~6%+h&w9#cf1vgM(JK zq;6nZS;oOpIb|7091zf&;UF$($wVtSu%j)4{y`RAM(+tJ+u$NUkjZ2#t(I|d*$!|) z)^E*lvHkFW7F)r^135k<1C?B$siaomK#f^M%9B1z>b>9LP zkEW9P?-tlP>`7lM+Nu}&PHs{G)GTd}#uR?I8cy=pz6`~*eYHLWQ>9ErC>51IW${;O zKDny0V_nX5mnv1)-Z(33-?c)OaWu2!?GIO}P9C^4>2$`csy1VFbHlby>C{=@^s9S1 zrRZO_&dYeWQz{u9;NEz+Q(Ds|J=*oBPU-QLYp=zB(kc0AH{{h8$Ecsw)eyRRXBeRk z%}tNv^19<(8@w8$x+K2|qonY#E-7&Bn3Mw{UDD;xPKHibbV*0IB%MDK)Fm~x zzNIVk?~-_Dx5&2H_7f|^|pRtqM5p}Qr)dfVk{u?qjBEKrsLsV}PnisK| z&{cs=gudgxl~8Wai&3h5SN((4owFWZw!T53p4PVf{UM@4ty{H8zI^T&^`+gdA70-p zS95+!Ls;Gt^;5t9-uVS}q2B8Cd;d!4r%^iyy_`6o&^0st2;F^ap8@mrmhZ?pEa0$! z!vg=m7O>5n_cRL=?E6_rK8D(AAfJUuY=YUG_mh=tOb4yMn8`AoIanGvg`miBzDCNH+%EGUpClrqe)3k03I=n9h#6%-I_3y zQuEm0jJ$C;yb&kc;HZhm1NVue8au!diTN?Z(GIn7sU|xm3oyuJBm$J!2TWn4^H``A zkCHTn4w0oD#LTF9pd=G>8$s+m2#iIE$q{V|r;Gyp{CMM7O%8f0hSkifoSjoHQiLp6 z404pEj9)TMf*Av+&%C&$CLuRX1I`EYv~DmJaVy|o>9O8$gdE2IZkIP)K_>;nt0l)q zz;IML7Mh2&Dc~?F4FK1=VNoy#m4?9Pka{KHXx2xN%2TspHd4wDp1XKx4t$Ong5d4g zIdfq+Djf^rq=y#68N@pZZhiLAZ{Rg_sDQn@o>>cxh&K>kshsvYs8MMERCiQ0!W?wy z4=z);?SrXEf*fv1xy^vbs*y1A$A2G$;pos8>JP2`0G>k<6!62%71vRjBlTqg;(s0- zh+^UqKMA;}=TQ?QlhdR|Bml+FvP9j&zy-|vJOGh-)g+oChKCz6_w#5whO(htGDVuH zOHyR=P(p?{9zn=F&I1P-giPUj6v{kIkRi~EH$<_?g8~`4cq||T@$x@DvX%2caCETW zD{bje+uS=(#W0=FK!%g$_3FK2D91p~-AR`!+`J)IUb8f^UvE z;g@voZa?qj$BW2dYa5(}PNe#b>w_I<%!l`d*amm?lkXIF zUJA-NkmDPfbeQ2T;pIf-V8>ne_!c^edg%$~R@=vnj9hnK&4d$T8yvbm%4GI=pHx!t=y-5+W$>Eus?a8Fth)JB*V-7fXiuA zrh$pJJpVJ;fnCUBmziA*vU9tw;Bru`-FAQ4V1q=OrKN)}5uTInFtlH_cpMB8hxT$0dpaVIt#7j%Z}Mq;v-ew<~5jDDP@sK8iIrQv1h7f z&*r@B<`jNJX)Sx!;H7~$-n2&&<9cM!VQYNv{D!37`P{_b`57s_^Z!Ww+X78WQjgtf zH$WEX?SG`ip}cP*1k)~}VC0_~5kxkofmj%$Rr0q6c*6Q@SLStn`Iq!yo$agP?5wejoq< literal 0 HcmV?d00001 diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-adept/observer-netcdf/spectrum-electron.pkv.nc b/benchmarks/vlasov_cuda/results/2026-10-24-adept/observer-netcdf/spectrum-electron.pkv.nc new file mode 100644 index 0000000000000000000000000000000000000000..48145efefdce262b8beb1f88df34d2c8b9bbb2aa GIT binary patch literal 8843 zcmeHNeNvKKp;>OG!RHOMEp3oR=Tw< zU}f#vMNkVI)>7N9Ew!x*qWz*$KeqMQ0$p{jt5%VttEWBaTGpL;AL6RWw#R>ZPKJ{= zGk5O2bMNoonU|BxO9?S?UK6HG0JsT-zzvA$A@{h$wwhpu!p*t$d~PsGpPCSp&KI5K zGvWewmW@+$e?Nf_c$mdQf}cD$KJHNCITzA&v@(O9y9sDI0G=fkMzhsi=ycf1USruk zf*2k*mPf=fDD5Q`&$Bj62|&&QAmD-fjo{{D8j9Jh=Vn$J@a6{h^gRK21nsI<0*~j1 z7kE6zV9w6Fy##wWNvZ`xTAl(bPoTPkmYMCmS)<=!wY!NhKf{YjH`fU#;~v?~66x&_ zg>z`!7!EaiV0i#eF&z+vBWuy02T~{m`ea>323qk!AM8I1JBUeK6q}Nfn3`hD(Cd<8 zfe68ZgXK<3rRWyLVs%&w2Z57v0+9+JoOxg(5Ft)uirHCcDpJK*N-ZYDCq_On4olH& zL=KPh_Q)F;Oo{t2mEmR7xLz;@rUzL}yBRSsI;MUsrb5;M4pWa&F~$C2yMLa4Sp6@= zJ#;kOMYJ8*8UXG&6hKz{Z6ZS*`%Hq zvH8vdcQ*qtvnUOqJmCRPq!Ve3vskUaEwHX&Pt?QC0+!-V(8Dz1bMWKVL^HORIX6Ix zW$wx9$La+gFXRQzv<*&VLwamnY6RW%_+u#vB0r=EWFp%lfrUe_4A%7P+pn{{}n1 zrOk_8qF7UWho$qW6|Z+(?ihBb82<58NBhyPC*wpE!{<$be_ZX!iJwF6YJU_vv-<*BeL+QgcC>aaBmcGvy(Uq9UiUBBV$< zPs&n@6bfmWw5&ogXHJBI;b-4mfPUvA61spb6=rF<&01l0NS8V2w#272VI9D<#TB)@4BUxL{-3w5Qh53X&o?U?cXXV9%`>2n7Ad+@c6 zO0p%Y30_ULUhM070iKGgKDN=#0Szzq{c{U16Ta|P-&wUc35x~c*PCtx!&lV;&}DLi zKYgy{75ZN!Bww_X>(+M@=T8%VvIMmfp_ys>UT>`;>b-UhOf}>Y{Nk*aT9j(SdTG|? z){~P6-Ji$JfMI=Qzl)4t%% z$)u<|xBg_efZTBAoYCO82|tlaq)Gp}1Pu*M_FPFXH2AK%cwot4cxThqj^6%u_~nih z10~T-aPHfNhRbhP!*+Faflsd;F8RZR%5T~1vQB$5F!gq&NO%!N|1GR1M zZ>^t990xnoYICn@GK7tmH`@+rsynx4)kJO940~4fB%NETS$`y;RyUs_JDq{qi@$+n zmtUWUY(PUETo~SaVp}vRNgWC+?1~^wKO9f@-lrr#$ymH`^GZ3X3jIzMaAP_tU(@t~ z&t4z0_xs4Ux0U&#`^`GGb!gl4ob`OdwL^ z^*-qr{E5yhe>>Am-qaAQw^Z61ztpTcWPhr^uu)Td{z7B6u8PTv2ak)3;zGcMz^@2_ zN0(P{(aszEn#wEO@AFaemg-0NhKjbhc>VK=wm7V~l!=R@kGBXYN*F0E3W3R7WGS;a z@7AFFQAr7xAMl6qkEt?nzT(;fS2n}-2CNN7ZXzll2|b!)c)l!am5DFnS$x0ko?F7tvru=<)?+bj=^m iS4J!6hb}i2Er1cqaAi1Jv9E@a%K1^s@Wu/dev/null || true + source /global/common/software/m4490/archis/venvs/vp-turbulence/bin/activate + export PYTHONPATH="$vp_source" + srun --jobid="$SLURM_JOB_ID" --overlap --nodes="$nodes" --nodelist="$node_list" --ntasks="$ranks" \ + --ntasks-per-node=$((ranks / nodes)) --gpus-per-node=4 --gpu-bind=none \ + --cpu-bind=none --mpi=none --kill-on-bad-exit=1 \ + python -u "$source_root/benchmarks/vlasov_cuda/adept_timing.py" \ + --prepared "$prepared" --config "$source_root/benchmarks/vlasov_cuda/reference/latest-nv4096/nx$nx.yaml" \ + --vp-source "$vp_source" --output "$report_root/jax-nx$nx-g$ranks.json" \ + > "$report_root/jax-nx$nx-g$ranks.log" 2>&1 + ) + python -u benchmarks/vlasov_cuda/measure.py "$EC_SESSION_WORKDIR/cuda-build/vlasov_bench" \ + --launcher "srun --jobid=$SLURM_JOB_ID --overlap --nodes={nodes} --nodelist=$node_list --ntasks={ranks} --ntasks-per-node={ranks_per_node} --gpus-per-node=4 --gpu-bind=none --cpu-bind=none" \ + --placements "$nodes:$ranks" --nx "$nx" --nv 4096 --steps 10 --warmup 3 --repeats 3 \ + --expect-cuda --velocity cubic --layout halos --integrator sixth --cells --nu 1e-12 \ + --length 12600 --vmax 15 --dt .02 --input "$prepared/initial.bin" \ + --v-grid "$prepared/v.bin" --background "$prepared/background.bin" --max-mass-change -1 \ + --source-state "$EC_SESSION_ROOT/source-state" --output "$report_root/cuda-nx$nx-g$ranks.jsonl" +done +sha256sum "$EC_SESSION_WORKDIR/cuda-build/vlasov_bench" "$EC_SESSION_WORKDIR/native-build/libadept_vlasov.so" > "$report_root/binaries.sha256" +sha256sum adept/_vlasov1d/native/api.cpp adept/_vlasov1d/native/core/* benchmarks/vlasov_cuda/adept_timing.py > "$report_root/source.sha256" +cp "$EC_SESSION_ROOT/source-state" "$report_root/source-state" +cp benchmarks/vlasov_cuda/run_adept_comparison.sh "$report_root/launch.sh" diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-adept/pinned-one-node/source-state b/benchmarks/vlasov_cuda/results/2026-10-24-adept/pinned-one-node/source-state new file mode 100644 index 00000000..84e90391 --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-adept/pinned-one-node/source-state @@ -0,0 +1,5 @@ +version=1 +base_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4 +dirty=true +workspace_fingerprint=c0a2abe12f3d288f697c8b183b00313aa4c6ba063d5122a570dfae4a37751380 +synced_at=2026-10-24T19:31:46Z diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-adept/pinned-one-node/source.sha256 b/benchmarks/vlasov_cuda/results/2026-10-24-adept/pinned-one-node/source.sha256 new file mode 100644 index 00000000..1ce74b5f --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-adept/pinned-one-node/source.sha256 @@ -0,0 +1,6 @@ +11ec35339aa737893312c0b2618097a8035d8bfa6bdd3debb900bf07d546f59c adept/_vlasov1d/native/api.cpp +39956b49c55c68572f85fda5b7bea073dbc96bafe708c7c8abcd0a29ba6a54fc adept/_vlasov1d/native/core/backend.cpp +06dac8cb9d13f2850fdc404aa27c42035d657e2c9320e6b42e711424a76d0e97 adept/_vlasov1d/native/core/backend.hpp +f87f6aa9731f64e501b94e95c20a53f45ae7d1ea5ec39c7e859c6b1a4b8c40c3 adept/_vlasov1d/native/core/local_cubic.hpp +a13890d45196df9a0f3da8f309fc5024c0f574174701a645e9558c3df08dcb66 adept/_vlasov1d/native/core/solver.hpp +3470b5039a1f4ee5b21930de121325e909b52ee7c8100836d880b51181993f00 benchmarks/vlasov_cuda/adept_timing.py diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-adept/slurm-accounting.txt b/benchmarks/vlasov_cuda/results/2026-10-24-adept/slurm-accounting.txt new file mode 100644 index 00000000..9dbd68e2 --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-adept/slurm-accounting.txt @@ -0,0 +1,35 @@ +JobID|JobName|State|ExitCode|Elapsed|NodeList +60221398|ec-adept-20261024T190159Z-99199-22637|CANCELLED by 71501|0:0|00:32:24|nid[001600,003745] +60221398.extern|extern|COMPLETED|0:0|00:32:24|nid[001600,003745] +60221398.0|bash|FAILED|1:0|00:00:32|nid[001600,003745] +60221398.1|bash|FAILED|1:0|00:00:25|nid[001600,003745] +60221398.2|bash|FAILED|1:0|00:02:48|nid[001600,003745] +60221398.3|python|COMPLETED|0:0|00:00:12|nid001600 +60221398.4|python|COMPLETED|0:0|00:00:37|nid003745 +60221398.5|python|COMPLETED|0:0|00:00:17|nid[001600,003745] +60221398.6|python|FAILED|1:0|00:00:10|nid001600 +60221398.7|bash|COMPLETED|0:0|00:07:08|nid[001600,003745] +60221398.8|python|COMPLETED|0:0|00:00:12|nid001600 +60221398.9|python|COMPLETED|0:0|00:00:15|nid003745 +60221398.10|python|COMPLETED|0:0|00:00:19|nid[001600,003745] +60221398.11|python|COMPLETED|0:0|00:01:05|nid001600 +60221398.12|vlasov_bench|COMPLETED|0:0|00:00:15|nid003745 +60221398.13|vlasov_bench|COMPLETED|0:0|00:00:13|nid001600 +60221398.14|vlasov_bench|COMPLETED|0:0|00:00:13|nid003745 +60221398.15|python|COMPLETED|0:0|00:00:47|nid[001600,003745] +60221398.16|vlasov_bench|COMPLETED|0:0|00:00:11|nid[001600,003745] +60221398.17|vlasov_bench|COMPLETED|0:0|00:00:10|nid[001600,003745] +60221398.18|vlasov_bench|COMPLETED|0:0|00:00:11|nid[001600,003745] +60221398.19|python|COMPLETED|0:0|00:01:47|nid[001600,003745] +60221398.20|vlasov_bench|COMPLETED|0:0|00:00:18|nid[001600,003745] +60221398.21|vlasov_bench|COMPLETED|0:0|00:00:17|nid[001600,003745] +60221398.22|vlasov_bench|COMPLETED|0:0|00:00:17|nid[001600,003745] +60221398.23|bash|FAILED|1:0|00:00:37|nid[001600,003745] +60221398.24|python|COMPLETED|0:0|00:00:20|nid[001600,003745] +60221398.25|bash|COMPLETED|0:0|00:00:34|nid[001600,003745] +60221398.26|python|COMPLETED|0:0|00:00:16|nid[001600,003745] +60221398.27|bash|COMPLETED|0:0|00:01:09|nid[001600,003745] +60221398.28|python|COMPLETED|0:0|00:00:30|nid001600 +60221398.29|vlasov_bench|COMPLETED|0:0|00:00:12|nid001600 +60221398.30|vlasov_bench|COMPLETED|0:0|00:00:13|nid001600 +60221398.31|vlasov_bench|COMPLETED|0:0|00:00:12|nid001600 diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-adept/source-state b/benchmarks/vlasov_cuda/results/2026-10-24-adept/source-state new file mode 100644 index 00000000..35ccb776 --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-adept/source-state @@ -0,0 +1,5 @@ +version=1 +base_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4 +dirty=true +workspace_fingerprint=37396dd0852f9e6ff790f730c645eaa6681006a0a135e7039737d9472112a7d3 +synced_at=2026-10-24T19:19:12Z diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-adept/source.sha256 b/benchmarks/vlasov_cuda/results/2026-10-24-adept/source.sha256 new file mode 100644 index 00000000..1ce74b5f --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-adept/source.sha256 @@ -0,0 +1,6 @@ +11ec35339aa737893312c0b2618097a8035d8bfa6bdd3debb900bf07d546f59c adept/_vlasov1d/native/api.cpp +39956b49c55c68572f85fda5b7bea073dbc96bafe708c7c8abcd0a29ba6a54fc adept/_vlasov1d/native/core/backend.cpp +06dac8cb9d13f2850fdc404aa27c42035d657e2c9320e6b42e711424a76d0e97 adept/_vlasov1d/native/core/backend.hpp +f87f6aa9731f64e501b94e95c20a53f45ae7d1ea5ec39c7e859c6b1a4b8c40c3 adept/_vlasov1d/native/core/local_cubic.hpp +a13890d45196df9a0f3da8f309fc5024c0f574174701a645e9558c3df08dcb66 adept/_vlasov1d/native/core/solver.hpp +3470b5039a1f4ee5b21930de121325e909b52ee7c8100836d880b51181993f00 benchmarks/vlasov_cuda/adept_timing.py diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-adept/source.tar.gz b/benchmarks/vlasov_cuda/results/2026-10-24-adept/source.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..4ee632448ee5a9de3c4c7220eda4acfed3e5ca94 GIT binary patch literal 58287 zcmaI7Q*ZQHgwwr$(CZQDu5#v9wVoxJD!_ZVki?sdDysLQpg=6s4U z8U`c~X4?t`{Hj-1=TNeVk+XZQd5-_Gcc*vOEnkHcOMDFq0n+rOH1ML1?|A^U%KXIC zLv1iA7EZ@WrYW#;f9pnnt-5zzpt;||-u;kNWirJCT+L-B&yII7l+INA|34jh0#X_| z>*I1oTAMnJx$|hFJHBmRi5owd283!O^jN%_U-kz zU<m9fdKibBV8RO%PD#0S) zGSn9jN&mA6eC!AI15dAjej)Pwz9+t0gI@wJ_5Kh0hCL4+PJ6TOyBn0#z*sx?XXJwq zCfxqZN1l9OE|yUalyU#+jt5|U%c50d^=byII(Me^#mm-Lq{s7n=mxN(Q18#*P`}L@ zdtJ4@vHAVFt^2wxerjYo@tS04dmYLdCT;)IRguI#Vhyxm{<0God7o?cJ6784u>Lvp zVLAnJr+fx$`m55O`1qYZ`p>>?&Z?p7e>C!SzJIU>{N8SD^*;B{b@P_{7c{SZ%X{v> zd-PZW{H_ylKJl-E;t#^_aQ<9m|1`4gHLedNS=+v9%R~RS z`(r-)d^@-MYvBz8G-bD* zz_b%=BSqe{4mnO8_Yg5I4hZhQe!G;ZS9|^l4lm9}ee0L`C9$_zRUl0y?Yr=HqS%oP zO2y`lDWyc^C4}0Z)ex3F{mFdu^$6~5lt{cJhbtl&8oVg^akXN@Rb`d75y*jbM9*6Z z9G~6K-cHuIjowD!t&#=iE72SUxfbO5KHc|sh!^2ejzhim zI?EgAr6ng!*ozAo%UJux-FTp<7ym=<;}mgUEj1Yju};(IDlpQ-suw#px@v2G{XWT1T_ zG@6uMm2Kx^LJY0sVC^;TxVoE$psPWYMC12i6Nnta%F{Y*bd6ZfNJ>B`D7hNHd7zjY zim#&Dz)No_2E#2ZohE36J8qv&h|biM?D-Gvk)0jIe?>~^@; zOSQL+o|aT4=X)l8!&OU;yIjMh^f2!QT|2Z4QsvJ;pN)f>x!pFs^+0ZxRFFKJe%#!1V8_*rATJNm5M774T^gCVSvubQjUimCxTVDs}T~U=3tbF_IhkmftZkFAHOtfGk zn}e8QE}aZWG84s+N?2F8)Y%6rdrL4XF~=D}Y3mm*hwsv085r832E$5peNw`PAin|= zKRgIsI z21AG#ofYL1lw^PC<919E4{EDIU4t!09s|zYV(CH1z4TpBxGUTswdch_`&F^Ju>7f+ zNn(hc4FnD{k|iTfoFlI7qqDHQa(`E?YCjc1@vV@z(hgIgpkM&X??MBUNDWE@bWr}H zv1NIWc}BebR{Di zKxK1e2$0il5gINEmBHbQ__M}#Eg@VJb7VS@8~;=3o~HIl0aKK`a>&5oQ{!tl!d`8C~*l6*jmnlxcf6v3( zL_B#cZsWFFr&j2tJv`+17&(pVA!UST&uem0Y|f)U*sjT)=#%7k(kna(1&ODx7aB6I zY^xy5z5t59$=wUZ{xoD z=V7EuBl^?F(uCDDKb13OvI)TgUNf1d;P654tPt^;mp;07qKE#V5Umsf;u?xl_o>Ia z8AAXDdkBXhcn(8&)#Q4EH`Y%3gS+g3u2~PjFKp1{Gggi_rUoBYF(lgSdV?ob(qls- z5rpx~9Fn_yGSSRCDc0D}4yO~HWjJ`fu;@mP^iRty9vzE>Cbq-Ct<(tuTn-f|-P*ot zsHtPeS>_r@0(-2NWTMzxk#cF=V|>w(821)}Oogs3zXYZlu7gzT(gB9$W=6-cn1vXytbtKa1lp(|O1#*G(- zhmG33hAN2sGR48{_S|`7X$u;xg|1^vVV2w`YU>SndUOqVsA8iT;OXamA%ms!@V{S3L^bzJn4p3`0O@;u)-`Z@EjA^jFUdD{{E6wi07I2YQ? z|6MhtX}8d5!hoH^+O8R-G0KKu78iTrQg@jTpBb8D)6^)F?sUR(<`Gq6oK!1q$j% ziH%R73>1v!Q}HQvhoK}Ua=C7}Vlw*L66^G;mqB>O2JRj3x$)d`8u3EnR8G~$D?!zm zj`s|1M~A33f{@Q+kKd&PkYMWb4n3519dWhgwnDh-YK)dkU%*Layfs}^_ZH3~h^*%c zYrw&V)MsJ!Z`s*afvdR?ZvM=k5Ln5dpTIT_?n^wnC!p44Ti%y{-^A$z@YVgha}oH8 za`Z3vQGXk#{Mf&E)3W;CZU4wv4{%vG9JpRPGt(+x-ur&^^SKF}&IHDHeC)CTH?6I$ zYY#qlwY zc=ws}P=?v>27e~ELz(UuzRgGgF~-vs#s>Q6%ZQ2coPMv%q6PkwdkclH>hn1cZyP&|g|oESuGb(|qf5$j;5ARdgHb-dZ|4DUhL|S&-WM z&DKXsT-Y^T&O@mHGfkKIwK^qB@n>=yQa1x0m``m1e5k^FAG>X%NRJJL5m3G)J?7hW z9CgX^HCr#qf-*x8?QRyS!*qdvC&S3S1dg-%RWbxaeOPX@JS{k341eJiQ98Zx1&k4B zYp+W(v1}sfLgU12V}zh|G-2jSoix*EbM4*8y)5F3z0ULpV9azI$V~9$%2BnAb(-DZ z7OZrG3cAes(q~vP(zJbf&p+hZ61fyUBCq2q&JjQQCnl$O&XlTC8!Yzb_9c9b)da`2 zNv8oLCVI_lg9+0>XYCnyRraiY`ZiD7mdx%X`l9iLq}lMVoH>;rktWcX0!=-3X(8A> zjXZXN&O(VY`8*`n^vK21!c9W&Pm zh4b*En_k5(sClpnH(AO2rz-ch*Bp}KbLky>qz|diMFSuiKHX);Pb-Mqg||2nrXF6T z;jsO5i3BR=Yv<*wUy_=0eEvt}ep?N2|MEDV(vU1fo1v!7aB9%=lB$KGtb>xc2Tn(F zo69bUe9GBjlCkd`(DaS|4vOD7A1RJ%a(8l_{rO1(c=s7QRvr`7mLndm5V2-}m;a%) z0Dn6Bc%Qks|I8mKHMBRf z7Cs?1Tek>+frum%h)^VFme0?%tezx$xi5-&m?G8S^VA8wG7OWIW$QdUW-usq6XZMB z*S1I2=c#`rSz)R5kt(^o4ft(JnRX?j;tC~bQ3dKyT7QJ=*RY|-`kSW%0Yr=;0^S&` z{q^Ts=XCDB`SgmTa%%tp_$)R)J=;4P0(1)#K9=@uaRn({I-^{$xFyWdT)=@C$gqpZ z;E9}Q{~jlZ7H^WWwPy4S;d+ooDC8C{45jwxN-;-sCyjC2W*+=20C4=BG{PwfpF2uf!(S0o6R5aneM3lCO*!HalL z^1~cw1Um*n_fT`Qm@6pWP_Ti)hQbtadGVm)7W>8h3^#JGm;!ZT$V%TPC|!j}{$(hO z<=S@{>4o~Y9Lx3oDzm}|9uM;gx#!kFZpef?(JJ_XwL~sjx%Wv(X=}#n#^@(MCMYG; zFO{)im%^Wq*CEb-gO7Xd)N=e*uiyK5$Mbc61}}rLMggltOzB&njHRLm>QlXQ`|tRthErp!Q)*tTWT_CQJI==+k%692K-yGdqs_Sa7C!A_OUPgmdPwm2!E;zB zt*t|z%qn>+(Zt5`WgL%OvH1}k@T;>g%Gq9!h}ocavFGj>70~Yb-K&Yjbew#x;-q5+ zx}+Ke^Q3aEr?R*NpT(+swu|#K1!=B+sJ<%$ss;cbPOMu7aVoB)EEul>$ac?(cLlhR z5CTUS<^B=A_{X>^)`Uren^H;>7==K|MUui9 zEOycR1|~+QPXo!du1U6GxG97x3=8MNSCZA0;K7+)2nb!GQ)M#PMF$~)aKm%+77}S_ zhP+BO?2gP`uKIB6L+ge84DgD8x5wZ(;*gg!I^Q7KDf~=5Ns6=ddsW+{H{P5XL|<$h z-%6L2O8xNi+=9JZrK(ut+L1B<-mLP)5^el8+5c7e#~)i_rv(}DnbhT;c#g-jai=tK zU_{v~5vC;V#353y3NZ?Ox`+6^1{SfEMQIx#%EE7%BJWvqAZCU!0xqNbXJ z$7EH)2lT~Qqb56&{}p;;j_ZFgc-f_iAryxV~wz^H|H;aYohhG?Go~#hp}=N zkZ^DNF^O;s*y*x`3zfw4!{Vjkch4&7UO~5|H2c-0MJALict2!j*zBd#$SpvVa5ROf z<0i{|nuaAe7x?VI4Y~Rk#7%;9p4z}mGYf(YOW(wq(K=caJ_NX!EG9ZtA| zfA`wn?O&;xISH1etNNt3+nqiq#@~OlsF1>^YV2tacUb90=AmG=4>o*>^ho7Z3|Wj) zAZ-Y8mQk$w2s5{e00=jszI@_z^lhh{ZE!vp!qG` z*|R;^(}aBeV!hSjz>h%g%IL<$;`=VgJMTO2t4_fE1IYUyWpw}QxA}2jVc$^r5!gLY zi2#oM2;fz3zie!XpH6_DaR4(}fP{|E{mwr_R{eT^fvXz%D?s~)^W=9o{Ez_;ZZ{J~ z^tq4sy|cIHPj{o`n;JGNMstBe8G-I~#(GJ6ynD0P?RY{re_l>ZZ~u3h;!zz}8J;cl zXXJvr!}iplr{!ui!(RXADB)61SWjf7wKSZH$GoW%b= zkNIAA1MnUVZE_!QneVQEBnte%eHqJJ+!?2i3%XmFppRD70AFfvvnUSiD1ww#hdLPS zW3R1v`?=dYJ7QCOx%l)Ed zP-bpoU}qwnl(KnUxFZzs@CBNCQRJ>dehfcMrK%C9l&S_iKJM=kW%vF*`KR=~DJ;mN zGd1OL^n38fBhw!C#^T@Y&@UJ>Yk2U&5z7~2MoBz2-p6+7Lst*3tzto|Q42Zx@E$t1 z(L|2W{U!@;)8r$z>BQpXVBb@d*R!!;G*HC9wfw&pYmRH_Adp~+HUpnw+7~~oyI%?oKkePNPe9w+dVyNtf0Vif2-Nkz*#ZjuA4YBc>7ND6Rd0Vz0iI|3 zM12C6u?imC+4tryv_CC@!0i9E#wBM0>Ui!mK=o=y;O3p{GZk>L{cU;~$SL~MJ`03^ z>`nnzK9>M}v*(h3Ub}y`DuI0e)qZ_fU29W0zIO=7+oddi^oCLnK`~W}ly?S>f6vX?INC?Jf@9V1Yc5SRDa+M3>4Dg!o zNCXmzdzM|PVagy%6 zgrPmF*Xz)p1cL-?X-Wmqy~idq5WTok z+sXhFDHSgW6^C;lq-oeu9>h;(zw3Dg$JX|xFRLi0Js!2KuoX#B)<+5Km$V0i2vq9M zc3a*YLznYMZaUp5TFMK#`O*UB#cFNBb@P*eqOlTutX4zl%m9nfZSf2=RCnUMi|8^D#9-#P zje2%FomHB~GbNHU68_m?eE5?HWbfZIsKzmDp&!3gTN~VFov5=rPx-G4Wkfl9ncrSw zvoa+cU6%o2E;^G_3c|GuU0hPcB&cOtH$OcU779kWQx_@h4y77-_{?jjhUWCwR$|0O z*L3Y04&LWk4n_Nl8~c27=Y^#NC9RFXzE{zW8YQ$$=^~}uD9;N$y&VWSh^~sblv4Ra zc)$tJ2EO}=6aq8g9rJRs7O(N+YScPxtl`v1mtOEA-VsB@%V-uP&8!X_{hTMQ7sM)# zDe<>y^LJ@qiygvH&JAIO=?yhVgc|E7`YE64RlY}egliLTBq>*@8k&40cHf;8<|ymI zkBLaAhXYkgy72$u+k{FUpKi30u&I{9X&+RmL!wsMPY3YHx4-mIjo!49k}ge#=S|X5 z_^Wg{Digrr@KJE6Exbdcax#yAChF;d?03+Xv>)oeh16HrEZ&)|$y1Q-PVOvT;}j`mex#oBBHECbIe`zuMEBttMdzdoRj`I~c_qUzE)c2glS)Sk7@oep?J&4JY*@-Zh`*b8Pg}* zoKldUS%54`%A)N$tELd>^vI)OT!{Vs_zB5QNSS%8Fg$3<%-1(o9q_cMJyMN3t8({f z9%IOH|50mfD5F*;r{I=A9#Lj0H7NqgYad}`rzXdtA=YwI%{t(Xry*Ou08Woqcb*DJ z)AI)*p6#3pyuSmjJ|})2ZTS^W1^O@Na!&=~iJ)(Mf0AeXe}Qb5Kjz!tXC49mJs@e_kNX_Zty||WPzNvfKUJsWa|aPtQ*OSxPGtO>D&Tsr=Lhc-iw01Ddu`j_+aSKT-u@8uN58sy`$pv_-u~Tl z0qDP)&+`nFNB~E^10MXR1>OJChd_?^j1QoAb#IrMebJ+Tb??s+5cq$o1s8CYr+WK; z(87NK*gtXVeV$y};^*1ftzil z#fBKNq>cS<5Q-=#{&NsfHUxhUhz|!0xIO_e$NCfs>THI}YWa*RqY}%)Gxp#LgeCHwZoy@oDFOwhuv%-~km%}v zM2Vk1i^0rHR_-p8OPRt41dz$H%f$D6$?X5z0?e@Qh z=i)HP0Hw9cZal5wwHO8>hvKdRJ6T}Qq5+;*5o2y6;%adz-kSM577IF~p-i1e$^MAc zMG;^0o|CE}IN^Jp=j&g#A%f_5$7B%OtR=!@*cUa^0FRkD6mBoa_wQ0?V>iz714Zz; zu5bAJ!+mH2gFUblWbOp33yL(@mTEEVnFFnAhd?{wFLBsXG!Px<&<9aXdGVXh07PJ@ zTqDd|fcm%x(ZQ`BKHcGZjSGw^w{d!F;T{+K_dvrAdN2B$U^L)xn6sbO)c(D(IJa}8 zvMxv7%oD6!@Oy>Lvfd2*)VpEvKFqam5nmV%Q6wUe;DpcQ8Lgz;YrjEX&d;DYPv)jI zsWoYec^*v59$Ae<1^(myC&hbL;`3i*hDOr#;GjqGa}s{aiwpUfUGgNU@)T%t=(o=D zn)mmdxQQ;rQaN4!LiZb9?6~=R5=HEz!O6kv&nG5%1hzN*OBYzGosCe&1JmsK=l=*A zYlR+)gb|;yo@#e>!H8>;Cw54+fNp1G_-DPfd3>5NM%C>p;UaALWi6>XQM@s!B{6^o zsnAB3Wuc<0qA@~s3Pgp;g8GKg|26ol*(~WdQe`OJ8GO2g+RKK!PN zBR|HhA_xv~3H+5O;J{Qz{qN4u#ctdmf(+d`Jy($_T%^^nBsBOiRl4#ra1X+<2bx=) zbrzC4YaUe~PpOdzlR2B_R}7=5P$@3|>*o!+sWt118uoAB#z-SF4l#!^A1b1uf)?*s z)VFVghG~?lPG2eXyamQrwb5JC|LB))BEHKkU!=IxfzQ!J&T{6jt%Iv+!q8~*lW}Bv zc!s>|HFt2a5rs%x7@XQP(#|N6Vjcv!+eKCc#XGn%c2}elNz?(+F;xN6gfmG{$tA{_ z$SSeito^R=^QkpFNNppVN6@{(lmq=qb`M^FQ@$Ky8e`MNu-G|$InQ+HA3Vwb#9oW{ zfE3=#1{=AF++@)2z(|C&;x!SZF&9uOA+8v(0?~1PF-B+ySrW_Q<0X2#4EixnGw-v^ zrXDOq7lOw4k5t0)GqGY+tF?pRv|T}0aH5q0WYAF9q-Guza1QKqR5|>IsPL}n6fU$V zb-Ag$_&8qT&f_Hqwdj$`L^%1lh`M&QOB9->J?Zl0bNY6jsoYl6jnsgym${WV|Ihr+ z7yXkcdtV?HUBAmu^NR`an?0Z6^Y-?`x%~Toa2Wim-|44)wOiq3*ZzU(&A&%e*M9xK zw|Mm9;^HJ3IA#xIG5--bxbR>7(zkZq4`%{y;%zVgSApYw>GIiqQc6Ed29S|~Bei}i zVvC|WYQENzH7e~I%D{93g8X@bXE$%yT<7N}z2*|WTP*4;LtpT%^a6iNaME9P>J*>B z*E!K=)TbSzrah#ZWb%AhuhO$yeM9nycAh<33Uf$_ORh!S?!V75LYc??lp4JsONO{W z{59S6I?=1HKXi`mJ{Ji?@pmqH1^fNyZ0~I-JNd9oaTMo`olgL2;kt~iPKX6CYer(q z3EK5Z3K=|ho)Hs-AK%*Y31x)e2N5g4#wGno2R|e5s&8V?CHx($OY6%7u$2 z1cbs=@8ru~#xAa8y3HJfF$^<$xU9q*?GhU&hsLrsMe|AU>&H(`lM359P8h&)HmZ%X z*rvDGr>_t*>yYdcOOn7Eg`1fpYiTa*)1_E}p^Wvp)9MfC;n68^{vKdn(y z69!S|Z0yA124#$GQ#;jv{`14UR~3!@AP%}x^4}j5v6x>qi{@yP2}e0Ge+1J^65#vt zjJb}YaQ>+!l9(aEc&2}#2Kfqgy>l@bU(#yysKol*LG+rA%;y*M5&p#5)mg$V9yz`&$2?)dRkSI$K;l3h)g$rd<&ddQ96@Q@M1=*5c?C~nQ zQIn(aPy^5xj;D4UdZm+>+901a7U_%YFh_GYTW1!^>fKLqYMxNKt@nBN@+QBSW^qp* z#cNTau>z7CqBytgMq6ywNAUH~yszTNZbaKc!3s3`wW~O|f}N4!5I{Q7nY~0!J0O3(ow%wv zt}G71)&C14a3}`d&7b%l6?r&H8aw0Lk3ekMob4}-dW&>uDg_QJUn5>b96XqRxmUN| zY}d`Zv@}`}tj|8$NppOOC-&Pv#-Q-GT5}~>Kp3uDO{Q~E@vnZfYJtd3vskg)Y zpG@DL4F6o{cLd9L#q=5AHKk@`-#&ajzly(UVQD9e9FKr?NAz}90Bot|nLq5#^@l8`iqvSXPGw=_4M$qVX7q8ibpdZ>EfC@u+b z78OUtfXj?(+GrhUE>aYZsUV&_T=_#oA!$@i7N61Uoq?B*{9Xk6GsyGs5U7B>QJR&q znOMb9shvE@p7j@Ne(bY51e~^@-1S1}J4PxuakOW_xe-ckf7dEh{ zz)^E}*cFXd?~g757iaqfcg}*3bIBUD+JM6B5~nLaOSWC>O3&u;oKsGlcJ&V3@7U$) z(EncX49ch8ID}!;gdaM#tqT08Nc0HN7$@lap_NbRhwW&Wi{zLRi2dlQr$w)uAhQ`^ z(0(e_roeir=iax#*(Qt7a|txw&Z5Z3@UsUOMR(fyBvDLQ5bg7|Ia4J=d3A^6kNLHH zZB@cD`q#QZcyN@f6ulg^=9os93gl@4zX>of``d$bIMV{3SJwjitA{{hP;6*e2V-DR zSEQ_)4mP&u$GLofK3}xoITM=VFEoq-nTqUxR(qR^4vbEV=oKY0+&-ay|NTO|mJg$0 zBK_U?^UdCYH25X{naTrT*Y)w4l&CqTb2IwbVK;-Q$8ahWk~~8xA$a zf_vzI%u=#)8p|JD`-`eb38@Q1+kf{F&`wNw z@~zBeqwJT>112K=QOurDwM^8;qC$H7%odcdV%J0TCJH~ugZt*J>v=!4l9mrp=MQEB zNM#Q?_G87sCuZ3hQ)q8XZ(^!R5U^hN$~rJlgI!^92TTc4TZaQ3xvObM(K{o`W-0%P za^L;h3sODA`rA68+;zW<#t}VEaa#NDh?Obp)I=luVE32xG|`ow3=n(?3Z@ZtovaXB zjlm(@P;`f*3A*RIIcj5YsIm7RbP(lB8Op7f!G6{?1*QkjbQ!$Q;V0W$RzT8oK(zIGQ^avX9H+2_Vf0s zP3p^RwE@>0D)XAsQU81~@`H4x!YkN9U{eJkR}C1m=|Xm{F}{lcg<k^YH^?P2g_WZoT>v+Y`1~+;1*GDb3i+ge+d)~))e7J?D zcVww`jprS)U>a1t2rb3%j0ixh;4|-Kp zRf~_Jpo>tsi9xAQujzJC(R^t&*=|hC%>X2HMMx&d-XLL&Ft$;NCV2%{d0W1YRIdr{ z&=Ic&`HA9mFRic>b!qN&MRC2Gk0c9@>KBu;&8ntG16shyozRw;;b!yEP#I^b+iX-= zu|dKs1_uoWcFJGr%6y2rqV!JhGQ*J34Yw>+1Kp)h=>W3u@;YmY%4e80pMT4^(Sr z-!T=ZB&?F};!Rqv4`tQkt16)rG9ANtY9}zF*x~bd#*e+>IHn3Md2-*X0guG%xS(aY zk=)qN47~daW|;CaTZIRBWj893%g%W-em@5iYGZk~1*=jih2KmFgTL?Rh-h(2(I3<9 zspK`=SXYI3*A`(jnYXN$1v6flH*ueMAj8fYkBpslE@Se9A-}2ecFtZ9xUr{W*fZzq zq+ck9p7xV8QOfF7zpR&C+tlBSj(GmA@quucTKIm56p`}3S+R_;Cc4`$VK94kgD)M@ zhBSozwM(r^kbZ|v>`|LnIktU#E&`Q*bu_c5u8v@M&kFxkS0FyJfmQNg4~>@UN44RyZmWyk=2d%z95C-| zv#R!b-2KiJYb5k?8K6nuibReg;u6j|SV!M0?|iJMbe#WQ);fm=*WK&T(-H@lLB zPFUZlNz~)eoQ=Y$;#O`vV{{nYfWJ!$;EWGA2LV0ma3~{o$0;gXL(6xrrp(g2BaHuX zJ*WAOZdb!^_p?SwWWWh(xq?Cf6&coLWZ1cgSH@`?eCQZ5$?}ZFRA40>z@*@zHC4M| z6kO%yV@M$OT;n`|-G*W@|Kr#Ary5h0mwCNPu^}Z)60N}y+_<=GlhXQM*MoHoT3k>d zn>+=M2Tnji9a%25OKhZGpnQp*WtLKCSQ@xRsx3(qT~C*+dt3^rb>sH+DxhL^dw4Ys z^U3gGJio4`<*UKlXD6Dox5K0ULbh8+LXTt0~DS^x=Z6*|H4J74Z<5xVbYuMN^l>PEZCkc6wv;Q zT<(09$^fn4KzZK5a+@TkL;*5FTb1Cfr+32Jn4*gozyH*o&^r^p$d_c>+wKJehc1C6 zc(2Qxj8=1W=$hz8sra3k>^Ivhb}mAf?H{E#J$wMMp&MlKm9CQlvT`K7*Mnt`8wOfA zg+9dz_@s~VY`C>D#w4yB+&R7=cG{F6p}gU~E^7ID^WT~yJC(_Jv7PX?Iq;OnEZW(T z{yyzT?V~eou=Aa_fZ+^v5V?0;bmR2dbvh>$2P=eLwS1}&R*&*$9ByoMX_Arf623EE z#)Kft^Sc2}F<)+(%8NdwHiHVfPTf@90Y%E>1xhMilZ;iZth|ivzo{f~pgBspjNnLX zMp(9*0Y?$Uq@QW~u0*Oa(CNZ6!OZmY_i<}U;*Az)Mt>)btrdzQQ)BD* z*AhnlsxvNh{EFIgL73tx{S4EcxksGx&~PuW*>YAn`<~?3Vp_L~$5QKD?lGRQnGKb| z>#}Qwn=WzGodq+Qgj+sL8Q?^Gj~BPhqS%2K{ATH7?WLm;JGG){vMSGH5_Vq z;I=-XWY`GADh-$*@3=N+xTyg|4p=-;y8kFCbB?H>Y&g$1q7dBWttja&?$eF;o^E)1 zjv8Sy#uu9e`0=!JNd-#l9O?0!a~OM6=3bX*6uU$R7| zUa##+17)B2XW{FjnG55w)j{ZZFsPEjN$rAGYWtpyk~ZL-R;f=%l8>@GMoPs6i=J6v zP?cK7+%n~?vc_RlX5CN0dV*FEx4-2olR=wr?HMn=r=@%v46^IrY|mJ@s2SvJM`6T2 zffyGVc6RE9lz?}IciGsj}x31%t)y% zYj|_D#z-~Hosz|Y@1pC0*+=nsWxa7_XYpH>FitqhW;sQt#V#bR)Lm9?lK%DS+N?ae$(;Li%=Kc&BAh3ZqBSQ|!}b|GGbLza&!k+HyPf6(eW9-wBl$ zDo%J~afYdKue?P6AiP~I!vyk9-Bzy?H2WTfZA9R~eZS?f3zkV$(Jb9P&QI^RPK5uH4qo|` z?X*z8tD6o3+x%X#9WIS7#4)d<`N@ANMnee9wFIqvXU+ClZ{w`R=o<%B{aU8uEHhm9 z^ER;&-E{8W>aL9x9YIY`@(dr$}ZL;3%l8%=y zn7-fgV{yEu>NmJg)7PZMpL)krYe3O<`=<0-;ozqkBF-IqD&(f^%N83TBGX21u>yQtG>K?<) z^k6M&$-?eeW6KNxUhfRI{T=i)WZTJNr8*>Gao+G2*Fc*T8ZQ)6^`x<9&9aedTSc^6 zUThhJl_5R(rJoB^z*PRgwy9B@BQ|n@HEkqFEc}*h6WbxWhP}y{W`U zx8#V&O8;$DkMZfAc3(ZfCpolXis!t5K+(Ubm zg{8>KFN!{SZzldEvM;(7ICug-ENdr2MikxXG>%8xAKG=+({W?XS?Nu$-GPG$kS>^;wZwN20oqbB1@`qGkiI3bK!@U_+@eKSz@&65Rb$Rr! z_t4+;vA?;o?*$!rspAv%zPsn|X6k$Qf&IZzE0ppwhm;PMUxxCCh_&^5w zYXkd7PQ5V#y6eP0jhHJ}AI5V0b7s5W)P8PCd2q!WV0*A{Es#GO$~zfK{=5TP zCZ1k7V7&kykN=x;o(Gb=?T&CGmy;^{1NWeL!%_1Iq`uX#(wO~~b^LQ5&K97fgNl=x zfG6{PRt|B)IkK6CPzYG7+g2w6;nWFvN7-CHY43mC|1Xkp`Zlz)GjM+ohH&vsg>E8P zM4#3niwhw5g+}0Fg4EcMH%QQgT6(-D6Bj_rw1njTYSFgDd!T<*gSfRN;`q zr10l`}L!l3MRr3Y=ys*p!vq4`YLpEK!JE1rjGx_cN$0y#%yXAP+Yr`W!aPyD&Sn@$$y(7)oL4C_}lo;do5qg7kTC zGNSd)9tZ*7u>ui}i{?lM+$?UAADn$Hm&;>9Ynz(A{UCq3WQ}+sp3fodMGH^%=0o}i zjk;G+aYWgltQB;ZXn6J$VqxD3uhp6=q2yotxol4S^`gKw^-v@|y-BL`G63gUKO}@v z{`GWYKDX~)efr`r1wBR#F?6K!{8JvxX6aIBLTDspSFG{9v?r?WV2?_!kVp{ z*4NKix%9FpGgx>G>O9ri*4wr?i^tHDGpCZc-X{rNd^tx_!VA0PcCxrv1^UGcp>sb75I3$Yw2AfP`sg-M$b zF|)r8wZIkTeW$6LYg0Y{+|}`UK|Nz)mDkOT)6!G5g4Z}>C|6WK=}WDSc2sfZ=V(gK zjs}Zw6TVV#q`B;kycJ*~FuUkNq0z)CsvNbxNbPb(GM*V`!otY##ybep1h1lt%lz@Z zHx}hxFvXDDKM#={gs3DMV!@DXWF%D$5ze7U=LtA#5p3Uo5|;(T0I&KX4=MdGmxSSZ zOGbUU8u!l@qH{*bD-ba^hkzSUX0?i_@+3C^#u-Q*R8Ijg%VE6 zan!?}#nL{_oaz~wd`mwM!1PWrI>$9$9J!f4t&t^CrOe1Tmq`rYD5kpIgRQM3zQj9~ zeEy3O=xUV$JYbbyb(eEhc^~m>;k(2s@HrmUL(?(xb?<-F&)zDa4?a;Hvz06S@xQ*( zSXza>IG_eIbyBM+f>hj#OO#@P$nS)|Qz~`#$;-VQX&yKLXSP^}#Mti{SCq77z@{j# zaP&d4cLqU4R25aEEfS%DxxE{`hpI$#|4{FiV#~Iw@rWT)VY_eg(BxgEr)7Z9MEX;M zY^4b=sEBbgb(--y#l^|*`R(?4BzRd}B*a$tE&q4;SqVT5LW5>Vztb{J#SJ_pJm6VQ z6g!AyUNJ<5gya_2AbfeO(4ddztiSn-Ryg*_Ul9+8J>bv1e_F)}OSLW^E42WjCQ6;c z-h-=1UXl>^7A1CYxPCS4I(3dQOw$bbl%=~)#V({xt2BOKcV-v3u`))=o*{>Pa4dW* z3-IBj5_GZzJ2F(=^V<%_jCc(aKbd6*(-%U}ZAdG{A77QipEiJRV2j6Is?9eL!+V#W z^zE}XQ$BjE-6@qVm}ZcE+qas?NQ8x`hYIj28q6-vFrC8UmZrm|q+N?IJ4#>AYM(Q| zHb9QnnyY9@`XW<$n*c*W70*dg8pXqECW1Xf#RCr%1A4H=j-^Z|pb06j@zcjy3vm5X z`Nx2DdnL)p9OZ>=ILZ===nhp38)ZJniW`yp+g-KWiv??+>BQ)Wv+62QrgDXBBV(Ld zsbkS=)Z+!6xba#b67wqa;e`xdeZF_vSja_JsT zy5OXyY_eG(k27qO7({azBHMaR=MjgET5u*?iv&=*52DW7@^h_N-HClTAOIf1^-hCqygOn9B(6lVJr%pwv zS?3|;!*2?un<`KHXfOC!EK5^a`fA$+rF_E>5fus?#-C|3=h&+fIk$e)rye&R3hY(Z zNYUrp20c)kUx8U3sY=P!4IaJkM2yu8P?5$8TADwU;xu{@Td0SN2A#f8CHlekZ&*LH zt&XIdzBjH=G&@)Qt`AcKqx&M+a$Uo=Q*Y6<-2wj}0Fgj$zX}3^*LDj=)mpPLb_6}e zQfI}jo58#d#niU6RKd@pIiS$>S;iSdMNfkS8Oop73jIl`4yS;xuBXvxbL?STs_9P4k#dfkSmR+&wZ2=sip zF%uZOM}7^bit}0fU@GAKp5>i=nam|C zU>wjOTb=gK)akl$_HnJt6G@DO4;j%}7L{22$Z_5@$|jMi^GSe8M;x5$BKQ*u-4IXO zuFp*U4-r`EakcFn3-t)9;an{$h+2bV-^IX1SkJa*Mv_ z#nJg*_}&mZsb)txvdD=7kE%F$_FFSCyFy zeaO*Wm8YmK@6~Bock|uP5U+G-x8F zXt$i9Pv|YFab7VP8b3OfvsxC0XQQe}sLHyxC04@VUqLiY93@3TU}eu(Y<|@w#|_%~ z2rrkw-`MO0b^sV^85K4z*VvX1RfX!I#z0|mrYxy-2Kw*D01JRr5KLAaMOW0x<*X8D zZdkf0E`lLnmCjzB#}$Bz$MAEcQf2Gt8yZl6SLy&Pf1ucHS4^=T3@ENjFZCqaR33U5 z76rYUmufa@HD#~3>QcEX$hWtYPKLrE{xOlPhu76*xI6*E>4M_hxEx(c#a153J6xk1 zyQ;IXp9J2NatvWN0W*o#oCz65V_e`_V?gpOXWce=)U03PU)kdo9ps!U%-&XpX+?`r zq_GGVwaV&w0ZY_53NPTvf^{kMuhRvyP3;Lt?!wetUM@|!FXBzvti*OX@6mI5Wflen ziTgC*y^UEyT9_KTZRiTK{3#8uX)0M{Z8A02APeGT-A7SXqyu(E(%D6quIepHNmZN8 z->?v9<&OvA+gj$eX@>tHO>x78fmobA2AP0a_v{(YM#`c_%Ve*Y?dTeEg`tIkH_NCi z>s|##&3IPSj5VzsHPUeAs8n9Yrg_$)CuAsmo|URpv1RPBd1|(y;k)Ar!G*u0Af0-C zg`%99ROai;fnf`U9$|+9@vj2AG~=Xq_+W)ucBV}VZO91oQ!cK_wE?qed9YS^G;yV* zl>QatfeuAtGr#Woeh*pfnC6?_l-YZy^ zLL7a`5NAwxDYsFYrDXdqw#4$f3CE^mM%e#dHY%7ymr1zSer~euy&2|GG0f!x)*uGD zBnCPw6?4NOr4U=b#Ie7WY~rP*f_*x)Ni3>>-ZCg!gE0F|`-s8iDGo(&))bU+OZ_Ir zRpU)n3`H((!TM;+L7q&?g{UsBP?woFu_(lChkt{lp*PG+4VEr=qC}7}Ha<)o2z&3j4pIf72w9q{X&;;ur{>DCGT?RgZ91clO~ z*SaiUWU`o5Ss5aJX;^k2*PLMAsf~|#m8&uZEZjs}-!Tz*{aPZDW{a5mS+$Pk2mt|3 z$xB%6q~{A!GRb}gH{krwW$0H)M>$&N(6Rk}o>}T#VyW3HOWfdd5#179C?KV6**g6| zcA*LA^Qw0~WHnCq12-7@%+E~aB&9LxYo(ONbM*U(V%T~FbEOl#F(kf}R7K6z*TkY< zkbVZs-*SAx%B?2JVZR`U4QNeTHb~wF+j9{J(9(5T{azup9Q$UW+oFukDUI^g2ZH*Q zl)l%vE)cAl)>=09V!)RX8kZ;$8p*kW1xvqgBPX4Cdhl7=y7N{5ya~4mvS{?V@7&g9 zx@l={ifdE6Uq3*+-`mt2gwoioPE1M@c6^7Cxx&N@`|J0iFE5XflZwjaho?oH{0KMH z*PI4qIE8&8e04{Dg3c?x3tRK#3JNJ`Zab0c#uHndTm*?w(KyA0OBo+2qAecQ zQU-}W8e#Be0|A?H!&Qtpy61v4W*I*=78A8F(CF}zQkc30gVVyrtgRL^bq@9=d)?7o zLkzmSc1h*8lu2cTLX)=va&N0FCZjXlIauDTXd5OH=Do4b0fPFC7~m1zr-HquU@N$X zIbbf0h&UZ-5NLm?L7;+2$|ArE3HPGX8ZhS0EHVPzdBkGQ!MSVZBW^H(%|B(E%Tak- zAbLfS2HMWDe|p_ww%#)2U#b2tP#=yE~77Y759t^+%t+a_GyX> zcs=6TlGo4rqn=3qrU$kJecZwt}egBeEUGWL>u zvSs{Q&g~28BO7KeG+GD1^V+_x5S5zp?a(Q`MT6Ajfq)5&)4C~%+~o7#hAH=uTu7}( zK?2UIB_LLzJ=HzsrR1GVVvgNwC*RE9&za?|KgO&^*)B;shH;av=A1j2#f#$FeRV6_ zy^uX)!Cm!?)fEk4D@xXOKb8D- zpG%eoJmp}F7%aFh5rq9H#sg!a_(<~x|-D^_N&0lO5lVPlIVu}@&3&J=`ENDCh1}PtF0fT7H!1ranvdiXN76f`- zTBP)*k#qe}+i!(^sHER!7Gk}+&PZu6%C1>tHc8SBD$fjX6F(u14>4Y>oSFejl&MHJ z6n(Xd=uuc))C#d5)jPEybYv${nKa8Gj2UTKTxNy0!`IrQe=Y6dCQ3JvBp`|yI?gOR zNI(R03y{J5h0j(699esYwP(61Y5p<7H2lxZRA?>q`xPfi84qb}cdfDA>=BtNiTmG} zsVK?|4VGjtopOjyB@+-lX2u*8X|5Gea+Z|v<^j3Yl-lRIFbutFy&#*?CGo~A!`X4D z(mef}hq78BL4K%iIOCM6YUvbtq&+V?%@UZAw&F>za~x<-(rmd6b#sEhfV zdjrY08dVD9mjF!IUB5KMUjj=zR=jR6-eC(IU~WySw+E1mgKz3f6yT2KZ!*8 zTROwh-e0gznMsx%FLmIf}TGF-6sFl@~Rb?P@if!|rGXz#gwc#kr{sa%c zJFOF@*`eRbNVLnJ-xG&!}l>>WFL$h^OJIRJB&>Ru|mdmeMD&HBg< zJ$G7f3QROh;R$3uXM71|-EdfMvK^~AFR%?o%H(5rsGWs})tY-IFsqpwVtpI#hZ-TW zw1;`HA*=@@0>!i=XUu2yU5{YO!0M-V?;g2A2W_tq2je66jdoI92sk8 zW(cKpRHk#@rH_6f2;Z=-A|>cmPZ-Il(OJ#b_!;Juzs(-3;nX*L)(i;pkcz64VZ^vN zcYin+{%BIKx}DB(u`p=33Q9XoOQ#SoUy3@b0?xS~_AkUKW}z4F&gjcbeqjN?-=zTH zA299r&-B!NI$VJzXdd$<{s+tPe`FHE{|Ck|bJcPPIcu_+4{C6(*zPPI(ulLV{wTHO_m-ix!lPC70DPAT{ zZ?^MPeG;oisfm>pu{+8_0lJqy#T`sA5*=n7-xeS>=w>V3LUczXN=QC-gV4M`9ocFf zAU3abG>WPmhxsnXl4xTmamp_eOyhuZA<9r71DP=crdTOhFiQv(e1v2KNRqiyBD{u~ zDW`YHhd{gl6+_g~(M_T5OshM4SY6Iri`CWA{ea3m!Krq(*gv4Umr_+y2xx`6M;(3Q zQ|*EhabGy$lxmZ@ao;*DNOD?d+Sq*opzz77lPG#X&y~2!A>t3M$uK zmJXmW)#XA_N9WA^DJ>iIMe=AGUWUW%G!a4gbD#StAk z^VK9=Iscg8W@<{oM?fV+AW?{Z&6EqBfy=F>z#_3-^TX*_9H$u8|1l> z9Ky6|ii2xP2@Jnq;W<`Q?5P-$O>q*X7<+~QQ1D{(%jJ9k$FPu0N;y#EQ%fFJK2zCz zoAQiO>f@MeBTX=v+_@!7G@2r^_~~aogtkJ4a<5y4e+G;?%XU&*^xZao)a^3GtDtBE zn7ka7iX=;e5RBz$Wesr__HBHt!ve)i<#WCa`k=*36Q4cOFoOb;1`{e9gNZ5(jK>Ok z;mCaT8AQI|>v8!;!MUwFd2i(&kY-or#cJZ*+9FG0rDBL81dN`sqJ(08Im8$bbJMNh(S|$& zzEi!zEsKunf=M))D;PZtLrsCzF43x@Zbds z-gWrFDDMO&gX=Vf%!o#XO&GaV4#gg9FxXAUdM2-lJkzUE=u<5B`rS!W$w~f_$E0o) zuv8c>zyU^|=&Fk9e&*VnCc`Sg_a*a5cj;o0+mci$&Yp@J$oHQ^np7wtl8uf&$8*2I zXXH4@TxITcRET2?;a-->xpyJj+*45<&ILX5qY04`3oVsHz2Hap&cQ)Ef=#HOq&u`5 zg)d2UQRSdS78;JdO<&Omrg#bHImL$tL~<#Wd4844A_OXH_|2Q-$t^=g;hEt>AT&p) zaw&4x!N6R9_bcKnR|Xv}fRXKFZOUXv+o}(FGx9}+^gOqXV6Ax(y6FTfE)FlSDf^$Y z+ZJq=*+w_<{kWN@PRO%j6v7YVM~$aFnI;$Hg-z4_=-}T!92}oG?>@YFQz03%1{(!2 z@tOj4le?j88=vmK|M0_`gQi4mqrF;`3?NN1L_eCCRQ-8gjJMh z`m&U_TO7xn;qZh&iAO&B`T-O`$ht@KBS|gh5kb^pc77CmgVx z(vzaTWYQRsuY`a#zl^|o!jxiP=a7f=Iz=BjlSqaw6)s+5Y)|a;9JxpR!@)}^LM^)q z%HB=^ukIQ^k;z!f9(>hg&%c>-|Icm!o2L;@Pb-_v^6;mb3pJHyj!q6wc27PW9~?Wc ze|-1;=%9FiU0*!TZR4I*Ca{f_eNJjksY%eTFiGhYP|v7|pSi+a-64|8H+2-@>176? z{%bBM+7X=+0%tf18Z0%8B{jv+_DdHVazc^1t7{^;uB3d>tD*=nWq4n#G%UJ9L9LjZ z51ozGiH%IBkSXmfgCDTrNJj4Y{g~72qzo^zh+`M4ANb)b%0(4KI)RrbQk7OkYis6R zY!#PC{FIdTG{R}3sUek%rKXLj_a)Hy(d#VM?U^}?Zspn}QlHAWN(;II9IxMysO-+U zZCqxk?n_0z#a+^*QfcR4a<#l0RN!lErjZrIF(Z*N#WF|!d(830Oh*itM;Oz1uA&Y6 zRI!{)1|n-z^WD6SqN=3$O8W7upV>)Gd3cIra{S@#U4xVHn(7?4rfYUpj5ZJVY#=m7 zL$0d6s8a4h-noZ%M&AQS1}U>o(OUOaD+;p_pJlL{hwqNj&@5LnwreWhmN7`tN>-5B znYCC0mK(GAW=hG1RcT_XGC^xVrKPUkC>byd$duJ`4T|)2K_k0)B7-!*gW!^#9zU22 z4i8pgbI|vfp^6uvsp=@J%*H$;yI-Rw(&Gw5f zrYl+9*w|ceuW!ESbXqSuo2#wW&CS+Ew_7bWms@u0#cF$Fy|wXT9ig_@);rzxjTfD@ zg}}soByP(qM`k$f(E3k#JlHPlrE8)`FE9{D2Oo6l!5Tl9535c7QLVKXt(?XCo0O>%*K>)24N3l-Kkx|=UH*ETlW z-Loo;&reNHzwHaEL4@SRnEd7Y7iMmFKwn*CyP9cYe4fQh=B z?M{3B#VXLtU*4dS-TB>McJ?rPEsp%E{13mkE4V1zz;nVrfqSejZrU%aW;T9UkidU=+yY=yC2edufruv))E+PXQsIXK)_NP>B?QGMpF8ag?WQ#z+w zyou01MYCiXyJ|sPH6{y*!V$I?(B$ExY2s;)sX7-b;a#;8D0lA5yE1z_Yi8y zWXZi)s(||W1ewj zxWf>1M*?gyLJNGRHw4fNGqGL z4Mj8ME4C(ed{AkB)^HP)21Vk>KO}O>ZWK<(J_fX5r);IDWBvldJJafsp4=1i#-wMJ z(M?+HsX3Sz*hEEr%$jc&$4NQX8neIesv_AVbyHXEI=9*qdmjVC^EpF3Hpp z&t)GWnq&4{r(~v(!7_PpZ>U#WL>l<$smzDxF_G z%h7I~Wdn~fq#0CG=wwBaWf`<++-A{A&`HM-R)%O&KMj9&^cXc{mI{(rmRnT@gTZt( zlIN5>fB#gE$PxnOA2gY*&@lMc1M_t$7TRpS4Q& zms}cBXCir*2IGHNNH7`hEQpe=t@9vsW8Ik*3fWgNyX_Y<%IArgcPy<74cQ@i0ibAm zENtGtgBBL~h4tMqj8Z{35KFM*6^*nKLS{6DU#3xNMuC`r-GOUxbjteQ>UC{0(7l{K zjI&-#VckdC{;~c?l<@rhZ-4tE1!lpyfb)hOe;2 z?OJtM4UClS@bFc2T^ne*d`f%q=ee_igH=eovNmY~#Xxw_R>?o&AKD|(e>BegE(nok zL1?G2W)AhRXaqDG3}l-565re>x%^d(A-g3OSii(A&~oDYmqV+SKS0`5@Hca|jGUJ7 z_8jbtF5ZKoDYAJ1H#yeaVkQT*F(NTxIyQe3NDjs*BSb2r< zhXz@|B2&cSODCH6p;bJbRxS4NKAJkadi#F=z&YA~|L)CS)iU?$Jsr-&LWpfB;^DjZ ze6C9R_1w0HFhnF>ur>gp()tjvdj4Hc18HSe%?9mOl(&3oAn(Ct?8!{A2E}-aA&Gqv zyH{cW3k>WO*xxd_a3{X7!dVlVtS?w~XEDrtd3 z`3m;-9H;g@{KkY@dmnyyz32S25AVpq$AK?0w521%_ZN$*`1yFvjrZa1wY<#OllK$1 z#9mwcDi0xBTW6FoiGIUQ^OTPv&a#{zTsOCrb5b43Uc&i2 zFoli94|Y_BBzVl(kubdtzLNX5s|&Omps(!?ry0rd7)8+L^71GumVBu|WyN=TRGv}> zHpJ4Bsv~|8;WUA^U%x|DBC)`)U9GEk4ElKm5;b zt-e@qy?C*)`LzH4&3_in|HoPV@0|Z@-PO|i-|9T=|G&w{*#8&ev%>aW50*Oi$5DK9 z&w#41GvABd8$2%54!0A!BJktk2NjqlhD|bW-RaO63c?m7!oz(}41zdGZ9X5L#2A0V zR~Cjr9E}}6oCR?dj(t2!-L5ION{U$eIos`0ptZFYUL~NoX~0@*O&N-=hfZ1SMWcMA z)K*`G{c3mY^=@}{qt$J%t**ZS4s8k<)N9jjHH99Yt7W(3eumsM3lSXUEmku=lYFRR z9?74XLv8jf-@O;(RCU+%oxM!fR7@)ZL|75>O89M|_^hJ%tXRC{R8p>F=6J$ATkd*k zIuUvmP4cFYC<|L#gLGtx*ixCB3$QZxP4WUQf#q0`OX#e*R07LhnlNe&&X>%H-fk23 z{yL?47-dG7d)S$#!~1YG>D1wX)yA(iOI=FAel~>r#bi&*_Qx_O#mZcD87E0 zh)ds}a1cb9^PV%j=9%vGYc*Vs(%7G4iaU%X#Fi@QD^du=h=N5{axXj zAU!kuAdQGeUzV|21N0aPur+WhoKqPN26i(2uCJ^7JD-a=VM(%#&BffJI$a--JJf_) z5#NL$J}~oVx#oTIdKP-|u*{u>$`8uwsMRf`*3&yvOuJX&pzI2|L003^u=&ckD{Q_h zUi2(htiBF9-%sg29<(hKpTI43Pv z;d+QvQC@@6px-?SgI}h;GfCr@_-02;=$gnk(f?Tgw;h-XiSO8xn_r=g2iX6*p!s?g*{z43g1g1>7D-Y553`)UtHAUXWd znWn)=JrAd20N^GfoTx`P?s&w&yLpydo`8${s z$SeS)*WsrP-JL^vvH-p8)D15z3y8UleL)`->>fI^l-?f6w?~Zzki`f$ZqZOm0@UlA zI^_hWEq3%18sJ>KJ~+NnH)v~AVha<&1M>}rc}?CY1I;O90h(!=!0hojfj=$yuiX?k zXx@#7Nt1&z#B88wiZPJT`2^h1$engPe>AUDR^|p>Yp4x-Hn6Z2)j~sq6M`TP!U6hd zc|kw5ZdyGocGK?NwDWS3!Z(2Cy9gr!=M~ix$^UjT?f3m88B9kRIVqqIvwQ_DxeR^@T3=RT3wNirWl0XM=L&B{;@ED z!@?i^co-dAM;|2yeGFbCA&r$GHl{oJ6vD{nVci9i=0k(S4jsc#{NW%NPGuSqBrg1Q zz#KIZ#mJ*r{8 zK|ZK_fnv@7g|CADGIcR#Q&lm#yGHMWH1M-B8R{5G(*flRc|oEdPeX^(cyAFofbBgD zSQ=BO@JZe&#UUA&3^JLKV4AK8+BnH~54Z%|T9$1o12(7fN<=hZdv-vH3;;kLA__PN z*$)QT_k=qD{wWltwx%J)tkPkkjozKP0j)HW3CfH>^Cs)Eu=b&Y4Q37bsDztQH}xWmR{4_CLP?3Ue2rc17c7b9e9y5t)w$hk_U`G$cG%6Cs zV2Ywh`h1)799`&krZ8og<-V4$5T4>Eg4&b`qk_NVW3ss61j+kK(apGzwEAL)G7n&F zwm>IcZ3Ju+Cv9k0EQWMT9ma&}p$bA5X{k9_*oXW^Y6Ista!iyolj6+&0+yg&*D9%3 zL4Oifn*4Z_LC6kp)JRYRG01N`#{#{I`G{9T=(SYKbUF6%qcQ1`J;jvI!%&QA1H zfuZ65q3@$(JOT%y7-r{;xm-pum*|5bya*8~ad-y%i#a`n;rC=(n?;NC-Sn4ASdsjX zK=u+#d_$_gdw-%)K7*>s7*Nrtl-bC?p+zTxskLvpT&hqvw znzUa+^f|bVP%zy@Q;|&1*$e;wWAELz8#j_g!FlFapiyMFk-mUtXFfic_Wu1>~sKSxeeDXlN$D zHBmNwz=|sFJ~YoXMUr6ytl5|%kFs7cbz}lb;wUKIEngf7%#}I@wS;JEyCT(VR)i~P zIH3>1CFG{lP+biqK$6^e5i&~<{!(kOYzG(GQhFmf4>7LZCVbkJitSF>|Ihu37jMfv(Sqw!!W}? zWy`!AGs-D?>HM@X#A1ahWfLEybL=+dC#5}>Tp@?5fOL%~K!njC|2Ed;Wi1i7sJ^q9z-L;8vE^G=ydc;j4BPeGi1LXN=eL=G zAiep{tpq2!aW|nhW+5iTz%C(w_n@DmfMEa&vYpz%3x+qno;JCQu2sTw7D0G}UT&0| zR;*DuK>;~yeYKrdnpW?62n=m>iSb9ND>Aar8867Eu=`07RgEHPO}ENpO=P26c=ixXBb znipa7RrHmnk+;bu96i+*ZQC>7#gq@lTLFBH0k@3Q5O}}O>O25w7907IHc^u#^+yO{ zS2$hp&b{(kioLw#0}o|^RD)pKU~@z~b7YgKN`r6S`c)Nr7hX5v!nlX?S(KgFq*C2Z z<)wIo$5WsVpiNiEdMS%I^Y;RSe-(oqfbeqbf}AvPq+yJ*FQ@G7s(3_}{X|5{ThTfN z>zU}#bWU{d_+nsN1;8#nWH4pIcSN-? zkAD>-#%}_(isVW3(#YDnL}wKp2d75uqM4OBNutcT*5I>TmgLsB<7ff_Rw}H`UOU3)}=FAt!3*u#*9i6I{Wd~Zop5|Ve8kT+sUIUn-5PoQgeabP(CC5 zl*j+#19s>Icj(2%#}`S*)iEcgCUkbAtG7nRbQn+Isgvq5ADxw7q`Us2N}fX%*unw6 zBfJkt*e3tGJnTwo`7~0Zx>2o}oj|gOF~Dflw55mPH$ePnx4~;~=Gx7DqJBU+fwpM| z`Aka8eQl3^0O~qk`HL^oyRqXH_KBkgtn8NzETH!7dw#dNXWXCxR9DtTAOgLr=`hWD zg&OBb1$rJyRAvl?!N1epcWHla+CQX~R#nc?WK3;)_OPUHNc&*l2 ztZYvfN@M=1eEZ-=uQ?`S&6(Uju-${>A3nXgv9-8$Jnw-`zU5jEZ+7AiS2hLT(-V69 zX7AGn=Q&qE*HdR@lcKOG3!CWmC%NyDJD zA3eg&pB5}*P;RGOZw-?7M{d50?As_`va7R+ z7$p}r1xEEPB?W+9go&5r3cKAWPXPgb;0!1ODIT}9Xl9ltKjbC;g*1AyvKD&>mR1fo zp}*p_N-I|@4VU-Lp6vTCjYz5~_@3#^y9-u689As~Ik?G<7~5d^;P|T5`oqhYDe}{* z+*e+82rfErTvAPXA!R$H9@!+fCGc6QN=sjq6rids$eU62q%d(O$$oJb$Zz}9HUgua zC_*=ab9v{hD)u+nn%47`QamXzT+p33k6G$V5yN>CkpmHV?kzD_3$F-P{4K>pTJ{!C zRR%u-t73zH#?~wy`crMZBt#V*PUh`3QufOC6#-5`K?qoklf!b)2m8uS_EqL zxk}o@C0$xfG0uvZj`9aQmoagGdQao9QOc0D8r>c3R6eGKCc;I$*HU1HzIW;#@gNMJ zC6rpUG^rbdst-|Q=gsr5WV{a-pD#S7oT#*pr&gY*6|Y>&OB_poap{UI=u)&=E^s|W;v(Nz+TLQf-W3-{ljC^0Sph{{ z&q8@6xIqpY0~eZLu`9j=HkjQbMj7)fs5hnDzqq&fOA98={1O7z{Sp>SQlH_Fzybz3 z!mAF>(i8qSA!#wDfXG8wNRn0DPa#XuOBSIYD5Rwt{zQT86NI!p_A``>@TN*sc?3;l z`z&|`j3BoV)`|_5fEEOGBMmCgdeUGacYAd4PXhi=DE(o6S2fyzAk~bxR={p6tHd)A zAI9O`uBw>;FB>-orhe{t=HxD#I!+nOJ)MYmZ#yt6&;+qBoFTLpd950DGJfq(3=dei zcV*x`_i}p3m6QZ=?<#Br$tB)JzXLfkHA8WNaVE2>R{k`xQav!j?!l)?XqM?~FAnTP zf~Did*S^RanZE-Di~Uo$28qCjydK_%+3Rwt znHT-0ZG?WOE^aQ10`ilnl8IEXtZw0mZ>QE|Lf-z!u@;_1n(Zv2Dm=rN z9*g;wfr@aBGZtO+Z%P0dR%l*(fjlZw8u3VS;tqN4fI2mpy7m8*0RppayuAhJB|TldO(eRy=t z98|!nYfWc2!?3dm_G`ys`?Ve!|5u(~{t__&A0GdAtJ&J*|9YBd6aV)Q{bb|+{t|rP z6XO4Nw>J5|p5!Tv|J&}ib~^3WCja;E|6%(7C3pWd=zpt~jsLsd=x+A^r+7dsIi}P> zIX?%KlD`)Y-utNt5C$D|a%kH`aE$MS+R*>gJRAA{5B+56KhXppj}JUR{%87cHQ?Wk{QneB zM*iPv?rs^oo!y=7-EMn(Bme*IpDg`9{_cMt{ljM8Ze;iWMst(@<4GQr|L<%(#298A zmc@|>ssf1`kQ{7;iWU&vO(>EL%khV9Nua-#(oVy~bRERZicD z5%OFcGm47CrR0SG$~SJ3o6_>Co4DGzCql-YPOPDGhF67CA4;XtGwtkX|EPR*U2APM zDm`uL;7MX2FHCEeI4NHSLT1Xqvntp3Ts*P|pmE|28b@eor;ejtN6{qg)$3PobUhm& zE39Ld1ziJU zAb#M6ju|=_)C&|!yu8S*1-ljO#4=PAW`dD8Jo4Hbv)4aIa)EFbzu%- z6;g3%GwB3mhC|E48;F>bdPyaspRx^TJoBQyaO(41E9~pBTNHs2sjYSH2ssQ!=F0x8 z01eeKurXH`Ag8dR_l@k@Wna+bWIca4~6vJ~%(QanGi(#68XeTvFLlYAqcV#de zPt0IyM$=$+b-gTt9F@9uFuK+9wG9#Ks--_=lA8vU2%s#x#)v?^7lNatyGWF3ifX+K z2U}|m2trLj(35DPf-x-T9D8xbCgfsEIRiT){xpMu_|6PCfQO$&M6n064Z$BW;P@tK zI+l3c0dY+gSxl0hq?kdb{0rEN5Lr3Yw3iQ%<7|s9CccNq12PK2>J~R`q;@J!wIJTn z1e{|pE|TT>4zCg}Uz}r^q=N4&wcJb*N|;v5)!!XK}Qk{W}&c`!5e zY?@-eNdYk{{89E1^M$V&$=by? zYmu+LSn@Ulhg+IcO9iw%PeI9lRiS0S;y8h-U^`=fXmSoU4C4aIs=+OJ2L2@@?v!;| zUuVguud`R6V6Fc`%ffz=s?RrJoGI&;hv*kFPdnSZ6KgG&NYr7+-3i?;^Wx6STQ1fD zSeNI9QHly&Xc8SQX&lIf=f(H41+}E3I6`GNbSX{C&?(QWxJ8E2d%RLfL!e&5jD64- zcpZDLe@x#J@nr#EM&&s=MA3`vRT5EdSu~mN(dm}2-H+n=hq24_R0BtQv3P;gR)@j9 zn7^oMS3v}HdE*DSfC?W3{EvE}9~O-adRJPfRoP%Xw-$`;rOxMCu$x^haqCvlytih!eVvR#HH;l6w4z={rvB zwz{A3zD_w6(w)3N#8|klDnc~-wU4HZ+;_<+tdz8W(^w&qulQ!i9|fb$ zsF7u6u9Ra;kS(&(;k7foF>g_1yh*Yx=$R?eGg1a!qyVuFWz1~gQ1}z(Xb^z-C*~`e zgMukgxE9F|Hx0>KMHp9jciULPZ?k+Q@ipg#xkFLib1Zm)0D+$Xc+9HRFcj7$*cLVw zEW7FvAFBS5APwdC1M8gUkULI=qkfGCm13Z;5iboP0{Zb zir*@{gtGTgZB9zjGxD^U|76-RED(hRnmXfPjusb`>R!J-6J_#gFz^AX1FEAHihGQh zISi)L*#y*$5eO`Xb_7eOOnUzuB##Te#0HzBW4|Op=)?SDCY)W4napG4dmY@|IMZVf z#J`(Q!afF2V+cej6s<2@j$vto6P|Ts)$BwOFZ=?!{S4-+_N_GwLzjYO3&l}fG73nw z3Z^eYN-ONSgQ+_kYh^@z*4zen0+Q$q!ZWV~yqDUz-@<7XVYA>>C2Q+S>;EmI0kWUd zZyJq6?EC992tI92sU{|Tv=$i^GB=kGo4r!jnz?@GtVHyYk{qDv7)I~b7L{N~Cd2_`7`M@8Zalm>C$iUBUP{ zb!DuYiGbP?)sz7f$4f@+*du$M#drCam^snU)UsR@jCMrCW=@FMdIiMGLK>n^PsKla zeI3k9&o!9`_43;eG?d?p2;ur9kqWU4r20^Zs_72O34VL{8o1H&8Gd_6R{)iCG2%)k zqDoHpQTJO}A(=nC_m;Ad?8i#SiZwoxtC*-L@H6CeVrV-_QVUS(xZvu_Qx&e|O3Nw} zJT@lI^-REtD3SM5yhP;M5XMBTR1*Oyv67ywf;pNz5YA{KVaVx{&Rxsr>igIXml$>P z%Bhv}OvB*B@r04htf!jTRD$iLO{2opju}Q0dlyp+tVT~!QH99JL3Sld2=_Eae8jME z)i`DxiH`V6a3_4(({y%r<0!Q$W3R3fAYV@DrEr{w!b$6Tc@dPU9+^&0NJ;7W#iR8{Lin=hHkJ|Ia`4 zvxNQ=H_%7m1MB;LO8Regn;ZYnCwbETpY5$~ciY(AZFF~9?RIKEcgJYqH9O*Dy5O@+a{<-H`bL?etY~zJ307Yzk73Xu&?bM zy*)m9cku2^v~StA>5v+V@kwN-sk;=>*BDMFvDyj0E2RjHgmH99bL5btOqi~EeDdb| zS7!&>aIpZb%Qlp;ZUGYR^<|R{ISQUz^KL6_Z zSBMR74i8K@z8(Vlx>u{o(op2KS?Q%z+)b8|F6~T;{D(4?d=W!f0}2b|NWt#d-cCZ-~$i1|Ht&-YHl^R zH~QaGJem7{TkTyC`C8js+l}`2M*sW$KTGKU5qJN4@BekTnw{+azt!z*_W!4No{4<& znr;|%_+R!uC?D36h*I|`> zF~0Z$|Cm>_4}5Smy%WFinM@r_lWX#!oryR7Ovj(Q)4-=3%$1i^^rJSkB9M2>bQwYe zU98u07DI!+aTk+Y82HlX6N=r`MKmd(N~>?~mTTHGeod zIoz*Oxr39Fqmxtf;N#B4e6TcoK41#s*};a7kcU-x=rYf-LL*6pSjkV_Q? z4b_lnV7I13fp~K*3eY|6MM%9vq@%8A)LOQmfk6kwFpEu`v%t8ursLTJ{%-K!cBFmP z=Rd9tbh1+ZWG~+M1@+S!gfRHnS-gTX*j#s1!fvHepT2IOr#bfK>puLnqdGQU=SIJ7 z&Ct`DgTmHK*Z%xxO?eweQ^y)--&(_= z9ev$#ic2TtBY~{yO93o50K5!vMV?}^!Lf^U zD(PCyv+jafq$zN-D*|jz&_-GEJnR(A7GRA>H*lwpz4p8)OjJ&eANam=#m+Ug8Z?2E zQ>!8P9Gg*DOJ;d0+84|ee;EdZZN;4D9!4WMy+#PNdM_ZGH1e;)ZI%8$`}^^MdHR=E zCkOky+3~kG?_M97@4h=cgw2B*N^Kq3FGA;gwkG(7blYWjrZ^NC~XU z2doO0weiaH06}I$r~GFD0-B*3RjnyUa>hSSzk92Uo903721;V&&x7LIIhK3->Tl-p z$FLpl`R>)*gNt;p$@_;{=&}xe>%@V@gDJTqmSr2zQw4z9l@BSj`9>TF$Y|47 zRb`TtTKJ^JIxNs9*^TBCUs0d=9V2pt?(#~X)Gp=x% z5ZJa!P$?KgzFfTScm5E#xzEymsz{ptzpcKQF`;;sNpM?Zky%v4TsvRc!Hm)m zoJQ7_vxghV*R+b^sy?1D2Te~s#yl!ls#^N51a39Q91zt0(Rbe*9x$5)_E|*+zIy0{ zEhD-I_E20Q`V@#}&;go*qQsvFns~8Fg!A!0h-#^h=Xx<0z=X63{Zo3{xQD=zxj@mv z%wS!+9W$Z}LPPT6?#Fl(zhlJ2DMl2GU$24}<0@5MV}bv?gl=FXO2dlx&hKFJgtpEe zvY^FH$0HJrDSv-PTevo{s8Gr5oA|Ibgp4d~mN~zmyYsB9nB$PVd-1xrBmTgnO7Rm% z!oTr1qa^ar7Okz-g z#lr`9?{)Hx?snee7gQfIjR-~VeyE8SmV=OdS_6(hll4&_xxc*ByfUq(wY~p}4)yRu z*LrjVVm7rreT~kr{*(jFx*Gtc9|Uxn>Q)F=RjnlOa$4g;=+SiM5bzPYw66m=y?IK| z#arwB(2%YhW`eQ4#5;%Bk_p|Dz|+j|ewZ2QB8HjmY^K~gSUt(DKO!`y z3tI!1;{mamPrRKgZ|CY{hsg)}bBz{I(UP_3k9bN@bV*1_n8o-T3$MvX{(B8J6eV@+ zC?*n8Rg$htGn$}JWm(zcc7Hpu6)dS3zlzI9vlcTl(JS@s8mK7lj>I$=81mJp(I4l$ z&(Mh?F_5`eSTg4D4;nFoQJH1NHR(Geu;g%kh4g|?k)WH%lq~BOb|kfxVrLlVXiIl5 z#Ky*5Ac+-UexWrR4bUF6dMr$@-lCc2NS-wDO00w^WlVf0%aM?oD#c)IFKf)N@ZB6`7KdLeje+zJMUd7$zW4Psek3P1A5oFscR*?k?#Ry# zv(=JoVW=FIaU(M1*ea{o2s1C{?P{4rr0m&R<1vosMgD)?-F9~q{r`{fETR8L-2LyT|K@hq|F;4EZubADctF_4K*k`)4_pfDEc^jq za-#b&5H0%NiHL63?nDH$rl>1M=^^fe3Qp-8ivNu~_saj*zmELh+1&qmnr9>b|Dm6? z<^Rv%0}sFdvyuOw>REFCr?Io!Zf-Py-~V%u{Qn5M|EJ0St@dXBe~RZ5Bu_^E-`Q?$8{KYmYkRBJ+1USo z|IZTof5hGY-uPdgt?ljX{@?6w?Eg>lV3u7Ve{JIWzGI7IN^zDk4TOFXmCM#J(Nq}K zFmo39_)}`kuuA#>-YUV=GNDl_9a4TARueD}%<&z3M>rH3Lc@>;-yVObaR4z&W6o{Q zC$(rc9n3sRcMMEoZj|;Y>@R5_)%c=4-3pU2e&WQ1&M@*3J_X4TDb%Kp20aU1P*4Oe zmnhGWi}{Z!E*V9^l7A0_$({Vo|9I}e7(0=Lm%`;NcclhSu-I;lOa`5W# z@M!PV*#X_t6<4K7CA>jPq1HqeCK?hZOP9klUQMrNsKp)Am$L1I!>LPmDEg*p2Sd}W zC^ZcXplyjNWxZCD7y!*;fR#RWyeh-a*dpiHiu#<(=~HV1TJZgoPXz!N0QmB$wK*S> zs0;{S)#CI@jg=J{Jqv5LS5}1CcC4tlO;HnyhcblQ)S!P5L|7KNq3%Khkg;+Da8sDRo*%wV9+~j z?AWK`)9MO9b`eE9$ApX6!f{d_mLp+$25q|{GPtb${qfmta7vf>oBPWu(lGE^s;reP zgore5Qm-7FSDB+9amc$qby~1z)Aem z>|!nCorV`!c38&J21SvHoJE9*dXa^jv#DobDZNrLt{iwjA7ZCQ0pQ(-kK!G}Gvh39 z?npL)5zTDd-fj$bx3=wJ+aB#~jT(b3XSma`M!T);VQb57?Aoo4)6g;dH4f7>F_|95 zIWP%8!)P{)mXhb2v-F;DObwA;+x5k+Blj;qPUWcttrQ)UbCjL}rpS1Mtz*hHDIJSR zP-dtSj}?}Zt1_Whu+y@IbDdw}dbhT^`o-!|h&+3?I~rjMcSbE32BmRNs7nmPUsN%; zq1u|i&@Za$-*b`icsRoFk4UaYXpI~bhphHGKQcyCsFXtaGVaQT?Kl(sQ3kN7;>PXN zjU1CwK9*A-FkU`ruB%zujUP7IWJP;jt9WThs>o-twfu)HKy zGZe##!_*7;C7l}s*Vi!)A#!sp%7Gv%Rkgkq5@gZ30i!(EF>t1SajuK3ttS2*;tP>B z+{fQA-2&3%by*1mwXE;=^%;qYlke(+t&6RR)4lhx{@59htx3Lld0Up-#;_QDeOl0q zDiQ*`Df@Hi&qAUFuxEgeT)K>%q%P7vMOzUd0Q9~rB7NvpQ7h)z{WNDUkBQqKClx5U zVI_q{f`w8l6`pO1T??$h!>KHnnQki8xS6WPStvAOw>>c}#@RXjeUZXr4%UgGT;g@_ z$(BE=1@WCpl>8Hg>J_LirFkSB_$ zaEMVPOb&Jc+^4ac%glV-dxsbhD0I9LN`xV>e@wiEP z%a3jp&$MOFaZBP^1dWgKhLVbhsA2YdHfC@rlbUZ4+ zL>}B?56@Xhz#ti{?_hd&3{oq$BcTK-81zg#aX>@XqJWGBA@B!9K+zm)cr8G^)SNgA zF=dr<$74P~$fi}XuY4Dk)M&08FggWA^cVD2_!W{~Nc&oN-ngLf zc}Z>9H~b*N%5KOZC|ED7U^uvPhCdYT97*6hVVYuPKGQ!loQ*R>J*N)_d?>TD62eY5 zjhh$mxzHGmC-EN`&h|o?`W)!OSxov-L7|;C4hq-?9HEn%kT{B4X7s@=g#{c&<pL4XW_MC`@#pgwA`KvnduA4HyQL-x#vo5Li5jdQB=lc;N5b?a6osq@rU>049cJ= zKhWPdbQDeE4F+R5?Nn6dp_q6$m}?>J;N0}_Kj{ffwvE0NjnW19_=8Ofef}q1AWp&7 z|A0g46`4bXO`(hhDzww#FWl$Ob)sMm|A8v_3q$u!C;S%0^C|qi_?T{#)}nvT;WP0M zU^HlPnUs0*K*Ql_rN&4IhQ_E?mLq`*cafQsv=GtH{GBc=Hp>1vniXws&&I}5dRC@| z5cGG@tS|Hmn&LP*W||Yn6!7WgDsZ;`7 zH7Qi2X;R2}lQQa=rm!lp!T#4j_FSI-&#}VU)cNegzyIYy{y(klZ9M;Lv^VE}PxNg3 zfBw+V{rUespAS61|3}b&tBLo2HvT`q{FCwj+1=UMHoA@7#&&bJ)!F#}{O+G+`Trho z_rKTwr?uV8?f=cK&Hn!+kFM({GhZnDXr4#2=~XQR{SQw=1Rmf?rf~eXe2V|+k&}-z z)gos+LC+wgR66#op(uI{vOJmTKriZL3d(I;MV>S@F@Csj_U)oiN)3KXt~Yy4RBD^O7XQ+2!t?T$%N zyH{4_L%$mCD4$SUCLGW)b!fo$hB&7{RVEs^P>?sBa9}CcW2%7^&?KioRi+wH!Ddqq zECH;f$5Vle+)yN0pM zYa9e!u~sQy)$>#yp;OyQ-eMM3Ky;bsZ|V$#sU1@C7iT(WgIHofgZ=F;_Rocl)*6I? zH;Wu?IE#YONLL+4?(r#17=8NmE`E=8ICkZTOD1MDk*7{NoT1wXalN>#K?(fIdQVr* zp6K+5QY~mrwEA)N)F1yv2p4IqH;+N&K2my430X-t0djU}2B~_TfuSog`N;_2=&Fq0 zmE)!an6$Z<)v+Z$TIwk#b~))`x_7b&=ZZ}_CW$*%msm;&4QApdF>%$0DsEIQpkbCA z@>n5g&$I(fj27a+hf|dJ=!l)>#u^S?n|vbyXYd@@8lnV<;+Zw|T$l<)7~~_=)uvT` zmyDY%;3^&f!W7|(EW75puzb)=j|&Lcnfr#%7bLFNo zrItfMkFO;#WJ2Ej*Nfa6Ne?p9NFHVI5ojZ+SrvHl={aO!^hklig#e4OPYf-QX_l5^ zFKtn7c`-$jSq+Lytpub>E~|khC07FUi&jWJ2{z7_;DCT4H~#w}i6ovYq4Cy#KP)V_ z8j`(g&SN-sv;;~XQ}c)>ahHU@brDx&`puO_W_sh?@yVkU3-!4vBtFI6Q}q^BAM%C* z&U)oUXwIVnRuN`#1{>%-)hGt0c3u$DIrJX0Sp!6aw;rdgo7lISUg66N?7??YNRt!K zVW+`F3pb`Szbsm*Vv1=fUB&jn&?oqRk!JxJd?;Muy$%!pfe?3IYZgo% zo~Q@6=bYY3zd%<`mbh^L#SfgMo<7je2aGzA499Lb#yl=P?E?TU8k59#))>hK$%$IA z55oI}dqD{R^|ia9$Bw6HjiP)u5?o;~n0C>(mGTy`q>h=zId42>si!^_Wi`?-D%#gt z5rB$DvQI?gDc;!F%Sq4gYY&)l*9R(bm8`Zx1Y0`o@LJyTNxS?=0`12nM1D42aMjPE zXjM}M&VC}W=9?2zB6;j*ol=1M_4j^3A4Mf{St;i~vxiIJ94Rd+; z-|9448~gv0Je%|XKlF3&`TryEfrp>}cbe_S=KTLDo+ao1&8@BO_U8Qm_x~)R|3}>Y z?>+x-Z#O$R|DWwm{=X-AbX`AKIMX3&6R06^z&^fgQFo?S!NDS+!|W=kFzyv*?9}8H zIX0NTbE3Wd*BS}+)6iHqzQ&R$8h5|u`FX9ze%*JUdcgX5$Fow+P>#{_=fmgCs@AD$ zJF5@T3xj5?tie}9Qzw@Li%&%76S5KonZrwZvDVp<8uZ~`U;WKIK74id`sn2CDX8?Q z(NAW2VY!2Muf91vF#p!=a6vls%$0n5`1&O^%n$X7VTJTv9E1%$yoAn2hOv=H-t+GhYVi@(x!T=} z)FE0Nu_UpCJS4zLJWUS`wV@tEs{KYI6p=>SG9!j6?Xf9XVlr?e%^aT&Jvs*ymD7i1 z@hKUfeC5q=g&`J@(#tx=00`^&UlaRU=a>s&-C1CQcgMg=MbRY`(DE|Bv=I4=e!GSD z*!y^2$EJgO_&XX~3;elL)zEaFoTH~e*%*gm8WK@{(4B?e(>{CzuHxL$uaEf};zBz7 z0P&QGnH?xr6deUOH_p_YcpzWiBt3US6ZpTvighx}nZp2Yp}@8rh!ofBH4TujL9YDoVfr%(q!>jjX9uME&OOZUSyeAq+x6o7x9Hj z+X{!kBhA6gzH*`^#!038&D_KZh5dHir_QB_j$cJ%*T(j!K9Ny3PZIJ`t-Q+*`X*iwndk@$;>?NzKQ1Sz>)v8B&zm}-iPy&NFAY2I z`i03$4Lun^(paws^*LU(CB^fnm1#Z@YXyBo?4`kavc(I+_T-AJ4B%5(nj9>qYZRtX zDRf|Af)1m~s>Ma~QB~x_$7Ox3C2yyO+Sgi#g_?qmAUVqd!WS%LId73nDYpiE$Fk3d zVflp>a)Wddg{hIiR#F`6_sY-sQ0m(26FmCNrg8XJx_K!1e9EANWiC zUuw1CJ;gG&ZVPrUmg$piB#DdH7noP$I0K3m2zA<--yi88)a~nIE$t*OA7>E&aG@1t z2u8}>eJq8F#cWc*HLBtcHka;0qZ#s~&9#c|auIdSbZrJH!ck?)3zlD{D5c2wv2nGD z!k|c4*osJ4m<=#G67Z~Xn2lS34}Qm(Hj-0l4;PSXt5Yo-_hF%cjsl8Zo93Ub^JX-$4iUJQkh?||fJcTV=Rc(Mx|I?jN zj5H819m8`llCJ{$j=_nbkZKt*+2SkuoeWu4EFh74hJsxIN|qgND4ho}k=$|t8>K`E z(`=5ztNv0SB$9MFaD@-D4u`+PHpo#{jc!yiDCC5374 z695PEz(9A_g5qKf!ZLNMQlU=JZzFX7Dcp1$SNSK5%##H*i%^h!FNeg8jytOc) zzb1{Sj|1Sb6~!*0?Zjjr+~29?* z{vW^lX9@j3;_iPh{bSn44E@9J?q>ggis#v%>a%cK2lct`_;YPwh1aEL=vS@V zB~P*7&s&UfAKeCI$C8l~C@+gOia?7W%v_HmXro^Md)h}B|2qne62@(-&(x}i)1hAC;BEy)j=noP{_gCw{`>>E!I*nT zZ;xM{yg7|O^@M#ap$%w}Odrq}1*~3@Va>Uf`Xn4q(HWo~+I6eWD95>$YM~Qiw7{_5 zYP7mdppP?ey|E=cF zJB{6L&0jRzolc`s+ZsBZk<;wp{9qd%uHR#|(dzE(KpVSltJB=Ex2X+h!5KA<-|YRx z{Py@e^Ypvp7&nw1bw(~!VY%w>g4a{@vF1H^z}N& z`mN)|hQP0f!6>?g6{`ErG#u{M{o&B7w~W@V(W(t+&31j_&8~nrJa^Em2a~Ao&t2QK z>erK@8QM4XL)V`z%*9UE>~`v{uCY}o{@H9BEv(b+JpYjCLhsAo^UN#29?{)*dwYlG z>CxU_56;Xtuiw3Td(giQeSW=na&&rXzI(IRZ}E$F2WNf#)eooIp*t8^gQ+{z;}7Py z`zPk%!S@G;eV8ik^&4~VZ*R=~H>Wg9e>plmGfxh_eRFztaMEwe)^V<1AHMqb6zJhY z{73K0=kagtW39IEEc0qIGp$(^L^IzpJ!kHC{RWVLHNk`eS}lqrtJtQz!TnV^90qXtk;4FT}BpxKRGpSI`G+j?J6+%|zov|9KfJR$pq)@H#*~ zlNr|e7Z<%oPn-bA3)@WbFoh}N7pZW85T5%BZYTCsfwGdKb?N$o+z|@m$r6o6a>P<% zgSf9DL2*im%EWw|X`3jWzXGSlf1YV)Nop=7p~;I7w*nE3nJ%s0!W!X$40O*8@np*J zf(hOGB3niv18zS00Jf8VvG4P~JlP6-58hA5<{ZO8E#i5V@G3aKJctM!T z?+9!4Bt#PIM&whdy-%61^2b@v%Vlw(CGWIrxsxj`b3~N`nmU<+zsD0-{=9KLuNL^V zfo0clO9#fDu07!-akUwsy|Q=L*UL|sA7GqMB%uzR7m@B}tu|sTtBcHcyp`37g4uy8 z*KF)}si_OCNzc%w9Z?!Icik{bxRJ7JMk4(0fC?X({RF zhoh6f?!P&qeXK@8ovcQH)ZQf_r1-Qq#^DqS7hteBmsJrs!QWhjuL69{2KQP6TeByp zsc|SVLIid7BDE`CV(ax()0NI^lsjrQtnw&1keV5ei75gu0qx zCk2bc0I5YeeUYYy{82w=QUz(bC^uTInQB3eL8TQ)=4yzlWpqQRnI|0BJZ+40!kMqa z+4zBoU07P|rd%E*I_5lF(Fg-QN1k*N={lhy5^yQqp}O_h%~!y#TNBq9PA1Df4+GeL zzF0=a8BUn%j>s$|J2RZzJLyIB#W$?}FpEJ38uJTyuWu|qzlrDGz0WXHyWgj@fi|A4 ztu@YnnvHG?I2Ydk?QZV>KFzZ^|M^2d_tJlI67AKnf<@r0{Y*2IP8~N`^o{jwXhkhPN|A{pC06ws`|4&T+-45n|*vNms{F9OY zcAM>v(QI{g8jWUm6aVG+|2&ZXKVkR3m;OQ6+sf|$Tg{FC?~^=ue^wt_7;jM37 z_-jNh7)4EbbHpwpbl*(Ba%FN%tkD{Fb{n1DF7S+9ySuyPw7RXWkqu0I)Y#g#?cL!{ zr`u@na$P=wGle}Gs&84lqpfyl+kskbt8I5$jpjD+|K`q4r@gbaJ>2T<4BFc}+r!ar z)7c$4J58(GYTJ#WJs7l_e4$R~gKASv+=;+#+um*rcA*=?wmsU}8Z`!64)n(w?Y6du ztt}h6-0C=u6c9|#33VGrvthLO%UFc)@GKiPKfD)DM;f}1C3$N2hJt>GFYqWvUcrVL zg;zMK;%^&!MzIt4Z@j5a_ssEfzPwA>X~`Q8oi_crkkIH_dQ6`t^VVo!s~ zgo;G*Z4wNQ-&1EY zbwZ3wkNLX+dw!f4DCAtr8|3-5%eite_F+b%dT%W8C5yC}uLj@htxt=+~5iKn@}-5LSr#aA>1 z>JyO(a^h3uOo7Lw4T3Dv$fgI%&LIyO=x(zMRJ4oMdAl=gcC1cw+lE2z?l!H~PP0`! zP;TeGfp#r>&}r>#ZFSnM!S=4T-5GV;Fst3|oo1swY{E#}J23j~4zxHJHFjI}2wv>8 zp;*V-X^VlD@V@%TjWDtP^I-iyg}Upt@c{k5BlrK-_GbTol4qm;|Dm5x*8d;C2OgmR z3;KucZfm3eKgE+h|7n8E3o`s}quJ?hZ}k7){qtbo#|Q8J_tO8?_Esyq|F>J4_^(g$ za0rf`hR)2lL5(Af)tYvRZ}sE-@+3SA7mIzK;TvcNJx!PlMy~JVEe7%eNCYZz4fVfH zt3*+s5VX`TF{-49%+|x$cTTHO9}Y7GFw?9v#T1VY6oN^1zAvG-A^}=R)eZlcvnm5W z7h~5aWt)Cmi&!^@za!;aMBn6>IhWj`KkSIL^%8|nwLH)9u*I6Tp#RQGNn|#BKQ{+G zt<5!Mur&hErr`*kn@0&MH@w8)E~aDReXE}INiPh-mi?-PT;n(Z6mqSX__fE$*VLi% z-|BGj!Nu)1OL7t5?N2i+gcY4dFlJg80TyiuwW)h`EuaseSlH8GO)>bGxIiI8=fOg> zlvfyMqX#v7+w|PAi}xt<^;Fx2Fl{JbGm{4s-jMUHX=e=@Wuw#B;=l-mDB`0Gu!u$Y z^I$ffn8DPHrWgUTM}N}?v?L%^$f{Fl8LcN064twRXb$2sx2_#sGh)U~rfo68{vziP z;|{Jjh3zMp!rFhXIiy&aIC~Vyr>9Mz6h@Q=c4d!jU;yM?UE|y=C5~j)V}EM+>pGah z?3p9i!!D|{Gm3)-THfT^0@Uh?egsn+htuXMvymavl{c==>_CK+Wd4Acn|hiIt;ut8 zjrMbxY#HAZ9i@A5w9gdZN%1jcCPbO!T^smG#>ARJkm?J>E}R}U-slo8K^RpW5tR$z z+zCFBAOImyvnAgzLxVgbOOv&I5$4d6t{TF{r#B=uGaGK!Ve zK%hxQCG2O>%VcX#e^g_7X6vcJK{MZS(LLpNlpbO}D0spH_x(0SES8Ll>vA_^I7u|G zEXX3Du{gnhERx!>Jl5&%Y;P&ASv1?)-8FW%+VCTR1Rq+B&W?C7PhianR;}4Fc6Yn& z_V&(hqpMUD%yh|D**K|@&6M3 zzgz8%|JTzz8~g7c`uPm|@7jFeA^u-X|J_!1ha*e*9#-t}f`W6Ryxa)0B- z$Biv_W6Ryxavzs6Q)6aVFDo{jzY5B(I;zjDduUV014WpUmaaKj-BcTBhOC^!*E(qnX@Olru?MHti-^cHgSCc!$Z~51P z^0{G&#Z4 zWkp%x8&KZA=Xd*6jaMxBTiAm|ZrM_)^e;R}W8^N-T#k{~G3z-;#?~t(+ZoA9A~S3` z0@WQS+Z@4?2Gcu{z>jZg&M=a4UrTbeVz%9~&T-k_y*h*4!8|)SJ(E%GMJ#kob{*GG z^dCU6+0>`CEGOUNDt1Se=dBA&WLF%imM`IE z<{%!EvR<`Qo%3jNyA^!(jd(@fqBr~BAHF(0`rgEd@%^LMuc_w#-rHAyJutud?#p1!DQa0e5@4pTJ5ZusciRtY+)u*rKzGO7`Q-q zA5;C|k`Nx#55!|$FqqYX=)zTP=~w#>-sl&Ib_mK~)GAZHzzqEpy)zodbS~+8bOplzde!B&H zYToa5s+t}3Dg1%V0^iuFlF_5TBNJc}2w@TAH1E2Dm)1Xggqe5l=-0=jBpo1NUR~UQ zIJGvN4(J6%QbzQohpBA!cgQS@jp%yieweZ!nC+WwuGA+yZfUMlo6NSk>e6bPu1tn$ z{6Pbtf=S+cDvDb(&tXgpGW{(~qaM+Jef($Z=d(nj-bJoZ6Cwc=WScS9QA&v9@)R$nq0YEqm(W9CWyrcjl5j7q3N9v?m&o)1i{oKtU70?d zi&T^%BG)SU9h6faU|go{Ag8f8Pq!PvJm$-=pTyE47*OA&80jXS*<#+;f&xlX@N#Nr z;6h@w96ue8cQGw?SeB{0rJN@Q&-s&aZsHO~6jm@lEWQ)^TO`s`D`}>c(uu^eA~Ca4 zUSySyVXw$CExH#cj0P;24M`Y{;mO3&*yVz9q7_*xhJ0V%>Ky!h+UgAGRhg@QnT*xE z(#Zk7B^7Ryl&EEE{>fS9#qw8KQG{k5y+HQWsp%8p1-;Tx?gV)sR_C8i zS?7!#RqY(6w{^j4jAE=H7b0MDxWq(;tlSBPP|avO)mTEVavrFZ#*^^Ib6r#*Hi^tE z4X3iC7Izlw&@U>LT~_#5&NDY!EqFoF;VR8f5CsuK{~_2Ue-tkPk0;H9D06xml?XQW zaR5u6N0JqUqa;!%AQg~rSt5illv;aPiAtW?2u9;Mei=K0+v3Qgfe>TiLqU;7B+-gc zqUM(@B$ksb-ITxs1U-W?`jv&|qloh?JrOb_+PmZ_uxqiVkOv4;h%84+Ul&!4JDmJ2 ze{wr?Hk>q*YGFWRmb4>)r>4XWNjC(HC^OviM5u$t7E@#2iU7T$R4r;XDQQSK?t(Ja zhd8G!`Jog>LKwsyLKQm?pvRwUJN-6L?F<%ZOl*hgt*=vZ zAhC)ur8A+YZ8haRs(f}EoRSA&C1wIDzkdL;F5N?V#FY~gAuKv40UQ3)#s4-1SGgC+ zsFAJTk@ZWZA+V@W%Z#q9GBYvFxFk8@2-KcYYQF&;tXx?pL{Sl_vDO4}zf&7mYQG0c zN9IGUCmvXPyUH!#`YJ*oC}K_h0tE=z@H}fguq`dBYQP*(0_>qB(il6IU%pF|o&~yk zzmfDl`E>(5!&=rLEdQu*L26ymtU>cz08W3>1WX+3=D}0Yxn~Aa5Q=Bx=A1?}y#i&d ztPXNCitzt5OE{>CV#@(!l4ydiJf^VH(1cXc|5%t?U_Y`|lnFfC664RsyBt3J0tzSm zFg+a~5Mz2;xeQznSsgNR0LH8waH*0}B!@vTwZlF$Nh}zNf>qW{^GhkTHxvJ+6-sdB z9I9M=tQKGj_uGg9(|4kwJz}d3Oak&&=1Igwvu4*1;-lUZAU>*$5v|zHA}|G0W5vXz z5R6k4tK##3HKeL>;Dewr!O)pvW%5?@i3qI8?&iqx`4_J`|2AMr&}nLTgKprnsar0q z+xXbSERe8YTOoanHNUuZ1OSR`S$})@`taxnnJf16@26)6Z_T#{XD4s=PR+xkZ@+!> z?prd>q6tM`CfThc4@!|M=KC2|nCA1c4Y4b*2}?Dq$^twgu}Mri=2NIeHtQ;8?HaBq z4hQB3OC%2dI&aXh&;UprT%`0>+;!lqA%s)0)NFs`68-GXRZ5VuN~Frv>DmmcEC9;^ zr^U;4WLNBu+Jj0}G?%08dpEEp^%P1?vHoOJ%u80;URAT3UaBG*?6WnDA32=v#PuAt zEY9J%B-Wht)fU{O+u^merjd)5{!m)+1?`~m0xjCS7PyCwpm|Xk>BX2j<6w?aR!8ol ztfT3_SFh`pRKLp!_%wHk&Cbh9i!^q4x`JvQrMP@RJwaA7gPl@SVOxT5(4CSfp47jq znFWkEE!ByZ(8ODg5#KOzmee&dV=EFKv#4Urdh}X=Cu=m8)$h{GVTHjMb`-!MKZJ?F zn5_dRvLui2F>WqeVNrik$1v4np3iGzUH{K+v$M6e z)zHu(tkK-~e>};v@&EioKg;Mp@d5pOKCrI;XF~riivPUv|NP~jl>g_}Zfm>KF}mB0 zcDvJUZ*BZPfA`NiP9C4z#y#}k-fA~9^p7zCHv9imJn>b0ki$WzxV7dE=aUBwbF1m& zb>pRX@LPNbf9W;;EvMDQYy72ku9fHgDOe#$-z8TsIP+cCD{B^puH`Eq=;pJ$)irM< zzwy#@d|Ma#pImp&Ud$#JoEq>dQip|R^4qyV8OJ@~-~@C%3@?Ul(H&mAdDg}s^3GdH zUUuv3r~wYT2F4=gJ8a}uQ&Iyae9CP^37^&iy0+WR-$q35p$FX0Q$Y%#&@~ygf^UH~ z?b5P%6~aTYBtlh?>u=_+DFJ`ccp>{jN6w0Cghg26)Vhh0o^Yxi|9wj(np?X&@hkhb zA!Z&Q#|N^)rA`K?TpCu9rK65kV$4?Roj``RS~7LhSA|yuuGuJp*J&j8bH&BmPrRA$ zIo4z}4X#S7uB@M>udMH{cT4?OrMlciXD-lpf92lgFR3S!yqVC4GueCZoxd);cg}Br z?TvlS#}0;Jo!zkO%vEI|D7pyD;-E3MKc|fc-2ZL2yBPn!+1%d5e|nl{WB>U>KP%}! zF#>%SA9%q1UqSz^?N)PR|9OfhegAj6f%$)Twp%-`_HJim|M~qtE9w7HcK`e6zq{St z&h7tDV6*=}#q&)2Ug*&K+Mc+J$=3`|q)SGrbOtk{0jA7Qsn%>K98TSV6KdDNEv9J2 zlV|eU*R1%yC&NJN2{{Z$*N%1>YwMSM*?4M{O3$8YXThX)=zxN(ol4*TQt9&YlCI$v zX!^0q>X~+YeHXez%R}2u6q})_?J#9^dZwMSeF+`O(2bBwNHYq2cE>Z1JB=?j;dc(? zkQhRqX-Dk5ni`~PdU&QC#Ks3rdMu^v4Ojx8qGynJo6|F%Y5VaDdDU7`#i6h4wUX`y zOc>4P!NM8N)O(^Zm*Q6s??GImLjx{{@BuZnGx#HzU=`?8QkGBQFVSv76-vU4gST9u zW5RXD^|ie(zpUdO%{pF~y}iaI=S$=;c6no*raWGoDotr}x4bH@Vb|X|(cb=RO^MkB zFsgJ5_fj}eT*mWiXm29z8t--oC2JN1W9k>VKu#&B7&c`HcO30f^)|Rf&suU(1&o;B zKQtmyteS_wD4ik z@TxJq(E<;V4|5? z_5SW@O|qQxAbQ$4j;aci2_nwavnKcxhZOc(RqX}*j;!z|?0=w!KfcJ^U6kKzz(J^5 zzk>j-CbNQ9wSnu~eLwJ>9F(E@^@3OMWab0%Imk|61qD$u1HudBFcw-bu}nZWFX?YN zGz~Y&DAa)QmT;o@cj_6PXDZEH9ahHyUubP)`%G(&2J zE^5K8uc5(p2;9FGde#7#9mU!VT^`$|@TjMSxI|70j14c`L_obiOUTR=wV$k ztz!{e`?>UUtyYWw4|Og@=P#iYa7+hFpQfoFQRpSiG4X$F!Mj7}Q0CI}n63o@2lw*O9`Hf9-!n0b}0~^xGwO6XvV7` zcAyYth|BTB#g)`*KI08+$B~O{fa-FCf=a35-@H1-JE(7u_urgcVt59@P4R>(py^I< zD_#je<11Lcy10ZAm#D@(c4^xw!cV3S{pST!(3E{#;A@0})7bLwWSkW8X~oMSRc+$U zLTwPh`~Y8@1g;;6mU8z=s5R+=jei`A-|;RMZ>8J-AUHJwn>QY$A(8~hk%NF+xVF5& z99k0#351|`JNVCAYXM@=IHny93$k7GCD7T>aco3>5Vb~@3*_P~CY}og0go9jI_Sef z5=io)rYVXr@(JP>bS;G6*dMawYFGi@j&YH2;t2@>Wng`Xap27lauX-;;=-^>VLvd& ziZ3XQgh1&yr>vhXzPYWCX(jwjiJY{|6Zvu!_+&5#~w~QxS?g)r8+KN?0^0 zgF8w1wB)gf00JNSd8)!+Fb!Wq$3`_;i6j(}4Af#lCE$+%Y>;&Knz~1^gpg68C;W60 zctQ^R4uqXNZQ=EFqLkZfhlR6L`x-{K2<57Zt0Elds3Zi-bbkSa6+uB}!InH!mMj7< z{xDkw>Ig_qV%Q~~oVXVwx;!9i(MW*5qv#%913e(h0~%zxl2Ib85W+TIv0#s0-W3Eb zg(L{bGg*1=4ssiC&@o-L9hQcU=hg6N1vz5O4Nx(vG8u~S+aT^;7MujaYmS%;<3fLI z> zcrXTw;#mu!fBYqw{l|a*KM!4v!-kWV;mlE`u{C>Cx1$s@#~#CAT=J!kZ%D9cjyMZ^ zfF_>iKnuu2f^Y}$!kz(U#No71yz;cYruG+5=iJE2ASr2Zq~DfGt908~rw(Jal?0d9 zIIF-G(bt;f7Sz;*6hjhAO-L-sKk_9q*d?AG$Z!$EQ0__00-92Sryn~C(`a0Q+H|S2 zcZe=zg#s6mC1AzZB9O652|+C#u%gh5m4w2)5})LF0f^ZZ?t^1wCle4xwDQT@Q>~6-9c}bAl$NtA#Ud{|HDMEn)C4qC&kTcaESewo zGfnP0gF7b%0I<+WN^H^quUQ9(SOJY+ASf$efn7hp$pS8w_JD$U0svcJ-$mS5GV=1j z|Bs3?7FgUGsLCLx-2#VpVF$1xZdE15=V!QR&#C#5_8yqf1SA0GA0KLSSfCbyB?O#X zQx^^K6$79m|N17l9s^tab{c?4iC1Z0(Ok0La3e%t1eT}nA1!Q1`%-IbH~(lomtYj+ z=VgL1dU)E-RdItHV$ha)ZnQ|~N{Qx{vAgpe$B1;RIp0H1OZzfg1MYA>NjPO@BVocb zLS`D;rHIhoXw1f-{D=(&kP+n(Dh-nlf&tZeiTH`JF+hWXT~I>DN|AMg8*>2LHcA9{ zsGX)~1&I?aNm8~gp>Bkdn$28YP%}uS!VrlSB(F=E@(k_o!Hk4HvQ;{XIwCKM>L^4p zUjY#^)cEd}9MPq8rMrBLJt5<-hfI4A1Rl!X*A7x6&Whv1h6KDjQy7sQnW!9c!jFJ? zTf=K9d>~%%Sj7-5id92K2OMN(Sd1stq!W_?Vx~=D9FQr6*a8&wfSmvrnE#{meo8;<$*Y!l1zfZJ zzKU!3ery3N8h^ln!(vj@BoJAikDs%e`GuUGFE~_Dm`oGT4ot`;ddb49y@tvokTYvh zFiFh=@ZQU@_B9l$usllIcTIF$-INk}8pzr6NU;LV^6EO)We_uX2l)T~@21h}$PToD zi1AH(TfXT?I~IBFrU>X%s2jp{m%Ixx#ttaSfEKqE!IGZgolcl@qug2S(1GSx4RCPI#yau%Am14mT<@oRzm$D5qyAx zvwfyP!lYi)ApZjMJ{<%eU_*qPB+Rv3u`*(+9QoDMvRw?nQ(14^*gz?hhNsb#J$ws% z1MOSM(2G0^p`ecc>d1g~{(`r9rK@3!+ympuJQn@t&zdx#msC$g#Xb+aVdOXy-w9dv z!(1?~sE~0?+}+)!3V)$DSQ-n|xkshM5qMYW53U2grX@(QmS!Jk#G2ClAJWgtB|k2!|gib2S;do-|tg0R#T;W5NkJ4(r4 z(#%~>ls2)q(lL*aYv09eJ!K5k#IZ#gg*_jpT0K7z>ms2oil9m3pbCsl{bScx`+170 zfUryOcWaT7+k9L3I?sIlZh}Wz1G4iAeLFplH3l%x^yo0ST?sR>Xl4$&YTUbWY;C3D z6*5p`0cv)HCTz`0%EYjp1%`N7-D88hyvTHUKft>;+F0y@3t1{XS+bNk3lgPd>QnV- z2b#rGK9bk<(N6PC@m$}pJ1 zPvG!$^c8D6Aon6lqlyNEQo+Fv96BrIBkRFk<}Bx5U;Ryv-H->$j1WAVxMpNTPGcs> zsPYJb1}_cs4o1`mYMx^VNyI2jMGGs7(VviVY1hQkV($oXYJ4&P5-Mw7YWBR|x}+Y6 z`t-ukUZoI^>4WS1sjo5iQdE>#=!|AwRh;6`?!$jrB&XCc0FmW-X}#%|G(iug9||td zZ!wfT<8r`$3h0plJrNciCTI+{9^g6VfDbiq@mSjN=^&9#T`ou3|E7Tw0YAUMgH=#$ zY&>lY1OCnk!simnG{=aApa1DhgWAZOEfh4P=I}rnRU7skgT51ySi>P2Z5#td3)1Mm z_=1!rl^hi#eOM)Qsm3EBlPkAH-c0qC(Q6XO&`%*dL zK~EWNQRR$~{MS6*5Am3e%%z+n0*}XF88MP1F-knb#i@XI^g*1r!O<~{otR!LDW$ANR zQ2>ccR7-)pru&4oW4cec1Xdy(qs=sT>K~GSix@-34jDQ_$&W0k$uslC7d$sq8R)xw ziA#<|j7Q}7?>GSUB$&caCmI?RqZGH!4H6TJ9TOGCCyo|zq?qTM4g}EfJ8;y*E5Lq0 zfg$dNxld$gZ=5?rqwdGYj22E~7}eqxu7BlO1IPQ?@Q0V|jf?C=L|(ltS)4`$*ciH6 z;=W~tvC9I+H_QRjFVL#|3zO6{?KRA7P1zu9*ifp%ufg@j`BCKYZP5p2{LE9J%&u9K z@l$!D@;OmivQ5$$av46ES;|)l`xKSq3)?v2y#T!_rZiIIXu>p>9F>uj$B2eb#l(8} z&(GsNja8{#?;U-!yL9wzr8?Zv*@AvlsNmpeV#NmcG#TEqz)?JN4D<*CO|HVHjHGW5DcEN6m6Iy5 zYv9Z;Dhsb=p}b^KeD@yJ<6+I2+}2>OYMA*LEjV-y-w%9YM3P&5>9cw($!?AUif}auPfzPfMVMNEq8J_0S48H?)dmO!WZpCIwKjF|$W#>g(&2U%{ol z>}xO@g-%o-Uc1wYAaj$|r0m&LfU1Iw-XKl9q6S*>R+q;Ts?owETj1(CR77az1)Y-T z8-;yZ5~p8kWnuED@F6B~Kpnlj=wJy1 zA#`0P+K9>eR4%+BWjOZVO1nNW%>$OPvK%8XW1hfjmXpT9;RPvbkmnilQVz&>QQB!? zIjVy6rywhzMM}ADh&LAKN)&a)#$LfbBg_bRK8~G1nr5sALCirq#=Q<^j$Av)J{@-r zWle86pQ-e?eV(O62K2MMg^$!@a=ZNZf7AZw|EobKtC~oC!@~9Dfd&fxfBYZ+`>Q(q zC!(`f7!m^!Rs>r*5lkJ9#K;_u4IZ*gYg?p5A!6i>}ysi+XJlE#da5fuj-%cIJe-SE* zp<-DHX8(VCSGU_XlBMtY6oez7C66gtcH(p#crj_mjrPD!j3hHzZ0we4iI$j9q=uwq zdpo_@%iZgJleyXZd4PS0d6GTfIaS3fib|Ymr+a@aAc*-_Kc}iro!_s@ab~)Om1s>l znrvPF1Wi}{Dtx#;5VOn9=Y-=Cc$La*!d}PmIK#-M279f-$|NIG`=ZG53sn8xKAY5# zT*aZcrYsr@Ua`Vhv;YHNE9ewmtNCYnPd*6|q#=Q{=~J5*Ume^7_?lOYZ2bq6|vpqMJ#yX%VQp z0y(z2kDq*=Je$h|+S**E(BmhYO*DG6Oe8UUIW*Gk`li1$zB?XPKe>Ge zj2vBsEt8zc0n12y;K1e4Qd;SqIFuR;Kq^9v?O75|-Dm=8wEL|1c7^4qNgeq`xILPX zY;yHHm1`R&V^X`z$@K1ZG~QkvN$SAezI6&T^L|9jizpu=wkE>gEMiUcwNg98Yhso9 zC*$N8EtHuHC*L5JAG@OQVXvvcGdrMPmkL?dDU^+L`pAKd8!0Dm((by0D1@9O4F894 z8Yg9z#$l=iLJbk73wChAP=kPeLuz-%y}|k8W}J)8fOvnW6v+U%-PeS`kcOf37h|;lD9*Bve*2{{Rvsz$nKtP;ttplqd$uB|g=GDx zeQdwzr1BH<{x^-UZ(&ls)UWYOMP=hTU zv^A-HzT?C;UP5-^w46jy#1xGn0ZD%}B&NIA*yZj&?e#Z(Z3aN>B1h|0wH<@3c3mcj zoe8e1X|Cnl?HzreWahG}<%`|vBCaD@$@Ua{sRw6(sMmjYKWt5#{D^oQEbXU$AqReUkL4i9BCFU!ETz(pzNviI!zE=MB4LF<9ur z9VaiXXL~jXu}lVhM+Le%`+|}A52>HehS{8@!ME&Z{RM$zzXocr@BAJpDGADQ6=i(; z2rVtm*Q3m~8v?ZYR*&P)cDDbEJ4=7}Yn?rG`{}obBqi&}X~V@kU#tJ02ZD8jAgvV9 zigrS*fEnYiX|z&iWFwR?IduzLmaTRf51K=nBYbBuD1^a_cScf<3~{v2(ek?a{xXNE z+*rY@g%+Wa z32kZ&Rsqhbk?KVpc7%7&Qxdy1JC~$)z|z5W5?wf#a-}7H|er#{8vm)2&$WTfU*ZF&cB?W@8@PAEv*9&v|#m%4RkdN?Kwnd>F zTk+1{Q*|$t?+(f<;}?)PL=+MfDfk`bgOFp!E=&HBp4+)~4YI#RYq_Ns(4Y_SM7FFE z$aI49*RiLR6^X_71+v)^?zu?RXd%Ca9C$={E60IqC%@(Y?ce{+50Y%$=8tzC(t;mO zppB_?+Riy?Ld9H%CEBC~xe>U=AXer(nwUdTcDZppfR!Xnm0Atshfj6Pfr=*^g9U{g zxCMDXC*?UHy;dYwb0Q?QAm^d77YO?_(o!3ERLrC;H=kGGVG@O;-;w+$w92f5^LJ+u zu<(WYW5IvGlHe56U<)!Em%|vEL9%)Lc6|&7Uo%y3%~w@7?ASJ05QmpbW&;`%C1XN^ zjftn~W@7w-rC!O%Mn?b0zA+7F$Nbt7A^QqtFoI4ilFK(pvUCBTkX!-H{X~i=*rEt&6 zDAeYrhNA!-8qV|x))*wy`mQ!$kJx3~-5jrLw$R=YpXJR<59$3MXwRzDvGmp4$qj9tjoJy_B>AM4lK%Qv-TAOe%ULTLt5R8X0LTQ*mTmE+rcRq#_7h# zMIa>j_~W$G(jZZwQ)rEe*KSSswpxsR#a_czlS*f|z{o{ats{UI0e%RHk*hAaQXL1Iq+c$(~59 z4G9?F_#BebM)eH)SOLwdih*dsmRVh;vkYm7hdI&mpY31}cf0ah+y>6B2VR^?%E7RW zZvwSO6<$n|LAX|5Q0voPdwRN|6720>s6_jY;3N1l2~2pBwA8C}_+G_}Obz{Bd+SZ7 z#m88SnUkaU)V|D;lrB#7)ihmYsHk)lK@D56^;6f?8NgkIJfpjebx$o~%M3^EsMJ|x z#mW%0t8|!#B$$rG`y6nS@KKt1M)~Bn^f5NDF1kaXMnk$v8UJ zTDo(DhiLUWE>)@)&B1g&N9w&APn<=xkf5lS?a|_|f9b%cIZ{?MS`_h!;rckQ1Z^(- zHyDP0c@zbYktl$oD!C8xE4`U6_WE1jihFKvTlkB@g-N4v5|25Qex7og6T?56GL&(Q z1|R&Q|}6gs&OU*IFe&*&NW-sX(a0G zYo8%#LlrJr;Wo(x14k=viR01Q@Wcqhd!`waO6_iFBoDC;*|O>WP2sV3NCAEz#qXy6 zu_FRemF;jwG`xm}d zRGwv}*y2~7yA=gG=|oJV-13!2VL7kr0ImmqUE&5fmJ$Vn>@uLHScqtP4tcHpcb7OE z{xqGRZuoEL-zhu#!(>47&p(Bst$pbS{^j8K9gU?dAbRucG>1vO)9QMMrA+lI*p^pl zp&1fbO=0URfVmSc4OB{i#;P3-oZQn$Tt0f?$Rf(F)t(84jOty-q(IiS3J=2{RKss; z5Rd}q7s$3?t4sx6TLn6DU-WM4RofrR8pa7vY?7LfTRKHd?aQ{Xf~K`n;N+dO#Om2P zXn7jT603LA^}maTp+-rsf>CGz$1Gh6AwsbdtAJ25lp9{mLK-v#SM{K&*jgYHcyLNG zSc`WiNz>(E3LPNJjG2^+gauDerfkB}hP}yh8w8Pu3T7DesO-(ct0XAO8}KHuYIv!= zD(?fKAhk09sv)r3YRuPW2#uE0u!23$7%q?vq|6AaKZzc`+uJm}%BJyOe-VHqyk~14 zYE*d3m~!W3zOU<1Uw2%O4K2YL&inRkT(z4dZv_ayiSmEBkSLSv@0AMMUx2pCCBXQ z@QS-WW497b|0kXb`x`fH`HbBv*x`TUA-lh6({!x;zK-co3#U-aNXNdM2yFCQP7pmf zdD~IZBsy}$91hltW>+d&B#=~=DmqGoPjhRb8Ky2Os#O4r)kQZ?x5WdZo z+o(6D6*$(YF}z#a0FYWoS67w4q0e43Wd5zHrdkAG-_$$I<`}o2!reE7K%Es%iUdp2 z*77Qx5}$uP3`;@8=NzEbq%QKpRJJ-{SpoRinVuS0U8R8gG@$arNA- zm7B<*fV6Q0mRpHzswf+U(pVrO_XIZ8prwdNH>AFE2N6u}Y-pPS`JK-?twps_H049} zL|bqp_XSk9dpAdBb_cT?b@E2H%%;hm6k7Y;`uykqUrXXZz9P`}3cFjh}n|^UmY#hmX1sA8kL}?myXja)18w|M}n7 z-~B(k{vY-q_v`!r!>!Hx{r}hay;H+~<}td{p1;QDq^@}jb@mJ0+3b>{xaN&43mn~j zrVqUhp1Vix1_#QGuhh0T+Diay)gPYy@ zNjK;jkBt9^Y@9?N!n}8okFycU=toEUYa2d*IFr3K8NS~~@~j(YYd>#zXI>JM`dV-? z$A@p$^mQQVWmMJL(c%8Vi`NJ1YiFJgS`B7$J}Q0m4}06)ejk08v{shDbqm*6fw^9j z6Ij>a$H=Qoq75*wlK4X3DTeXvMqi|)ESih*&!egSTaH3Dr7*OXzM4ED{c1S6)YsXp ztSY?7%4}O-%zzT2|K@S1YMT|LX(swgP9O`f{oniJxcuFuEN8`TuSb7R<`-bxy~|Bx z4HRWhzuKjTrs?4Y>yM`Jn-NaJ8d{x(m6PGW^u71o=rhmLv#&ePlj6+#A)7`s92QBG zZ)k@a+&Js0Ir>~3NgnDssXyZ9T}-ob65)t@nh?Dx?V+L6k2R}iox2^_Lwx29XR!0W zJ5vV@^Hdoo*?csMhaZxNX3JRKFg<#Q^I7{_XJ&vG@hA?1X+qO!jXH~FR38+xNt~LY zldAy|4e2#b8=D{_we}oG8(z%IGE3bDL0*pF?VU!o`$9bN*6BxzcGRF~QMG55M*(7P zs)40NK(unN=#@q7P9BYs0Ry{TQ)&Z2o(YVVAARHGN%fUbuN$ZFKxQD-`M`g} z5It~s@bLRrJ>L)$Hl2B%Gwh|WTtOrHCLE2Z1fg9JnAd|D5eSE^?#X~t!_15HUd-9W z0n$hIS2H*220jk<7amm{-+d@jh+l~1*}ilSLT zl?p4x$;bRrm&Xw%6ncjL@kGkgcO`-1D$G+pOUTU7D9^K8&8P={JWjJ*LgrLVyy9kj ze|Pu$Ih?X@r4+;J)Whcyl4F=_ta#(qJcU%e;^`lYC|~jDBx0cO&(|R7-R{E&ejdgu z17M#Cx8ctVfk&Ez`4sUsPAR;JbVod63V^Juw5VET^4c7*O--*o@C#r}AuMgfcXqlY z3O_JhEkiarWp>w#j8N```WV{&)ZT PZGZm_cwhoz06qi&=RdZ& literal 0 HcmV?d00001 diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-adept/summarize.py b/benchmarks/vlasov_cuda/results/2026-10-24-adept/summarize.py new file mode 100644 index 00000000..69a582ad --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-adept/summarize.py @@ -0,0 +1,82 @@ +"""Rebuild paired ADEPT/CUDA timing comparisons from the preserved raw reports.""" + +import hashlib +import json +import math +import statistics +import tarfile +from pathlib import Path + +root = Path(__file__).resolve().parent +pairs = [] +for nx, ranks in [(134400, 4), (134400, 8), (268800, 8)]: + reference = json.loads((root / f"jax-nx{nx}-g{ranks}.json").read_text()) + jax_rows = reference["rows"] + cuda_rows = [json.loads(line) for line in (root / f"cuda-nx{nx}-g{ranks}.jsonl").read_text().splitlines()] + assert len(jax_rows) == len(cuda_rows) == 3 + assert reference["adept_commit"] == "ddd70b957dc3df85f0b5ec84af927c25d09d24e0" + assert reference["libraries"]["jax"] == "0.10.2" + assert len(reference["hosts"]) == ranks and len(set(reference["hosts"])) == ranks // 4 + for row in [*jax_rows, *cuda_rows]: + assert row["nx"] == nx and row["nv"] == 4096 and row["ranks"] == ranks + assert row["steps"] == 10 and row["warmup"] == 3 + for row in cuda_rows: + assert row["finite"] and row["nu"] == 1e-12 and row["integrator"] == "sixth" + assert row["velocity"] == "cubic" and row["layout"] == "halos" and row["cell_centred"] + assert row["dt"] == 0.02 and row["length"] == 12600 and row["vmax"] == 15 + for host in set(reference["hosts"]): + assert f"host={host} " in row["rank_log"] + jax_mass = jax_rows[0]["sum"] * (12600 / nx) * (30 / 4096) + assert math.isclose(jax_mass, row["final_mass"], rel_tol=3e-10) + assert math.isclose(jax_rows[0]["minimum"], row["minimum_f"], rel_tol=1e-10, abs_tol=1e-11) + a = statistics.median(row["step_ms"] for row in jax_rows) + b = statistics.median(row["step_ms"] for row in cuda_rows) + pairs.append( + dict( + nx=nx, + nv=4096, + nodes=ranks // 4, + gpus=ranks, + adept_jax_ms=a, + native_cuda_ms=b, + speedup=a / b, + adept_jax_ms_range=[min(row["step_ms"] for row in jax_rows), max(row["step_ms"] for row in jax_rows)], + native_cuda_ms_range=[min(row["step_ms"] for row in cuda_rows), max(row["step_ms"] for row in cuda_rows)], + hosts=sorted(set(reference["hosts"])), + repeats=3, + native_max_allocated_bytes_per_rank=max(row["max_allocated_bytes_per_rank"] for row in cuda_rows), + endpoint_aggregate_check="mass and minimum agree; pointwise validation is reported separately", + ) + ) +archive = root / "source.tar.gz" +with tarfile.open(archive) as snapshot: + for line in (root / "source.sha256").read_text().splitlines(): + expected, name = line.split() + assert hashlib.sha256(snapshot.extractfile(name).read()).hexdigest() == expected +checks = json.loads((root / "integration-checks.json").read_text()) +assert len(checks) == 7 and all(row["passed"] for row in checks) +report = dict( + date="2026-10-24", + system="NERSC Perlmutter", + allocation="60221398", + device="NVIDIA A100-SXM4-40GB", + precision="float64", + dt=0.02, + nu=1e-12, + libraries=reference["libraries"], + adept_commit=reference["adept_commit"], + timing=( + "Median of 3 repeats, 10 steps after 3 warmup steps, maximum rank elapsed time; " + "no concurrent benchmark workloads. Both paths reset to identical t600 input." + ), + scope=( + "Steady-state physics operators. Excludes compilation, preprocessing, save callbacks, I/O and " + "postprocessing. Not a full-run wall-clock speedup or a full t0-to-2000 reproduction." + ), + comparison=pairs, + integration_checks=checks, + local_tests=dict(native=5, existing_regressions=33), + source_archive_sha256=hashlib.sha256(archive.read_bytes()).hexdigest(), +) +(root / "summary.json").write_text(json.dumps(report, indent=2) + "\n") +print(json.dumps(pairs, indent=2)) diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-adept/summary.json b/benchmarks/vlasov_cuda/results/2026-10-24-adept/summary.json new file mode 100644 index 00000000..4c53a664 --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-adept/summary.json @@ -0,0 +1,306 @@ +{ + "date": "2026-10-24", + "system": "NERSC Perlmutter", + "allocation": "60221398", + "device": "NVIDIA A100-SXM4-40GB", + "precision": "float64", + "dt": 0.02, + "nu": 1e-12, + "libraries": { + "jax": "0.10.2", + "jaxlib": "0.10.2", + "interpax": "0.3.14+10.g84aaf30", + "diffrax": "0.7.2" + }, + "adept_commit": "ddd70b957dc3df85f0b5ec84af927c25d09d24e0", + "timing": "Median of 3 repeats, 10 steps after 3 warmup steps, maximum rank elapsed time; no concurrent benchmark workloads. Both paths reset to identical t600 input.", + "scope": "Steady-state physics operators. Excludes compilation, preprocessing, save callbacks, I/O and postprocessing. Not a full-run wall-clock speedup or a full t0-to-2000 reproduction.", + "comparison": [ + { + "nx": 134400, + "nv": 4096, + "nodes": 1, + "gpus": 4, + "adept_jax_ms": 223.53556799935177, + "native_cuda_ms": 174.2416196000249, + "speedup": 1.2829057059529296, + "adept_jax_ms_range": [ + 223.41581310029142, + 223.62807510071434 + ], + "native_cuda_ms_range": [ + 174.2364576000227, + 174.2776944999605 + ], + "hosts": [ + "nid001600" + ], + "repeats": 3, + "native_max_allocated_bytes_per_rank": 11155281976, + "endpoint_aggregate_check": "mass and minimum agree; pointwise validation is reported separately" + }, + { + "nx": 134400, + "nv": 4096, + "nodes": 2, + "gpus": 8, + "adept_jax_ms": 383.94048050104175, + "native_cuda_ms": 140.0714161999531, + "speedup": 2.74103375918584, + "adept_jax_ms_range": [ + 383.0053849000251, + 384.9415123986546 + ], + "native_cuda_ms_range": [ + 140.015343999994, + 140.3418015999932 + ], + "hosts": [ + "nid001600", + "nid003745" + ], + "repeats": 3, + "native_max_allocated_bytes_per_rank": 5650233400, + "endpoint_aggregate_check": "mass and minimum agree; pointwise validation is reported separately" + }, + { + "nx": 268800, + "nv": 4096, + "nodes": 2, + "gpus": 8, + "adept_jax_ms": 756.6989740997087, + "native_cuda_ms": 275.9853768999847, + "speedup": 2.741808216795238, + "adept_jax_ms_range": [ + 754.9265112000285, + 757.4678515986307 + ], + "native_cuda_ms_range": [ + 275.6066252999744, + 276.0648734999677 + ], + "hosts": [ + "nid001600", + "nid003745" + ], + "repeats": 3, + "native_max_allocated_bytes_per_rank": 11300409400, + "endpoint_aggregate_check": "mass and minimum agree; pointwise validation is reported separately" + } + ], + "integration_checks": [ + { + "module": "BaseVlasov1D", + "ranks": 1, + "passed": true, + "max_abs_error": 1.0302869668521453e-13, + "mlflow_run_id": "10db23e28ed8428d9849bce2dea93d3f", + "stats": { + "native_seconds": { + "stream": 0.000619517000018277, + "velocity": 0.0003302759998859983, + "pack": 0.0, + "mpi": 0.0008770139997977822, + "field": 0.0017125090001854915, + "collision": 0.0005039860001261331, + "total": 0.004283324999960314 + }, + "solve_and_save_seconds": 0.7859312820073683, + "steps": 5, + "ranks": 1, + "device": "cuda" + }, + "keys": [ + "fields", + "electron.main", + "default" + ], + "log": "integration-v3.log" + }, + { + "module": "BaseVlasov1D", + "ranks": 4, + "passed": true, + "max_abs_error": 1.2434497875801753e-13, + "mlflow_run_id": "c15bc81503a44c02bb7f0d22043aa3df", + "stats": { + "native_seconds": { + "stream": 0.0005902340002421624, + "velocity": 0.0003807449994610579, + "pack": 0.00023884999984602473, + "mpi": 0.005609786999912103, + "field": 0.0067621719998669505, + "collision": 0.000502029000017501, + "total": 0.013536288000068453 + }, + "solve_and_save_seconds": 1.881096900004195, + "steps": 5, + "ranks": 4, + "device": "cuda" + }, + "keys": [ + "fields", + "electron.main", + "default" + ], + "log": "integration-v3.log" + }, + { + "module": "BaseVlasov1D", + "ranks": 8, + "passed": true, + "max_abs_error": 1.3145040611561853e-13, + "mlflow_run_id": "fb178ca1d165436c98457fd4ba0db80f", + "stats": { + "native_seconds": { + "stream": 0.0006939400003602714, + "velocity": 0.0005676090001998091, + "pack": 0.00025299700007508363, + "mpi": 0.01576393800007736, + "field": 0.013597956999888083, + "collision": 0.0005164489999742727, + "total": 0.02774611500001356 + }, + "solve_and_save_seconds": 3.1865445940056816, + "steps": 5, + "ranks": 8, + "device": "cuda" + }, + "keys": [ + "fields", + "electron.main", + "default" + ], + "log": "integration-v3.log" + }, + { + "module": "VPTwoStream1D", + "ranks": 1, + "passed": true, + "max_abs_error": 1.0457412713549274e-11, + "mlflow_run_id": "ac6e44af25584f74add8c11f040061c7", + "stats": { + "native_seconds": { + "stream": 0.0005986490004943334, + "velocity": 0.0003280549998407878, + "pack": 0.0, + "mpi": 0.0008833609997509484, + "field": 0.0016982180000013614, + "collision": 0.000500530999829607, + "total": 0.004253136000102131 + }, + "solve_and_save_seconds": 0.8431226150132716, + "steps": 5, + "ranks": 1, + "device": "cuda" + }, + "keys": [ + "fields", + "electron.main", + "default", + "electron.f0" + ], + "log": "integration-vp.log" + }, + { + "module": "VPTwoStream1D", + "ranks": 4, + "passed": true, + "max_abs_error": 6.3788974102862994e-12, + "mlflow_run_id": "d16748ce2f364431bb4fa17896661f44", + "stats": { + "native_seconds": { + "stream": 0.0006106250001494118, + "velocity": 0.000375073000213888, + "pack": 0.00023151799985043908, + "mpi": 0.005738596999663059, + "field": 0.006173489000275367, + "collision": 0.0005022090000466051, + "total": 0.012953435000099489 + }, + "solve_and_save_seconds": 1.350150696001947, + "steps": 5, + "ranks": 4, + "device": "cuda" + }, + "keys": [ + "fields", + "electron.main", + "default", + "electron.f0" + ], + "log": "integration-vp.log" + }, + { + "module": "VPTwoStream1D", + "ranks": 8, + "passed": true, + "max_abs_error": 1.1592504733926035e-11, + "mlflow_run_id": "e3e58e103a6747489eadadb244806865", + "stats": { + "native_seconds": { + "stream": 0.0006200880002324993, + "velocity": 0.0004060329994217682, + "pack": 0.0002525769999692784, + "mpi": 0.008024029000125665, + "field": 0.007057047000103012, + "collision": 0.000508114000012938, + "total": 0.01669021899999734 + }, + "solve_and_save_seconds": 7.212047922017518, + "steps": 5, + "ranks": 8, + "device": "cuda" + }, + "keys": [ + "fields", + "electron.main", + "default", + "electron.f0" + ], + "log": "integration-vp.log" + }, + { + "module": "VPTwoStream1D", + "ranks": 8, + "passed": true, + "max_abs_error": 1.1592504733926035e-11, + "mlflow_run_id": "a31a0009b9a440c79965f3e70332ccde", + "stats": { + "native_seconds": { + "stream": 0.000619704999962778, + "velocity": 0.0004217190003146243, + "pack": 0.00027711899952009844, + "mpi": 0.008916388000216102, + "field": 0.010889343000826557, + "collision": 0.0005323509999470843, + "total": 0.02131122699961452 + }, + "solve_and_save_seconds": 3.074192091007717, + "steps": 5, + "ranks": 8, + "device": "cuda", + "libraries": { + "jax": "0.10.2", + "jaxlib": "0.10.2", + "cloudpickle": "3.1.2", + "diffrax": "0.7.2" + } + }, + "keys": [ + "fields", + "electron.main", + "default", + "electron.f0", + "electron.pkv", + "electron.cascade" + ], + "log": "integration-observers-v2.log" + } + ], + "local_tests": { + "native": 5, + "existing_regressions": 33 + }, + "source_archive_sha256": "a457bc1ff3919eb30c64ceba61a503c478aa94bbaa06cd0fef3428017380946e" +} diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-adept/unmatched-node/README.md b/benchmarks/vlasov_cuda/results/2026-10-24-adept/unmatched-node/README.md new file mode 100644 index 00000000..1acf8ac8 --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-adept/unmatched-node/README.md @@ -0,0 +1,4 @@ +These initial four-GPU measurements are excluded from the reported comparison. +Slurm placed ADEPT/JAX on nid001600 and CUDA on nid003745. The audit caught +this before publication; the final pair was repeated with an explicit node +list, using identical physical GPUs for both solvers. diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-adept/unmatched-node/cuda-nx134400-g4.jsonl b/benchmarks/vlasov_cuda/results/2026-10-24-adept/unmatched-node/cuda-nx134400-g4.jsonl new file mode 100644 index 00000000..d112a9c2 --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-adept/unmatched-node/cuda-nx134400-g4.jsonl @@ -0,0 +1,3 @@ +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "integrator": "sixth", "nu": 1e-12, "velocity": "cubic", "layout": "halos", "cell_centred": true, "max_halo_width": 4, "rank0_halo_sent_bytes": 215040000, "halo_sent_bytes_per_rank_max": 430080000, "slab_sent_bytes_per_rank": 16515072000, "input": "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/initial.bin", "nx": 134400, "nv": 4096, "ranks": 4, "nodes": 1, "steps": 10, "warmup": 3, "dt": 0.02, "length": 12600, "vmax": 15, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 4404019200, "max_allocated_bytes_per_rank": 11155281976, "stream_ms": 96.76722870008234, "spline_ms": 29.388790800067, "pack_ms": 6.512254699987352, "mpi_ms": 20.99453199996333, "field_ms": 6.691103699904488, "collision_ms": 14.49035960004039, "step_ms": 174.2564972000082, "initial_mass": 12597.6196507818, "final_mass": 12597.62016522545, "mass_relative_change": 4.083657516783715e-08, "initial_kinetic_energy": 55713.43281125277, "final_kinetic_energy": 55714.15660222915, "initial_field_energy": 787.6538181371966, "final_field_energy": 786.9554651665927, "minimum_f": -0.1916308738890627, "finite": true, "repeat": 0, "command": ["srun", "--jobid=60221398", "--overlap", "--nodes=1", "--ntasks=4", "--ntasks-per-node=4", "--gpus-per-node=4", "--gpu-bind=none", "--cpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T190159Z-99199-22637/workdir/cuda-build/vlasov_bench", "--nx", "134400", "--nv", "4096", "--steps", "10", "--warmup", "3", "--mode", "landau", "--velocity", "cubic", "--layout", "halos", "--integrator", "sixth", "--nu", "1e-12", "--length", "12600.0", "--vmax", "15.0", "--dt", "0.02", "--cells", "--input", "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/initial.bin", "--v-grid", "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/v.bin", "--background", "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/background.bin"], "rank_log": "rank=2 host=nid003745 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid003745 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid003745 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid003745 local_rank=0 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=afdc248c4553148cd3ca5a63cb0431b8e474acd835d818af386e898beb75eb2d\nsynced_at=2026-10-24T19:13:57Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "integrator": "sixth", "nu": 1e-12, "velocity": "cubic", "layout": "halos", "cell_centred": true, "max_halo_width": 4, "rank0_halo_sent_bytes": 215040000, "halo_sent_bytes_per_rank_max": 430080000, "slab_sent_bytes_per_rank": 16515072000, "input": "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/initial.bin", "nx": 134400, "nv": 4096, "ranks": 4, "nodes": 1, "steps": 10, "warmup": 3, "dt": 0.02, "length": 12600, "vmax": 15, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 4404019200, "max_allocated_bytes_per_rank": 11155281976, "stream_ms": 96.76718900000196, "spline_ms": 29.3890182000041, "pack_ms": 6.513456599805068, "mpi_ms": 21.01721300018653, "field_ms": 6.694112699869947, "collision_ms": 14.50337810001656, "step_ms": 174.2706530000532, "initial_mass": 12597.6196507818, "final_mass": 12597.62016522545, "mass_relative_change": 4.083657516783715e-08, "initial_kinetic_energy": 55713.43281125277, "final_kinetic_energy": 55714.15660222915, "initial_field_energy": 787.6538181371966, "final_field_energy": 786.9554651665927, "minimum_f": -0.1916308738890627, "finite": true, "repeat": 1, "command": ["srun", "--jobid=60221398", "--overlap", "--nodes=1", "--ntasks=4", "--ntasks-per-node=4", "--gpus-per-node=4", "--gpu-bind=none", "--cpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T190159Z-99199-22637/workdir/cuda-build/vlasov_bench", "--nx", "134400", "--nv", "4096", "--steps", "10", "--warmup", "3", "--mode", "landau", "--velocity", "cubic", "--layout", "halos", "--integrator", "sixth", "--nu", "1e-12", "--length", "12600.0", "--vmax", "15.0", "--dt", "0.02", "--cells", "--input", "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/initial.bin", "--v-grid", "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/v.bin", "--background", "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/background.bin"], "rank_log": "rank=2 host=nid001600 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001600 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001600 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001600 local_rank=0 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=afdc248c4553148cd3ca5a63cb0431b8e474acd835d818af386e898beb75eb2d\nsynced_at=2026-10-24T19:13:57Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "integrator": "sixth", "nu": 1e-12, "velocity": "cubic", "layout": "halos", "cell_centred": true, "max_halo_width": 4, "rank0_halo_sent_bytes": 215040000, "halo_sent_bytes_per_rank_max": 430080000, "slab_sent_bytes_per_rank": 16515072000, "input": "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/initial.bin", "nx": 134400, "nv": 4096, "ranks": 4, "nodes": 1, "steps": 10, "warmup": 3, "dt": 0.02, "length": 12600, "vmax": 15, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 4404019200, "max_allocated_bytes_per_rank": 11155281976, "stream_ms": 96.7677092000713, "spline_ms": 29.38927480004168, "pack_ms": 6.511935900039134, "mpi_ms": 20.97058130004825, "field_ms": 6.669697600000291, "collision_ms": 14.48755779995281, "step_ms": 174.236390100009, "initial_mass": 12597.6196507818, "final_mass": 12597.62016522545, "mass_relative_change": 4.083657516783715e-08, "initial_kinetic_energy": 55713.43281125277, "final_kinetic_energy": 55714.15660222915, "initial_field_energy": 787.6538181371966, "final_field_energy": 786.9554651665927, "minimum_f": -0.1916308738890627, "finite": true, "repeat": 2, "command": ["srun", "--jobid=60221398", "--overlap", "--nodes=1", "--ntasks=4", "--ntasks-per-node=4", "--gpus-per-node=4", "--gpu-bind=none", "--cpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T190159Z-99199-22637/workdir/cuda-build/vlasov_bench", "--nx", "134400", "--nv", "4096", "--steps", "10", "--warmup", "3", "--mode", "landau", "--velocity", "cubic", "--layout", "halos", "--integrator", "sixth", "--nu", "1e-12", "--length", "12600.0", "--vmax", "15.0", "--dt", "0.02", "--cells", "--input", "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/initial.bin", "--v-grid", "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/v.bin", "--background", "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/background.bin"], "rank_log": "rank=3 host=nid003745 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid003745 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid003745 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid003745 local_rank=0 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=afdc248c4553148cd3ca5a63cb0431b8e474acd835d818af386e898beb75eb2d\nsynced_at=2026-10-24T19:13:57Z\n"} diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-adept/unmatched-node/jax-nx134400-g4.json b/benchmarks/vlasov_cuda/results/2026-10-24-adept/unmatched-node/jax-nx134400-g4.json new file mode 100644 index 00000000..00c59da8 --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-adept/unmatched-node/jax-nx134400-g4.json @@ -0,0 +1,120 @@ +{ + "rows": [ + { + "backend": "adept-jax", + "nx": 134400, + "nv": 4096, + "ranks": 4, + "repeat": 0, + "steps": 10, + "warmup": 3, + "step_ms": 223.53141740022693, + "rank_seconds": [ + 2.2350591710128356, + 2.235074671014445, + 2.2353141740022693, + 2.2350723760027904 + ], + "sum": 18346614.114405707, + "sum_squares": 2101439.806997545, + "minimum": -0.19163087388906355, + "maximum": 0.3687389216845132 + }, + { + "backend": "adept-jax", + "nx": 134400, + "nv": 4096, + "ranks": 4, + "repeat": 1, + "steps": 10, + "warmup": 3, + "step_ms": 223.51721489976626, + "rank_seconds": [ + 2.2351721489976626, + 2.2348791630065534, + 2.2348948220023885, + 2.2348771590040997 + ], + "sum": 18346614.114405707, + "sum_squares": 2101439.806997545, + "minimum": -0.19163087388906355, + "maximum": 0.3687389216845132 + }, + { + "backend": "adept-jax", + "nx": 134400, + "nv": 4096, + "ranks": 4, + "repeat": 2, + "steps": 10, + "warmup": 3, + "step_ms": 223.62216420006007, + "rank_seconds": [ + 2.2362216420006007, + 2.2361008370062336, + 2.2359991599805653, + 2.236146445007762 + ], + "sum": 18346614.114405707, + "sum_squares": 2101439.806997545, + "minimum": -0.19163087388906355, + "maximum": 0.3687389216845132 + } + ], + "checkpoint": { + "checkpoint": "/pscratch/sd/a/archis/vp-turbulence-sessions/20261023T081121Z-nv4096-nx134400-5ce4fe14/workdir/campaigns/production-nx134400/checkpoint-state/step-030000-1a19a5ae15f8", + "manifest_sha256": "9ab7e963d2699777bd935ca84fad446b89846e64c416d10db6d3c14829af8627", + "actual_time": 600.0, + "config_sha256": "5a9f5347e6253a3d4201745318843857c568b3787cf91e9fe81a623d0cdbb214", + "adept_commit": "ddd70b957dc3df85f0b5ec84af927c25d09d24e0", + "jax": "0.10.2", + "nx": 134400, + "nv": 4096, + "dt": 0.02, + "nu": 1e-12, + "devices": [ + "cuda:0", + "cuda:1", + "cuda:2", + "cuda:3" + ], + "milestones": [ + { + "steps": 1, + "advanced_steps": 1, + "seconds": 3.020330819010269 + }, + { + "steps": 10, + "advanced_steps": 9, + "seconds": 2.0284883639833424 + }, + { + "steps": 100, + "advanced_steps": 90, + "seconds": 20.115412370010745 + } + ] + }, + "adept_commit": "ddd70b957dc3df85f0b5ec84af927c25d09d24e0", + "libraries": { + "jax": "0.10.2", + "jaxlib": "0.10.2", + "interpax": "0.3.14+10.g84aaf30", + "diffrax": "0.7.2" + }, + "hosts": [ + "nid001600", + "nid001600", + "nid001600", + "nid001600" + ], + "compile_seconds": 1.1488750469870865, + "devices": [ + "cuda:0", + "cuda:1", + "cuda:2", + "cuda:3" + ], + "scope": "Warmed operator loop; no SaveAt, diagnostics, I/O, initialization or compilation in timings" +} \ No newline at end of file diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-halos/comparison-nx134400-smoke.json b/benchmarks/vlasov_cuda/results/2026-10-24-halos/comparison-nx134400-smoke.json new file mode 100644 index 00000000..f1ff7afc --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-halos/comparison-nx134400-smoke.json @@ -0,0 +1,446 @@ +[ + { + "nodes": 1, + "ranks": 4, + "layout": "transpose", + "steps": 1, + "max_abs_f_error": 4.9779624866630456e-14, + "relative_l2_f_error": 5.802416626972201e-15, + "relative_l2_e_error": 8.618125582264429e-15, + "max_abs_e_error": 7.389922007661198e-15, + "hole_counts": [ + 132, + 48 + ], + "reference_hole_counts": [ + 132, + 48 + ], + "hole_counts_match": true, + "passed": true, + "limits": { + "max_abs_f_error": 1e-07, + "relative_l2_f_error": 1e-08, + "relative_l2_e_error": 1e-07 + }, + "benchmark": { + "backend": "cuda", + "device": "NVIDIA A100-SXM4-40GB", + "precision": "float64", + "mode": "landau", + "integrator": "sixth", + "nu": 1e-12, + "velocity": "cubic", + "layout": "transpose", + "cell_centred": true, + "max_halo_width": 0, + "rank0_halo_sent_bytes": 0, + "nx": 134400, + "nv": 4096, + "ranks": 4, + "nodes": 1, + "steps": 1, + "warmup": 0, + "dt": 0.02, + "length": 12600, + "vmax": 15, + "amplitude": 0.01, + "acceleration": 0.2, + "host_mpi": false, + "state_bytes": 4404019200, + "max_allocated_bytes_per_rank": 11017656376, + "stream_ms": 96.85286999956588, + "spline_ms": 23.19442900079594, + "pack_ms": 45.6734180006606, + "mpi_ms": 139.9384269998336, + "field_ms": 6.371654999838938, + "collision_ms": 14.55919499994707, + "step_ms": 326.5107229999558, + "initial_mass": 12597.6196507818, + "final_mass": 12597.61992816154, + "mass_relative_change": 2.201842476587016e-08, + "initial_kinetic_energy": 55713.43281125277, + "final_kinetic_energy": 55713.52390600385, + "initial_field_energy": 787.6538181371966, + "final_field_energy": 787.5827421915643, + "minimum_f": -0.1915654276559724, + "finite": true + }, + "command": [ + "srun", + "--jobid=60205333", + "--overlap", + "--nodes=1", + "--ntasks=4", + "--ntasks-per-node=4", + "--gpus-per-node=4", + "--gpu-bind=none", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/workdir/cuda-build/vlasov_bench", + "--nx", + "134400", + "--nv", + "4096", + "--length", + "12600", + "--vmax", + "15", + "--dt", + "0.02", + "--nu", + "1e-12", + "--cells", + "--velocity", + "cubic", + "--integrator", + "sixth", + "--layout", + "transpose", + "--steps", + "1", + "--warmup", + "0", + "--input", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/initial.bin", + "--v-grid", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/v.bin", + "--background", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/background.bin", + "--dump", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/comparison-nx134400-smoke/1n-4g-transpose-0001.bin" + ], + "rank_log": "rank=0 host=nid001024 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001024 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001024 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001024 local_rank=2 device=NVIDIA A100-SXM4-40GB\n" + }, + { + "nodes": 1, + "ranks": 4, + "layout": "halos", + "steps": 1, + "max_abs_f_error": 4.9779624866630456e-14, + "relative_l2_f_error": 5.802416626972201e-15, + "relative_l2_e_error": 8.618125582264429e-15, + "max_abs_e_error": 7.389922007661198e-15, + "hole_counts": [ + 132, + 48 + ], + "reference_hole_counts": [ + 132, + 48 + ], + "hole_counts_match": true, + "passed": true, + "limits": { + "max_abs_f_error": 1e-07, + "relative_l2_f_error": 1e-08, + "relative_l2_e_error": 1e-07 + }, + "benchmark": { + "backend": "cuda", + "device": "NVIDIA A100-SXM4-40GB", + "precision": "float64", + "mode": "landau", + "integrator": "sixth", + "nu": 1e-12, + "velocity": "cubic", + "layout": "halos", + "cell_centred": true, + "max_halo_width": 4, + "rank0_halo_sent_bytes": 21504000, + "nx": 134400, + "nv": 4096, + "ranks": 4, + "nodes": 1, + "steps": 1, + "warmup": 0, + "dt": 0.02, + "length": 12600, + "vmax": 15, + "amplitude": 0.01, + "acceleration": 0.2, + "host_mpi": false, + "state_bytes": 4404019200, + "max_allocated_bytes_per_rank": 11155281976, + "stream_ms": 96.81700500050283, + "spline_ms": 29.46111599885626, + "pack_ms": 6.616684999698919, + "mpi_ms": 43.50261599984151, + "field_ms": 6.872578000184149, + "collision_ms": 14.55399099995702, + "step_ms": 197.2929050002676, + "initial_mass": 12597.6196507818, + "final_mass": 12597.61992816154, + "mass_relative_change": 2.201842476587016e-08, + "initial_kinetic_energy": 55713.43281125277, + "final_kinetic_energy": 55713.52390600385, + "initial_field_energy": 787.6538181371966, + "final_field_energy": 787.5827421915643, + "minimum_f": -0.1915654276559724, + "finite": true + }, + "command": [ + "srun", + "--jobid=60205333", + "--overlap", + "--nodes=1", + "--ntasks=4", + "--ntasks-per-node=4", + "--gpus-per-node=4", + "--gpu-bind=none", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/workdir/cuda-build/vlasov_bench", + "--nx", + "134400", + "--nv", + "4096", + "--length", + "12600", + "--vmax", + "15", + "--dt", + "0.02", + "--nu", + "1e-12", + "--cells", + "--velocity", + "cubic", + "--integrator", + "sixth", + "--layout", + "halos", + "--steps", + "1", + "--warmup", + "0", + "--input", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/initial.bin", + "--v-grid", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/v.bin", + "--background", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/background.bin", + "--dump", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/comparison-nx134400-smoke/1n-4g-halos-0001.bin" + ], + "rank_log": "rank=2 host=nid001025 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001025 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001025 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001025 local_rank=0 device=NVIDIA A100-SXM4-40GB\n" + }, + { + "nodes": 2, + "ranks": 8, + "layout": "transpose", + "steps": 1, + "max_abs_f_error": 4.9710235927591384e-14, + "relative_l2_f_error": 5.7939964374065035e-15, + "relative_l2_e_error": 1.0312384960988757e-14, + "max_abs_e_error": 9.159339953157541e-15, + "hole_counts": [ + 132, + 48 + ], + "reference_hole_counts": [ + 132, + 48 + ], + "hole_counts_match": true, + "passed": true, + "limits": { + "max_abs_f_error": 1e-07, + "relative_l2_f_error": 1e-08, + "relative_l2_e_error": 1e-07 + }, + "benchmark": { + "backend": "cuda", + "device": "NVIDIA A100-SXM4-40GB", + "precision": "float64", + "mode": "landau", + "integrator": "sixth", + "nu": 1e-12, + "velocity": "cubic", + "layout": "transpose", + "cell_centred": true, + "max_halo_width": 0, + "rank0_halo_sent_bytes": 0, + "nx": 134400, + "nv": 4096, + "ranks": 8, + "nodes": 2, + "steps": 1, + "warmup": 0, + "dt": 0.02, + "length": 12600, + "vmax": 15, + "amplitude": 0.01, + "acceleration": 0.2, + "host_mpi": false, + "state_bytes": 4404019200, + "max_allocated_bytes_per_rank": 5512607800, + "stream_ms": 48.57247100017048, + "spline_ms": 11.7405529995267, + "pack_ms": 23.03244600125254, + "mpi_ms": 412.1097049996933, + "field_ms": 10.8350719997361, + "collision_ms": 11.40117900013138, + "step_ms": 511.398970999835, + "initial_mass": 12597.61965078436, + "final_mass": 12597.6199281584, + "mass_relative_change": 2.201797209846199e-08, + "initial_kinetic_energy": 55713.43281126581, + "final_kinetic_energy": 55713.52390601106, + "initial_field_energy": 787.6538181371961, + "final_field_energy": 787.5827421915658, + "minimum_f": -0.1915654276559724, + "finite": true + }, + "command": [ + "srun", + "--jobid=60205333", + "--overlap", + "--nodes=2", + "--ntasks=8", + "--ntasks-per-node=4", + "--gpus-per-node=4", + "--gpu-bind=none", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/workdir/cuda-build/vlasov_bench", + "--nx", + "134400", + "--nv", + "4096", + "--length", + "12600", + "--vmax", + "15", + "--dt", + "0.02", + "--nu", + "1e-12", + "--cells", + "--velocity", + "cubic", + "--integrator", + "sixth", + "--layout", + "transpose", + "--steps", + "1", + "--warmup", + "0", + "--input", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/initial.bin", + "--v-grid", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/v.bin", + "--background", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/background.bin", + "--dump", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/comparison-nx134400-smoke/2n-8g-transpose-0001.bin" + ], + "rank_log": "rank=7 host=nid001025 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001025 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=6 host=nid001024 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001024 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=5 host=nid001025 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001024 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=4 host=nid001024 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001025 local_rank=1 device=NVIDIA A100-SXM4-40GB\n" + }, + { + "nodes": 2, + "ranks": 8, + "layout": "halos", + "steps": 1, + "max_abs_f_error": 4.9710235927591384e-14, + "relative_l2_f_error": 5.7939964374065035e-15, + "relative_l2_e_error": 1.0312384960988757e-14, + "max_abs_e_error": 9.159339953157541e-15, + "hole_counts": [ + 132, + 48 + ], + "reference_hole_counts": [ + 132, + 48 + ], + "hole_counts_match": true, + "passed": true, + "limits": { + "max_abs_f_error": 1e-07, + "relative_l2_f_error": 1e-08, + "relative_l2_e_error": 1e-07 + }, + "benchmark": { + "backend": "cuda", + "device": "NVIDIA A100-SXM4-40GB", + "precision": "float64", + "mode": "landau", + "integrator": "sixth", + "nu": 1e-12, + "velocity": "cubic", + "layout": "halos", + "cell_centred": true, + "max_halo_width": 4, + "rank0_halo_sent_bytes": 21504000, + "nx": 134400, + "nv": 4096, + "ranks": 8, + "nodes": 2, + "steps": 1, + "warmup": 0, + "dt": 0.02, + "length": 12600, + "vmax": 15, + "amplitude": 0.01, + "acceleration": 0.2, + "host_mpi": false, + "state_bytes": 4404019200, + "max_allocated_bytes_per_rank": 5650233400, + "stream_ms": 48.52810200009117, + "spline_ms": 12.26694700017106, + "pack_ms": 3.326494999782881, + "mpi_ms": 106.7915769999672, + "field_ms": 5.192836999412975, + "collision_ms": 11.28559500011761, + "step_ms": 186.7928990000109, + "initial_mass": 12597.61965078436, + "final_mass": 12597.6199281584, + "mass_relative_change": 2.201797209846199e-08, + "initial_kinetic_energy": 55713.43281126581, + "final_kinetic_energy": 55713.52390601106, + "initial_field_energy": 787.6538181371961, + "final_field_energy": 787.5827421915658, + "minimum_f": -0.1915654276559724, + "finite": true + }, + "command": [ + "srun", + "--jobid=60205333", + "--overlap", + "--nodes=2", + "--ntasks=8", + "--ntasks-per-node=4", + "--gpus-per-node=4", + "--gpu-bind=none", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/workdir/cuda-build/vlasov_bench", + "--nx", + "134400", + "--nv", + "4096", + "--length", + "12600", + "--vmax", + "15", + "--dt", + "0.02", + "--nu", + "1e-12", + "--cells", + "--velocity", + "cubic", + "--integrator", + "sixth", + "--layout", + "halos", + "--steps", + "1", + "--warmup", + "0", + "--input", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/initial.bin", + "--v-grid", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/v.bin", + "--background", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/background.bin", + "--dump", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/comparison-nx134400-smoke/2n-8g-halos-0001.bin" + ], + "rank_log": "rank=0 host=nid001024 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=6 host=nid001024 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=5 host=nid001025 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001025 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001025 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=7 host=nid001025 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=4 host=nid001024 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001024 local_rank=1 device=NVIDIA A100-SXM4-40GB\n" + } +] diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-halos/comparison-nx134400-window.json b/benchmarks/vlasov_cuda/results/2026-10-24-halos/comparison-nx134400-window.json new file mode 100644 index 00000000..bb4408e2 --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-halos/comparison-nx134400-window.json @@ -0,0 +1,914 @@ +[ + { + "nodes": 1, + "ranks": 4, + "layout": "transpose", + "steps": 10, + "max_abs_f_error": 1.2218004385999848e-13, + "relative_l2_f_error": 1.8566744087723585e-14, + "relative_l2_e_error": 4.172291687802881e-14, + "max_abs_e_error": 3.424344141578217e-14, + "hole_counts": [ + 130, + 50 + ], + "reference_hole_counts": [ + 130, + 50 + ], + "hole_counts_match": true, + "passed": true, + "limits": { + "max_abs_f_error": 1e-07, + "relative_l2_f_error": 1e-08, + "relative_l2_e_error": 1e-07 + }, + "benchmark": { + "backend": "cuda", + "device": "NVIDIA A100-SXM4-40GB", + "precision": "float64", + "mode": "landau", + "integrator": "sixth", + "nu": 1e-12, + "velocity": "cubic", + "layout": "transpose", + "cell_centred": true, + "max_halo_width": 0, + "rank0_halo_sent_bytes": 0, + "halo_sent_bytes_per_rank_max": 0, + "slab_sent_bytes_per_rank": 115605504000, + "input": "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/initial.bin", + "nx": 134400, + "nv": 4096, + "ranks": 4, + "nodes": 1, + "steps": 10, + "warmup": 0, + "dt": 0.02, + "length": 12600, + "vmax": 15, + "amplitude": 0.01, + "acceleration": 0.2, + "host_mpi": false, + "state_bytes": 4404019200, + "max_allocated_bytes_per_rank": 11017656376, + "stream_ms": 96.79199040010644, + "spline_ms": 23.18394020021515, + "pack_ms": 45.59686090110517, + "mpi_ms": 137.6495783999417, + "field_ms": 6.097852600396436, + "collision_ms": 14.47472449995075, + "step_ms": 323.8134939000247, + "initial_mass": 12597.6196507818, + "final_mass": 12597.62016522545, + "mass_relative_change": 4.083657516783715e-08, + "initial_kinetic_energy": 55713.43281125277, + "final_kinetic_energy": 55714.15660222915, + "initial_field_energy": 787.6538181371966, + "final_field_energy": 786.9554651665927, + "minimum_f": -0.1916308738890627, + "finite": true + }, + "command": [ + "srun", + "--jobid=60205333", + "--overlap", + "--nodes=1", + "--ntasks=4", + "--ntasks-per-node=4", + "--gpus-per-node=4", + "--gpu-bind=none", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/workdir/cuda-build/vlasov_bench", + "--nx", + "134400", + "--nv", + "4096", + "--length", + "12600", + "--vmax", + "15", + "--dt", + "0.02", + "--nu", + "1e-12", + "--cells", + "--velocity", + "cubic", + "--integrator", + "sixth", + "--layout", + "transpose", + "--steps", + "10", + "--warmup", + "0", + "--input", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/initial.bin", + "--v-grid", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/v.bin", + "--background", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/background.bin", + "--dump", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/comparison-nx134400-window/1n-4g-transpose-0010.bin" + ], + "rank_log": "rank=0 host=nid001024 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001024 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001024 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001024 local_rank=2 device=NVIDIA A100-SXM4-40GB\n" + }, + { + "nodes": 1, + "ranks": 4, + "layout": "transpose", + "steps": 100, + "max_abs_f_error": 6.725869861057276e-13, + "relative_l2_f_error": 9.515283499791307e-14, + "relative_l2_e_error": 4.921388303823267e-14, + "max_abs_e_error": 4.907185768843192e-14, + "hole_counts": [ + 129, + 47 + ], + "reference_hole_counts": [ + 129, + 47 + ], + "hole_counts_match": true, + "passed": true, + "limits": { + "max_abs_f_error": 1e-07, + "relative_l2_f_error": 1e-08, + "relative_l2_e_error": 1e-07 + }, + "benchmark": { + "backend": "cuda", + "device": "NVIDIA A100-SXM4-40GB", + "precision": "float64", + "mode": "landau", + "integrator": "sixth", + "nu": 1e-12, + "velocity": "cubic", + "layout": "transpose", + "cell_centred": true, + "max_halo_width": 0, + "rank0_halo_sent_bytes": 0, + "halo_sent_bytes_per_rank_max": 0, + "slab_sent_bytes_per_rank": 1156055040000, + "input": "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/initial.bin", + "nx": 134400, + "nv": 4096, + "ranks": 4, + "nodes": 1, + "steps": 100, + "warmup": 0, + "dt": 0.02, + "length": 12600, + "vmax": 15, + "amplitude": 0.01, + "acceleration": 0.2, + "host_mpi": false, + "state_bytes": 4404019200, + "max_allocated_bytes_per_rank": 11017656376, + "stream_ms": 96.7996663899794, + "spline_ms": 23.2147768099685, + "pack_ms": 45.58691152003121, + "mpi_ms": 137.3947532199827, + "field_ms": 6.027191200023481, + "collision_ms": 14.49148772999251, + "step_ms": 323.5432447600033, + "initial_mass": 12597.6196507818, + "final_mass": 12597.61289309191, + "mass_relative_change": -5.364259347188723e-07, + "initial_kinetic_energy": 55713.43281125277, + "final_kinetic_energy": 55718.73788187712, + "initial_field_energy": 787.6538181371966, + "final_field_energy": 781.820420166659, + "minimum_f": -0.1919615815105848, + "finite": true + }, + "command": [ + "srun", + "--jobid=60205333", + "--overlap", + "--nodes=1", + "--ntasks=4", + "--ntasks-per-node=4", + "--gpus-per-node=4", + "--gpu-bind=none", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/workdir/cuda-build/vlasov_bench", + "--nx", + "134400", + "--nv", + "4096", + "--length", + "12600", + "--vmax", + "15", + "--dt", + "0.02", + "--nu", + "1e-12", + "--cells", + "--velocity", + "cubic", + "--integrator", + "sixth", + "--layout", + "transpose", + "--steps", + "100", + "--warmup", + "0", + "--input", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/initial.bin", + "--v-grid", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/v.bin", + "--background", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/background.bin", + "--dump", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/comparison-nx134400-window/1n-4g-transpose-0100.bin" + ], + "rank_log": "rank=1 host=nid001025 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001025 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001025 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001025 local_rank=2 device=NVIDIA A100-SXM4-40GB\n" + }, + { + "nodes": 1, + "ranks": 4, + "layout": "halos", + "steps": 10, + "max_abs_f_error": 1.2218004385999848e-13, + "relative_l2_f_error": 1.8566744087723585e-14, + "relative_l2_e_error": 4.172291687802881e-14, + "max_abs_e_error": 3.424344141578217e-14, + "hole_counts": [ + 130, + 50 + ], + "reference_hole_counts": [ + 130, + 50 + ], + "hole_counts_match": true, + "passed": true, + "limits": { + "max_abs_f_error": 1e-07, + "relative_l2_f_error": 1e-08, + "relative_l2_e_error": 1e-07 + }, + "benchmark": { + "backend": "cuda", + "device": "NVIDIA A100-SXM4-40GB", + "precision": "float64", + "mode": "landau", + "integrator": "sixth", + "nu": 1e-12, + "velocity": "cubic", + "layout": "halos", + "cell_centred": true, + "max_halo_width": 4, + "rank0_halo_sent_bytes": 215040000, + "halo_sent_bytes_per_rank_max": 430080000, + "slab_sent_bytes_per_rank": 16515072000, + "input": "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/initial.bin", + "nx": 134400, + "nv": 4096, + "ranks": 4, + "nodes": 1, + "steps": 10, + "warmup": 0, + "dt": 0.02, + "length": 12600, + "vmax": 15, + "amplitude": 0.01, + "acceleration": 0.2, + "host_mpi": false, + "state_bytes": 4404019200, + "max_allocated_bytes_per_rank": 11155281976, + "stream_ms": 96.76741280004535, + "spline_ms": 29.37165490020561, + "pack_ms": 6.529228900035378, + "mpi_ms": 26.30602839999483, + "field_ms": 7.745825100028014, + "collision_ms": 14.48409960007666, + "step_ms": 179.6518952999122, + "initial_mass": 12597.6196507818, + "final_mass": 12597.62016522545, + "mass_relative_change": 4.083657516783715e-08, + "initial_kinetic_energy": 55713.43281125277, + "final_kinetic_energy": 55714.15660222915, + "initial_field_energy": 787.6538181371966, + "final_field_energy": 786.9554651665927, + "minimum_f": -0.1916308738890627, + "finite": true + }, + "command": [ + "srun", + "--jobid=60205333", + "--overlap", + "--nodes=1", + "--ntasks=4", + "--ntasks-per-node=4", + "--gpus-per-node=4", + "--gpu-bind=none", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/workdir/cuda-build/vlasov_bench", + "--nx", + "134400", + "--nv", + "4096", + "--length", + "12600", + "--vmax", + "15", + "--dt", + "0.02", + "--nu", + "1e-12", + "--cells", + "--velocity", + "cubic", + "--integrator", + "sixth", + "--layout", + "halos", + "--steps", + "10", + "--warmup", + "0", + "--input", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/initial.bin", + "--v-grid", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/v.bin", + "--background", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/background.bin", + "--dump", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/comparison-nx134400-window/1n-4g-halos-0010.bin" + ], + "rank_log": "rank=2 host=nid001024 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001024 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001024 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001024 local_rank=0 device=NVIDIA A100-SXM4-40GB\n" + }, + { + "nodes": 1, + "ranks": 4, + "layout": "halos", + "steps": 100, + "max_abs_f_error": 6.725869861057276e-13, + "relative_l2_f_error": 9.515283499791307e-14, + "relative_l2_e_error": 4.921388303823267e-14, + "max_abs_e_error": 4.907185768843192e-14, + "hole_counts": [ + 129, + 47 + ], + "reference_hole_counts": [ + 129, + 47 + ], + "hole_counts_match": true, + "passed": true, + "limits": { + "max_abs_f_error": 1e-07, + "relative_l2_f_error": 1e-08, + "relative_l2_e_error": 1e-07 + }, + "benchmark": { + "backend": "cuda", + "device": "NVIDIA A100-SXM4-40GB", + "precision": "float64", + "mode": "landau", + "integrator": "sixth", + "nu": 1e-12, + "velocity": "cubic", + "layout": "halos", + "cell_centred": true, + "max_halo_width": 4, + "rank0_halo_sent_bytes": 2150400000, + "halo_sent_bytes_per_rank_max": 4300800000, + "slab_sent_bytes_per_rank": 165150720000, + "input": "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/initial.bin", + "nx": 134400, + "nv": 4096, + "ranks": 4, + "nodes": 1, + "steps": 100, + "warmup": 0, + "dt": 0.02, + "length": 12600, + "vmax": 15, + "amplitude": 0.01, + "acceleration": 0.2, + "host_mpi": false, + "state_bytes": 4404019200, + "max_allocated_bytes_per_rank": 11155281976, + "stream_ms": 96.76782525001272, + "spline_ms": 29.39105241994184, + "pack_ms": 6.511805909976829, + "mpi_ms": 21.13449154009231, + "field_ms": 6.609109270007139, + "collision_ms": 14.49049542000466, + "step_ms": 174.3556135900099, + "initial_mass": 12597.6196507818, + "final_mass": 12597.61289309191, + "mass_relative_change": -5.364259347188723e-07, + "initial_kinetic_energy": 55713.43281125277, + "final_kinetic_energy": 55718.73788187712, + "initial_field_energy": 787.6538181371966, + "final_field_energy": 781.820420166659, + "minimum_f": -0.1919615815105848, + "finite": true + }, + "command": [ + "srun", + "--jobid=60205333", + "--overlap", + "--nodes=1", + "--ntasks=4", + "--ntasks-per-node=4", + "--gpus-per-node=4", + "--gpu-bind=none", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/workdir/cuda-build/vlasov_bench", + "--nx", + "134400", + "--nv", + "4096", + "--length", + "12600", + "--vmax", + "15", + "--dt", + "0.02", + "--nu", + "1e-12", + "--cells", + "--velocity", + "cubic", + "--integrator", + "sixth", + "--layout", + "halos", + "--steps", + "100", + "--warmup", + "0", + "--input", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/initial.bin", + "--v-grid", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/v.bin", + "--background", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/background.bin", + "--dump", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/comparison-nx134400-window/1n-4g-halos-0100.bin" + ], + "rank_log": "rank=0 host=nid001025 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001025 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001025 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001025 local_rank=3 device=NVIDIA A100-SXM4-40GB\n" + }, + { + "nodes": 2, + "ranks": 8, + "layout": "transpose", + "steps": 10, + "max_abs_f_error": 1.1882161921050738e-13, + "relative_l2_f_error": 1.854096980472526e-14, + "relative_l2_e_error": 3.8024486397472564e-14, + "max_abs_e_error": 3.226585665316861e-14, + "hole_counts": [ + 130, + 50 + ], + "reference_hole_counts": [ + 130, + 50 + ], + "hole_counts_match": true, + "passed": true, + "limits": { + "max_abs_f_error": 1e-07, + "relative_l2_f_error": 1e-08, + "relative_l2_e_error": 1e-07 + }, + "benchmark": { + "backend": "cuda", + "device": "NVIDIA A100-SXM4-40GB", + "precision": "float64", + "mode": "landau", + "integrator": "sixth", + "nu": 1e-12, + "velocity": "cubic", + "layout": "transpose", + "cell_centred": true, + "max_halo_width": 0, + "rank0_halo_sent_bytes": 0, + "halo_sent_bytes_per_rank_max": 0, + "slab_sent_bytes_per_rank": 67436544000, + "input": "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/initial.bin", + "nx": 134400, + "nv": 4096, + "ranks": 8, + "nodes": 2, + "steps": 10, + "warmup": 0, + "dt": 0.02, + "length": 12600, + "vmax": 15, + "amplitude": 0.01, + "acceleration": 0.2, + "host_mpi": false, + "state_bytes": 4404019200, + "max_allocated_bytes_per_rank": 5512607800, + "stream_ms": 48.5531004999757, + "spline_ms": 11.72833750001701, + "pack_ms": 22.98895100011578, + "mpi_ms": 392.2544488003041, + "field_ms": 12.02276010003516, + "collision_ms": 11.25639949996184, + "step_ms": 491.9087526999647, + "initial_mass": 12597.61965078436, + "final_mass": 12597.62016522508, + "mass_relative_change": 4.083634327505363e-08, + "initial_kinetic_energy": 55713.43281126581, + "final_kinetic_energy": 55714.15660220217, + "initial_field_energy": 787.6538181371961, + "final_field_energy": 786.9554651665925, + "minimum_f": -0.1916308738890627, + "finite": true + }, + "command": [ + "srun", + "--jobid=60205333", + "--overlap", + "--nodes=2", + "--ntasks=8", + "--ntasks-per-node=4", + "--gpus-per-node=4", + "--gpu-bind=none", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/workdir/cuda-build/vlasov_bench", + "--nx", + "134400", + "--nv", + "4096", + "--length", + "12600", + "--vmax", + "15", + "--dt", + "0.02", + "--nu", + "1e-12", + "--cells", + "--velocity", + "cubic", + "--integrator", + "sixth", + "--layout", + "transpose", + "--steps", + "10", + "--warmup", + "0", + "--input", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/initial.bin", + "--v-grid", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/v.bin", + "--background", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/background.bin", + "--dump", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/comparison-nx134400-window/2n-8g-transpose-0010.bin" + ], + "rank_log": "rank=4 host=nid001024 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=6 host=nid001024 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001024 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001025 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=7 host=nid001025 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001024 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=5 host=nid001025 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001025 local_rank=0 device=NVIDIA A100-SXM4-40GB\n" + }, + { + "nodes": 2, + "ranks": 8, + "layout": "transpose", + "steps": 100, + "max_abs_f_error": 6.686595721561162e-13, + "relative_l2_f_error": 9.400228503663682e-14, + "relative_l2_e_error": 5.6328186846468416e-14, + "max_abs_e_error": 4.96269692007445e-14, + "hole_counts": [ + 129, + 47 + ], + "reference_hole_counts": [ + 129, + 47 + ], + "hole_counts_match": true, + "passed": true, + "limits": { + "max_abs_f_error": 1e-07, + "relative_l2_f_error": 1e-08, + "relative_l2_e_error": 1e-07 + }, + "benchmark": { + "backend": "cuda", + "device": "NVIDIA A100-SXM4-40GB", + "precision": "float64", + "mode": "landau", + "integrator": "sixth", + "nu": 1e-12, + "velocity": "cubic", + "layout": "transpose", + "cell_centred": true, + "max_halo_width": 0, + "rank0_halo_sent_bytes": 0, + "halo_sent_bytes_per_rank_max": 0, + "slab_sent_bytes_per_rank": 674365440000, + "input": "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/initial.bin", + "nx": 134400, + "nv": 4096, + "ranks": 8, + "nodes": 2, + "steps": 100, + "warmup": 0, + "dt": 0.02, + "length": 12600, + "vmax": 15, + "amplitude": 0.01, + "acceleration": 0.2, + "host_mpi": false, + "state_bytes": 4404019200, + "max_allocated_bytes_per_rank": 5512607800, + "stream_ms": 48.54172135000226, + "spline_ms": 11.71508251993146, + "pack_ms": 22.96030381008222, + "mpi_ms": 390.6149536599469, + "field_ms": 12.35562636996747, + "collision_ms": 11.23707952998757, + "step_ms": 490.0134217499863, + "initial_mass": 12597.61965078436, + "final_mass": 12597.61289309384, + "mass_relative_change": -5.364259848226194e-07, + "initial_kinetic_energy": 55713.43281126581, + "final_kinetic_energy": 55718.73788187286, + "initial_field_energy": 787.6538181371961, + "final_field_energy": 781.8204201666606, + "minimum_f": -0.1919615815105861, + "finite": true + }, + "command": [ + "srun", + "--jobid=60205333", + "--overlap", + "--nodes=2", + "--ntasks=8", + "--ntasks-per-node=4", + "--gpus-per-node=4", + "--gpu-bind=none", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/workdir/cuda-build/vlasov_bench", + "--nx", + "134400", + "--nv", + "4096", + "--length", + "12600", + "--vmax", + "15", + "--dt", + "0.02", + "--nu", + "1e-12", + "--cells", + "--velocity", + "cubic", + "--integrator", + "sixth", + "--layout", + "transpose", + "--steps", + "100", + "--warmup", + "0", + "--input", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/initial.bin", + "--v-grid", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/v.bin", + "--background", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/background.bin", + "--dump", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/comparison-nx134400-window/2n-8g-transpose-0100.bin" + ], + "rank_log": "rank=7 host=nid001025 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001025 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001025 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=5 host=nid001025 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001024 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001024 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=6 host=nid001024 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=4 host=nid001024 local_rank=2 device=NVIDIA A100-SXM4-40GB\n" + }, + { + "nodes": 2, + "ranks": 8, + "layout": "halos", + "steps": 10, + "max_abs_f_error": 1.1882161921050738e-13, + "relative_l2_f_error": 1.854096980472526e-14, + "relative_l2_e_error": 3.8024486397472564e-14, + "max_abs_e_error": 3.226585665316861e-14, + "hole_counts": [ + 130, + 50 + ], + "reference_hole_counts": [ + 130, + 50 + ], + "hole_counts_match": true, + "passed": true, + "limits": { + "max_abs_f_error": 1e-07, + "relative_l2_f_error": 1e-08, + "relative_l2_e_error": 1e-07 + }, + "benchmark": { + "backend": "cuda", + "device": "NVIDIA A100-SXM4-40GB", + "precision": "float64", + "mode": "landau", + "integrator": "sixth", + "nu": 1e-12, + "velocity": "cubic", + "layout": "halos", + "cell_centred": true, + "max_halo_width": 4, + "rank0_halo_sent_bytes": 215040000, + "halo_sent_bytes_per_rank_max": 430080000, + "slab_sent_bytes_per_rank": 9633792000, + "input": "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/initial.bin", + "nx": 134400, + "nv": 4096, + "ranks": 8, + "nodes": 2, + "steps": 10, + "warmup": 0, + "dt": 0.02, + "length": 12600, + "vmax": 15, + "amplitude": 0.01, + "acceleration": 0.2, + "host_mpi": false, + "state_bytes": 4404019200, + "max_allocated_bytes_per_rank": 5650233400, + "stream_ms": 48.50393130009252, + "spline_ms": 12.2415565999745, + "pack_ms": 3.332494700134703, + "mpi_ms": 64.20001480000792, + "field_ms": 6.318310100277813, + "collision_ms": 11.24460030000591, + "step_ms": 144.2658396000297, + "initial_mass": 12597.61965078436, + "final_mass": 12597.62016522508, + "mass_relative_change": 4.083634327505363e-08, + "initial_kinetic_energy": 55713.43281126581, + "final_kinetic_energy": 55714.15660220217, + "initial_field_energy": 787.6538181371961, + "final_field_energy": 786.9554651665925, + "minimum_f": -0.1916308738890627, + "finite": true + }, + "command": [ + "srun", + "--jobid=60205333", + "--overlap", + "--nodes=2", + "--ntasks=8", + "--ntasks-per-node=4", + "--gpus-per-node=4", + "--gpu-bind=none", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/workdir/cuda-build/vlasov_bench", + "--nx", + "134400", + "--nv", + "4096", + "--length", + "12600", + "--vmax", + "15", + "--dt", + "0.02", + "--nu", + "1e-12", + "--cells", + "--velocity", + "cubic", + "--integrator", + "sixth", + "--layout", + "halos", + "--steps", + "10", + "--warmup", + "0", + "--input", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/initial.bin", + "--v-grid", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/v.bin", + "--background", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/background.bin", + "--dump", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/comparison-nx134400-window/2n-8g-halos-0010.bin" + ], + "rank_log": "rank=7 host=nid001025 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001025 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001025 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=6 host=nid001024 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=5 host=nid001025 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001024 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001024 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=4 host=nid001024 local_rank=2 device=NVIDIA A100-SXM4-40GB\n" + }, + { + "nodes": 2, + "ranks": 8, + "layout": "halos", + "steps": 100, + "max_abs_f_error": 6.686595721561162e-13, + "relative_l2_f_error": 9.400228503663682e-14, + "relative_l2_e_error": 5.6328186846468416e-14, + "max_abs_e_error": 4.96269692007445e-14, + "hole_counts": [ + 129, + 47 + ], + "reference_hole_counts": [ + 129, + 47 + ], + "hole_counts_match": true, + "passed": true, + "limits": { + "max_abs_f_error": 1e-07, + "relative_l2_f_error": 1e-08, + "relative_l2_e_error": 1e-07 + }, + "benchmark": { + "backend": "cuda", + "device": "NVIDIA A100-SXM4-40GB", + "precision": "float64", + "mode": "landau", + "integrator": "sixth", + "nu": 1e-12, + "velocity": "cubic", + "layout": "halos", + "cell_centred": true, + "max_halo_width": 4, + "rank0_halo_sent_bytes": 2150400000, + "halo_sent_bytes_per_rank_max": 4300800000, + "slab_sent_bytes_per_rank": 96337920000, + "input": "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/initial.bin", + "nx": 134400, + "nv": 4096, + "ranks": 8, + "nodes": 2, + "steps": 100, + "warmup": 0, + "dt": 0.02, + "length": 12600, + "vmax": 15, + "amplitude": 0.01, + "acceleration": 0.2, + "host_mpi": false, + "state_bytes": 4404019200, + "max_allocated_bytes_per_rank": 5650233400, + "stream_ms": 48.49508005004736, + "spline_ms": 12.23889673997746, + "pack_ms": 3.278673720083134, + "mpi_ms": 60.91060809000737, + "field_ms": 6.122944579915384, + "collision_ms": 11.23616700002458, + "step_ms": 140.7295966299989, + "initial_mass": 12597.61965078436, + "final_mass": 12597.61289309384, + "mass_relative_change": -5.364259848226194e-07, + "initial_kinetic_energy": 55713.43281126581, + "final_kinetic_energy": 55718.73788187286, + "initial_field_energy": 787.6538181371961, + "final_field_energy": 781.8204201666606, + "minimum_f": -0.1919615815105861, + "finite": true + }, + "command": [ + "srun", + "--jobid=60205333", + "--overlap", + "--nodes=2", + "--ntasks=8", + "--ntasks-per-node=4", + "--gpus-per-node=4", + "--gpu-bind=none", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/workdir/cuda-build/vlasov_bench", + "--nx", + "134400", + "--nv", + "4096", + "--length", + "12600", + "--vmax", + "15", + "--dt", + "0.02", + "--nu", + "1e-12", + "--cells", + "--velocity", + "cubic", + "--integrator", + "sixth", + "--layout", + "halos", + "--steps", + "100", + "--warmup", + "0", + "--input", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/initial.bin", + "--v-grid", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/v.bin", + "--background", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx134400/background.bin", + "--dump", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/comparison-nx134400-window/2n-8g-halos-0100.bin" + ], + "rank_log": "rank=6 host=nid001024 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=4 host=nid001024 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001024 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001024 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001025 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001025 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=5 host=nid001025 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=7 host=nid001025 local_rank=3 device=NVIDIA A100-SXM4-40GB\n" + } +] diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-halos/comparison-nx268800-1node.json b/benchmarks/vlasov_cuda/results/2026-10-24-halos/comparison-nx268800-1node.json new file mode 100644 index 00000000..28872c84 --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-halos/comparison-nx268800-1node.json @@ -0,0 +1,462 @@ +[ + { + "nodes": 1, + "ranks": 4, + "layout": "transpose", + "steps": 1, + "max_abs_f_error": 4.1300296516055823e-14, + "relative_l2_f_error": 5.6935482508837695e-15, + "relative_l2_e_error": 5.236955259389934e-15, + "max_abs_e_error": 5.578870698741412e-15, + "hole_counts": [ + 118, + 54 + ], + "reference_hole_counts": [ + 118, + 54 + ], + "hole_counts_match": true, + "passed": true, + "limits": { + "max_abs_f_error": 1e-07, + "relative_l2_f_error": 1e-08, + "relative_l2_e_error": 1e-07 + }, + "benchmark": { + "backend": "cuda", + "device": "NVIDIA A100-SXM4-40GB", + "precision": "float64", + "mode": "landau", + "integrator": "sixth", + "nu": 1e-12, + "velocity": "cubic", + "layout": "transpose", + "cell_centred": true, + "max_halo_width": 0, + "rank0_halo_sent_bytes": 0, + "halo_sent_bytes_per_rank_max": 0, + "slab_sent_bytes_per_rank": 23121100800, + "input": "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/initial.bin", + "nx": 268800, + "nv": 4096, + "ranks": 4, + "nodes": 1, + "steps": 1, + "warmup": 0, + "dt": 0.02, + "length": 12600, + "vmax": 15, + "amplitude": 0.01, + "acceleration": 0.2, + "host_mpi": false, + "state_bytes": 8808038400, + "max_allocated_bytes_per_rank": 22035230776, + "stream_ms": 194.5556020002641, + "spline_ms": 57.92932199983625, + "pack_ms": 91.67665799964198, + "mpi_ms": 276.4357230003043, + "field_ms": 11.46622900000693, + "collision_ms": 21.33707100006177, + "step_ms": 653.1722329998502, + "initial_mass": 12597.62207264873, + "final_mass": 12597.62199624369, + "mass_relative_change": -6.065036546636737e-09, + "initial_kinetic_energy": 55701.51067944818, + "final_kinetic_energy": 55701.4580005624, + "initial_field_energy": 791.8606180599772, + "final_field_energy": 791.9073734470658, + "minimum_f": -0.09528937374122746, + "finite": true + }, + "command": [ + "srun", + "--jobid=60205333", + "--overlap", + "--nodes=1", + "--ntasks=4", + "--ntasks-per-node=4", + "--gpus-per-node=4", + "--gpu-bind=none", + "--cpu-bind=none", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/workdir/cuda-build/vlasov_bench", + "--nx", + "268800", + "--nv", + "4096", + "--length", + "12600", + "--vmax", + "15", + "--dt", + "0.02", + "--nu", + "1e-12", + "--cells", + "--velocity", + "cubic", + "--integrator", + "sixth", + "--layout", + "transpose", + "--steps", + "1", + "--warmup", + "0", + "--input", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/initial.bin", + "--v-grid", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/v.bin", + "--background", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/background.bin", + "--dump", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/comparison-nx268800-window/1n-4g-transpose-0001.bin" + ], + "rank_log": "rank=1 host=nid001024 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001024 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001024 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001024 local_rank=0 device=NVIDIA A100-SXM4-40GB\n" + }, + { + "nodes": 1, + "ranks": 4, + "layout": "transpose", + "steps": 100, + "max_abs_f_error": 5.11674036474119e-13, + "relative_l2_f_error": 7.512052460545957e-14, + "relative_l2_e_error": 4.687552521565683e-14, + "max_abs_e_error": 4.2334191707738e-14, + "hole_counts": [ + 129, + 50 + ], + "reference_hole_counts": [ + 129, + 50 + ], + "hole_counts_match": true, + "passed": true, + "limits": { + "max_abs_f_error": 1e-07, + "relative_l2_f_error": 1e-08, + "relative_l2_e_error": 1e-07 + }, + "benchmark": { + "backend": "cuda", + "device": "NVIDIA A100-SXM4-40GB", + "precision": "float64", + "mode": "landau", + "integrator": "sixth", + "nu": 1e-12, + "velocity": "cubic", + "layout": "transpose", + "cell_centred": true, + "max_halo_width": 0, + "rank0_halo_sent_bytes": 0, + "halo_sent_bytes_per_rank_max": 0, + "slab_sent_bytes_per_rank": 2312110080000, + "input": "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/initial.bin", + "nx": 268800, + "nv": 4096, + "ranks": 4, + "nodes": 1, + "steps": 100, + "warmup": 0, + "dt": 0.02, + "length": 12600, + "vmax": 15, + "amplitude": 0.01, + "acceleration": 0.2, + "host_mpi": false, + "state_bytes": 8808038400, + "max_allocated_bytes_per_rank": 22035230776, + "stream_ms": 194.5113329900028, + "spline_ms": 58.0104210499826, + "pack_ms": 91.58659695997585, + "mpi_ms": 273.2179411500283, + "field_ms": 11.19875519001198, + "collision_ms": 21.1204347399962, + "step_ms": 649.6034333900138, + "initial_mass": 12597.62207264873, + "final_mass": 12597.6175104671, + "mass_relative_change": -3.621462523818079e-07, + "initial_kinetic_energy": 55701.51067944818, + "final_kinetic_energy": 55695.27553237718, + "initial_field_energy": 791.8606180599772, + "final_field_energy": 797.6672297926372, + "minimum_f": -0.09642050941611767, + "finite": true + }, + "command": [ + "srun", + "--jobid=60205333", + "--overlap", + "--nodes=1", + "--ntasks=4", + "--ntasks-per-node=4", + "--gpus-per-node=4", + "--gpu-bind=none", + "--cpu-bind=none", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/workdir/cuda-build/vlasov_bench", + "--nx", + "268800", + "--nv", + "4096", + "--length", + "12600", + "--vmax", + "15", + "--dt", + "0.02", + "--nu", + "1e-12", + "--cells", + "--velocity", + "cubic", + "--integrator", + "sixth", + "--layout", + "transpose", + "--steps", + "100", + "--warmup", + "0", + "--input", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/initial.bin", + "--v-grid", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/v.bin", + "--background", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/background.bin", + "--dump", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/comparison-nx268800-window/1n-4g-transpose-0100.bin" + ], + "rank_log": "rank=2 host=nid001025 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001025 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001025 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001025 local_rank=0 device=NVIDIA A100-SXM4-40GB\n" + }, + { + "nodes": 1, + "ranks": 4, + "layout": "halos", + "steps": 1, + "max_abs_f_error": 4.1300296516055823e-14, + "relative_l2_f_error": 5.6935482508837695e-15, + "relative_l2_e_error": 5.236955259389934e-15, + "max_abs_e_error": 5.578870698741412e-15, + "hole_counts": [ + 118, + 54 + ], + "reference_hole_counts": [ + 118, + 54 + ], + "hole_counts_match": true, + "passed": true, + "limits": { + "max_abs_f_error": 1e-07, + "relative_l2_f_error": 1e-08, + "relative_l2_e_error": 1e-07 + }, + "benchmark": { + "backend": "cuda", + "device": "NVIDIA A100-SXM4-40GB", + "precision": "float64", + "mode": "landau", + "integrator": "sixth", + "nu": 1e-12, + "velocity": "cubic", + "layout": "halos", + "cell_centred": true, + "max_halo_width": 4, + "rank0_halo_sent_bytes": 43008000, + "halo_sent_bytes_per_rank_max": 86016000, + "slab_sent_bytes_per_rank": 3303014400, + "input": "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/initial.bin", + "nx": 268800, + "nv": 4096, + "ranks": 4, + "nodes": 1, + "steps": 1, + "warmup": 0, + "dt": 0.02, + "length": 12600, + "vmax": 15, + "amplitude": 0.01, + "acceleration": 0.2, + "host_mpi": false, + "state_bytes": 8808038400, + "max_allocated_bytes_per_rank": 22310481976, + "stream_ms": 194.4458430000395, + "spline_ms": 61.24707199978729, + "pack_ms": 13.11971299992365, + "mpi_ms": 97.95531999975537, + "field_ms": 12.17473299948324, + "collision_ms": 21.21514399982516, + "step_ms": 399.1295580001406, + "initial_mass": 12597.62207264873, + "final_mass": 12597.62199624369, + "mass_relative_change": -6.065036546636737e-09, + "initial_kinetic_energy": 55701.51067944818, + "final_kinetic_energy": 55701.4580005624, + "initial_field_energy": 791.8606180599772, + "final_field_energy": 791.9073734470658, + "minimum_f": -0.09528937374122746, + "finite": true + }, + "command": [ + "srun", + "--jobid=60205333", + "--overlap", + "--nodes=1", + "--ntasks=4", + "--ntasks-per-node=4", + "--gpus-per-node=4", + "--gpu-bind=none", + "--cpu-bind=none", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/workdir/cuda-build/vlasov_bench", + "--nx", + "268800", + "--nv", + "4096", + "--length", + "12600", + "--vmax", + "15", + "--dt", + "0.02", + "--nu", + "1e-12", + "--cells", + "--velocity", + "cubic", + "--integrator", + "sixth", + "--layout", + "halos", + "--steps", + "1", + "--warmup", + "0", + "--input", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/initial.bin", + "--v-grid", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/v.bin", + "--background", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/background.bin", + "--dump", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/comparison-nx268800-window/1n-4g-halos-0001.bin" + ], + "rank_log": "rank=3 host=nid001024 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001024 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001024 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001024 local_rank=2 device=NVIDIA A100-SXM4-40GB\n" + }, + { + "nodes": 1, + "ranks": 4, + "layout": "halos", + "steps": 100, + "max_abs_f_error": 5.11674036474119e-13, + "relative_l2_f_error": 7.512052460545957e-14, + "relative_l2_e_error": 4.687552521565683e-14, + "max_abs_e_error": 4.2334191707738e-14, + "hole_counts": [ + 129, + 50 + ], + "reference_hole_counts": [ + 129, + 50 + ], + "hole_counts_match": true, + "passed": true, + "limits": { + "max_abs_f_error": 1e-07, + "relative_l2_f_error": 1e-08, + "relative_l2_e_error": 1e-07 + }, + "benchmark": { + "backend": "cuda", + "device": "NVIDIA A100-SXM4-40GB", + "precision": "float64", + "mode": "landau", + "integrator": "sixth", + "nu": 1e-12, + "velocity": "cubic", + "layout": "halos", + "cell_centred": true, + "max_halo_width": 4, + "rank0_halo_sent_bytes": 4584652800, + "halo_sent_bytes_per_rank_max": 9169305600, + "slab_sent_bytes_per_rank": 330301440000, + "input": "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/initial.bin", + "nx": 268800, + "nv": 4096, + "ranks": 4, + "nodes": 1, + "steps": 100, + "warmup": 0, + "dt": 0.02, + "length": 12600, + "vmax": 15, + "amplitude": 0.01, + "acceleration": 0.2, + "host_mpi": false, + "state_bytes": 8808038400, + "max_allocated_bytes_per_rank": 22310481976, + "stream_ms": 194.4447632700076, + "spline_ms": 61.47456958001158, + "pack_ms": 13.08143660998667, + "mpi_ms": 41.6358091600182, + "field_ms": 12.248491260018, + "collision_ms": 21.12072830000443, + "step_ms": 342.950239549989, + "initial_mass": 12597.62207264873, + "final_mass": 12597.6175104671, + "mass_relative_change": -3.621462523818079e-07, + "initial_kinetic_energy": 55701.51067944818, + "final_kinetic_energy": 55695.27553237718, + "initial_field_energy": 791.8606180599772, + "final_field_energy": 797.6672297926372, + "minimum_f": -0.09642050941611767, + "finite": true + }, + "command": [ + "srun", + "--jobid=60205333", + "--overlap", + "--nodes=1", + "--ntasks=4", + "--ntasks-per-node=4", + "--gpus-per-node=4", + "--gpu-bind=none", + "--cpu-bind=none", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/workdir/cuda-build/vlasov_bench", + "--nx", + "268800", + "--nv", + "4096", + "--length", + "12600", + "--vmax", + "15", + "--dt", + "0.02", + "--nu", + "1e-12", + "--cells", + "--velocity", + "cubic", + "--integrator", + "sixth", + "--layout", + "halos", + "--steps", + "100", + "--warmup", + "0", + "--input", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/initial.bin", + "--v-grid", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/v.bin", + "--background", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/background.bin", + "--dump", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/comparison-nx268800-window/1n-4g-halos-0100.bin" + ], + "rank_log": "rank=1 host=nid001025 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001025 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001025 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001025 local_rank=3 device=NVIDIA A100-SXM4-40GB\n" + } +] diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-halos/comparison-nx268800-window.json b/benchmarks/vlasov_cuda/results/2026-10-24-halos/comparison-nx268800-window.json new file mode 100644 index 00000000..6afd1f12 --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-halos/comparison-nx268800-window.json @@ -0,0 +1,922 @@ +[ + { + "nodes": 1, + "ranks": 4, + "layout": "transpose", + "steps": 1, + "max_abs_f_error": 4.1300296516055823e-14, + "relative_l2_f_error": 5.6935482508837695e-15, + "relative_l2_e_error": 5.236955259389934e-15, + "max_abs_e_error": 5.578870698741412e-15, + "hole_counts": [ + 118, + 54 + ], + "reference_hole_counts": [ + 118, + 54 + ], + "hole_counts_match": true, + "passed": true, + "limits": { + "max_abs_f_error": 1e-07, + "relative_l2_f_error": 1e-08, + "relative_l2_e_error": 1e-07 + }, + "benchmark": { + "backend": "cuda", + "device": "NVIDIA A100-SXM4-40GB", + "precision": "float64", + "mode": "landau", + "integrator": "sixth", + "nu": 1e-12, + "velocity": "cubic", + "layout": "transpose", + "cell_centred": true, + "max_halo_width": 0, + "rank0_halo_sent_bytes": 0, + "halo_sent_bytes_per_rank_max": 0, + "slab_sent_bytes_per_rank": 23121100800, + "input": "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/initial.bin", + "nx": 268800, + "nv": 4096, + "ranks": 4, + "nodes": 1, + "steps": 1, + "warmup": 0, + "dt": 0.02, + "length": 12600, + "vmax": 15, + "amplitude": 0.01, + "acceleration": 0.2, + "host_mpi": false, + "state_bytes": 8808038400, + "max_allocated_bytes_per_rank": 22035230776, + "stream_ms": 194.5556020002641, + "spline_ms": 57.92932199983625, + "pack_ms": 91.67665799964198, + "mpi_ms": 276.4357230003043, + "field_ms": 11.46622900000693, + "collision_ms": 21.33707100006177, + "step_ms": 653.1722329998502, + "initial_mass": 12597.62207264873, + "final_mass": 12597.62199624369, + "mass_relative_change": -6.065036546636737e-09, + "initial_kinetic_energy": 55701.51067944818, + "final_kinetic_energy": 55701.4580005624, + "initial_field_energy": 791.8606180599772, + "final_field_energy": 791.9073734470658, + "minimum_f": -0.09528937374122746, + "finite": true + }, + "command": [ + "srun", + "--jobid=60205333", + "--overlap", + "--nodes=1", + "--ntasks=4", + "--ntasks-per-node=4", + "--gpus-per-node=4", + "--gpu-bind=none", + "--cpu-bind=none", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/workdir/cuda-build/vlasov_bench", + "--nx", + "268800", + "--nv", + "4096", + "--length", + "12600", + "--vmax", + "15", + "--dt", + "0.02", + "--nu", + "1e-12", + "--cells", + "--velocity", + "cubic", + "--integrator", + "sixth", + "--layout", + "transpose", + "--steps", + "1", + "--warmup", + "0", + "--input", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/initial.bin", + "--v-grid", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/v.bin", + "--background", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/background.bin", + "--dump", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/comparison-nx268800-window/1n-4g-transpose-0001.bin" + ], + "rank_log": "rank=1 host=nid001024 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001024 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001024 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001024 local_rank=0 device=NVIDIA A100-SXM4-40GB\n" + }, + { + "nodes": 1, + "ranks": 4, + "layout": "transpose", + "steps": 100, + "max_abs_f_error": 5.11674036474119e-13, + "relative_l2_f_error": 7.512052460545957e-14, + "relative_l2_e_error": 4.687552521565683e-14, + "max_abs_e_error": 4.2334191707738e-14, + "hole_counts": [ + 129, + 50 + ], + "reference_hole_counts": [ + 129, + 50 + ], + "hole_counts_match": true, + "passed": true, + "limits": { + "max_abs_f_error": 1e-07, + "relative_l2_f_error": 1e-08, + "relative_l2_e_error": 1e-07 + }, + "benchmark": { + "backend": "cuda", + "device": "NVIDIA A100-SXM4-40GB", + "precision": "float64", + "mode": "landau", + "integrator": "sixth", + "nu": 1e-12, + "velocity": "cubic", + "layout": "transpose", + "cell_centred": true, + "max_halo_width": 0, + "rank0_halo_sent_bytes": 0, + "halo_sent_bytes_per_rank_max": 0, + "slab_sent_bytes_per_rank": 2312110080000, + "input": "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/initial.bin", + "nx": 268800, + "nv": 4096, + "ranks": 4, + "nodes": 1, + "steps": 100, + "warmup": 0, + "dt": 0.02, + "length": 12600, + "vmax": 15, + "amplitude": 0.01, + "acceleration": 0.2, + "host_mpi": false, + "state_bytes": 8808038400, + "max_allocated_bytes_per_rank": 22035230776, + "stream_ms": 194.5113329900028, + "spline_ms": 58.0104210499826, + "pack_ms": 91.58659695997585, + "mpi_ms": 273.2179411500283, + "field_ms": 11.19875519001198, + "collision_ms": 21.1204347399962, + "step_ms": 649.6034333900138, + "initial_mass": 12597.62207264873, + "final_mass": 12597.6175104671, + "mass_relative_change": -3.621462523818079e-07, + "initial_kinetic_energy": 55701.51067944818, + "final_kinetic_energy": 55695.27553237718, + "initial_field_energy": 791.8606180599772, + "final_field_energy": 797.6672297926372, + "minimum_f": -0.09642050941611767, + "finite": true + }, + "command": [ + "srun", + "--jobid=60205333", + "--overlap", + "--nodes=1", + "--ntasks=4", + "--ntasks-per-node=4", + "--gpus-per-node=4", + "--gpu-bind=none", + "--cpu-bind=none", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/workdir/cuda-build/vlasov_bench", + "--nx", + "268800", + "--nv", + "4096", + "--length", + "12600", + "--vmax", + "15", + "--dt", + "0.02", + "--nu", + "1e-12", + "--cells", + "--velocity", + "cubic", + "--integrator", + "sixth", + "--layout", + "transpose", + "--steps", + "100", + "--warmup", + "0", + "--input", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/initial.bin", + "--v-grid", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/v.bin", + "--background", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/background.bin", + "--dump", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/comparison-nx268800-window/1n-4g-transpose-0100.bin" + ], + "rank_log": "rank=2 host=nid001025 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001025 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001025 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001025 local_rank=0 device=NVIDIA A100-SXM4-40GB\n" + }, + { + "nodes": 1, + "ranks": 4, + "layout": "halos", + "steps": 1, + "max_abs_f_error": 4.1300296516055823e-14, + "relative_l2_f_error": 5.6935482508837695e-15, + "relative_l2_e_error": 5.236955259389934e-15, + "max_abs_e_error": 5.578870698741412e-15, + "hole_counts": [ + 118, + 54 + ], + "reference_hole_counts": [ + 118, + 54 + ], + "hole_counts_match": true, + "passed": true, + "limits": { + "max_abs_f_error": 1e-07, + "relative_l2_f_error": 1e-08, + "relative_l2_e_error": 1e-07 + }, + "benchmark": { + "backend": "cuda", + "device": "NVIDIA A100-SXM4-40GB", + "precision": "float64", + "mode": "landau", + "integrator": "sixth", + "nu": 1e-12, + "velocity": "cubic", + "layout": "halos", + "cell_centred": true, + "max_halo_width": 4, + "rank0_halo_sent_bytes": 43008000, + "halo_sent_bytes_per_rank_max": 86016000, + "slab_sent_bytes_per_rank": 3303014400, + "input": "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/initial.bin", + "nx": 268800, + "nv": 4096, + "ranks": 4, + "nodes": 1, + "steps": 1, + "warmup": 0, + "dt": 0.02, + "length": 12600, + "vmax": 15, + "amplitude": 0.01, + "acceleration": 0.2, + "host_mpi": false, + "state_bytes": 8808038400, + "max_allocated_bytes_per_rank": 22310481976, + "stream_ms": 194.4458430000395, + "spline_ms": 61.24707199978729, + "pack_ms": 13.11971299992365, + "mpi_ms": 97.95531999975537, + "field_ms": 12.17473299948324, + "collision_ms": 21.21514399982516, + "step_ms": 399.1295580001406, + "initial_mass": 12597.62207264873, + "final_mass": 12597.62199624369, + "mass_relative_change": -6.065036546636737e-09, + "initial_kinetic_energy": 55701.51067944818, + "final_kinetic_energy": 55701.4580005624, + "initial_field_energy": 791.8606180599772, + "final_field_energy": 791.9073734470658, + "minimum_f": -0.09528937374122746, + "finite": true + }, + "command": [ + "srun", + "--jobid=60205333", + "--overlap", + "--nodes=1", + "--ntasks=4", + "--ntasks-per-node=4", + "--gpus-per-node=4", + "--gpu-bind=none", + "--cpu-bind=none", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/workdir/cuda-build/vlasov_bench", + "--nx", + "268800", + "--nv", + "4096", + "--length", + "12600", + "--vmax", + "15", + "--dt", + "0.02", + "--nu", + "1e-12", + "--cells", + "--velocity", + "cubic", + "--integrator", + "sixth", + "--layout", + "halos", + "--steps", + "1", + "--warmup", + "0", + "--input", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/initial.bin", + "--v-grid", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/v.bin", + "--background", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/background.bin", + "--dump", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/comparison-nx268800-window/1n-4g-halos-0001.bin" + ], + "rank_log": "rank=3 host=nid001024 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001024 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001024 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001024 local_rank=2 device=NVIDIA A100-SXM4-40GB\n" + }, + { + "nodes": 1, + "ranks": 4, + "layout": "halos", + "steps": 100, + "max_abs_f_error": 5.11674036474119e-13, + "relative_l2_f_error": 7.512052460545957e-14, + "relative_l2_e_error": 4.687552521565683e-14, + "max_abs_e_error": 4.2334191707738e-14, + "hole_counts": [ + 129, + 50 + ], + "reference_hole_counts": [ + 129, + 50 + ], + "hole_counts_match": true, + "passed": true, + "limits": { + "max_abs_f_error": 1e-07, + "relative_l2_f_error": 1e-08, + "relative_l2_e_error": 1e-07 + }, + "benchmark": { + "backend": "cuda", + "device": "NVIDIA A100-SXM4-40GB", + "precision": "float64", + "mode": "landau", + "integrator": "sixth", + "nu": 1e-12, + "velocity": "cubic", + "layout": "halos", + "cell_centred": true, + "max_halo_width": 4, + "rank0_halo_sent_bytes": 4584652800, + "halo_sent_bytes_per_rank_max": 9169305600, + "slab_sent_bytes_per_rank": 330301440000, + "input": "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/initial.bin", + "nx": 268800, + "nv": 4096, + "ranks": 4, + "nodes": 1, + "steps": 100, + "warmup": 0, + "dt": 0.02, + "length": 12600, + "vmax": 15, + "amplitude": 0.01, + "acceleration": 0.2, + "host_mpi": false, + "state_bytes": 8808038400, + "max_allocated_bytes_per_rank": 22310481976, + "stream_ms": 194.4447632700076, + "spline_ms": 61.47456958001158, + "pack_ms": 13.08143660998667, + "mpi_ms": 41.6358091600182, + "field_ms": 12.248491260018, + "collision_ms": 21.12072830000443, + "step_ms": 342.950239549989, + "initial_mass": 12597.62207264873, + "final_mass": 12597.6175104671, + "mass_relative_change": -3.621462523818079e-07, + "initial_kinetic_energy": 55701.51067944818, + "final_kinetic_energy": 55695.27553237718, + "initial_field_energy": 791.8606180599772, + "final_field_energy": 797.6672297926372, + "minimum_f": -0.09642050941611767, + "finite": true + }, + "command": [ + "srun", + "--jobid=60205333", + "--overlap", + "--nodes=1", + "--ntasks=4", + "--ntasks-per-node=4", + "--gpus-per-node=4", + "--gpu-bind=none", + "--cpu-bind=none", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/workdir/cuda-build/vlasov_bench", + "--nx", + "268800", + "--nv", + "4096", + "--length", + "12600", + "--vmax", + "15", + "--dt", + "0.02", + "--nu", + "1e-12", + "--cells", + "--velocity", + "cubic", + "--integrator", + "sixth", + "--layout", + "halos", + "--steps", + "100", + "--warmup", + "0", + "--input", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/initial.bin", + "--v-grid", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/v.bin", + "--background", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800-distributed/background.bin", + "--dump", + "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/comparison-nx268800-window/1n-4g-halos-0100.bin" + ], + "rank_log": "rank=1 host=nid001025 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001025 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001025 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001025 local_rank=3 device=NVIDIA A100-SXM4-40GB\n" + }, + { + "nodes": 2, + "ranks": 8, + "layout": "transpose", + "steps": 1, + "max_abs_f_error": 4.1272540940440194e-14, + "relative_l2_f_error": 5.693709254174748e-15, + "relative_l2_e_error": 5.726948268097046e-15, + "max_abs_e_error": 5.842548667089886e-15, + "hole_counts": [ + 118, + 54 + ], + "reference_hole_counts": [ + 118, + 54 + ], + "hole_counts_match": true, + "passed": true, + "limits": { + "max_abs_f_error": 1e-07, + "relative_l2_f_error": 1e-08, + "relative_l2_e_error": 1e-07 + }, + "benchmark": { + "backend": "cuda", + "device": "NVIDIA A100-SXM4-40GB", + "precision": "float64", + "mode": "landau", + "integrator": "sixth", + "nu": 1e-12, + "velocity": "cubic", + "layout": "transpose", + "cell_centred": true, + "max_halo_width": 0, + "rank0_halo_sent_bytes": 0, + "halo_sent_bytes_per_rank_max": 0, + "slab_sent_bytes_per_rank": 13487308800, + "input": "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/initial.bin", + "nx": 268800, + "nv": 4096, + "ranks": 8, + "nodes": 2, + "steps": 1, + "warmup": 0, + "dt": 0.02, + "length": 12600, + "vmax": 15, + "amplitude": 0.01, + "acceleration": 0.2, + "host_mpi": false, + "state_bytes": 8808038400, + "max_allocated_bytes_per_rank": 11025158200, + "stream_ms": 97.36238300001787, + "spline_ms": 23.15364699984457, + "pack_ms": 45.73831100060488, + "mpi_ms": 795.7124910005859, + "field_ms": 21.12155600025289, + "collision_ms": 14.52817399990636, + "step_ms": 983.8031960000535, + "initial_mass": 12597.62207265221, + "final_mass": 12597.62199624691, + "mass_relative_change": -6.065056761443461e-09, + "initial_kinetic_energy": 55701.51067942529, + "final_kinetic_energy": 55701.45800056863, + "initial_field_energy": 791.8606180599771, + "final_field_energy": 791.9073734470651, + "minimum_f": -0.0952893737412273, + "finite": true + }, + "command": [ + "srun", + "--jobid=60206959", + "--overlap", + "--nodes=2", + "--ntasks=8", + "--ntasks-per-node=4", + "--gpus-per-node=4", + "--gpu-bind=none", + "--cpu-bind=none", + "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/workdir/cuda-build/vlasov_bench", + "--nx", + "268800", + "--nv", + "4096", + "--length", + "12600", + "--vmax", + "15", + "--dt", + "0.02", + "--nu", + "1e-12", + "--cells", + "--velocity", + "cubic", + "--integrator", + "sixth", + "--layout", + "transpose", + "--steps", + "1", + "--warmup", + "0", + "--input", + "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/initial.bin", + "--v-grid", + "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/v.bin", + "--background", + "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/background.bin", + "--dump", + "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/comparison-nx268800-2node/2n-8g-transpose-0001.bin" + ], + "rank_log": "rank=6 host=nid001024 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=4 host=nid001024 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001024 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001024 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001025 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=7 host=nid001025 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001025 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=5 host=nid001025 local_rank=2 device=NVIDIA A100-SXM4-40GB\n" + }, + { + "nodes": 2, + "ranks": 8, + "layout": "transpose", + "steps": 100, + "max_abs_f_error": 5.144495940356819e-13, + "relative_l2_f_error": 7.864106656264876e-14, + "relative_l2_e_error": 4.782235170485902e-14, + "max_abs_e_error": 4.3395842475035806e-14, + "hole_counts": [ + 129, + 50 + ], + "reference_hole_counts": [ + 129, + 50 + ], + "hole_counts_match": true, + "passed": true, + "limits": { + "max_abs_f_error": 1e-07, + "relative_l2_f_error": 1e-08, + "relative_l2_e_error": 1e-07 + }, + "benchmark": { + "backend": "cuda", + "device": "NVIDIA A100-SXM4-40GB", + "precision": "float64", + "mode": "landau", + "integrator": "sixth", + "nu": 1e-12, + "velocity": "cubic", + "layout": "transpose", + "cell_centred": true, + "max_halo_width": 0, + "rank0_halo_sent_bytes": 0, + "halo_sent_bytes_per_rank_max": 0, + "slab_sent_bytes_per_rank": 1348730880000, + "input": "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/initial.bin", + "nx": 268800, + "nv": 4096, + "ranks": 8, + "nodes": 2, + "steps": 100, + "warmup": 0, + "dt": 0.02, + "length": 12600, + "vmax": 15, + "amplitude": 0.01, + "acceleration": 0.2, + "host_mpi": false, + "state_bytes": 8808038400, + "max_allocated_bytes_per_rank": 11025158200, + "stream_ms": 97.34923378998246, + "spline_ms": 23.1298673800211, + "pack_ms": 45.69342769000741, + "mpi_ms": 777.1080535100737, + "field_ms": 23.60504804998754, + "collision_ms": 14.4846032800001, + "step_ms": 965.6051862700042, + "initial_mass": 12597.62207265221, + "final_mass": 12597.61751046275, + "mass_relative_change": -3.621468742758491e-07, + "initial_kinetic_energy": 55701.51067942529, + "final_kinetic_energy": 55695.27553237438, + "initial_field_energy": 791.8606180599771, + "final_field_energy": 797.6672297926364, + "minimum_f": -0.09642050941611634, + "finite": true + }, + "command": [ + "srun", + "--jobid=60206959", + "--overlap", + "--nodes=2", + "--ntasks=8", + "--ntasks-per-node=4", + "--gpus-per-node=4", + "--gpu-bind=none", + "--cpu-bind=none", + "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/workdir/cuda-build/vlasov_bench", + "--nx", + "268800", + "--nv", + "4096", + "--length", + "12600", + "--vmax", + "15", + "--dt", + "0.02", + "--nu", + "1e-12", + "--cells", + "--velocity", + "cubic", + "--integrator", + "sixth", + "--layout", + "transpose", + "--steps", + "100", + "--warmup", + "0", + "--input", + "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/initial.bin", + "--v-grid", + "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/v.bin", + "--background", + "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/background.bin", + "--dump", + "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/comparison-nx268800-2node/2n-8g-transpose-0100.bin" + ], + "rank_log": "rank=2 host=nid001024 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001024 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=4 host=nid001024 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=6 host=nid001024 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=7 host=nid001025 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=5 host=nid001025 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001025 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001025 local_rank=1 device=NVIDIA A100-SXM4-40GB\n" + }, + { + "nodes": 2, + "ranks": 8, + "layout": "halos", + "steps": 1, + "max_abs_f_error": 4.1272540940440194e-14, + "relative_l2_f_error": 5.693709254174748e-15, + "relative_l2_e_error": 5.726948268097046e-15, + "max_abs_e_error": 5.842548667089886e-15, + "hole_counts": [ + 118, + 54 + ], + "reference_hole_counts": [ + 118, + 54 + ], + "hole_counts_match": true, + "passed": true, + "limits": { + "max_abs_f_error": 1e-07, + "relative_l2_f_error": 1e-08, + "relative_l2_e_error": 1e-07 + }, + "benchmark": { + "backend": "cuda", + "device": "NVIDIA A100-SXM4-40GB", + "precision": "float64", + "mode": "landau", + "integrator": "sixth", + "nu": 1e-12, + "velocity": "cubic", + "layout": "halos", + "cell_centred": true, + "max_halo_width": 4, + "rank0_halo_sent_bytes": 43008000, + "halo_sent_bytes_per_rank_max": 86016000, + "slab_sent_bytes_per_rank": 1926758400, + "input": "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/initial.bin", + "nx": 268800, + "nv": 4096, + "ranks": 8, + "nodes": 2, + "steps": 1, + "warmup": 0, + "dt": 0.02, + "length": 12600, + "vmax": 15, + "amplitude": 0.01, + "acceleration": 0.2, + "host_mpi": false, + "state_bytes": 8808038400, + "max_allocated_bytes_per_rank": 11300409400, + "stream_ms": 97.3275939998075, + "spline_ms": 30.09822499984693, + "pack_ms": 6.571679999979096, + "mpi_ms": 151.4998850000211, + "field_ms": 8.81993999973929, + "collision_ms": 14.53518300013457, + "step_ms": 308.1012009999995, + "initial_mass": 12597.62207265221, + "final_mass": 12597.62199624691, + "mass_relative_change": -6.065056761443461e-09, + "initial_kinetic_energy": 55701.51067942529, + "final_kinetic_energy": 55701.45800056863, + "initial_field_energy": 791.8606180599771, + "final_field_energy": 791.9073734470651, + "minimum_f": -0.0952893737412273, + "finite": true + }, + "command": [ + "srun", + "--jobid=60206959", + "--overlap", + "--nodes=2", + "--ntasks=8", + "--ntasks-per-node=4", + "--gpus-per-node=4", + "--gpu-bind=none", + "--cpu-bind=none", + "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/workdir/cuda-build/vlasov_bench", + "--nx", + "268800", + "--nv", + "4096", + "--length", + "12600", + "--vmax", + "15", + "--dt", + "0.02", + "--nu", + "1e-12", + "--cells", + "--velocity", + "cubic", + "--integrator", + "sixth", + "--layout", + "halos", + "--steps", + "1", + "--warmup", + "0", + "--input", + "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/initial.bin", + "--v-grid", + "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/v.bin", + "--background", + "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/background.bin", + "--dump", + "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/comparison-nx268800-2node/2n-8g-halos-0001.bin" + ], + "rank_log": "rank=4 host=nid001024 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001024 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=6 host=nid001024 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=7 host=nid001025 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001024 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=5 host=nid001025 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001025 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001025 local_rank=0 device=NVIDIA A100-SXM4-40GB\n" + }, + { + "nodes": 2, + "ranks": 8, + "layout": "halos", + "steps": 100, + "max_abs_f_error": 5.144495940356819e-13, + "relative_l2_f_error": 7.864106656264876e-14, + "relative_l2_e_error": 4.782235170485902e-14, + "max_abs_e_error": 4.3395842475035806e-14, + "hole_counts": [ + 129, + 50 + ], + "reference_hole_counts": [ + 129, + 50 + ], + "hole_counts_match": true, + "passed": true, + "limits": { + "max_abs_f_error": 1e-07, + "relative_l2_f_error": 1e-08, + "relative_l2_e_error": 1e-07 + }, + "benchmark": { + "backend": "cuda", + "device": "NVIDIA A100-SXM4-40GB", + "precision": "float64", + "mode": "landau", + "integrator": "sixth", + "nu": 1e-12, + "velocity": "cubic", + "layout": "halos", + "cell_centred": true, + "max_halo_width": 4, + "rank0_halo_sent_bytes": 4584652800, + "halo_sent_bytes_per_rank_max": 9169305600, + "slab_sent_bytes_per_rank": 192675840000, + "input": "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/initial.bin", + "nx": 268800, + "nv": 4096, + "ranks": 8, + "nodes": 2, + "steps": 100, + "warmup": 0, + "dt": 0.02, + "length": 12600, + "vmax": 15, + "amplitude": 0.01, + "acceleration": 0.2, + "host_mpi": false, + "state_bytes": 8808038400, + "max_allocated_bytes_per_rank": 11300409400, + "stream_ms": 97.31272944003194, + "spline_ms": 30.11586632999069, + "pack_ms": 6.524751499971444, + "mpi_ms": 120.3591580900047, + "field_ms": 10.87078800999961, + "collision_ms": 14.4880603899901, + "step_ms": 276.4345721300037, + "initial_mass": 12597.62207265221, + "final_mass": 12597.61751046275, + "mass_relative_change": -3.621468742758491e-07, + "initial_kinetic_energy": 55701.51067942529, + "final_kinetic_energy": 55695.27553237438, + "initial_field_energy": 791.8606180599771, + "final_field_energy": 797.6672297926364, + "minimum_f": -0.09642050941611634, + "finite": true + }, + "command": [ + "srun", + "--jobid=60206959", + "--overlap", + "--nodes=2", + "--ntasks=8", + "--ntasks-per-node=4", + "--gpus-per-node=4", + "--gpu-bind=none", + "--cpu-bind=none", + "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/workdir/cuda-build/vlasov_bench", + "--nx", + "268800", + "--nv", + "4096", + "--length", + "12600", + "--vmax", + "15", + "--dt", + "0.02", + "--nu", + "1e-12", + "--cells", + "--velocity", + "cubic", + "--integrator", + "sixth", + "--layout", + "halos", + "--steps", + "100", + "--warmup", + "0", + "--input", + "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/initial.bin", + "--v-grid", + "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/v.bin", + "--background", + "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/background.bin", + "--dump", + "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/comparison-nx268800-2node/2n-8g-halos-0100.bin" + ], + "rank_log": "rank=4 host=nid001024 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001024 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001025 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001024 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=6 host=nid001024 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=7 host=nid001025 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001025 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=5 host=nid001025 local_rank=2 device=NVIDIA A100-SXM4-40GB\n" + } +] diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-halos/compiled-source.sha256 b/benchmarks/vlasov_cuda/results/2026-10-24-halos/compiled-source.sha256 new file mode 100644 index 00000000..ba6ebaa9 --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-halos/compiled-source.sha256 @@ -0,0 +1,5 @@ +4dab281aa96caf3589f56bf0b6f313686e70c7f3b0c10788ea071b4edb6a1ef2 /pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/src/benchmarks/vlasov_cuda/backend.cpp +fdea36e5d804fd24df663e6c48194dba4e42dec3771cea8167b526db91b52927 /pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/src/benchmarks/vlasov_cuda/main.cpp +73ee9d9f649eebf405873f87c1eece597e8bd57a32636d2ec047eef156ea0cb3 /pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/src/benchmarks/vlasov_cuda/backend.hpp +f87f6aa9731f64e501b94e95c20a53f45ae7d1ea5ec39c7e859c6b1a4b8c40c3 /pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/src/benchmarks/vlasov_cuda/local_cubic.hpp +d5f11680925c336c63ea39e306230371f6dcb3e41ba423c8c09599c7c592664c /pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/src/benchmarks/vlasov_cuda/CMakeLists.txt diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-halos/executable.sha256 b/benchmarks/vlasov_cuda/results/2026-10-24-halos/executable.sha256 new file mode 100644 index 00000000..077a5f64 --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-halos/executable.sha256 @@ -0,0 +1 @@ +7d88657b9597e939f310c2c488d81842793ec361ad190b8f60595c47c8c6ea51 /pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/workdir/cuda-build/vlasov_bench diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-halos/nx134400-halos.jsonl b/benchmarks/vlasov_cuda/results/2026-10-24-halos/nx134400-halos.jsonl new file mode 100644 index 00000000..eaac1851 --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-halos/nx134400-halos.jsonl @@ -0,0 +1,6 @@ +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "integrator": "sixth", "nu": 1e-12, "velocity": "cubic", "layout": "halos", "cell_centred": true, "max_halo_width": 4, "rank0_halo_sent_bytes": 215040000, "halo_sent_bytes_per_rank_max": 430080000, "slab_sent_bytes_per_rank": 16515072000, "input": "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/initial.bin", "nx": 134400, "nv": 4096, "ranks": 4, "nodes": 1, "steps": 10, "warmup": 3, "dt": 0.02, "length": 12600, "vmax": 15, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 4404019200, "max_allocated_bytes_per_rank": 11155281976, "stream_ms": 96.75858740024523, "spline_ms": 29.37038770005529, "pack_ms": 6.511311200029013, "mpi_ms": 20.98856150023494, "field_ms": 6.72662069978287, "collision_ms": 14.47656629998164, "step_ms": 174.2806366000877, "initial_mass": 12597.6196507818, "final_mass": 12597.62016522545, "mass_relative_change": 4.083657516783715e-08, "initial_kinetic_energy": 55713.43281125277, "final_kinetic_energy": 55714.15660222915, "initial_field_energy": 787.6538181371966, "final_field_energy": 786.9554651665927, "minimum_f": -0.1916308738890627, "finite": true, "repeat": 0, "command": ["srun", "--jobid=60206959", "--overlap", "--nodes=1", "--ntasks=4", "--ntasks-per-node=4", "--gpus-per-node=4", "--gpu-bind=none", "--cpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/workdir/cuda-build/vlasov_bench", "--nx", "134400", "--nv", "4096", "--steps", "10", "--warmup", "3", "--mode", "landau", "--velocity", "cubic", "--layout", "halos", "--integrator", "sixth", "--nu", "1e-12", "--length", "12600.0", "--vmax", "15.0", "--dt", "0.02", "--cells", "--input", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/initial.bin", "--v-grid", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/v.bin", "--background", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/background.bin"], "rank_log": "rank=1 host=nid001024 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001024 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001024 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001024 local_rank=0 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=35a370ebe958e3df17ad798c54efedf85892f8029da920ff437ff54953bea6b6\nsynced_at=2026-10-24T11:25:39Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "integrator": "sixth", "nu": 1e-12, "velocity": "cubic", "layout": "halos", "cell_centred": true, "max_halo_width": 4, "rank0_halo_sent_bytes": 215040000, "halo_sent_bytes_per_rank_max": 430080000, "slab_sent_bytes_per_rank": 16515072000, "input": "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/initial.bin", "nx": 134400, "nv": 4096, "ranks": 4, "nodes": 1, "steps": 10, "warmup": 3, "dt": 0.02, "length": 12600, "vmax": 15, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 4404019200, "max_allocated_bytes_per_rank": 11155281976, "stream_ms": 96.7698514001313, "spline_ms": 29.39963990011165, "pack_ms": 6.513359599830437, "mpi_ms": 20.97919940019892, "field_ms": 6.686601499905009, "collision_ms": 14.49262569999519, "step_ms": 174.3251663999672, "initial_mass": 12597.6196507818, "final_mass": 12597.62016522545, "mass_relative_change": 4.083657516783715e-08, "initial_kinetic_energy": 55713.43281125277, "final_kinetic_energy": 55714.15660222915, "initial_field_energy": 787.6538181371966, "final_field_energy": 786.9554651665927, "minimum_f": -0.1916308738890627, "finite": true, "repeat": 1, "command": ["srun", "--jobid=60206959", "--overlap", "--nodes=1", "--ntasks=4", "--ntasks-per-node=4", "--gpus-per-node=4", "--gpu-bind=none", "--cpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/workdir/cuda-build/vlasov_bench", "--nx", "134400", "--nv", "4096", "--steps", "10", "--warmup", "3", "--mode", "landau", "--velocity", "cubic", "--layout", "halos", "--integrator", "sixth", "--nu", "1e-12", "--length", "12600.0", "--vmax", "15.0", "--dt", "0.02", "--cells", "--input", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/initial.bin", "--v-grid", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/v.bin", "--background", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/background.bin"], "rank_log": "rank=3 host=nid001025 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001025 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001025 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001025 local_rank=1 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=35a370ebe958e3df17ad798c54efedf85892f8029da920ff437ff54953bea6b6\nsynced_at=2026-10-24T11:25:39Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "integrator": "sixth", "nu": 1e-12, "velocity": "cubic", "layout": "halos", "cell_centred": true, "max_halo_width": 4, "rank0_halo_sent_bytes": 215040000, "halo_sent_bytes_per_rank_max": 430080000, "slab_sent_bytes_per_rank": 16515072000, "input": "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/initial.bin", "nx": 134400, "nv": 4096, "ranks": 4, "nodes": 1, "steps": 10, "warmup": 3, "dt": 0.02, "length": 12600, "vmax": 15, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 4404019200, "max_allocated_bytes_per_rank": 11155281976, "stream_ms": 96.7597968001428, "spline_ms": 29.36039309979606, "pack_ms": 6.510793699726491, "mpi_ms": 20.96922610003276, "field_ms": 6.668245399851003, "collision_ms": 14.47216869996737, "step_ms": 174.2041793000681, "initial_mass": 12597.6196507818, "final_mass": 12597.62016522545, "mass_relative_change": 4.083657516783715e-08, "initial_kinetic_energy": 55713.43281125277, "final_kinetic_energy": 55714.15660222915, "initial_field_energy": 787.6538181371966, "final_field_energy": 786.9554651665927, "minimum_f": -0.1916308738890627, "finite": true, "repeat": 2, "command": ["srun", "--jobid=60206959", "--overlap", "--nodes=1", "--ntasks=4", "--ntasks-per-node=4", "--gpus-per-node=4", "--gpu-bind=none", "--cpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/workdir/cuda-build/vlasov_bench", "--nx", "134400", "--nv", "4096", "--steps", "10", "--warmup", "3", "--mode", "landau", "--velocity", "cubic", "--layout", "halos", "--integrator", "sixth", "--nu", "1e-12", "--length", "12600.0", "--vmax", "15.0", "--dt", "0.02", "--cells", "--input", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/initial.bin", "--v-grid", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/v.bin", "--background", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/background.bin"], "rank_log": "rank=2 host=nid001024 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001024 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001024 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001024 local_rank=1 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=35a370ebe958e3df17ad798c54efedf85892f8029da920ff437ff54953bea6b6\nsynced_at=2026-10-24T11:25:39Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "integrator": "sixth", "nu": 1e-12, "velocity": "cubic", "layout": "halos", "cell_centred": true, "max_halo_width": 4, "rank0_halo_sent_bytes": 215040000, "halo_sent_bytes_per_rank_max": 430080000, "slab_sent_bytes_per_rank": 9633792000, "input": "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/initial.bin", "nx": 134400, "nv": 4096, "ranks": 8, "nodes": 2, "steps": 10, "warmup": 3, "dt": 0.02, "length": 12600, "vmax": 15, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 4404019200, "max_allocated_bytes_per_rank": 5650233400, "stream_ms": 48.49465079987567, "spline_ms": 12.44509260009181, "pack_ms": 3.281850700113864, "mpi_ms": 60.40269479999552, "field_ms": 5.99238590011737, "collision_ms": 11.23696820000077, "step_ms": 140.1605807999658, "initial_mass": 12597.61965078436, "final_mass": 12597.62016522508, "mass_relative_change": 4.083634327505363e-08, "initial_kinetic_energy": 55713.43281126581, "final_kinetic_energy": 55714.15660220217, "initial_field_energy": 787.6538181371961, "final_field_energy": 786.9554651665925, "minimum_f": -0.1916308738890627, "finite": true, "repeat": 0, "command": ["srun", "--jobid=60206959", "--overlap", "--nodes=2", "--ntasks=8", "--ntasks-per-node=4", "--gpus-per-node=4", "--gpu-bind=none", "--cpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/workdir/cuda-build/vlasov_bench", "--nx", "134400", "--nv", "4096", "--steps", "10", "--warmup", "3", "--mode", "landau", "--velocity", "cubic", "--layout", "halos", "--integrator", "sixth", "--nu", "1e-12", "--length", "12600.0", "--vmax", "15.0", "--dt", "0.02", "--cells", "--input", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/initial.bin", "--v-grid", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/v.bin", "--background", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/background.bin"], "rank_log": "rank=6 host=nid001024 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=4 host=nid001024 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001024 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001024 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=7 host=nid001025 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001025 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001025 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=5 host=nid001025 local_rank=2 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=35a370ebe958e3df17ad798c54efedf85892f8029da920ff437ff54953bea6b6\nsynced_at=2026-10-24T11:25:39Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "integrator": "sixth", "nu": 1e-12, "velocity": "cubic", "layout": "halos", "cell_centred": true, "max_halo_width": 4, "rank0_halo_sent_bytes": 215040000, "halo_sent_bytes_per_rank_max": 430080000, "slab_sent_bytes_per_rank": 9633792000, "input": "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/initial.bin", "nx": 134400, "nv": 4096, "ranks": 8, "nodes": 2, "steps": 10, "warmup": 3, "dt": 0.02, "length": 12600, "vmax": 15, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 4404019200, "max_allocated_bytes_per_rank": 5650233400, "stream_ms": 48.50223350013039, "spline_ms": 12.23678170017592, "pack_ms": 3.282552000064243, "mpi_ms": 59.93244790010976, "field_ms": 6.357290200094212, "collision_ms": 11.2380733999089, "step_ms": 139.8805341999378, "initial_mass": 12597.61965078436, "final_mass": 12597.62016522508, "mass_relative_change": 4.083634327505363e-08, "initial_kinetic_energy": 55713.43281126581, "final_kinetic_energy": 55714.15660220217, "initial_field_energy": 787.6538181371961, "final_field_energy": 786.9554651665925, "minimum_f": -0.1916308738890627, "finite": true, "repeat": 1, "command": ["srun", "--jobid=60206959", "--overlap", "--nodes=2", "--ntasks=8", "--ntasks-per-node=4", "--gpus-per-node=4", "--gpu-bind=none", "--cpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/workdir/cuda-build/vlasov_bench", "--nx", "134400", "--nv", "4096", "--steps", "10", "--warmup", "3", "--mode", "landau", "--velocity", "cubic", "--layout", "halos", "--integrator", "sixth", "--nu", "1e-12", "--length", "12600.0", "--vmax", "15.0", "--dt", "0.02", "--cells", "--input", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/initial.bin", "--v-grid", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/v.bin", "--background", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/background.bin"], "rank_log": "rank=6 host=nid001024 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=7 host=nid001025 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=4 host=nid001024 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=5 host=nid001025 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001025 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001024 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001024 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001025 local_rank=0 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=35a370ebe958e3df17ad798c54efedf85892f8029da920ff437ff54953bea6b6\nsynced_at=2026-10-24T11:25:39Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "integrator": "sixth", "nu": 1e-12, "velocity": "cubic", "layout": "halos", "cell_centred": true, "max_halo_width": 4, "rank0_halo_sent_bytes": 215040000, "halo_sent_bytes_per_rank_max": 430080000, "slab_sent_bytes_per_rank": 9633792000, "input": "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/initial.bin", "nx": 134400, "nv": 4096, "ranks": 8, "nodes": 2, "steps": 10, "warmup": 3, "dt": 0.02, "length": 12600, "vmax": 15, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 4404019200, "max_allocated_bytes_per_rank": 5650233400, "stream_ms": 48.50200009991568, "spline_ms": 12.23685909994856, "pack_ms": 3.279367700179137, "mpi_ms": 60.16711680003937, "field_ms": 6.355414500239931, "collision_ms": 11.24243100002786, "step_ms": 140.0107500999638, "initial_mass": 12597.61965078436, "final_mass": 12597.62016522508, "mass_relative_change": 4.083634327505363e-08, "initial_kinetic_energy": 55713.43281126581, "final_kinetic_energy": 55714.15660220217, "initial_field_energy": 787.6538181371961, "final_field_energy": 786.9554651665925, "minimum_f": -0.1916308738890627, "finite": true, "repeat": 2, "command": ["srun", "--jobid=60206959", "--overlap", "--nodes=2", "--ntasks=8", "--ntasks-per-node=4", "--gpus-per-node=4", "--gpu-bind=none", "--cpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/workdir/cuda-build/vlasov_bench", "--nx", "134400", "--nv", "4096", "--steps", "10", "--warmup", "3", "--mode", "landau", "--velocity", "cubic", "--layout", "halos", "--integrator", "sixth", "--nu", "1e-12", "--length", "12600.0", "--vmax", "15.0", "--dt", "0.02", "--cells", "--input", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/initial.bin", "--v-grid", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/v.bin", "--background", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/background.bin"], "rank_log": "rank=4 host=nid001024 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001024 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=5 host=nid001025 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001024 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001025 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=6 host=nid001024 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=7 host=nid001025 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001025 local_rank=0 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=35a370ebe958e3df17ad798c54efedf85892f8029da920ff437ff54953bea6b6\nsynced_at=2026-10-24T11:25:39Z\n"} diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-halos/nx134400-transpose.jsonl b/benchmarks/vlasov_cuda/results/2026-10-24-halos/nx134400-transpose.jsonl new file mode 100644 index 00000000..80c1f0b4 --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-halos/nx134400-transpose.jsonl @@ -0,0 +1,6 @@ +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "integrator": "sixth", "nu": 1e-12, "velocity": "cubic", "layout": "transpose", "cell_centred": true, "max_halo_width": 0, "rank0_halo_sent_bytes": 0, "halo_sent_bytes_per_rank_max": 0, "slab_sent_bytes_per_rank": 115605504000, "input": "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/initial.bin", "nx": 134400, "nv": 4096, "ranks": 4, "nodes": 1, "steps": 10, "warmup": 3, "dt": 0.02, "length": 12600, "vmax": 15, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 4404019200, "max_allocated_bytes_per_rank": 11017656376, "stream_ms": 96.80129259998012, "spline_ms": 23.18175969996901, "pack_ms": 45.59191170001213, "mpi_ms": 137.4090791999834, "field_ms": 6.102015500027846, "collision_ms": 14.47260410000126, "step_ms": 323.5450839000123, "initial_mass": 12597.6196507818, "final_mass": 12597.62016522545, "mass_relative_change": 4.083657516783715e-08, "initial_kinetic_energy": 55713.43281125277, "final_kinetic_energy": 55714.15660222915, "initial_field_energy": 787.6538181371966, "final_field_energy": 786.9554651665927, "minimum_f": -0.1916308738890627, "finite": true, "repeat": 0, "command": ["srun", "--jobid=60206959", "--overlap", "--nodes=1", "--ntasks=4", "--ntasks-per-node=4", "--gpus-per-node=4", "--gpu-bind=none", "--cpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/workdir/cuda-build/vlasov_bench", "--nx", "134400", "--nv", "4096", "--steps", "10", "--warmup", "3", "--mode", "landau", "--velocity", "cubic", "--layout", "transpose", "--integrator", "sixth", "--nu", "1e-12", "--length", "12600.0", "--vmax", "15.0", "--dt", "0.02", "--cells", "--input", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/initial.bin", "--v-grid", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/v.bin", "--background", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/background.bin"], "rank_log": "rank=2 host=nid001024 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001024 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001024 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001024 local_rank=3 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=35a370ebe958e3df17ad798c54efedf85892f8029da920ff437ff54953bea6b6\nsynced_at=2026-10-24T11:25:39Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "integrator": "sixth", "nu": 1e-12, "velocity": "cubic", "layout": "transpose", "cell_centred": true, "max_halo_width": 0, "rank0_halo_sent_bytes": 0, "halo_sent_bytes_per_rank_max": 0, "slab_sent_bytes_per_rank": 115605504000, "input": "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/initial.bin", "nx": 134400, "nv": 4096, "ranks": 4, "nodes": 1, "steps": 10, "warmup": 3, "dt": 0.02, "length": 12600, "vmax": 15, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 4404019200, "max_allocated_bytes_per_rank": 11017656376, "stream_ms": 96.80772089986931, "spline_ms": 23.22596530000283, "pack_ms": 45.59232870010419, "mpi_ms": 137.3818458999949, "field_ms": 6.081885599883208, "collision_ms": 14.48917420000271, "step_ms": 323.5631748999822, "initial_mass": 12597.6196507818, "final_mass": 12597.62016522545, "mass_relative_change": 4.083657516783715e-08, "initial_kinetic_energy": 55713.43281125277, "final_kinetic_energy": 55714.15660222915, "initial_field_energy": 787.6538181371966, "final_field_energy": 786.9554651665927, "minimum_f": -0.1916308738890627, "finite": true, "repeat": 1, "command": ["srun", "--jobid=60206959", "--overlap", "--nodes=1", "--ntasks=4", "--ntasks-per-node=4", "--gpus-per-node=4", "--gpu-bind=none", "--cpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/workdir/cuda-build/vlasov_bench", "--nx", "134400", "--nv", "4096", "--steps", "10", "--warmup", "3", "--mode", "landau", "--velocity", "cubic", "--layout", "transpose", "--integrator", "sixth", "--nu", "1e-12", "--length", "12600.0", "--vmax", "15.0", "--dt", "0.02", "--cells", "--input", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/initial.bin", "--v-grid", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/v.bin", "--background", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/background.bin"], "rank_log": "rank=2 host=nid001025 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001025 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001025 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001025 local_rank=0 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=35a370ebe958e3df17ad798c54efedf85892f8029da920ff437ff54953bea6b6\nsynced_at=2026-10-24T11:25:39Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "integrator": "sixth", "nu": 1e-12, "velocity": "cubic", "layout": "transpose", "cell_centred": true, "max_halo_width": 0, "rank0_halo_sent_bytes": 0, "halo_sent_bytes_per_rank_max": 0, "slab_sent_bytes_per_rank": 115605504000, "input": "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/initial.bin", "nx": 134400, "nv": 4096, "ranks": 4, "nodes": 1, "steps": 10, "warmup": 3, "dt": 0.02, "length": 12600, "vmax": 15, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 4404019200, "max_allocated_bytes_per_rank": 11017656376, "stream_ms": 96.79299130004893, "spline_ms": 23.18371020005543, "pack_ms": 45.59643329980645, "mpi_ms": 137.4355132999881, "field_ms": 6.073187200104257, "collision_ms": 14.47208049996789, "step_ms": 323.5294349000242, "initial_mass": 12597.6196507818, "final_mass": 12597.62016522545, "mass_relative_change": 4.083657516783715e-08, "initial_kinetic_energy": 55713.43281125277, "final_kinetic_energy": 55714.15660222915, "initial_field_energy": 787.6538181371966, "final_field_energy": 786.9554651665927, "minimum_f": -0.1916308738890627, "finite": true, "repeat": 2, "command": ["srun", "--jobid=60206959", "--overlap", "--nodes=1", "--ntasks=4", "--ntasks-per-node=4", "--gpus-per-node=4", "--gpu-bind=none", "--cpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/workdir/cuda-build/vlasov_bench", "--nx", "134400", "--nv", "4096", "--steps", "10", "--warmup", "3", "--mode", "landau", "--velocity", "cubic", "--layout", "transpose", "--integrator", "sixth", "--nu", "1e-12", "--length", "12600.0", "--vmax", "15.0", "--dt", "0.02", "--cells", "--input", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/initial.bin", "--v-grid", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/v.bin", "--background", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/background.bin"], "rank_log": "rank=0 host=nid001024 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001024 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001024 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001024 local_rank=3 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=35a370ebe958e3df17ad798c54efedf85892f8029da920ff437ff54953bea6b6\nsynced_at=2026-10-24T11:25:39Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "integrator": "sixth", "nu": 1e-12, "velocity": "cubic", "layout": "transpose", "cell_centred": true, "max_halo_width": 0, "rank0_halo_sent_bytes": 0, "halo_sent_bytes_per_rank_max": 0, "slab_sent_bytes_per_rank": 67436544000, "input": "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/initial.bin", "nx": 134400, "nv": 4096, "ranks": 8, "nodes": 2, "steps": 10, "warmup": 3, "dt": 0.02, "length": 12600, "vmax": 15, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 4404019200, "max_allocated_bytes_per_rank": 5512607800, "stream_ms": 48.55348729995512, "spline_ms": 11.72295309993387, "pack_ms": 22.98062660006508, "mpi_ms": 391.2288103000037, "field_ms": 13.02523969998219, "collision_ms": 11.2371266999844, "step_ms": 490.7239440000239, "initial_mass": 12597.61965078436, "final_mass": 12597.62016522508, "mass_relative_change": 4.083634327505363e-08, "initial_kinetic_energy": 55713.43281126581, "final_kinetic_energy": 55714.15660220217, "initial_field_energy": 787.6538181371961, "final_field_energy": 786.9554651665925, "minimum_f": -0.1916308738890627, "finite": true, "repeat": 0, "command": ["srun", "--jobid=60206959", "--overlap", "--nodes=2", "--ntasks=8", "--ntasks-per-node=4", "--gpus-per-node=4", "--gpu-bind=none", "--cpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/workdir/cuda-build/vlasov_bench", "--nx", "134400", "--nv", "4096", "--steps", "10", "--warmup", "3", "--mode", "landau", "--velocity", "cubic", "--layout", "transpose", "--integrator", "sixth", "--nu", "1e-12", "--length", "12600.0", "--vmax", "15.0", "--dt", "0.02", "--cells", "--input", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/initial.bin", "--v-grid", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/v.bin", "--background", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/background.bin"], "rank_log": "rank=3 host=nid001025 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=7 host=nid001025 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001025 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=5 host=nid001025 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=4 host=nid001024 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001024 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001024 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=6 host=nid001024 local_rank=3 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=35a370ebe958e3df17ad798c54efedf85892f8029da920ff437ff54953bea6b6\nsynced_at=2026-10-24T11:25:39Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "integrator": "sixth", "nu": 1e-12, "velocity": "cubic", "layout": "transpose", "cell_centred": true, "max_halo_width": 0, "rank0_halo_sent_bytes": 0, "halo_sent_bytes_per_rank_max": 0, "slab_sent_bytes_per_rank": 67436544000, "input": "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/initial.bin", "nx": 134400, "nv": 4096, "ranks": 8, "nodes": 2, "steps": 10, "warmup": 3, "dt": 0.02, "length": 12600, "vmax": 15, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 4404019200, "max_allocated_bytes_per_rank": 5512607800, "stream_ms": 48.54444679997414, "spline_ms": 11.72291799998675, "pack_ms": 22.9713284998752, "mpi_ms": 390.2375082999924, "field_ms": 11.86256170008164, "collision_ms": 11.23720970006161, "step_ms": 489.5213302999537, "initial_mass": 12597.61965078436, "final_mass": 12597.62016522508, "mass_relative_change": 4.083634327505363e-08, "initial_kinetic_energy": 55713.43281126581, "final_kinetic_energy": 55714.15660220217, "initial_field_energy": 787.6538181371961, "final_field_energy": 786.9554651665925, "minimum_f": -0.1916308738890627, "finite": true, "repeat": 1, "command": ["srun", "--jobid=60206959", "--overlap", "--nodes=2", "--ntasks=8", "--ntasks-per-node=4", "--gpus-per-node=4", "--gpu-bind=none", "--cpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/workdir/cuda-build/vlasov_bench", "--nx", "134400", "--nv", "4096", "--steps", "10", "--warmup", "3", "--mode", "landau", "--velocity", "cubic", "--layout", "transpose", "--integrator", "sixth", "--nu", "1e-12", "--length", "12600.0", "--vmax", "15.0", "--dt", "0.02", "--cells", "--input", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/initial.bin", "--v-grid", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/v.bin", "--background", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/background.bin"], "rank_log": "rank=2 host=nid001024 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001024 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=7 host=nid001025 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=5 host=nid001025 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001025 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=4 host=nid001024 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=6 host=nid001024 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001025 local_rank=1 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=35a370ebe958e3df17ad798c54efedf85892f8029da920ff437ff54953bea6b6\nsynced_at=2026-10-24T11:25:39Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "integrator": "sixth", "nu": 1e-12, "velocity": "cubic", "layout": "transpose", "cell_centred": true, "max_halo_width": 0, "rank0_halo_sent_bytes": 0, "halo_sent_bytes_per_rank_max": 0, "slab_sent_bytes_per_rank": 67436544000, "input": "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/initial.bin", "nx": 134400, "nv": 4096, "ranks": 8, "nodes": 2, "steps": 10, "warmup": 3, "dt": 0.02, "length": 12600, "vmax": 15, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 4404019200, "max_allocated_bytes_per_rank": 5512607800, "stream_ms": 48.55127589999029, "spline_ms": 11.72265499997138, "pack_ms": 22.97926750006809, "mpi_ms": 391.0327226998788, "field_ms": 12.39774919995398, "collision_ms": 11.23903609998251, "step_ms": 490.5058394000434, "initial_mass": 12597.61965078436, "final_mass": 12597.62016522508, "mass_relative_change": 4.083634327505363e-08, "initial_kinetic_energy": 55713.43281126581, "final_kinetic_energy": 55714.15660220217, "initial_field_energy": 787.6538181371961, "final_field_energy": 786.9554651665925, "minimum_f": -0.1916308738890627, "finite": true, "repeat": 2, "command": ["srun", "--jobid=60206959", "--overlap", "--nodes=2", "--ntasks=8", "--ntasks-per-node=4", "--gpus-per-node=4", "--gpu-bind=none", "--cpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/workdir/cuda-build/vlasov_bench", "--nx", "134400", "--nv", "4096", "--steps", "10", "--warmup", "3", "--mode", "landau", "--velocity", "cubic", "--layout", "transpose", "--integrator", "sixth", "--nu", "1e-12", "--length", "12600.0", "--vmax", "15.0", "--dt", "0.02", "--cells", "--input", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/initial.bin", "--v-grid", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/v.bin", "--background", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx134400/background.bin"], "rank_log": "rank=3 host=nid001025 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=7 host=nid001025 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001025 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=6 host=nid001024 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=5 host=nid001025 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001024 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=4 host=nid001024 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001024 local_rank=1 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=35a370ebe958e3df17ad798c54efedf85892f8029da920ff437ff54953bea6b6\nsynced_at=2026-10-24T11:25:39Z\n"} diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-halos/nx268800-halos.jsonl b/benchmarks/vlasov_cuda/results/2026-10-24-halos/nx268800-halos.jsonl new file mode 100644 index 00000000..0bb5ec17 --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-halos/nx268800-halos.jsonl @@ -0,0 +1,6 @@ +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "integrator": "sixth", "nu": 1e-12, "velocity": "cubic", "layout": "halos", "cell_centred": true, "max_halo_width": 4, "rank0_halo_sent_bytes": 438681600, "halo_sent_bytes_per_rank_max": 877363200, "slab_sent_bytes_per_rank": 33030144000, "input": "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/initial.bin", "nx": 268800, "nv": 4096, "ranks": 4, "nodes": 1, "steps": 10, "warmup": 3, "dt": 0.02, "length": 12600, "vmax": 15, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 8808038400, "max_allocated_bytes_per_rank": 22310481976, "stream_ms": 194.4373317000554, "spline_ms": 61.42588710008567, "pack_ms": 13.08217789996888, "mpi_ms": 41.11955449998277, "field_ms": 12.09190620002119, "collision_ms": 21.10292339998523, "step_ms": 342.4020051000753, "initial_mass": 12597.62207264873, "final_mass": 12597.62136206869, "mass_relative_change": -5.640588611556955e-08, "initial_kinetic_energy": 55701.51067944818, "final_kinetic_energy": 55701.0068872211, "initial_field_energy": 791.8606180599772, "final_field_energy": 792.3077830250793, "minimum_f": -0.09580882788649099, "finite": true, "repeat": 0, "command": ["srun", "--jobid=60206959", "--overlap", "--nodes=1", "--ntasks=4", "--ntasks-per-node=4", "--gpus-per-node=4", "--gpu-bind=none", "--cpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/workdir/cuda-build/vlasov_bench", "--nx", "268800", "--nv", "4096", "--steps", "10", "--warmup", "3", "--mode", "landau", "--velocity", "cubic", "--layout", "halos", "--integrator", "sixth", "--nu", "1e-12", "--length", "12600.0", "--vmax", "15.0", "--dt", "0.02", "--cells", "--input", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/initial.bin", "--v-grid", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/v.bin", "--background", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/background.bin"], "rank_log": "rank=1 host=nid001024 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001024 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001024 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001024 local_rank=0 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=35a370ebe958e3df17ad798c54efedf85892f8029da920ff437ff54953bea6b6\nsynced_at=2026-10-24T11:25:39Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "integrator": "sixth", "nu": 1e-12, "velocity": "cubic", "layout": "halos", "cell_centred": true, "max_halo_width": 4, "rank0_halo_sent_bytes": 438681600, "halo_sent_bytes_per_rank_max": 877363200, "slab_sent_bytes_per_rank": 33030144000, "input": "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/initial.bin", "nx": 268800, "nv": 4096, "ranks": 4, "nodes": 1, "steps": 10, "warmup": 3, "dt": 0.02, "length": 12600, "vmax": 15, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 8808038400, "max_allocated_bytes_per_rank": 22310481976, "stream_ms": 194.4473017999371, "spline_ms": 61.46278019996316, "pack_ms": 13.08220940004503, "mpi_ms": 41.15909789966281, "field_ms": 12.18012080012159, "collision_ms": 21.11939120004536, "step_ms": 342.4947891999636, "initial_mass": 12597.62207264873, "final_mass": 12597.62136206869, "mass_relative_change": -5.640588611556955e-08, "initial_kinetic_energy": 55701.51067944818, "final_kinetic_energy": 55701.0068872211, "initial_field_energy": 791.8606180599772, "final_field_energy": 792.3077830250793, "minimum_f": -0.09580882788649099, "finite": true, "repeat": 1, "command": ["srun", "--jobid=60206959", "--overlap", "--nodes=1", "--ntasks=4", "--ntasks-per-node=4", "--gpus-per-node=4", "--gpu-bind=none", "--cpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/workdir/cuda-build/vlasov_bench", "--nx", "268800", "--nv", "4096", "--steps", "10", "--warmup", "3", "--mode", "landau", "--velocity", "cubic", "--layout", "halos", "--integrator", "sixth", "--nu", "1e-12", "--length", "12600.0", "--vmax", "15.0", "--dt", "0.02", "--cells", "--input", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/initial.bin", "--v-grid", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/v.bin", "--background", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/background.bin"], "rank_log": "rank=2 host=nid001025 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001025 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001025 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001025 local_rank=3 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=35a370ebe958e3df17ad798c54efedf85892f8029da920ff437ff54953bea6b6\nsynced_at=2026-10-24T11:25:39Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "integrator": "sixth", "nu": 1e-12, "velocity": "cubic", "layout": "halos", "cell_centred": true, "max_halo_width": 4, "rank0_halo_sent_bytes": 438681600, "halo_sent_bytes_per_rank_max": 877363200, "slab_sent_bytes_per_rank": 33030144000, "input": "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/initial.bin", "nx": 268800, "nv": 4096, "ranks": 4, "nodes": 1, "steps": 10, "warmup": 3, "dt": 0.02, "length": 12600, "vmax": 15, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 8808038400, "max_allocated_bytes_per_rank": 22310481976, "stream_ms": 194.4254820000424, "spline_ms": 61.42703189980239, "pack_ms": 13.08142329990005, "mpi_ms": 41.25540480013115, "field_ms": 12.30976880001435, "collision_ms": 21.11400799994954, "step_ms": 342.5728927000364, "initial_mass": 12597.62207264873, "final_mass": 12597.62136206869, "mass_relative_change": -5.640588611556955e-08, "initial_kinetic_energy": 55701.51067944818, "final_kinetic_energy": 55701.0068872211, "initial_field_energy": 791.8606180599772, "final_field_energy": 792.3077830250793, "minimum_f": -0.09580882788649099, "finite": true, "repeat": 2, "command": ["srun", "--jobid=60206959", "--overlap", "--nodes=1", "--ntasks=4", "--ntasks-per-node=4", "--gpus-per-node=4", "--gpu-bind=none", "--cpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/workdir/cuda-build/vlasov_bench", "--nx", "268800", "--nv", "4096", "--steps", "10", "--warmup", "3", "--mode", "landau", "--velocity", "cubic", "--layout", "halos", "--integrator", "sixth", "--nu", "1e-12", "--length", "12600.0", "--vmax", "15.0", "--dt", "0.02", "--cells", "--input", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/initial.bin", "--v-grid", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/v.bin", "--background", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/background.bin"], "rank_log": "rank=0 host=nid001024 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001024 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001024 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001024 local_rank=1 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=35a370ebe958e3df17ad798c54efedf85892f8029da920ff437ff54953bea6b6\nsynced_at=2026-10-24T11:25:39Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "integrator": "sixth", "nu": 1e-12, "velocity": "cubic", "layout": "halos", "cell_centred": true, "max_halo_width": 4, "rank0_halo_sent_bytes": 438681600, "halo_sent_bytes_per_rank_max": 877363200, "slab_sent_bytes_per_rank": 19267584000, "input": "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/initial.bin", "nx": 268800, "nv": 4096, "ranks": 8, "nodes": 2, "steps": 10, "warmup": 3, "dt": 0.02, "length": 12600, "vmax": 15, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 8808038400, "max_allocated_bytes_per_rank": 11300409400, "stream_ms": 97.31439470001533, "spline_ms": 30.11662399994748, "pack_ms": 6.547028900104124, "mpi_ms": 119.097262600053, "field_ms": 10.73557600002459, "collision_ms": 14.48864589997356, "step_ms": 275.1715809000871, "initial_mass": 12597.62207265221, "final_mass": 12597.62136207489, "mass_relative_change": -5.640567068345301e-08, "initial_kinetic_energy": 55701.51067942529, "final_kinetic_energy": 55701.00688716807, "initial_field_energy": 791.8606180599771, "final_field_energy": 792.3077830250799, "minimum_f": -0.09580882788649095, "finite": true, "repeat": 0, "command": ["srun", "--jobid=60206959", "--overlap", "--nodes=2", "--ntasks=8", "--ntasks-per-node=4", "--gpus-per-node=4", "--gpu-bind=none", "--cpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/workdir/cuda-build/vlasov_bench", "--nx", "268800", "--nv", "4096", "--steps", "10", "--warmup", "3", "--mode", "landau", "--velocity", "cubic", "--layout", "halos", "--integrator", "sixth", "--nu", "1e-12", "--length", "12600.0", "--vmax", "15.0", "--dt", "0.02", "--cells", "--input", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/initial.bin", "--v-grid", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/v.bin", "--background", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/background.bin"], "rank_log": "rank=4 host=nid001024 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=5 host=nid001025 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001025 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001024 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001024 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=6 host=nid001024 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=7 host=nid001025 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001025 local_rank=0 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=35a370ebe958e3df17ad798c54efedf85892f8029da920ff437ff54953bea6b6\nsynced_at=2026-10-24T11:25:39Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "integrator": "sixth", "nu": 1e-12, "velocity": "cubic", "layout": "halos", "cell_centred": true, "max_halo_width": 4, "rank0_halo_sent_bytes": 438681600, "halo_sent_bytes_per_rank_max": 877363200, "slab_sent_bytes_per_rank": 19267584000, "input": "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/initial.bin", "nx": 268800, "nv": 4096, "ranks": 8, "nodes": 2, "steps": 10, "warmup": 3, "dt": 0.02, "length": 12600, "vmax": 15, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 8808038400, "max_allocated_bytes_per_rank": 11300409400, "stream_ms": 97.31809059990155, "spline_ms": 30.11344969995662, "pack_ms": 6.52626839982986, "mpi_ms": 118.8481177000085, "field_ms": 11.04300180018072, "collision_ms": 14.49239670005227, "step_ms": 274.8684877000414, "initial_mass": 12597.62207265221, "final_mass": 12597.62136207489, "mass_relative_change": -5.640567068345301e-08, "initial_kinetic_energy": 55701.51067942529, "final_kinetic_energy": 55701.00688716807, "initial_field_energy": 791.8606180599771, "final_field_energy": 792.3077830250799, "minimum_f": -0.09580882788649095, "finite": true, "repeat": 1, "command": ["srun", "--jobid=60206959", "--overlap", "--nodes=2", "--ntasks=8", "--ntasks-per-node=4", "--gpus-per-node=4", "--gpu-bind=none", "--cpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/workdir/cuda-build/vlasov_bench", "--nx", "268800", "--nv", "4096", "--steps", "10", "--warmup", "3", "--mode", "landau", "--velocity", "cubic", "--layout", "halos", "--integrator", "sixth", "--nu", "1e-12", "--length", "12600.0", "--vmax", "15.0", "--dt", "0.02", "--cells", "--input", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/initial.bin", "--v-grid", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/v.bin", "--background", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/background.bin"], "rank_log": "rank=4 host=nid001024 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001024 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=6 host=nid001024 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001024 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=5 host=nid001025 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001025 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001025 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=7 host=nid001025 local_rank=3 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=35a370ebe958e3df17ad798c54efedf85892f8029da920ff437ff54953bea6b6\nsynced_at=2026-10-24T11:25:39Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "integrator": "sixth", "nu": 1e-12, "velocity": "cubic", "layout": "halos", "cell_centred": true, "max_halo_width": 4, "rank0_halo_sent_bytes": 438681600, "halo_sent_bytes_per_rank_max": 877363200, "slab_sent_bytes_per_rank": 19267584000, "input": "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/initial.bin", "nx": 268800, "nv": 4096, "ranks": 8, "nodes": 2, "steps": 10, "warmup": 3, "dt": 0.02, "length": 12600, "vmax": 15, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 8808038400, "max_allocated_bytes_per_rank": 11300409400, "stream_ms": 97.31957070011958, "spline_ms": 30.1135374000296, "pack_ms": 6.523464800011425, "mpi_ms": 119.0691504999904, "field_ms": 10.25784469979953, "collision_ms": 14.49078770006054, "step_ms": 275.0057376999848, "initial_mass": 12597.62207265221, "final_mass": 12597.62136207489, "mass_relative_change": -5.640567068345301e-08, "initial_kinetic_energy": 55701.51067942529, "final_kinetic_energy": 55701.00688716807, "initial_field_energy": 791.8606180599771, "final_field_energy": 792.3077830250799, "minimum_f": -0.09580882788649095, "finite": true, "repeat": 2, "command": ["srun", "--jobid=60206959", "--overlap", "--nodes=2", "--ntasks=8", "--ntasks-per-node=4", "--gpus-per-node=4", "--gpu-bind=none", "--cpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/workdir/cuda-build/vlasov_bench", "--nx", "268800", "--nv", "4096", "--steps", "10", "--warmup", "3", "--mode", "landau", "--velocity", "cubic", "--layout", "halos", "--integrator", "sixth", "--nu", "1e-12", "--length", "12600.0", "--vmax", "15.0", "--dt", "0.02", "--cells", "--input", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/initial.bin", "--v-grid", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/v.bin", "--background", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/background.bin"], "rank_log": "rank=3 host=nid001025 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=7 host=nid001025 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=5 host=nid001025 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001024 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=6 host=nid001024 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=4 host=nid001024 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001024 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001025 local_rank=0 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=35a370ebe958e3df17ad798c54efedf85892f8029da920ff437ff54953bea6b6\nsynced_at=2026-10-24T11:25:39Z\n"} diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-halos/nx268800-transpose.jsonl b/benchmarks/vlasov_cuda/results/2026-10-24-halos/nx268800-transpose.jsonl new file mode 100644 index 00000000..b6bb5075 --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-halos/nx268800-transpose.jsonl @@ -0,0 +1,6 @@ +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "integrator": "sixth", "nu": 1e-12, "velocity": "cubic", "layout": "transpose", "cell_centred": true, "max_halo_width": 0, "rank0_halo_sent_bytes": 0, "halo_sent_bytes_per_rank_max": 0, "slab_sent_bytes_per_rank": 231211008000, "input": "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/initial.bin", "nx": 268800, "nv": 4096, "ranks": 4, "nodes": 1, "steps": 10, "warmup": 3, "dt": 0.02, "length": 12600, "vmax": 15, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 8808038400, "max_allocated_bytes_per_rank": 22035230776, "stream_ms": 194.4949301001088, "spline_ms": 57.92510880005466, "pack_ms": 91.58833139995295, "mpi_ms": 273.2574189000388, "field_ms": 11.27814200003741, "collision_ms": 21.10967729986442, "step_ms": 649.6461631000784, "initial_mass": 12597.62207264873, "final_mass": 12597.62136206869, "mass_relative_change": -5.640588611556955e-08, "initial_kinetic_energy": 55701.51067944818, "final_kinetic_energy": 55701.0068872211, "initial_field_energy": 791.8606180599772, "final_field_energy": 792.3077830250793, "minimum_f": -0.09580882788649099, "finite": true, "repeat": 0, "command": ["srun", "--jobid=60206959", "--overlap", "--nodes=1", "--ntasks=4", "--ntasks-per-node=4", "--gpus-per-node=4", "--gpu-bind=none", "--cpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/workdir/cuda-build/vlasov_bench", "--nx", "268800", "--nv", "4096", "--steps", "10", "--warmup", "3", "--mode", "landau", "--velocity", "cubic", "--layout", "transpose", "--integrator", "sixth", "--nu", "1e-12", "--length", "12600.0", "--vmax", "15.0", "--dt", "0.02", "--cells", "--input", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/initial.bin", "--v-grid", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/v.bin", "--background", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/background.bin"], "rank_log": "rank=0 host=nid001024 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001024 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001024 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001024 local_rank=1 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=35a370ebe958e3df17ad798c54efedf85892f8029da920ff437ff54953bea6b6\nsynced_at=2026-10-24T11:25:39Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "integrator": "sixth", "nu": 1e-12, "velocity": "cubic", "layout": "transpose", "cell_centred": true, "max_halo_width": 0, "rank0_halo_sent_bytes": 0, "halo_sent_bytes_per_rank_max": 0, "slab_sent_bytes_per_rank": 231211008000, "input": "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/initial.bin", "nx": 268800, "nv": 4096, "ranks": 4, "nodes": 1, "steps": 10, "warmup": 3, "dt": 0.02, "length": 12600, "vmax": 15, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 8808038400, "max_allocated_bytes_per_rank": 22035230776, "stream_ms": 194.5085770000787, "spline_ms": 57.99670299984427, "pack_ms": 91.58881610005665, "mpi_ms": 273.2411245995536, "field_ms": 11.20944810036235, "collision_ms": 21.11824810003782, "step_ms": 649.6359549999852, "initial_mass": 12597.62207264873, "final_mass": 12597.62136206869, "mass_relative_change": -5.640588611556955e-08, "initial_kinetic_energy": 55701.51067944818, "final_kinetic_energy": 55701.0068872211, "initial_field_energy": 791.8606180599772, "final_field_energy": 792.3077830250793, "minimum_f": -0.09580882788649099, "finite": true, "repeat": 1, "command": ["srun", "--jobid=60206959", "--overlap", "--nodes=1", "--ntasks=4", "--ntasks-per-node=4", "--gpus-per-node=4", "--gpu-bind=none", "--cpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/workdir/cuda-build/vlasov_bench", "--nx", "268800", "--nv", "4096", "--steps", "10", "--warmup", "3", "--mode", "landau", "--velocity", "cubic", "--layout", "transpose", "--integrator", "sixth", "--nu", "1e-12", "--length", "12600.0", "--vmax", "15.0", "--dt", "0.02", "--cells", "--input", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/initial.bin", "--v-grid", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/v.bin", "--background", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/background.bin"], "rank_log": "rank=2 host=nid001025 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001025 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001025 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001025 local_rank=3 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=35a370ebe958e3df17ad798c54efedf85892f8029da920ff437ff54953bea6b6\nsynced_at=2026-10-24T11:25:39Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "integrator": "sixth", "nu": 1e-12, "velocity": "cubic", "layout": "transpose", "cell_centred": true, "max_halo_width": 0, "rank0_halo_sent_bytes": 0, "halo_sent_bytes_per_rank_max": 0, "slab_sent_bytes_per_rank": 231211008000, "input": "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/initial.bin", "nx": 268800, "nv": 4096, "ranks": 4, "nodes": 1, "steps": 10, "warmup": 3, "dt": 0.02, "length": 12600, "vmax": 15, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 8808038400, "max_allocated_bytes_per_rank": 22035230776, "stream_ms": 194.4883884001683, "spline_ms": 57.92602180003996, "pack_ms": 91.58432890003496, "mpi_ms": 273.2619063001039, "field_ms": 11.26907970001412, "collision_ms": 21.11227130003499, "step_ms": 649.6297856999718, "initial_mass": 12597.62207264873, "final_mass": 12597.62136206869, "mass_relative_change": -5.640588611556955e-08, "initial_kinetic_energy": 55701.51067944818, "final_kinetic_energy": 55701.0068872211, "initial_field_energy": 791.8606180599772, "final_field_energy": 792.3077830250793, "minimum_f": -0.09580882788649099, "finite": true, "repeat": 2, "command": ["srun", "--jobid=60206959", "--overlap", "--nodes=1", "--ntasks=4", "--ntasks-per-node=4", "--gpus-per-node=4", "--gpu-bind=none", "--cpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/workdir/cuda-build/vlasov_bench", "--nx", "268800", "--nv", "4096", "--steps", "10", "--warmup", "3", "--mode", "landau", "--velocity", "cubic", "--layout", "transpose", "--integrator", "sixth", "--nu", "1e-12", "--length", "12600.0", "--vmax", "15.0", "--dt", "0.02", "--cells", "--input", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/initial.bin", "--v-grid", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/v.bin", "--background", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/background.bin"], "rank_log": "rank=1 host=nid001024 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001024 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001024 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001024 local_rank=2 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=35a370ebe958e3df17ad798c54efedf85892f8029da920ff437ff54953bea6b6\nsynced_at=2026-10-24T11:25:39Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "integrator": "sixth", "nu": 1e-12, "velocity": "cubic", "layout": "transpose", "cell_centred": true, "max_halo_width": 0, "rank0_halo_sent_bytes": 0, "halo_sent_bytes_per_rank_max": 0, "slab_sent_bytes_per_rank": 134873088000, "input": "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/initial.bin", "nx": 268800, "nv": 4096, "ranks": 8, "nodes": 2, "steps": 10, "warmup": 3, "dt": 0.02, "length": 12600, "vmax": 15, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 8808038400, "max_allocated_bytes_per_rank": 11025158200, "stream_ms": 97.35810910015061, "spline_ms": 23.13156989971503, "pack_ms": 45.69633740002246, "mpi_ms": 775.7187433006493, "field_ms": 21.47743779996745, "collision_ms": 14.48875990008673, "step_ms": 964.4204498000363, "initial_mass": 12597.62207265221, "final_mass": 12597.62136207489, "mass_relative_change": -5.640567068345301e-08, "initial_kinetic_energy": 55701.51067942529, "final_kinetic_energy": 55701.00688716807, "initial_field_energy": 791.8606180599771, "final_field_energy": 792.3077830250799, "minimum_f": -0.09580882788649095, "finite": true, "repeat": 0, "command": ["srun", "--jobid=60206959", "--overlap", "--nodes=2", "--ntasks=8", "--ntasks-per-node=4", "--gpus-per-node=4", "--gpu-bind=none", "--cpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/workdir/cuda-build/vlasov_bench", "--nx", "268800", "--nv", "4096", "--steps", "10", "--warmup", "3", "--mode", "landau", "--velocity", "cubic", "--layout", "transpose", "--integrator", "sixth", "--nu", "1e-12", "--length", "12600.0", "--vmax", "15.0", "--dt", "0.02", "--cells", "--input", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/initial.bin", "--v-grid", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/v.bin", "--background", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/background.bin"], "rank_log": "rank=2 host=nid001024 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001025 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=4 host=nid001024 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=5 host=nid001025 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001025 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=7 host=nid001025 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001024 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=6 host=nid001024 local_rank=3 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=35a370ebe958e3df17ad798c54efedf85892f8029da920ff437ff54953bea6b6\nsynced_at=2026-10-24T11:25:39Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "integrator": "sixth", "nu": 1e-12, "velocity": "cubic", "layout": "transpose", "cell_centred": true, "max_halo_width": 0, "rank0_halo_sent_bytes": 0, "halo_sent_bytes_per_rank_max": 0, "slab_sent_bytes_per_rank": 134873088000, "input": "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/initial.bin", "nx": 268800, "nv": 4096, "ranks": 8, "nodes": 2, "steps": 10, "warmup": 3, "dt": 0.02, "length": 12600, "vmax": 15, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 8808038400, "max_allocated_bytes_per_rank": 11025158200, "stream_ms": 97.35549070010165, "spline_ms": 23.1311465998715, "pack_ms": 45.69765130004271, "mpi_ms": 777.1058046999769, "field_ms": 24.25418779998836, "collision_ms": 14.48958839991974, "step_ms": 965.7143639999504, "initial_mass": 12597.62207265221, "final_mass": 12597.62136207489, "mass_relative_change": -5.640567068345301e-08, "initial_kinetic_energy": 55701.51067942529, "final_kinetic_energy": 55701.00688716807, "initial_field_energy": 791.8606180599771, "final_field_energy": 792.3077830250799, "minimum_f": -0.09580882788649095, "finite": true, "repeat": 1, "command": ["srun", "--jobid=60206959", "--overlap", "--nodes=2", "--ntasks=8", "--ntasks-per-node=4", "--gpus-per-node=4", "--gpu-bind=none", "--cpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/workdir/cuda-build/vlasov_bench", "--nx", "268800", "--nv", "4096", "--steps", "10", "--warmup", "3", "--mode", "landau", "--velocity", "cubic", "--layout", "transpose", "--integrator", "sixth", "--nu", "1e-12", "--length", "12600.0", "--vmax", "15.0", "--dt", "0.02", "--cells", "--input", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/initial.bin", "--v-grid", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/v.bin", "--background", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/background.bin"], "rank_log": "rank=0 host=nid001024 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=5 host=nid001025 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001025 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001025 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=7 host=nid001025 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=6 host=nid001024 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=4 host=nid001024 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001024 local_rank=1 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=35a370ebe958e3df17ad798c54efedf85892f8029da920ff437ff54953bea6b6\nsynced_at=2026-10-24T11:25:39Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "integrator": "sixth", "nu": 1e-12, "velocity": "cubic", "layout": "transpose", "cell_centred": true, "max_halo_width": 0, "rank0_halo_sent_bytes": 0, "halo_sent_bytes_per_rank_max": 0, "slab_sent_bytes_per_rank": 134873088000, "input": "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/initial.bin", "nx": 268800, "nv": 4096, "ranks": 8, "nodes": 2, "steps": 10, "warmup": 3, "dt": 0.02, "length": 12600, "vmax": 15, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 8808038400, "max_allocated_bytes_per_rank": 11025158200, "stream_ms": 97.34848410021186, "spline_ms": 23.13205560012648, "pack_ms": 45.71225319996302, "mpi_ms": 776.3868526999431, "field_ms": 24.17435689994818, "collision_ms": 14.48879250001482, "step_ms": 964.9749842999427, "initial_mass": 12597.62207265221, "final_mass": 12597.62136207489, "mass_relative_change": -5.640567068345301e-08, "initial_kinetic_energy": 55701.51067942529, "final_kinetic_energy": 55701.00688716807, "initial_field_energy": 791.8606180599771, "final_field_energy": 792.3077830250799, "minimum_f": -0.09580882788649095, "finite": true, "repeat": 2, "command": ["srun", "--jobid=60206959", "--overlap", "--nodes=2", "--ntasks=8", "--ntasks-per-node=4", "--gpus-per-node=4", "--gpu-bind=none", "--cpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/workdir/cuda-build/vlasov_bench", "--nx", "268800", "--nv", "4096", "--steps", "10", "--warmup", "3", "--mode", "landau", "--velocity", "cubic", "--layout", "transpose", "--integrator", "sixth", "--nu", "1e-12", "--length", "12600.0", "--vmax", "15.0", "--dt", "0.02", "--cells", "--input", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/initial.bin", "--v-grid", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/v.bin", "--background", "/pscratch/sd/a/archis/adept-sessions/20261024T112505Z-77310-4969/outputs/native-nx268800/background.bin"], "rank_log": "rank=6 host=nid001024 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001025 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001024 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=7 host=nid001025 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001025 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=5 host=nid001025 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=4 host=nid001024 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001024 local_rank=0 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=35a370ebe958e3df17ad798c54efedf85892f8029da920ff437ff54953bea6b6\nsynced_at=2026-10-24T11:25:39Z\n"} diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-halos/reference-library-versions.json b/benchmarks/vlasov_cuda/results/2026-10-24-halos/reference-library-versions.json new file mode 100644 index 00000000..a8f88d36 --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-halos/reference-library-versions.json @@ -0,0 +1,9 @@ +{ + "adept": "0.1.0", + "jax": "0.10.2", + "jaxlib": "0.10.2", + "interpax": "0.3.14+10.g84aaf30", + "diffrax": "0.7.2", + "numpy": "2.4.6", + "scipy": "1.18.0" +} diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-halos/reference-nx134400.json b/benchmarks/vlasov_cuda/results/2026-10-24-halos/reference-nx134400.json new file mode 100644 index 00000000..772a04da --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-halos/reference-nx134400.json @@ -0,0 +1,35 @@ +{ + "checkpoint": "/pscratch/sd/a/archis/vp-turbulence-sessions/20261023T081121Z-nv4096-nx134400-5ce4fe14/workdir/campaigns/production-nx134400/checkpoint-state/step-030000-1a19a5ae15f8", + "manifest_sha256": "9ab7e963d2699777bd935ca84fad446b89846e64c416d10db6d3c14829af8627", + "actual_time": 600.0, + "config_sha256": "5a9f5347e6253a3d4201745318843857c568b3787cf91e9fe81a623d0cdbb214", + "adept_commit": "ddd70b957dc3df85f0b5ec84af927c25d09d24e0", + "jax": "0.10.2", + "nx": 134400, + "nv": 4096, + "dt": 0.02, + "nu": 1e-12, + "devices": [ + "cuda:0", + "cuda:1", + "cuda:2", + "cuda:3" + ], + "milestones": [ + { + "steps": 1, + "advanced_steps": 1, + "seconds": 3.020330819010269 + }, + { + "steps": 10, + "advanced_steps": 9, + "seconds": 2.0284883639833424 + }, + { + "steps": 100, + "advanced_steps": 90, + "seconds": 20.115412370010745 + } + ] +} diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-halos/reference-nx268800.json b/benchmarks/vlasov_cuda/results/2026-10-24-halos/reference-nx268800.json new file mode 100644 index 00000000..8df4fc30 --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-halos/reference-nx268800.json @@ -0,0 +1,37 @@ +{ + "checkpoint": "/pscratch/sd/a/archis/vp-turbulence-sessions/20261023T081121Z-nv4096-nx268800-93a418d7/workdir/campaigns/production-nx268800/checkpoint-state/step-030000-167819aeccca", + "manifest_sha256": "c9b9083f55cfcee8595556685d9e22eb9d87829fa6748726f7f44d5749fbec1f", + "actual_time": 600.0, + "config_sha256": "6adb42855432b79a74f63290678103c17c53842074d77bf092df38483f634a83", + "adept_commit": "ddd70b957dc3df85f0b5ec84af927c25d09d24e0", + "jax": "0.10.2", + "nx": 268800, + "nv": 4096, + "dt": 0.02, + "nu": 1e-12, + "devices": [ + "cuda:0", + "cuda:1", + "cuda:2", + "cuda:3", + "cuda:4", + "cuda:5", + "cuda:6", + "cuda:7" + ], + "milestones": [ + { + "steps": 1, + "advanced_steps": 1, + "rank0_seconds": 60.40102391698747 + }, + { + "steps": 100, + "advanced_steps": 99, + "rank0_seconds": 74.4436078449944 + } + ], + "process_count": 8, + "distributed": true, + "prepared_input": "/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs/native-nx268800" +} diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-halos/reference-source-state b/benchmarks/vlasov_cuda/results/2026-10-24-halos/reference-source-state new file mode 100644 index 00000000..48df1656 --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-halos/reference-source-state @@ -0,0 +1,5 @@ +version=1 +base_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4 +dirty=true +workspace_fingerprint=122fb1530212c155231e59f9d01d12c2566e24058627476aeee8c6310f3ab959 +synced_at=2026-10-24T11:08:15Z diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-halos/reference8-launch.sh b/benchmarks/vlasov_cuda/results/2026-10-24-halos/reference8-launch.sh new file mode 100644 index 00000000..a21ffb6b --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-halos/reference8-launch.sh @@ -0,0 +1,12 @@ +set -euo pipefail +source /global/common/software/m4490/archis/venvs/vp-turbulence/bin/activate +export PYTHONPATH=/pscratch/sd/a/archis/vp-turbulence-sessions/20261023T081121Z-nv4096-nx268800-93a418d7/src +export XLA_PYTHON_CLIENT_PREALLOCATE=false +export LD_LIBRARY_PATH="/global/common/software/nersc9/nccl/2.29.2-cu13/plugin/lib:/opt/nvidia/hpc_sdk/Linux_x86_64/26.5/cuda/13.2/lib64${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" +export NCCL_SOCKET_IFNAME=hsn NCCL_CROSS_NIC=2 NCCL_NET="AWS Libfabric" NCCL_NET_GDR_LEVEL=LOC FI_CXI_DISABLE_HOST_REGISTER=1 NCCL_DEBUG=INFO NCCL_DEBUG_SUBSYS=INIT,NET +export XLA_FLAGS="${XLA_FLAGS:+$XLA_FLAGS }--xla_gpu_autotune_level=0" +export JAX_ENABLE_X64=true +root=/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473 +cd "$root/outputs" +srun --jobid=60205333 --overlap --nodes=2 --ntasks=8 --ntasks-per-node=4 --cpus-per-task=16 --cpu-bind=none --gpus-per-node=4 --gpu-bind=none --mpi=none --kill-on-bad-exit=1 python "$root/src/benchmarks/vlasov_cuda/distributed_reference.py" --prepared "$root/outputs/native-nx268800" --config "$root/src/benchmarks/vlasov_cuda/reference/latest-nv4096/nx268800.yaml" --vp-source "$PYTHONPATH" --output "$root/outputs/native-nx268800-distributed" --steps 1 100 > "$root/workdir/reference8.log" 2>&1 +tail -n 8 "$root/workdir/reference8.log" diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-halos/slurm-accounting.txt b/benchmarks/vlasov_cuda/results/2026-10-24-halos/slurm-accounting.txt new file mode 100644 index 00000000..b06a1f3f --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-halos/slurm-accounting.txt @@ -0,0 +1,31 @@ +60206959|ec-adept-20261024T112505Z-77310-4969|RUNNING|00:15:16|0:0 +60206959.extern|extern|RUNNING|00:15:16|0:0 +60206959.0|bash|COMPLETED|00:12:45|0:0 +60206959.1|vlasov_bench|COMPLETED|00:00:29|0:0 +60206959.2|vlasov_bench|COMPLETED|00:01:59|0:0 +60206959.3|vlasov_bench|COMPLETED|00:00:24|0:0 +60206959.4|vlasov_bench|COMPLETED|00:00:50|0:0 +60206959.5|vlasov_bench|COMPLETED|00:00:16|0:0 +60206959.6|vlasov_bench|COMPLETED|00:00:17|0:0 +60206959.7|vlasov_bench|COMPLETED|00:00:14|0:0 +60206959.8|vlasov_bench|COMPLETED|00:00:14|0:0 +60206959.9|vlasov_bench|COMPLETED|00:00:15|0:0 +60206959.10|vlasov_bench|COMPLETED|00:00:15|0:0 +60206959.11|vlasov_bench|COMPLETED|00:00:13|0:0 +60206959.12|vlasov_bench|COMPLETED|00:00:13|0:0 +60206959.13|vlasov_bench|COMPLETED|00:00:12|0:0 +60206959.14|vlasov_bench|COMPLETED|00:00:10|0:0 +60206959.15|vlasov_bench|COMPLETED|00:00:10|0:0 +60206959.16|vlasov_bench|COMPLETED|00:00:09|0:0 +60206959.17|vlasov_bench|COMPLETED|00:00:26|0:0 +60206959.18|vlasov_bench|COMPLETED|00:00:29|0:0 +60206959.19|vlasov_bench|COMPLETED|00:00:26|0:0 +60206959.20|vlasov_bench|COMPLETED|00:00:25|0:0 +60206959.21|vlasov_bench|COMPLETED|00:00:25|0:0 +60206959.22|vlasov_bench|COMPLETED|00:00:24|0:0 +60206959.23|vlasov_bench|COMPLETED|00:00:21|0:0 +60206959.24|vlasov_bench|COMPLETED|00:00:23|0:0 +60206959.25|vlasov_bench|COMPLETED|00:00:22|0:0 +60206959.26|vlasov_bench|COMPLETED|00:00:16|0:0 +60206959.27|vlasov_bench|COMPLETED|00:00:15|0:0 +60206959.28|vlasov_bench|COMPLETED|00:00:15|0:0 diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-halos/source-archive.sha256 b/benchmarks/vlasov_cuda/results/2026-10-24-halos/source-archive.sha256 new file mode 100644 index 00000000..5aef810e --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-halos/source-archive.sha256 @@ -0,0 +1 @@ +df359db1f60990589a2fcac57643fc4770d51d636ecad7379d9bf777f4baf9b3 source.tar.gz diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-halos/source-state b/benchmarks/vlasov_cuda/results/2026-10-24-halos/source-state new file mode 100644 index 00000000..db22eba5 --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-halos/source-state @@ -0,0 +1,5 @@ +version=1 +base_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4 +dirty=true +workspace_fingerprint=35a370ebe958e3df17ad798c54efedf85892f8029da920ff437ff54953bea6b6 +synced_at=2026-10-24T11:25:39Z diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-halos/source.tar.gz b/benchmarks/vlasov_cuda/results/2026-10-24-halos/source.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..c0ec988883c2523e89d3a239c7a494b789197da1 GIT binary patch literal 33466 zcmV(uKNUUS0nYq5?{OaCoHywwIs4l?0tkR0MLRF6 zy34tiQv?wA*shqK{q{I@sW%@4je@TY(JUH)syf6M;1TASVMH{Q2fop$F>>gG>> zz<;wi@ut-MpZ+ob>+Gs=5{&&`duOk+yScT!z2ofdwYGOV-O8W7{P$n}zxl|Eqq*Ck z4ZMxRm)@2CB8ZdNNp6zg_WW;m<&5uaZ^>Ejq-Vapv(^1md#kg()20W!Z*OgOw*I7A zf8_c1rv1wxUS5uh4Tt~r_P;--J$J`J7>s9Qck2H<3#R^{_T$mp`tR?bzdd@U4qv}~^ZM1%t9QD8Z!mEE8^1qGyz`N7 z2F=4&V=oAu{$ygh#_xKCAmGAJ+1$6I2pL@ zowVX82(RdW=TmPQEUQLC-ruMHxje>TXfIE_qLd$EwC>0YFJ_pcd7qm%UcQT>(N&P7 z^GBhW1`$WIX`cp9Gl*T?Kj^4g|PFrO*O_`3+hMtdAyD*ouI%>Jc z@1DMT_Vn#D)!va8Pv0K?@ciA;;k);5smyK*?G_ey;M0r(Z0wSSDy&PPy1>?ws{i!( zpB@7mp1(M9&HVP!(3AaU)4xe*8T>(W|wb@mU%Kvpc?VXm>ZtZqDot-cI-#-}t_x9-NvzJHC zcRSKtzy8mgD2U@IR2g3=G%E-%)KlW@)%S1S z$BufQs6ph%l`u+_W;fN-XNp+9U_6P4wmyA!^yZzT-Vv)P#NHbTTWGjN{{|uobBW<> z>`#NfH>za(;(;ROD2Nq0@QCA_g0cQ}z_QJl8-bVh|4->BBR4QM8tzLME zA18{KzjLC^C?!)+G;?)Hb5}&QX)!RisZXmjnDtTH6r*0j;XZuW*;KFlNp$W{)vI~8 zwYRPM-gx2#7vaN)4IUZFMI$1)l^45o2ZY^3OJ5thk%oHF z?rh_EKK5?fTMacxT28CeP;qdRTsEUA+WWy92cskk122@l#11rQ)h4ucVG?+wjd()b z{FK)9MtOs|&vR&~v;OQn=r>U(^v~4XA4UBjxvf}5Fsj2#Vo{ri5m-d^ESgTEC=?6l7Jwsr>n&B1VYYuGy9 z^8348Z@Aal>36mUt-V2~>$lFh<9~koHwEJ4=w|&Jub))L{us;LXO1W4g2D-_V1Vfk z1Aj`d3Cd<6mgi*Z5B(`NcVooc-3)ngPisp)Y{=$19|*YX1Sbz49!ArtuQxuDoty?2 z0RS}x{%>M6_LIwKpu9MqjTw;lA3l_se-l^Ee$>l*h($lvYfg|nid7IM{&eEqP#eT8 zd$UoZ&ZMzkmG`K6Mq5E(MlDdEhwaJ_{&XB9zOEUO5Rm}Tu7l)K#Ut7&T7kJgCBAf^ zhSO+F+y24p)AA74->1T|ZUi6Am{x57j&TwN_{^2lQ_G-Bf*McyVicWI_aTHO45On2 zbh4vGy#-+fWrx9(4pzE4jv@{5)vyd76{)A)+5;QBk*0Kk@ zSOmXKS%2AKLdIb~7zxWom(CUD0T*g7G}-ya+h1gl9DlK?swn8CT3R%qU1 zI;ixO$ipc3H)*F2RObs!SVxPX>Bu923$s?yH85Vb)eJIbeAt_jPY06_Em18()PgV0 zy@aH|0lg;57FUS0-pW?X78_C;N258$PGm-pf>2tFUcBj)s(AfrM1#)2jJJVd9mA#d zOlV?E9L_|)qS22zVYJ_Uze(i=0gypI@`poOcHcnbAgT0IWEX3g3FsrWnRdG|1U$wG z5!x|;fo5|7oI#`L`-IQV2xdmqgaCp?HSx6xEQCXmPU2!347hW23vI~An`U2so_T{Q zt7jZcGn}1i%}H#a$h4l3rg1dp+0iMddW}2`m}d0$;ll%NPa`GfQzjm56cvx=dSU1c`3wpb zG)ui5XFNc*bwQk-6au5~fbl}1vq1B-&BPbdA!fMtra=sRqS>~ocddq_$V;}n0-Z5K zi@+iws54cYH*OU3DD?(_W8TD8pUEQAGx)oUX+&iDj8;}!%^h-@Y$Ypn02XXSCTWAheae))<_gNw1Z*&tGt2oniBpIpWp^Yj1r|7ie(0s@l$EQ!I9 zqL-GlRd>`YKM=_4PlHJU{@E;Jh52~wo{bVB!UhD{RD*!;L>--j z+!gjGx6rXS_RWqBz+9k(ffCqdsN_S>93^#5)nZ9R`RI0#-P3h95`C zF3B_BvPiREvGx`%rYBS5?><8uizAu{GQ3_S@w`LeIK3je+GlnJgx_K5g2&g?V^ir= z1&M>J`Ch~sp+RO(t%ZDbc=)37A~+v<=d?3wGMdpDI^d3})SPf8nnT6VPcR6gKNn0Y zH7`b!5e|W6otT)=7()vdMgyKGmBTWyCd3f-N3#JoSW5#I&Z5nHJr>b~=|l?5XDJ(F z^&{tvX3HH~%dHI)#80P4B55hsFN2EvQ+pKW%t!EcHi!a;gQie_V#o;nnJvfa5cdck zLt{SOoJM_&%!C=F)5d%bA3DQeJ2HuBsPvD=roG4EDJ=_-T-lc1S$bt=9LhM<6gB2)2F zF(CrnSOOoUek`tV%5)+sXJ=>eWo2?pe7mI)AZ%f&n-->^HbAyEl4!%$Ym;QIj&3|K zQW|MBAuQ88=QvDQa&HLWSc1D#Y|T5jUl~sVScgM##a(qck+Pn_J0pVxwV(PQJ2hasD z=Ts}QFL@D?Hjy}@AZGMZVKFXg{m1|MKOLw-1|*W+f)Dy2M6aPNf<^>_5c>rN)&Pls zOhtgiM(V@qd9x6K=7ucfB;8Juv+u6aSRsh!XQ0@UZO#j zJZAKaOu%am2qW}|88@`rX{;H1>i}EIr%J{@@kofz6I5d~7f@Ibqc|4<`sAG89Ybr% zcZWDf=i=X>vEU(YbXuM5X1mqwbT`V2)1+;OjI`ci?79Ua+Q<-*) z3VLAA2TGt`&h)0zaL+GA7OZGAlPV&Z+-v?T!LgLX!Ti_AB1t6e#X$Goc?9PDT>zdp zCO!ld!osjxP^NKTgb}eQec_s(Z@fOR1br&AA+ooC`2eg#!t)$?6W-U@aLPV&7HkRE zgBL*IH{J!9dgexGn|>iSL;ZjMw|X^K^b@=Y{l#L_pYQmfqyj>19B*J<>h$Rc#vb+? zV(>|DoTy)@$!4>u`2YK>|4>nl_Gko*^ymfs>(al?&Ze`4=bf&ziNDV7HkB%^)1iO6 zsJGXWI$h!l(T20t(seW|wb;*gchAA|X161CoxP=XOh0?uTikwUXUp`{<~o&^S{Fhv z&}3dIWg)R==eOXxW1y}fK;`^a?QS`3!bJ3E^UscgL^jN=I?GQ>Br%j1)=eJ|sMQI8 zP~W=(uXGMfzfHg{>?R?VeAU_mFVVf@bmhRY!u$+`fT{4@ z#7aXRC+bRN3AUcnKsv2uq8#x)P=RQrf=$qQT+DHjyv3NEdBG_pY$i`~WNSEl|M9>7 zukBW+4p~=L-aBXHP3x%q=`>>g=4{*9*(4&jc?RS@;_*~obnGG?gatYkXM4_8hl+N! zplFIKDt=T_e9PJHa&gT_GhST~cJKp?dC00Q!qsSirwkMo)8_#g1_fo;Ei32uD^!9e z*l7^$-`nW!Xpz9u*W>+GD<>*A>Zwp9wcCXWV#8QlDh4ZYO+!fXl!)d^lr@f1A}Twi zm8a9W#yc6Bs)(wq4^BwSjRNF_SMF@r=eqvaeRapWRcC zIAmsrr~Wy7j^b1i2SAweL^D-sst1Sq;-8OSzj~p0TXQHB`_qsdD?n#sk65wZ4su?9 z3K%+oB$(3jk>*ZieP~sKAz;v~8Lvk{&dJtGfH%4hS)&0iX2J}XmLR#=YoB*}M068G zjp=3VrAqX?(1s&IJiW!Wm2qhzuiV3rRcH!Kz0Q3<6L)9`w|*jg#`6L=A9f5=m5KuU zs?$wvWZ22I*bPseZFY9Hch!yBZSQr?*btf9luE6mz7M_yFUa_oVRGG z3vce{nU{)YUX;ze%z)>CN%1SsgJ_EgEeM^CF>(C_icHb-Wc(N+6p?DC(y8vvypW@> zpWYDfy)g^nRFoEgf*FJ*t&pSh`|8vGRD}BC-}bJv$CP*v!VIXrvx{e<->vLVoBrIS z^-ykW*V$nT+;z4<#oHZc7nFJ1+2qGfXJ?lyws*3M%vNkWds|Xrli7-vv$xA6yuAa; z?(7H?u&e8JbcgLuTgI}a$DM7bEhFBtYBGbdB?ZxANB6js^|-m~Y)O}`EmY)_$jUR( zY%6atFqFLI?BbL-Z3X?oYja!!AiV6PT1tqDI6kIYAHX!7Vrh+FltDK%8lDFDV!VQ; z2lQ7|R7?mfTg5M|J>mzAyVd5()XFh+FZJlmQJFC z@R=Wy2zqPR+H`2vJ7yTL}q-&>dHQ|le|Av zG8L9VHW+kR<*jjS!H)rAwl^Y1k#PNWEbJoI0?WgCD&hJ_&dO0T<{TC4fUt-K)^Q(A z+_DaRHYTFU(7LtvXJA=YQ(Z?*RNG+7B_Pn?TE{|g+B<@GC1#VUcqCK+atb2ph@oDH zP|5=JXH%h$GA$YrtwRQG!N8|wz18woMN5&)qiT`N(l&TG7Z8ieGpw^>enZ5tx5N+F zZ5j4Q)9;366WBHuijG#>1eP*7rSf{7xna68WIbl)(88zP8VgZ?*>tRVZewB8+FJhx z#wpt389`WUzjY>{XR#2K3?UHxi;D~daR%Wi@&*fB1nBx}qhu}csYdA_2*vQ1)F#|z zMp!b%7QiX9Eo_kK_vww+n4UF{n7Jod9!KXe+K@ZS5bG`O$(OQ`uUa0?J~{`B|`vD};_~ zaVuyCf6Pz$5-enV=}ibKYTum4U!7&OaL*MeFb?1rmVUVjSutb@wGQGo;++LV6QNo zmaAY=5lok3g8I?Kn0sw7p*??Q6>BK4=6oP$LlWaygwC8O8hm^vBQMkxBhUO*M3JL1 z=a@v4Di|e?X%;d^*YvKj#u!Lo96=ZZv6saqN&n3;A)ku4UT`@*9@C;iA*FXzv{otc zgMs3f0D2s!Is;*1D|)3=Jez>JIXwRC5E#K3fr9_zgk#dt2?$TH}-k*T_# zdK32dxMZOMki*V$u@X znJronpT(5JL`eM1uwZ;-jKdDr2JGCqc2AP8`YzOc;(x#Y7VUA>ct_xSxAL=Cw| zubzJU;^PU%c@`~}ya3-Nxg?lxQCQ11nwV}+u9sWS#dmE#jO*DVZ1}$c7rN;9&heQd(0g;Xh zpG@IxiR)yC642{Kdkzi&7sv%#mONhp#^0t>^t zB;JT7@rJJF#Ft9WRTE^0j?JjAzLDTK)%0gXkb;R17e#s(4AqIM{?o_f7w_M`bl<#v zefa#@ezSE~rBg{zqfQTmcVM2pbwGSQRg!I#UFPGyvFU8KoDO|j1Laiz>FCfsJ~~Eh zru)~|Z~x`l^S2w>k*_lH`E>Mic$jssvCO?aF&g?tPzW$D*!LI?m<5fcURlQP?2>b< zU3^*l-p~sYi;))+Rz=kELJ`tatm%mMTND&;5Rs8+!V4i2>H1YtgLCS61Ih>CZ=^m% z=US}!Q9B|S$9|)Z)hY(#Z7iw~qE4Mm6~Z0fW>!8}9Ee?;`jvu4MNE8U+7v>NhVwnM zoxvM=x#+}uquT^5u_3$-K6P5poabA#V2Y8;fu)1(PZ0QMw92<4NJ|mcelSX5)i5h8 zw01+(HO3>DHiaRA`8#TMhK^vNX+W%yN3Z!JwQ!6U8hiKvD zAij!L9cS!uy zPn`SOz$`SAFDLt@XiW%Swf;mbLo5)8D58B4j3$f8_b|8--!j(Xvx#lUIDzHmie&_a zwk`Dx6(K@XxwMd+E)R|oVqX%kI`KVdR4nUV&iaPgk3F{A$>h^mD8Z6ikC$1D)^xhc z*5P-8uaEraZha_o>Dk5eI^d?qlx=}(@3-k+hyHa{hyHfE2C^AJYWZ8Mvl$fh0!~D! zOSo5fpo)x*m<-{my)r%r4#L_ePwSXPNtrqf8%G6}aD{~j<3ohRKOibf8%x|c@zsim z2>UuQLDi;HCaC`c0bg5EC4!32iSGXltx<1h+BUEKi%?FT>2*585!VAlm#H5WADOYy zgIRWEI^5V=Fnc&NDmsVhkz`#=)E!fc+Gi(T5I`VnhkR&cLvwCCja0oE7Y-GW3?1jc z3J?UbwgLKs$~o3B-wWoG9rT>CAL@Eg<8SSJ77{SIOtPq|{B_57Z^FlUTXX<#V=dm!xy~orBvEJ$8 zJ1nj6HuNzv?Q>1S$_DK4@TbyoGbU`ZDZxtpD4G&b=_t)JwyAM5qT=)8)Op{yPV$xl zFjF0+?3-3d(nz3w$+uw?GE6Z-^Q5DBFa(j?rS>yO8 zF$seC(WgYSR-PwDhazAN9c{p71f%PgXxh{ZDD}X<(V?&f)q6$jTv%vD=t@2W>OdcM z(WOESpO631u~CkC>uV~L@yZu7kDw#*eLGMpGHwk< zTrY{cl9dt-E$l@RLPIMIA*?N=842yuT2Rd$NYIBX*Nfy5$OP%oU0w%|3t4OAkI#x?3+<}qmR6dy^%7SQ=fWlsMKa9 zp;*R}XP;_=Z#~Fy1O?}?RKaePDP=%8#)ckKNo(u}5Q{`G8Ec>B~{C zU?dR-4z?YqHb@Tb9C@^BnI{)Q5{$odVmT4Bp7`_9b$rSH@+JR^mH%&dyWMv7wp-gf zo%WafFaKcqU#v`Fe_;NXtyZVAWAguP?Qs4-P6P8L|H~hd|3xzys%m!sax$q_ve!&x zT_&>fDf{-=8(k1znp}>bSTFly5|eW;;$%QH)P5z7>h>EdgLDCt#C}PP23W8s=yN#m zhjwl_y8{AVSF<=yzH4M?<5!$^&R5Tletdp-q+C~+Dc6O`6uQ-U6lk?6pM@I!cLYw= z)f{VGePrD_3FI|VjegZrD0Mtz`*@u#t=Gg%4fgl-U|cT9ZGG=2+}v@3gbC6@y?#)+ zD;WTdxyB=xc%2pfAHFBP%RPR7czAStykZ1ZVHBlaRXvh2N#w>0!%6({1X}#Z#C#Bs z_1zO%Eb(h~T^Zz+KMnm+jhldrE3Hu2P~WLxXvPRuE%!Z8J#|iyboG32S}}jY&X>Q*4E}$y@7`(PoC7MHEK&HQh+Xkh!cInA@^TN*4`q!Vym(YI2YQVvsxKzl+%Je85@4@P0o zQ>}w+AJN*v{$=Z4*s|PCS*7gFNw+oYa7aIE`jM85CR~%*odl15_)4_r|n z8bN=bN0M41T*XItHsK|a1u0pE2bz8rPlUX6__)-; zb?;qREa%xjIcz_?d*vh=iZ z^CgR6Bi2PMNBDIz4dx^Q>@Qmkp4JcSC2FwT$Gb*~MLyoyQ%Vua@}fLGP)ivZt3b5N zv(^jM(%hRaOk68ADzoCHq+(Id7w+H?#cB_@w88w0QRK}E6cXy(Au_nsLAk0MA)+sC zvq;h@?#^sv)&|4O*I8#UHO*+BO8BBU>1bp!Um?8e!2@dz;P@y5O(AFbfXDD6$vnn& zsGjkwhCi|yJJT~6XU{Ao`u>Y*k<65JU%Ir%W@&4Kl>ah9gM4^1jW~_s4}dUxJr zcV0Z;3JsNSRCBxh&4CDMLtPCgzFr)zu2MyaE0ckmWt>K?zj`%;$I2ZxGL=m$6%m9b zd7aT$BF@USoak$*S}+H&1r?6);D~{G$Q{tieyoby@{8ApPhUKLR&A*2YrdW2^~;y= z_kHn9KDlq7zWNt@#YcXF7pPF8IG7~db%Xw%anO=vB_7K2tR1bw8M;~P|F@g zjo8ppmqEWF0qm}+m}@O{%VGZkE86k`F+;Jbo71!9;($P-Oey*)C9pCzS(8QqIx`Hi z&kI(&sGGx9Te|!?UDa@2yKop=kX(v|tbKLiNCGut%kU}9P4^d-mZa`lxn{8VwWU?- zK!6vHAh0Klla@Dy9sG@&5->npmhI(}=0yPXYpM(Ai;c_KsG0r# zi&~s2zPKjKwak_}FRoSYaMjxRb+k_ykD*~N83F#>5p|88q{Di@h1Rp$?QFrMW`2>l z{yo|&0tH^629kkTc$J7(7E~&?Gvv}l?5puf&}^U5(s$_>{?eaEZ3wd44+ov$#$f(G zJB}^SX2yrMsB{_Hns{Q(E)4|@^^mKBcD2@}KTUayQl~}HEeE`H1JsVzRMBHm9Z;D1 zJcW!4v!1ynNQeAEw9p^)kTKj3bu~ZvfWclk#_g^4!4}&x05l_-7i4Cgg;9S!zeo)P zU}jUAq<_>>g0z5t2?!^mpyu1(W?}I2%zq4w{iG%pY{aWR-erydyJ=nYq8x*HAgoGS z#&T#~CCU~!F=4}w?Yeb~JIZ&#m~7x{`AQ8EBUzCsZva@B2w4Rh?4{)N~}@26Fls*0Bx=o$he9SpK0JO3%X^C)L~)jCfMO-u<+sDKq29 zRW2vQVX+9Id?`Rb7Qe!IjK2zM&kE%1ECA?}-l;K?=S&zkHd@e&5sR&`p%bjzmcwPI zAw3Iu)=f8PSiRwijY&TMcZM*S-JIsxI1J!XPpTbcqcE?i8jf6%&$BI_K8Yp0tmX!q zY)E8k$Rf$Fnh$4=r^0;jo7&mhE|+%nXw{_Ym$YR4ov)Y`Y-+wF$w%j=4?>VwFjUT? z?KfH4Xe6_Dvt;%zlG)oS?DrMk^x7M>&ckog}?xm}3W)NzxV+CcuR7WH~ft;xR)wbLP*zFM%$ z{kO08@*0c8zx8v+*zxf)~X@p3s=QLmDasm040*3 z=Rq`!U4unijqIJE&0lrMA!nVBkjIPuPV=+C1PGv-;Jc(;{XoAo70S07K`Z*CJTi2K z=I>3ZEo!7>!E{9gtzeSPv=T938+_r*Wq|uI7$)?m^Jz&#MQi#JtW?^&Hz8KfhhCp2 zK;)13vxAZ&_Jp|Lv@{_<88(-v8HP8dl^&<3(W3SRwexQlHI>e=!1YBLN4^R z4~me}CFl3^B*l%17a8g}s?n`eHd(%A2+_zro+gb<4{OPR45Kj!yl@Oyo@P4jJdEb+ zFTk&!@rK>xk}3k;wREH}gR-87&+B4pzqt|RM z9J$YeP2H^?Z8MwEL;jR6Obn9DX(%E}zJyj3)om5ewKu7y99|>OWUdy*OnjVC! z8w!XnZq|nECMYFS4on8dr&u#vlYstr!<`zn_FV~#iryg;u0Bzsy3%`>8G^0t%x{~u zqa}NBg|ZzP_Xf&xqP5T2;uPqvv>0V6JPz&yOO&@YF%DY`N&dU4mjH*2lOSt9%!BRP z0CQyk=4u6KFaTXK0G%!r^Ejg-@d8)a`zsA5gJhv#pKUL3jVr+E1*kO$(`S|=hF2%p zir^GiK3UJqxLUAjiUH+91nWa31DK2#3vpdgsH+s6kQ7Rj?#MG?8vY`%R2S?5!w|O| zQ*~~+e~`n=3;P9&=J1L(tp53jSq|L>#&S@4Xta8=_F3u&L3CR0fS>oGyU}Xxg#GnA zibhIqCBbJKm#r$e7FzTR*e?h2cM3-3a#546ss(EHk|<{;F{W@GqRII!b;l*}b6W|FV@+DZ;hSV%l7asCjf?L?+& zvd~zPQE(Pf)0$3K;UR&?MCS%fQ-UIuu$SfT$>$iA#^%y-86b|a`D;d{G6K>RF+j_-`-5Td4mt)rU@#?GQ;pg9tqP zGc@Q9)61XL6HA4jDp#=QC7N?FQ)1{e_u+^Hx!TXgFcW;wxHK-U;iOV>27YFh7a{bP z)}}V$Qe6f;qXP7dA`5&P>XJT>QWlSU)nB zN;TAye>6h*aCur6MZebeZ<~YitMpGrYWFK>>iu+&YjbK25H5C=D3V1<4b+oTe=$B1 z7MWb2qxXwRXuwFs^fk9o&kV?izIR)HiBOo?6 z#&Q?wqQQ~XbPFa$<9fk;K<(Ge%`|mRVc>v>A*G*a2}7P(?5&$n!Gf5}MSv^JN-mgB zaKTjs^Py=C=2QUoBmHJS_tebv3x!xOfx=wx)bO&t?@Cam#eBPLrLXZOHDn{`0OMFU z^ddLJ-doS(JuDYit6so>^KuNx(hX#)HSwi}ofKn^y&E?Z=GSv(dFhX1KdGfslJ@VC zE?bQyp_IjVZ!7G)mP#+I&sY(1o<(-%PkBR?k0xt(%{X=28M@eF772L*g=FR+eQ${Q zjc9lfXPu1*j4_?4td@3KvSP*g4UF7{=@JWH7;Ngmn)(zOl(!&iy60(dsy$WFh^dl_ zOdLupS$JC&X|XtRxl9+=3Y4@OG%jNDX=&S>>Td0}X@>W;8zT(_zTO+?!9;u$E|H z7P!b%4{KMq0VxSazA9?|m<{#=SiF-Wa)hu8>n}<=z*y@;H&4>-+MDt#lN4Y!{ zgsz?>ShjpQv@vIXM#^l6(0OCM`tyRMG%*~2UIJc`949eP)%6%)re~?F^gvo{N7tUC z_S{z3ELYY`qR{`WKn0zg-=D95Aq(toEU=sZhCmfh`+qP{aY_{4EDc`v%3(OAK)~$@ zFlJ3Le^}B=29mnHWsTfftm8u)8HR&$z92o)s$4>sH0E4s8J8pFQz`rSISt0*6u+>= zpVO9QN?yAB`F2ZrKiX+OnEUI~Ul+6%X8Ftwj<{%B;ldv^45o47teFPmAkEFd!g`hzgQf0nYj0b-+YoK( zHgr~!9(L?8GYJyT8-?YXt8Q*KfToB zGWD59)XfOj57^=J=}S@9rH@vezJ8Y_OUaob;pNdiC?o*6S>%-2D~;!D%-A%}yf=&9Yh( zd@CG0i_MCIC!*NJ&@KbjY4pCeuAsmTyc|R0vz3pt=46+>9Wapdr6Br_mo)Q^zG&Ft z262msNcFMY?JQU4inElM1~V5a``&Boie0((J2q+gw4k#z73sZ^;0HPvrwQuTi9z(E zPJ}5>i-|M!`^C(eODgb5w&P!{*SVN+`+YGK!TOep$<{k9*IRFXovVY@&~M>6VKuih zguR%T+0D%>wfYy$Rr$|3qtrn$Gw0f7r^TA<8kOvQ{A9CHq4Fo06s)=hy{qRU>^O{7K=-OP8J|S_$p~fn~m?a=@dO-qv?3-uY zkz0Y-u5gR)C^%R8!y9@uh2q)yWE$Z%o%GFZT&YyxaEEJbYxR9&tByGpg6YM1I=z@- zsBid7ZQ#fKX~37e_HbWC-*xL&O~)GyTu)c2RnLhho!(X(D!HBbJq&0?;bs%5;j0bf zTr=pso6h`o>xv=hb8DNcn2YsSR~>qN zhU#qZ?zUQ|E1S#iYtA>DHdbx#cabBsv%gzi-4x%dTRI}+CbguZzSX(3>Za7hz~D|H z4e%c{K|Esm+0mxFF>XD^wI`UA-bDX~q&zpe>U{^5O+9r%3vZw_;M(EMzIA;9Un-%|cVK0b%^;VqIj&u@ZnUvhU zYu#PRiW9VKZl9JblB@MxP*8TOu35?qrt*F3Tj!02Z|qHKyw_=Y2f7pjiT!Fl?^lO@ zMC3y!C)MjT^z|g^DHa(w>ct&yoYyo6H4U}*LEjG5NPKf9^*=^`>ah}$Iv1={~@*|&UnS&c) zMb52iLQTwC6`5L5n9qk$wffRL&3SslRm^ik6>;{%!jW6mD$@=OgY&cE#T>K7_Eh;< zlrm8{avRaf<8E((t@hiV+$NW0XWR*m6_mF>D~IZX|{Is8ss zUS*^+9s;0ZWw(Y1rHjokWVUkeJlffmMl-^b7T2$&f`0=?jwV>}d*&*v16s))CaqG% zj#s70c@x^+va)ksYDGVprmd_2%g-b0kM^#Ki@BR}^Lww^UOAJL3rZJ6J1R2q6S`#( z3*(1TwSHQRn2~j7=0+5fi^xu&DLPCEG*c&w+)%cGRzvmEdLaZB!9k462D9-bu6+b9 z+UHb020!m-5QR2oF}=}86U6Ke=|V@d_|oFn%zt02L+dJaP7M}DaT4^!_%QYR(R5(l z0?lh|>;i&$3hfz%yraHLue;x19pJ_*a5u0V1+lt-7K|+q6IvcR*0|yNoDj29NUdSG za%VaVYkE-|LSyuNDizuhmobv-f*w^}7aY6mR`+#i3)uHBDupllzc2c~zhC{|=3b|@ z=d?SUyRCNT%VGTo)BiC-XPdQ~lKkI72e?}Q*WTXRPW^v&T3hh{+1c#0zUcq{ApPH4 zByRDTSqc(x1V-{p!GR@CZ!qUH((vHXf)J-vO}UNOsZ>aAVm?L0r7W-XeA2YUCChUM zmk@bjW|U*+Ai0s~bx8b(VHUx!Tu(8+@IV|Qm|;_WkHpj9BIF_3BWBLY8#Hmvj#J?* zG6R1UoMWs$H4G6hR^e>yPXoj}r?W_tzBW>hE1O7#1iAhdF<*#KrZo~fZOm6I3GW*# zWTXx_qbw_g*tzM&a72Q+sEeG+3OybiEs}kZ;2E8iB^i$s(*X&)P3`>?|yjw$~}DX{OHv?_s!d*r!R1E&$}bYHtd>2 zQty)b|NQiC?$N8K-@Z6<|F+HQy@Z&>YF%q|<4doGQEDv(gAI$h{L-Nf32W8qd9|** zSo?wO7w;a`OXRRzJP2s{lbRF-BYx?1wzjKv=hDB?-_pKm^W6CjcLm{h>fP#L(BAx& z;LKMyjk#hq2B#ddk{8VzHwyj9h|eJk`2*fK&UGG5iNW?o8P1}LtTaIy6`;04$#f)X z7QzW_<`_CE8OfL+zgN9x-6*RmsQ`hFmuz<%STK6SgLU}#$^L$`eVQp6=PhU(rU1iQ zOHWIeL~5|@uB8w{K>ob7KSv^k?Jg%VLGhD)5nG8>H;!j;rT#C^p+3kjetKPl4u%`G=*zQd>d^--4v^Bkc_U2Xgwsfg5_;kXJ@&3+a^J2=c0_Ajjh{>dN&SR$-kBtUej7H2+j>glGcre5O{0e_d(M z<)Qk63O9kV3+8oZlL1^2tMtnCL#WH$8)h{!?AIS6avQUE_{Io!niE7IZ7f(G0N=xl zlWO8m$8q(Pe$Xz8!9c5+ncd43@dv|!wnK=%xhWpItD(MXundEOwkvQKP3Zil8{#LP zGcTCY4WZ0f-;551fR8*km{;2Lq&Z3}Vatqq|SBT8SBQ zC1%a7a~Teoyr)&KpUk|G%WKZw*~H0-Dib$Rb~Y2^;mbC;hOTJtV^YaR?OcJs2+j6^icPTadId^;P`NRqvF zCqe&eC)VPz`j;*eGaq8GwWcPGE>yXg&z^qG`$t_ zf|B&(-~7;KhN)v1HD5YdxuHH5; zW|>ZxaB>I{(c{#*i{}E8q5U(+vSfQ`d9XSA`6E)=p&TBoC zG2W<#Y;d~Ndi-}uPAF(}2oSGYYDCm3V%q(f2zlK}&3xc>j=`+F2CwpiH-4U3Zf*+^D`$ov+Fzd5fQ_4LrTc9{Jd4Q~)ZEihTd306fPu_6bJafQ8PPE2$!5h?g#Z6< zSh){DqE%5tqng&)L>l^@`V2-;<~$nQ)?E6=DZ`Tjm*7Hi5|bR#P%aI_0ADS)fV%2k zV{ru<_B`rknKPyNQeIA%z^q2}kLXN5Qc5&|Xv+lRphCZ4b-qx)6dfH7mqO*x0hJ%qx)Y}tqO3kYMP$F|3@0oC%Sx8piR@d0Pe1Ke zr%ng0`jep~C0;5zYI;!JFM8U*Z71_#A>nABdF!f{x|ZxmnHP(l?LHNCz}jLlVv^>G zLqRH3%T3BUd(7}t*~h9~#k@nS^v^Zfdh6Q0q49XiE`zMLTl zw`Mu#@pVUXdWxroM`LBVw`w`(#9Carpfn>F!tAZpra$r~F(1Z~o^e?v#$}y&AcH*5 zG+gk)wb^^Aq51oDJQfX8pE++y6ZTq_@*cx#TBZ3B%dy|;4(>E3qKA=9gtH1W9^t+1 ziM*!~(!kALT8rAySKP-vy=qo_4B{u&W)q*{P2iuziH0-z@^?8yrRNC!mV0kmu#YMJ zXY_*llK%cCMH3Nmng_#Mp4E#E-_THI?T)4tt80k*T?@|j{KXFV8BX= zWlLNSE~Akzd{$gD={pKdh*=h2Od&f#b6Ptk^Dad|c8*bzI_<$G@y z$AK5>cH-b-?726+P4G4!JmipRj)}oSR{BW<6#|mDK=inbi5;1r|IdzSz55A?jY%x} zLK3e-e`MPVMt)ekN-Ynfmc6N4g-_c1&=I#k;4(ITc*vFuT>UCn$Rj}{f2luw*wH-s zWgwj(nPV`e|JU>b|27MStT)mNi8Vb8CHz`!9NH_gO*6zdP!m*CtV9BQmr8q zHLfiawyaQJC;S>2-{GbsIV&-S6RCDm3nWSVpY0)>lFVn735s9qcIwd+u$t63L$GVs zLx!-S_Gl>yc*7w3e0BdL?eEo+6mR^-{8MsXYhP*lntjF2drsOe@7&X%hg7d!JSbfLZC! z5CSdT&4NMKx_E@cC9S(PL`x{Z&kE1;*d9mSl8$AQ_zG52DW=n6&GDhKt1d>tE>`CI zu&knH+aAe5rV_LbG}gG+k_29hjk=3<&P(cC>6=uk2w#zHVawsv|5;4BJ)IPeZ_gIU z91>Ix@5kqjwGh5P)X3h()s|fFN*^~85_^Cf&vxBkHZf1c7r1iCjYXswrMAMJ+xVnZX?EiW8g(4NjEw<8%@-cW_| zC51}5Hxpdw+B&|gdoF9$TIt^6R@q;R8@RG9mpuo1W~+M(+h6~?OZQ;x>y%Wk6-FQJZ1*dh>uze!5V z?ANP;e@i^3ECUmUdvf}#GH0xp_2Kg}K74wJzVL=HL7UO=Q%T&hVV;eLmZe<($fLar zn;&_+cNb}-Gqmk!E>*WLi)Y5VK4e*0Pb?R_5BuR5cpv@r?6v9f|Mi1A@ux1n%3IvL z%w_A#W5ab`JUhKW=KQKd{Yzv2`Nh0m9{j(mw`F+&*yxtjvzUGEVOB53^Os}+xTm|N z2?feB2iR!Cz(KS*Sub-CiLE5~vEOZ=b zTOa+tHv!Ic<%k#(LdcO`^fp_CtIxD>Bpo=y-Sr7YUHr=%$YIf zK>B|}J$$J9Yv`HubwQPIW1;CnqFH@NG;H)z zki~@DQFPJE0Y<$9_L=sT8ZDO@FBg+DtukK5tE`vHb5;-t>~q?n5_X7F7NvgHSyl_q zW+6B(+pIJV)T*QwmKH;buYE#GB5AB_QFOq#(XsC%F4G_Q{SjOovil4LG^$^6Qhdq( z@WuZ7$@w34ciK+3v%Ryq-Th(!{s*)Frm=4R;5SVGagY6XtG(Sd`CnU|ogL(V*nog_Af$l>ORW3sf zVms}_nu!*EyUYqv|H@DP&hjk$uHpHqi|Jb(lI|=CM)808VCJ-ivcyZKx^~iK)3`YP1J~-n03Ai5bu2DNg5c6-`=!3F zDT~~nNqGA%^vp0va@>Kn=d2Du{NSj zF5Fy*#eMPOQg}8ssVe1;EI9@X%!3_BnnrBr^Z7Okd3?=~7pIz2g>rxsbUKO4cgspv zU|jc2|Jz-ABTVzc-k_xnK;&AuT&UUSSTWKZ;2y`hsgpuzpVrtm78I@p+XJEBQ4n4& ztP9$06pRdbx0sIEP=y>`=;`>Pb;DVfs?01}qE{mq_|ixh+hwT8LXFJ(`mguxH#1$P zA1lxg2X~I?`>0dYSCwv?eL6@<8Va}=B0-W3s(5dV{at}{&X+CG4W&6{`#@At7CA- z%-(%&bZwUMV`*qDFNpK*Q=@DDkofFh6B}*iz9O*<a6c6U#Y}`Psf5)50ZU&8%SyPGi$7 zn4p-NBJ<^#mZns@e@k-vQF@C`ZVo5t30DWv9}+UnQ#7fbo)(imt=qZ4SuPP)zW2|F zH0QzQTS3-YeqCAOhw>>Q=$JsuT^fjPOU`SAqyL^_>t_U}uXh%*)n9TPit+7bisSfl zmf&9B+82k5Fa952{6GGa`+w|g?QJ>RTdnQA*6x>c`VZ#+AqqtHlXfmAli&3GuekrW z)7kEr`2Wov;DYwnR=4xT|KpGN@9W9byBK>aA~vA%brAMPvw^Q3_s3px`J_T`hEN{3 zE|LQr9=dMjYkCesWDEHA`Jwy6Gy2AihD$%R&6o-yL)b!5dL0geVWqOM!72Zy6YmC0 z)@3y4Y4`knb=X!v_|q{YA}JF|I&2s@g5;8l2Yp;l9I9xDhC-Mh&m+EO4N`(fl3qND zCXjfDVQV6l@j$;pR)fjqZLFE2=|a4dx-dXQc0&ch-M_1D(if> zEL7F>ho5Ku^p=wjs|ymfI0Ic6C%WE4HD5z%i^|X#Naw^c<4?nMO)TToo7C0E)Ii34 zs>fM!(fU)AKrlb(|5^qPNk z=o^2kO9R)JxcQ<+Z9JiR4<4v&9HWT(Y&J(xvE*aZR9c+6L3v_xG0sPiOdUPjVGA55 zuK?8tkK49ipc+CzmDLKh&m+jSqrsu4YSciJnnh^kWv<8oU13FfycD`dCZDw=$wG~W z)U-y>Yq_&h@fC(yi3<9krz+8Z2K8YgTz#kNW*1`en_#11-w1opy~^Y4^OE z`CfVQvgS>*Ma3nEu>`aZ57i~f*%&5`^n~u z@{iP}DQ~LAyr8<4hSXZEk8Bp0b-z>Bzv7npHEbPJ?!Is(U- z=GJEW3;+L*#{WY-^BI=yXpIrB>xg}f&E5|>NFpyDa8-{V`Ux%14R>+)OR(`aSZ#6&2aMzux>BkJa>=Z zA08eZ9~&N9xET0Uz|SG}F8mXedI`tuw{Ks+b&ub@eg5kEQ;P|gTr$LU$~zQilaMPz zmhKv+xrTZmH8gvdTte+lQ{La7iaklf`fL13H<8xWRqJ4%=-aHHsMqY`9qajhKc)G@ zJtP~T_!AJ%*J>yUXr_nFJWHE!{-rjFX8pcD(uP`o*U>2$^z3jE3R~q}T4WgfV0}8! z*RiAnW}g!JS{-@ez(ZI!^xO?AJv=_f2-JtiI6IOx>_q=|xGc@}pu+bRpte7)iAZL3 z_khEY%@)fC_#Y(@4YP&Ff|;di6x2|Uc7%%Q?O^x_2QW_Jn}c@XMZ0vHszFE+WH2zJ z+gDX$aCtmAj#xU!zUeimb1?d>sigG_vl*;A8wi1etT$%5rjnK+VQ}NB0JQdjLIOFoazJusVCobh523Ie6fr*u^eHtf z!Jm4Feg^~w;fz^SheS3 zilo-dx=gQ+TN;b1hR2UpRnvWrpH?SG+@M~a@{hUxAqYTzYXl$7RZz`yLS&RDHF2QWka-)bVpC-2lCMqUY zdQ|m=E;cq4g0>_OAn~qzxu+M7&j@KbP(x(2g1a_yT6v?O#2*M;wu|#-7CsQA9I#s=t z0ExTXf#oa{`^CC*tM2^M>Pj#YsvFY%36*uq64XwK{U=l(BwUpl0;W)F)Ugjf*Df#- z>$VflxHhvJ>(Fmr>hvKBH7wth1>CES8OE?i}04f~!Q zs+Z%d)~U_m99(h2w-?`EN2jYnCHq3Ab*fr9YEV6wrRx6LITW0&GhVwXX8t4?Pjd;C z(Otm~2nN!Gq6%eq_ExMLtMJh*yb7c1FuTA?3?tS0K>;}mz-Fchvm883>BmBjrO>Y( z;a1KgONW)8iDbrFXC|hb(g|=gxuHwLWq7>|;UEzaqa0B8a)7vhgb{O&`vuH@SIHBa zJD8c?QFY|~5-HNZffr?v=(riB@6%&56^Fou5x<*B&NyUNAJ0PgUE&UTY7T+ZBX8oU zxBg&84IosGDd#sagsIthHe!!!G!lhKfR^w8xN$97a`x(}Q}M&uSRE%M=@DOftS>)5 z=G#XR@z}p=sG}RXoW7yHgEM(U9hz|LhI$tz@ZaSI4mmiG1Sf3>BSHc;;A3S%!b8v( zBZYoj`&JYN50us%7^|1&#~)yM;08|M2kpb)ts_<8R#>UGUKVq zoNP+pSQt)zx+|xg=L*Ykhm0qKe;!fZt58-aOMzS%kNOKu01*}(hfxDSGfqJ^)%;Wn zgEGrb&3KaJ+4S)+YhjSUAg5P+Ft61Y+4h3aZQ5gjJZ=0*IfZMg&Y~aaa>O`5v=67S z)E9l-^nHI2^BGX4(F34C!ZLI=7U*nY^wh+4FksFKzKbGt_`xWf1k9zKhSI)?TsYrG zrj-NQBLQQ|7_;_(%Qn;lyJ|zV8;l;Gzk1g&cv5~GzJB@A{p;(uFP_y4qckI2F$UUw zURP6;C!$kjTVEQ|R-t`g10NTRYsDgW3s|9edW6`2<{7fELSZ23?$~EsP&o8~4-}%V z1gi3~&V{2>ZT67;zIgVe1yLc2Q?ryOsJ}2%K|~{a3Fyac+Ik}bpc+|p(42^RBs)FFhr+E>z_O1bza%gFIGEJkN7uq2UtvnUlZyK8eI8LuR&2O)OMoh(QMA{QGD3({MSd&vtQHW(X= zjil4bn0CRzpCE9=Bkz1Y;LJ~w?;2VSv8&mSlyEPwAj}}tN)A#PSH-lIISLKwi`5pG z_9VhFvDlDOVrg+B=6k8*tX*d{Z!f^9kCCtov^&-zDk#c6t~(Wic$!Sg0hUhZ+(AcK znM!%0;e~pur*yYMa>zsVEG&a->!eBA0VCJim_gqTv(_134Rj1}*@a6M%$4-94CF*K ztLYl$aJNiQG?ffoNn7b-YN=V=p5mSyzkhk(=A^J@CMUhs7c{aT*BX1e5f-2Us_-%G zV+DX^ll#=k*fM|ykVWt*YU_R#MG-I(TZXuK_}&}gHsoqU5)S%) zld22?sH%SO#sMLyFz}FZ(;p6lKHkTi-94Hj`%ZYFytu4A$>@2EE?=W0we>t5PD87{ zIl&&aHvnDaELu?YIwdz%WbqHmJ3(aeM_eqsyR*5qx7FI(-t2DfbUU4HcelN_joaHc zcXoES+FQGOolXm$sIAT2-PX?bb~)B;ww%`9W_xF=wX?T{R@>dJ&i2;MUZ=aVF|~XU zcZ-xGRh)L1{?nu%D$&ahk}mw@6Nu*Yf4tb{7hQR=ysb9-N42_pt=;YJc5813m+f}C zTiv~Ox6AJF)B+WN&*a`tXLqMfBi-6;ZMAm0t-Zbbn(v7F)pobNvqP=#?(R|#JFQly z)oORQ?dG>TTRW}pZfB>v*=lun+Pi!0ZQXqNk}J%u<500}s<5@Qy}P&D-Pvt#ce`8L zdu_kD&9%GB2e&EyV!PYefX-fflU8A;wM7Hi?Y1|q!F4)qdbic?bg84w&eqm;dwX+x z#o!+C;5KPpaf5MZyS)Rq^4*=?-EG?V&ZgholEG0YyY#E;?CoyRnxhB-(e`e;)85+K zr1kQfJ6v*m`7{Ka2goIYo!`{|@Vi|CMX3OPCDJ=&tZfMGS7j$VBj(yvBN#77suwln zY2u2!;{W98GzHlbWqr0e=5s2@@`!@aE&dz%RCs_qgb?+HjGvF0dr*&(VWUw>Vbu5c14(C-a`jkGo!|!En3;#Bt%3^# zF~U=GsG0i(Z%s7x0d^#kyrK1BedMBDVS(qIvjHMCo_ts=aV-gKd#d);McyHkc(U3D z2@U+^7xh6DsS>$YN8BVGmwnZCTB3=9^-cz3?g`CnyLe%7$LUD`EGpe*pUrAN>gf=` zx&F7Ld|^YN$kVSHT*X$_$Ob&nMN?CvRqW0N0bWg^aydx7Gt$#ZN4vi*6I{3jQ@Q~Q z-zUf)dKB6oN0Z7xNx#!)W91MYq^t2DYbiAl+CzEy{ME9dENxsykhN`|7wnJA+tvrw z$rA(Xl;TZZWX=^bp~0a)y|~RV7@scljo_2()J?Gzx_tew6kN6Q%SXN2tuA#t$a6H>chY zZ?yV&PD|7(-R$EmsinqGO6s(Ed2LcgZ0jk;FVX)Obg%+{*iUkbkKkq)0x z&~;8YcfFkNDBj1mJ*a9Xcy9h$%Z5Daal@S}dQUg5I83^Kn8LabR3ItkIt ziyEiqGb5!&qv23`T!fpUDR6H3#?rUd<$gDGGws5 zWOTVHt^v^Kvp6=r*V1E(2dzsM?suNEW6!k9txi2xXH$m`*R$GOFn0j0e1|LyNH$!C z;_%UpCVp5e8mCQ~+tth0&yL);&tAWJ@vnx=efOG;^F#>Ihgm#)do6ZV=C7Cb zjhxVoT~M1E;nMaN2z&lj&uX%kYBlRDtr&0lg^gST%EXgd68es3wEpM5ntIntr}7hj zOU^f|kj|L8tgfIs`kdu0MT<+4TAUZ>Bv$%|x}{}^cjhKa577AHW>B}U6%#pEN zAN3LrjV+Z0J7r0y{ZSM%N0jZq0Zy$9=k+z2F>xC1__|pWfLW=C)J0gRG~~;igoH4` zM>{zkQd66xGn^;bx=A6`SF1j522?gGl{is4XAkmWp6R~!=`}~jX#ZY9L#+Bd_lcnR z0{V3BfRd;bx8Yo0;z|NMgE|}(v)i_9QeYaWJ&?&h;Av)nau|)rB%US!(@PTFN5qE| zPbf!>BDnO}eS7ro{o7ZT#3ej~9b3vR;&pA&4FxcFP_n{lV4b{1R7^X2tKgsd8WglOHqn1z1i_~@YB8!@-V+Gsm&Ps=}Nr>~4G`{&s zuLV8+2D;ynPM@@Rhki*r`sH!ES`PGI6>T}`TW!fIyv+h4i(m4Q7uJW=;I!sEO_&23 z5bjADTbO`lcKRxduwF1pePx-X?(fwOHY!4`M1MfN^3_EymW4Mw?|St(&Cq`TjHO+( zzgGCBw2r+@+a-wUtQc9j^>EE`q&4^+I*Dty0w{~2ql+L#XilGvn!$M(R6#+t_PSU| z)GSUIg`s~T*ZXnsCl)7-P|lBM4d>I}2?4R=xxqa5AR2?|MOCnx7T`F$z+a_gm7m}J z>zgC@_=l%&kDfL3om)kgmpiH#{DlOZ?=o`9=I;wkI0a$!Cm(-+rE#AC+jSLvC-@31!zC1e3kD8r91cfde;P=Gc z=&OjLDd!BXnLcQwRs;4>@}?OrLm6(fcG0jFsm!Y+W$XAd9_HP|=Mgc`d@B#XvvX)< zW3q>|B%L3(lMO*yv-XX@3ri{A44u4gB1&w*~D^2T(j@XO}Y8=|E)Kj1`A?MeXU() z#%+yFUdN2MYK#(`S<5fI5ux)>S-bc9k#S5?>poX9Z375D16H)|O0H|t#vf#F;nwtv zRV-X~DTRGOW3?O$K$DxDe^&LUBfMzn5Z$tkNsY0%n8I8zWZ)=$CxD411_-9FSGAYI zs-=5cxxo*XoEsuU`L-ksoRY>>oafyQ#D_KMlBj!FW zthi%bYg>&u5KTY8msJ16Oyn0Em-O$;0|C%_Rj?Rcy5I`TkZ56CwxiB+F~ecfWdclTC@z$d2PA8xn~}WOrA;yQ{0Jf2DjqsEu7U0rYx94tUU# zspzF{5@cPPsU*Z=9LCIR6Ugs3EJM2{O}48i2_03MqDwKFn4e0wvc&EAd}=9Jld!?9 zk#Ur3$xW%$uQv=uah0H+g9_H#C(S<+V4lts^~O!h9^h2la@+hqVOY+_`W5cw5~JJj zl+5dP7Nm{j1%=@;M_YPvFE%zV0vq9xF_CRqaC)FiOw3FVo~0}J$pkHeBt^+%l4+BO zM@(ib7PFoB<(8(bs-(=9B$!J05>-788bX@U${Vj^sqeR}AY2<^*_5zN%W1u&O;)RF z-ATA?BJB43nStugoeh9md3QyZTs?%`(7tRGl`Kv=M;MbenF03?QE;w3oguv`SCH@^ z{l^@*#?5;TIWh!uB-g2xqAZihZ#4hSYf`aAnFCLJCRaOABAx2itk0q9BX+qh6S_&i zN7A<0rre#067A7tE#NN&$Nrar|L&gSzuP_j`xn7~58L?fZfu#Z>x6;V!@vLL@Za;a z+LyVxeg7r#-)-B2$u98UeK!aY|2?q09{>GIJi4y`ZI2Ml+ngkDxRb9Rj0!ejLgeNl zv7$?aWDJ!O-kG*c_iNa@%Nl>c91Ol+rgIVXb$VdAHc_d$XBo|Mkc5z%T?r-WYY0n0oqw{yfo#U0OZD7b1w>sMpy*el>varW7KW z_;7rGGPPG|xlql^V!PAUIb={v9;=3JF$d5jgF&%EnLnDkkmZYfNAH^h(h{Jt-?Er# zBH(*?m658~%lufs>}NvFH5d#~e8o;M`63C?E*Vvpew*Kv4|{CF?__UQNiet@?zN>|ItLOErE*j$d!>a310Y9fxloB{k|f!u-=fM0deu zp3ph`YEt&WZQ$ne38`zqRWt_NpwC(uO;SoYmZHOZW$?B|h!;ma!IG*(8!6#f3J+DX zr$WRFgcXAqm3Zy1K-kWxlttK8vTTv@@y*9@y|?zV0P4Lpx&qF7Ym)U|MkU#H(+ZY( zy}uZ%xjU`xn0^qrp%vJk>&JF@eX0{|b2p{vdMUEqg>)eL2g_XzMQxg`e5&2u*LNxE?}&R!{;r#tjSD+rXCkn9V8`1IX^pGnnq+`8)F=9= z<$PU;N;MnZzH{={%kKp%F*tFZj&%^aOQqywP)VE!Ox}Xpj*`D*VyN1ORWy~5^=MtO z-n8uN8e-Dx_fV~yC_EILnnEn=SmdUJ?b698($-uSOitz6WjON$mB~vsTTrXs-Rn%% z2z>YCFPk;1x_*bazsr4v6-U5Mh=3lw84zMVFVNXIjTR^sRFt40fEi26Ws#RE>&GG+ zX8#%==O$eo&=Chctu+2NX~EXD#sLhNuCXHHa?4;WToUu4+M=ULhyHT4N(sg2u5;de zMNF6&XGBYKJ0`NE>5e;csupod|B9EIapbRLu#Ry@7jS&=ngr#;Oc-J};=Mkyte0F( z9cO(MUk|OXhYPe7CZ&!{ADBwbhtz@mg+mD@e%v_KIm_XG(18a)WTe!6asj!BxxeqmMzUNk|cbPgwn0k%1 zX5Xm8@SQ_wU#P!lyV3pVVsp|?ersC_L(EP^0pg&;M7h)D9kkdlYL`LoXWw80cu0FN~>VDSglSG6iSHBS;YN%v29X&DPatHEe67!>Hg92KHTLgxHQ zf+CnC5#x^KF~3!5cklnV_y4Qve{9<_gV>KPyH^1H*7QGRzR1fQwuF}`&<9q2eekR} zc)32>uIE{n`957OKfC_N@9qb<%WacdzRxGa?prDWb6X?UFCV^ zIfFg0OvHS$Bf#RH#ri>_nG&E*08c$c=m?)EKYTe% zG|Urz9#;hv&@wIhonkB}u=stagzYFyh8#GiPcPYyA|dYJHZ9ZTGGAhjhIXWX4+Cho zgxd|jAClPwfe0~WzTBk=eE28sxTg67CDViXRZ0VkSl%4AofpUVlo zGoV|J03uC`m5}p9b9_k&)4e7I(}q%vJIE{;>IR)zd#cj zzpgd}(kog*$nI@|{)(!Kj7#%uk--8$+989F2m=NnoM@zh8K8@mJ+xl3D3(}6b?%T& zDND$g+t(=!tw5B6;B)bv!MAWhq#L!IoR+gy#&IXb$>2SJE*7Vl>++GDxdJYE=F z-A!u!>}CE1H|GK7HOvm+MrimK>2i>junm?KGbt1HDTq=9_@P{w!(e|*`A;wZ5%M1=h@ziP{u6|?{3meifaO17 zFaPUM0U|jd2VZ|UY66#a(Y=#FU#p=IloR>j)Q*GF3325 z-w*zUq&Xj1)h{p4>Fs}d`OlA#|DaDk({?;KNIAXyr-e?&Cv;K%W7~ev%YVMagJ>xFJY8+m;$e+|CsJJFK+l|8^U(DX{h>hk*cIY0Lkef^ z#g-Qbux7+r5c|0U`_&?I0&ijYVV1@7$O|kt=CpIfmV8L6*C}k+nA}g}h3|S{4yoMK z%{<4lL(g~ZDDvFM59fXm&0rf0=Zo0R<3%3XY2dh-HP2=<$M%@)wpg!$oq|nltClUx zve25vP>i{oEh2wm&HNn7lP+Q>oI8F7MRvU0Y7u#p9x<(D+Lq~X2wA*C&@#tY zF3$d~2-036K{VS0vq0)(JJE_qH!-ltC?;rUu|k`8DS*o}U2v%~()+jj0ku_e+*LEAc7Y58L-c&

>ubRTUwNOS1aG8#ahGLG6rMxX>io9ayIan=wy|tLz$kqEA!P14jOe^OFW?S zzC7AZJ_T6`{7%Sn2dGY=G7nLSZRWz50g?-c_+ef$02 zwb{N{X6*Q4`5tdeHrF*N1gIAouxQ$!AhN()<>rH_ zy*NiIdVyKJsDNWpK5C|EzW512ox~R`9eTj4Ggw`c)IBHX%Wu^>|z<9Mo5B`CVTl9(zpIGz65rH1b|A$62)*>Rr zYAO!SkK3Vf`06VMQaUo^B?axubesQkQ)I){SQDuPekBy>FxFOglQF?eAe8LT*1pE$ zjLk)vZz|zO{x}-lVWhD(xfc}^e&!cMCEJObk*hjN($sHZlGnvM`yP=O9LPf8tSlb6 z{ENlTEa4v{mL7|0YLA!zRm^^aP88K*KGtP{mu%Onl^|Czy+>%PX_J3VUetkoFk=-& z@yplfAkRiEq1tv_NlGj3p_ivchm;(Wi;#cj<)%uO#bZ7!A4eA(XPK`tT&BIVCkWnO z@{gC$BX6j}@KytqDDC-sXijts&td_qC#@n|qj6&>LiI`f`?R`Fc5}1E%Ki--z@j5HG#Ig-RI?d`C&-Ql(s?N z&F}E?b)8`)$l+r2?)7Gwi#yBIMqo9(Pnn{% zs{L{&CKbBD;Qm{TYS5fxgKDBQVAK4p*aiy9N-#X27k6m7BMiXVoD2e{3i?%k#F0kp z$n-lrmX8#uSu7BzJns#9Hl8K7_Q@dLe{_;*BV#v>Zd-YIUF01Jk&wA%H&c>@{ zI@gE^)Rbon)2yO39f+i?70;K%$lC>ZHYd?ri+@RAL3#CG!4y0P%e%A38g!p`|5G8@ z)p&a+6@LaktK7U(fV1R`$#!Da;CA7_mQ4RxCRD;}5SAn-VZO?dKY(`sV^NZy;rGKW zOvd6k)rIz3FziqxgF%=w8MBSUM5CFI6H-*I)SvPu{WkSED7yFT$B|q zkToz>3-?KaLowh}H!R8T8W*S zDwO^cfLJNGRi6M%bqQ>$2CF*5sX~aVQ#dMtM0JT+jgSPxq77H26EJ9vP(@?uA{A(9 zNeBH&{d_Q1pADbLj(!dJMCAMKvq2QW`nxBGX2@K^sR$Q3>QG!YaEZn5?D}YgBNdq? zbi7_4Zdxl6));qLOfZC$=vx%ILx*sxru_}hiwZf6hjf(PeUT=BZQPOgcZ&w15=qw7w1Lz#*8M-^CZ5W~(5 zV$@v{&Yn?}Xp~$AQu=q)FWJsm45y1lxXAw11~<%PS{Jy?(+AJG3k!K&3lB!NY1!RX zS*OeQ=;&j5 zd+0A2+~ubPA#5*nC&G%fZ zwDNj+T0=v#MZ*La8-`KiK9FZPeZo7ZM9Z3J0>+Zv63r_1>ImQIFM0BVvY<*2_oJsS z;x*hULVi7u<%O}@s9Wmxdx{LILWmLr#*n6MdQkpijR2)nr@^i39c%J9<_+P(O{a-?wo6n~Od`AE0!GAUWm*=DYFR(4Y*Z+MX{ol86oI{cJw>Dq2 z(%6qG3MS}|l|i?qNJ_m!rj^B1tjJ1_FG)*i4hH01g>`ltx~GYHKVWlB?Jvak8Khm@ zZqUgg(yrYF(}6cF3)ufFl&ue{94}h#a_QuP(cFbQx%o0B*X&wz57j8ygH5@~_H!2d zpt|pxwom-tRG&}px2ep0eJD!EzRGt(LOIm``YsVKLytsjfOSPsm6b6z8bYDEa1y%WWMUwQjRqN1>R^Yom%bsm8#HNz<^ct zSz0y~URPh9wNsap?Nij6eA2v8WtkmhWyV?q6yDNfHlJBvSM67ISu89|cKDpl(9(R8 zV)Jn%WkD=wI-aOGPZg*?gVK|h#k29z(WE$i?o2y~Lf4Od%MbASkmmq@71=Qy?@Tug zBj5I;*l{f6zbrS3V2^|#!?a@84t*<(eN1nAutWO5qd8o_AzBOs*jzcF3ZJ^`3mTyI#6~{E_zWR#qHf0Wyastl|L!cBz z5l{C9vU+^CVUJa*bjp!j=V6kfpzeM7|+8Z zE-Oni7qhja0Bh;s#g*6!Egxt{p6xc-I*tvWeB1GWsO$KCUq1jRUWDfE zAs$GgFp2_bNyp6%UuFw5Blzu^F&e7I7zz3(h-}C9V;3r(8=;6B)Jo3+h6sw*@->>- z;Oc&t+3*duj;{D0f44NkZQUV`UlcM~_nkv+pehU1HJucSinW8773f;ne@3C1vYTLK z@$1q!(iWcO%1l`AMJ;ImA!~dyVII@YOa_~%%QAb@| zog5uZv=6#OV}KXZfDSsQbpZ{?tj8 zE;F~NQTxe)Kef#on*JCfOyBUwZfAdjW_i=t}L17JMFl`{6>azuFY25>$E?w zVp?iy@s6pS+AVE#WRA7ksEpUygV^aY4Vsx9x~NY2?PZ9L;HOgmG!phh-@uum0Y(D3{aEa@PPm3L+y{BA=q-dNs2AtG&)d29=xGvT!@HtNjEgQ z7fS*=H>-^&KGr6JWEYpsA$DB~`{wGkx!aIR5~ID;^aJVbn{b6Wropp8Ri~<&Y$BXc znmwG5BL7&h4UDDy|NIPw%PO)CG)CGRwsaBS&P0A~=OlfrL1rcjJ9ZVesqE(t<(^{c z!JKYg!pr<_+f-yJ@*^3&?D?x1i7W=sD?yb+%rs)4Tf}}AivG{{@n%Q_A zGZ~z?WV<#y*`;5z&T3sVizBI4T+jEO^nxZGps6!Qx-?ThJxfr!BEy7T%Mtg5^fIc% z5^L;p^jjWBmE>|^9iLbU@$4DhU$XhRXvVYp2os*zy67|Qb}OD+EO}Nx&Wh)v=?z!# zQ;ctZy#0*RZI06Dx3glt8)61i-Y9uGPUWmreKtPl_^+i3pWbRP#2(6pljRE~Wq#$V yzWmGR%()=F+Pr`Ir+@mVfBL6?`lo;Tr+@mVfBL6?`lo-?=l=ni?0Pf+FaiMQAPr^! literal 0 HcmV?d00001 diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-halos/summarize.py b/benchmarks/vlasov_cuda/results/2026-10-24-halos/summarize.py new file mode 100644 index 00000000..028c7d63 --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-halos/summarize.py @@ -0,0 +1,129 @@ +"""Rebuild the compact report from the adjacent preserved measurements.""" + +import hashlib +import json +import statistics +import tarfile +from collections import defaultdict +from pathlib import Path + +root = Path(__file__).resolve().parent +groups = defaultdict(list) +for nx in [134400, 268800]: + for layout in ["transpose", "halos"]: + for line in (root / f"nx{nx}-{layout}.jsonl").read_text().splitlines(): + row = json.loads(line) + groups[nx, row["nodes"], row["ranks"], layout].append(row) +timings = [] +for (nx, nodes, ranks, layout), rows in sorted(groups.items()): + assert len(rows) == 3 and all( + row["finite"] + and row["steps"] == 10 + and row["warmup"] == 3 + and row["integrator"] == "sixth" + and row["nu"] == 1e-12 + and row["velocity"] == "cubic" + and row["cell_centred"] + for row in rows + ) + result = dict(nx=nx, nv=4096, nodes=nodes, gpus=ranks, layout=layout, repeats=len(rows)) + for key in [ + "step_ms", + "stream_ms", + "spline_ms", + "pack_ms", + "mpi_ms", + "field_ms", + "collision_ms", + "max_allocated_bytes_per_rank", + ]: + result[key] = statistics.median(row[key] for row in rows) + result["step_ms_range"] = [min(row["step_ms"] for row in rows), max(row["step_ms"] for row in rows)] + result["maximum_halo_width"] = max(row["max_halo_width"] for row in rows) + result["sent_bytes_per_step_max_rank"] = statistics.median( + (row["slab_sent_bytes_per_rank"] + row["halo_sent_bytes_per_rank_max"]) / row["steps"] for row in rows + ) + result["max_abs_relative_mass_change"] = max(abs(row["mass_relative_change"]) for row in rows) + timings.append(result) +pairs = [] +for nx in [134400, 268800]: + for nodes, ranks in [(1, 4), (2, 8)]: + match = {row["layout"]: row for row in timings if (row["nx"], row["nodes"], row["gpus"]) == (nx, nodes, ranks)} + original, halo = match["transpose"], match["halos"] + pairs.append( + dict( + nx=nx, + nodes=nodes, + gpus=ranks, + transpose_ms=original["step_ms"], + halo_ms=halo["step_ms"], + speedup=original["step_ms"] / halo["step_ms"], + sent_bytes_reduction=1 + - halo["sent_bytes_per_step_max_rank"] / original["sent_bytes_per_step_max_rank"], + ) + ) +checks = [] +for name in ["comparison-nx134400-smoke.json", "comparison-nx134400-window.json", "comparison-nx268800-window.json"]: + rows = json.loads((root / name).read_text()) + assert rows and all(row["passed"] for row in rows) + checks.extend(rows) +validation = [] +for name in ["validation-local-cpu.json", "validation-local-cuda.json", "validation-natural-cpu.json"]: + rows = json.loads((root / name).read_text()) + assert all(row["passed"] for row in rows) + validation.append(dict(report=name, checks=len(rows), max_abs_error=max(row["max_abs_error"] for row in rows))) +archive = root / "source.tar.gz" +# These measurements refer to the preserved source snapshot, even after the +# solver is promoted into ADEPT or the working tree changes. +with tarfile.open(archive) as snapshot: + sources = { + Path(member.name).name: snapshot.extractfile(member).read() + for member in snapshot.getmembers() + if member.isfile() + and len(Path(member.name).parts) == 2 + and (Path(member.name).suffix in [".cpp", ".hpp", ".py"] or Path(member.name).name == "CMakeLists.txt") + } +for line in (root / "compiled-source.sha256").read_text().splitlines(): + expected, name = line.split() + assert hashlib.sha256(sources[Path(name).name]).hexdigest() == expected +report = dict( + date="2026-10-24", + system="NERSC Perlmutter", + device="NVIDIA A100-SXM4-40GB", + allocation="60206959", + reference_allocation="60205333", + nodes=["nid001024", "nid001025"], + precision="float64", + dt=0.02, + nu=1e-12, + software=dict(cuda="13.2.78", gcc="14.3.0", cray_mpich="9.1.0", cmake="3.30.2"), + reference_libraries=json.loads((root / "reference-library-versions.json").read_text()), + integrator="sixth", + velocity="interpax cubic C1 Hermite", + streaming="spectral exponential", + timing=( + "Median of 3 repeats; 10 measured steps after 3 warmup steps; maximum rank mean; " + "explicit GPU synchronization; no overlap. Input/output excluded." + ), + reference_cuda_executable_sha256="c5db5fa71e9d1119e7cb581a9e67624f42155c006da45184cb55de6542255b9a", + measured_executable_sha256=(root / "executable.sha256").read_text().split()[0], + comparison_scope=( + "Full native grids, mature checkpoint t600 to t602. Not a full t0-to-2000 trajectory " + "reproduction or a CUDA-versus-JAX speed comparison." + ), + checks=dict( + native_cases=len(checks), + max_abs_f_error=max(row["max_abs_f_error"] for row in checks), + max_relative_l2_f_error=max(row["relative_l2_f_error"] for row in checks), + max_relative_l2_e_error=max(row["relative_l2_e_error"] for row in checks), + all_hole_counts_match=all(row["hole_counts_match"] for row in checks), + validation=validation, + ), + paired=pairs, + timing_rows=timings, + source_sha256={name: hashlib.sha256(content).hexdigest() for name, content in sorted(sources.items())}, +) +if archive.exists(): + report["source_archive"] = dict(file=archive.name, sha256=hashlib.sha256(archive.read_bytes()).hexdigest()) +(root / "summary.json").write_text(json.dumps(report, indent=2) + "\n") +print(json.dumps(dict(checks=report["checks"], paired=pairs), indent=2)) diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-halos/summary.json b/benchmarks/vlasov_cuda/results/2026-10-24-halos/summary.json new file mode 100644 index 00000000..46daf241 --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-halos/summary.json @@ -0,0 +1,302 @@ +{ + "date": "2026-10-24", + "system": "NERSC Perlmutter", + "device": "NVIDIA A100-SXM4-40GB", + "allocation": "60206959", + "reference_allocation": "60205333", + "nodes": [ + "nid001024", + "nid001025" + ], + "precision": "float64", + "dt": 0.02, + "nu": 1e-12, + "software": { + "cuda": "13.2.78", + "gcc": "14.3.0", + "cray_mpich": "9.1.0", + "cmake": "3.30.2" + }, + "reference_libraries": { + "adept": "0.1.0", + "jax": "0.10.2", + "jaxlib": "0.10.2", + "interpax": "0.3.14+10.g84aaf30", + "diffrax": "0.7.2", + "numpy": "2.4.6", + "scipy": "1.18.0" + }, + "integrator": "sixth", + "velocity": "interpax cubic C1 Hermite", + "streaming": "spectral exponential", + "timing": "Median of 3 repeats; 10 measured steps after 3 warmup steps; maximum rank mean; explicit GPU synchronization; no overlap. Input/output excluded.", + "reference_cuda_executable_sha256": "c5db5fa71e9d1119e7cb581a9e67624f42155c006da45184cb55de6542255b9a", + "measured_executable_sha256": "7d88657b9597e939f310c2c488d81842793ec361ad190b8f60595c47c8c6ea51", + "comparison_scope": "Full native grids, mature checkpoint t600 to t602. Not a full t0-to-2000 trajectory reproduction or a CUDA-versus-JAX speed comparison.", + "checks": { + "native_cases": 20, + "max_abs_f_error": 6.725869861057276e-13, + "max_relative_l2_f_error": 9.515283499791307e-14, + "max_relative_l2_e_error": 5.6328186846468416e-14, + "all_hole_counts_match": true, + "validation": [ + { + "report": "validation-local-cpu.json", + "checks": 36, + "max_abs_error": 9.103828801926284e-15 + }, + { + "report": "validation-local-cuda.json", + "checks": 36, + "max_abs_error": 3.774758283725532e-15 + }, + { + "report": "validation-natural-cpu.json", + "checks": 12, + "max_abs_error": 3.1839523243379595e-15 + } + ] + }, + "paired": [ + { + "nx": 134400, + "nodes": 1, + "gpus": 4, + "transpose_ms": 323.5450839000123, + "halo_ms": 174.2806366000877, + "speedup": 1.8564603056989837, + "sent_bytes_reduction": 0.8534226190476191 + }, + { + "nx": 134400, + "nodes": 2, + "gpus": 8, + "transpose_ms": 490.5058394000434, + "halo_ms": 140.0107500999638, + "speedup": 3.5033441292889, + "sent_bytes_reduction": 0.8507653061224489 + }, + { + "nx": 268800, + "nodes": 1, + "gpus": 4, + "transpose_ms": 649.6359549999852, + "halo_ms": 342.4947891999636, + "speedup": 1.8967761714491342, + "sent_bytes_reduction": 0.8533482142857143 + }, + { + "nx": 268800, + "nodes": 2, + "gpus": 8, + "transpose_ms": 964.9749842999427, + "halo_ms": 275.0057376999848, + "speedup": 3.5089267313857793, + "sent_bytes_reduction": 0.8506377551020408 + } + ], + "timing_rows": [ + { + "nx": 134400, + "nv": 4096, + "nodes": 1, + "gpus": 4, + "layout": "halos", + "repeats": 3, + "step_ms": 174.2806366000877, + "stream_ms": 96.7597968001428, + "spline_ms": 29.37038770005529, + "pack_ms": 6.511311200029013, + "mpi_ms": 20.97919940019892, + "field_ms": 6.686601499905009, + "collision_ms": 14.47656629998164, + "max_allocated_bytes_per_rank": 11155281976, + "step_ms_range": [ + 174.2041793000681, + 174.3251663999672 + ], + "maximum_halo_width": 4, + "sent_bytes_per_step_max_rank": 1694515200.0, + "max_abs_relative_mass_change": 4.083657516783715e-08 + }, + { + "nx": 134400, + "nv": 4096, + "nodes": 1, + "gpus": 4, + "layout": "transpose", + "repeats": 3, + "step_ms": 323.5450839000123, + "stream_ms": 96.80129259998012, + "spline_ms": 23.18371020005543, + "pack_ms": 45.59232870010419, + "mpi_ms": 137.4090791999834, + "field_ms": 6.081885599883208, + "collision_ms": 14.47260410000126, + "max_allocated_bytes_per_rank": 11017656376, + "step_ms_range": [ + 323.5294349000242, + 323.5631748999822 + ], + "maximum_halo_width": 0, + "sent_bytes_per_step_max_rank": 11560550400.0, + "max_abs_relative_mass_change": 4.083657516783715e-08 + }, + { + "nx": 134400, + "nv": 4096, + "nodes": 2, + "gpus": 8, + "layout": "halos", + "repeats": 3, + "step_ms": 140.0107500999638, + "stream_ms": 48.50200009991568, + "spline_ms": 12.23685909994856, + "pack_ms": 3.281850700113864, + "mpi_ms": 60.16711680003937, + "field_ms": 6.355414500239931, + "collision_ms": 11.2380733999089, + "max_allocated_bytes_per_rank": 5650233400, + "step_ms_range": [ + 139.8805341999378, + 140.1605807999658 + ], + "maximum_halo_width": 4, + "sent_bytes_per_step_max_rank": 1006387200.0, + "max_abs_relative_mass_change": 4.083634327505363e-08 + }, + { + "nx": 134400, + "nv": 4096, + "nodes": 2, + "gpus": 8, + "layout": "transpose", + "repeats": 3, + "step_ms": 490.5058394000434, + "stream_ms": 48.55127589999029, + "spline_ms": 11.72291799998675, + "pack_ms": 22.97926750006809, + "mpi_ms": 391.0327226998788, + "field_ms": 12.39774919995398, + "collision_ms": 11.23720970006161, + "max_allocated_bytes_per_rank": 5512607800, + "step_ms_range": [ + 489.5213302999537, + 490.7239440000239 + ], + "maximum_halo_width": 0, + "sent_bytes_per_step_max_rank": 6743654400.0, + "max_abs_relative_mass_change": 4.083634327505363e-08 + }, + { + "nx": 268800, + "nv": 4096, + "nodes": 1, + "gpus": 4, + "layout": "halos", + "repeats": 3, + "step_ms": 342.4947891999636, + "stream_ms": 194.4373317000554, + "spline_ms": 61.42703189980239, + "pack_ms": 13.08217789996888, + "mpi_ms": 41.15909789966281, + "field_ms": 12.18012080012159, + "collision_ms": 21.11400799994954, + "max_allocated_bytes_per_rank": 22310481976, + "step_ms_range": [ + 342.4020051000753, + 342.5728927000364 + ], + "maximum_halo_width": 4, + "sent_bytes_per_step_max_rank": 3390750720.0, + "max_abs_relative_mass_change": 5.640588611556955e-08 + }, + { + "nx": 268800, + "nv": 4096, + "nodes": 1, + "gpus": 4, + "layout": "transpose", + "repeats": 3, + "step_ms": 649.6359549999852, + "stream_ms": 194.4949301001088, + "spline_ms": 57.92602180003996, + "pack_ms": 91.58833139995295, + "mpi_ms": 273.2574189000388, + "field_ms": 11.26907970001412, + "collision_ms": 21.11227130003499, + "max_allocated_bytes_per_rank": 22035230776, + "step_ms_range": [ + 649.6297856999718, + 649.6461631000784 + ], + "maximum_halo_width": 0, + "sent_bytes_per_step_max_rank": 23121100800.0, + "max_abs_relative_mass_change": 5.640588611556955e-08 + }, + { + "nx": 268800, + "nv": 4096, + "nodes": 2, + "gpus": 8, + "layout": "halos", + "repeats": 3, + "step_ms": 275.0057376999848, + "stream_ms": 97.31809059990155, + "spline_ms": 30.1135374000296, + "pack_ms": 6.52626839982986, + "mpi_ms": 119.0691504999904, + "field_ms": 10.73557600002459, + "collision_ms": 14.49078770006054, + "max_allocated_bytes_per_rank": 11300409400, + "step_ms_range": [ + 274.8684877000414, + 275.1715809000871 + ], + "maximum_halo_width": 4, + "sent_bytes_per_step_max_rank": 2014494720.0, + "max_abs_relative_mass_change": 5.640567068345301e-08 + }, + { + "nx": 268800, + "nv": 4096, + "nodes": 2, + "gpus": 8, + "layout": "transpose", + "repeats": 3, + "step_ms": 964.9749842999427, + "stream_ms": 97.35549070010165, + "spline_ms": 23.13156989971503, + "pack_ms": 45.69765130004271, + "mpi_ms": 776.3868526999431, + "field_ms": 24.17435689994818, + "collision_ms": 14.48879250001482, + "max_allocated_bytes_per_rank": 11025158200, + "step_ms_range": [ + 964.4204498000363, + 965.7143639999504 + ], + "maximum_halo_width": 0, + "sent_bytes_per_step_max_rank": 13487308800.0, + "max_abs_relative_mass_change": 5.640567068345301e-08 + } + ], + "source_sha256": { + "CMakeLists.txt": "d5f11680925c336c63ea39e306230371f6dcb3e41ba423c8c09599c7c592664c", + "backend.cpp": "4dab281aa96caf3589f56bf0b6f313686e70c7f3b0c10788ea071b4edb6a1ef2", + "backend.hpp": "73ee9d9f649eebf405873f87c1eece597e8bd57a32636d2ec047eef156ea0cb3", + "benchmark_latest.py": "9ab3ba1fc7413167bdb880b3520b50a791616a7f86ca929a79910c544387399a", + "checkpoint_reference.py": "6387baa7e5e56c909fc255ed4c9c822b84562f80fffc5386d00570a0e81dddef", + "compare_checkpoint.py": "0b333e43047049965ae9bdb792ea4578a48e6fb7e55e29406d755d8d18b7cd2d", + "distributed_reference.py": "746fef8b0b093eb964c78df99a2dac895ae618d2757cca38e27b90d9617e42a6", + "local_cubic.hpp": "f87f6aa9731f64e501b94e95c20a53f45ae7d1ea5ec39c7e859c6b1a4b8c40c3", + "main.cpp": "fdea36e5d804fd24df663e6c48194dba4e42dec3771cea8167b526db91b52927", + "measure.py": "fa044122a74245abd65e917c886fd2e6a12a8d926183427a75b2dbefceea77a4", + "validate.py": "36fd3383185f1a3ddba754d587ca8d76f94c790597a86859200683330907955f", + "validate_local.py": "9fc625fb020693efd292e9ead2388cbc0320b7eb1ee5d5f56c326706771c6f6b" + }, + "source_archive": { + "file": "source.tar.gz", + "sha256": "df359db1f60990589a2fcac57643fc4770d51d636ecad7379d9bf777f4baf9b3" + } +} diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-halos/validation-local-cpu.json b/benchmarks/vlasov_cuda/results/2026-10-24-halos/validation-local-cpu.json new file mode 100644 index 00000000..61cabd52 --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-halos/validation-local-cpu.json @@ -0,0 +1,254 @@ +[ + { + "test": "signed-positive", + "layout": "transpose", + "ranks": 1, + "max_abs_error": 8.881784197001252e-16, + "passed": true + }, + { + "test": "signed-positive", + "layout": "transpose", + "ranks": 2, + "max_abs_error": 8.881784197001252e-16, + "passed": true + }, + { + "test": "signed-positive", + "layout": "transpose", + "ranks": 4, + "max_abs_error": 8.881784197001252e-16, + "passed": true + }, + { + "test": "signed-positive", + "layout": "halos", + "ranks": 1, + "max_abs_error": 8.881784197001252e-16, + "passed": true + }, + { + "test": "signed-positive", + "layout": "halos", + "ranks": 2, + "max_abs_error": 8.881784197001252e-16, + "passed": true + }, + { + "test": "signed-positive", + "layout": "halos", + "ranks": 4, + "max_abs_error": 8.881784197001252e-16, + "passed": true + }, + { + "test": "signed-negative", + "layout": "transpose", + "ranks": 1, + "max_abs_error": 4.884981308350689e-15, + "passed": true + }, + { + "test": "signed-negative", + "layout": "transpose", + "ranks": 2, + "max_abs_error": 4.884981308350689e-15, + "passed": true + }, + { + "test": "signed-negative", + "layout": "transpose", + "ranks": 4, + "max_abs_error": 4.884981308350689e-15, + "passed": true + }, + { + "test": "signed-negative", + "layout": "halos", + "ranks": 1, + "max_abs_error": 4.884981308350689e-15, + "passed": true + }, + { + "test": "signed-negative", + "layout": "halos", + "ranks": 2, + "max_abs_error": 4.884981308350689e-15, + "passed": true + }, + { + "test": "signed-negative", + "layout": "halos", + "ranks": 4, + "max_abs_error": 4.884981308350689e-15, + "passed": true + }, + { + "test": "signed-wide", + "layout": "transpose", + "ranks": 1, + "max_abs_error": 9.103828801926284e-15, + "passed": true + }, + { + "test": "signed-wide", + "layout": "transpose", + "ranks": 2, + "max_abs_error": 9.103828801926284e-15, + "passed": true + }, + { + "test": "signed-wide", + "layout": "transpose", + "ranks": 4, + "max_abs_error": 9.103828801926284e-15, + "passed": true + }, + { + "test": "signed-wide", + "layout": "halos", + "ranks": 1, + "max_abs_error": 9.103828801926284e-15, + "passed": true + }, + { + "test": "signed-wide", + "layout": "halos", + "ranks": 2, + "max_abs_error": 9.103828801926284e-15, + "passed": true + }, + { + "test": "signed-wide", + "layout": "halos", + "ranks": 4, + "max_abs_error": 9.103828801926284e-15, + "passed": true + }, + { + "test": "sixth", + "layout": "transpose", + "ranks": 1, + "max_abs_error": 1.0547118733938987e-15, + "passed": true + }, + { + "test": "sixth", + "layout": "transpose", + "ranks": 2, + "max_abs_error": 1.0547118733938987e-15, + "passed": true + }, + { + "test": "sixth", + "layout": "transpose", + "ranks": 4, + "max_abs_error": 1.0547118733938987e-15, + "passed": true + }, + { + "test": "sixth", + "layout": "halos", + "ranks": 1, + "max_abs_error": 1.0547118733938987e-15, + "passed": true + }, + { + "test": "sixth", + "layout": "halos", + "ranks": 2, + "max_abs_error": 1.0547118733938987e-15, + "passed": true + }, + { + "test": "sixth", + "layout": "halos", + "ranks": 4, + "max_abs_error": 1.0547118733938987e-15, + "passed": true + }, + { + "test": "collide", + "layout": "transpose", + "ranks": 1, + "max_abs_error": 1.1102230246251565e-16, + "passed": true + }, + { + "test": "collide", + "layout": "transpose", + "ranks": 2, + "max_abs_error": 1.1102230246251565e-16, + "passed": true + }, + { + "test": "collide", + "layout": "transpose", + "ranks": 4, + "max_abs_error": 1.1102230246251565e-16, + "passed": true + }, + { + "test": "collide", + "layout": "halos", + "ranks": 1, + "max_abs_error": 1.1102230246251565e-16, + "passed": true + }, + { + "test": "collide", + "layout": "halos", + "ranks": 2, + "max_abs_error": 1.1102230246251565e-16, + "passed": true + }, + { + "test": "collide", + "layout": "halos", + "ranks": 4, + "max_abs_error": 1.1102230246251565e-16, + "passed": true + }, + { + "test": "sixth-collide", + "layout": "transpose", + "ranks": 1, + "max_abs_error": 1.0547118733938987e-15, + "passed": true + }, + { + "test": "sixth-collide", + "layout": "transpose", + "ranks": 2, + "max_abs_error": 1.0547118733938987e-15, + "passed": true + }, + { + "test": "sixth-collide", + "layout": "transpose", + "ranks": 4, + "max_abs_error": 1.0547118733938987e-15, + "passed": true + }, + { + "test": "sixth-collide", + "layout": "halos", + "ranks": 1, + "max_abs_error": 1.0547118733938987e-15, + "passed": true + }, + { + "test": "sixth-collide", + "layout": "halos", + "ranks": 2, + "max_abs_error": 1.0547118733938987e-15, + "passed": true + }, + { + "test": "sixth-collide", + "layout": "halos", + "ranks": 4, + "max_abs_error": 1.0547118733938987e-15, + "passed": true + } +] diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-halos/validation-local-cuda.json b/benchmarks/vlasov_cuda/results/2026-10-24-halos/validation-local-cuda.json new file mode 100644 index 00000000..e149fb4e --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-halos/validation-local-cuda.json @@ -0,0 +1,254 @@ +[ + { + "test": "signed-positive", + "layout": "transpose", + "ranks": 1, + "max_abs_error": 4.440892098500626e-16, + "passed": true + }, + { + "test": "signed-positive", + "layout": "transpose", + "ranks": 2, + "max_abs_error": 4.440892098500626e-16, + "passed": true + }, + { + "test": "signed-positive", + "layout": "transpose", + "ranks": 4, + "max_abs_error": 4.440892098500626e-16, + "passed": true + }, + { + "test": "signed-positive", + "layout": "halos", + "ranks": 1, + "max_abs_error": 4.440892098500626e-16, + "passed": true + }, + { + "test": "signed-positive", + "layout": "halos", + "ranks": 2, + "max_abs_error": 4.440892098500626e-16, + "passed": true + }, + { + "test": "signed-positive", + "layout": "halos", + "ranks": 4, + "max_abs_error": 4.440892098500626e-16, + "passed": true + }, + { + "test": "signed-negative", + "layout": "transpose", + "ranks": 1, + "max_abs_error": 3.774758283725532e-15, + "passed": true + }, + { + "test": "signed-negative", + "layout": "transpose", + "ranks": 2, + "max_abs_error": 3.774758283725532e-15, + "passed": true + }, + { + "test": "signed-negative", + "layout": "transpose", + "ranks": 4, + "max_abs_error": 3.774758283725532e-15, + "passed": true + }, + { + "test": "signed-negative", + "layout": "halos", + "ranks": 1, + "max_abs_error": 3.774758283725532e-15, + "passed": true + }, + { + "test": "signed-negative", + "layout": "halos", + "ranks": 2, + "max_abs_error": 3.774758283725532e-15, + "passed": true + }, + { + "test": "signed-negative", + "layout": "halos", + "ranks": 4, + "max_abs_error": 3.774758283725532e-15, + "passed": true + }, + { + "test": "signed-wide", + "layout": "transpose", + "ranks": 1, + "max_abs_error": 0.0, + "passed": true + }, + { + "test": "signed-wide", + "layout": "transpose", + "ranks": 2, + "max_abs_error": 0.0, + "passed": true + }, + { + "test": "signed-wide", + "layout": "transpose", + "ranks": 4, + "max_abs_error": 0.0, + "passed": true + }, + { + "test": "signed-wide", + "layout": "halos", + "ranks": 1, + "max_abs_error": 0.0, + "passed": true + }, + { + "test": "signed-wide", + "layout": "halos", + "ranks": 2, + "max_abs_error": 0.0, + "passed": true + }, + { + "test": "signed-wide", + "layout": "halos", + "ranks": 4, + "max_abs_error": 0.0, + "passed": true + }, + { + "test": "sixth", + "layout": "transpose", + "ranks": 1, + "max_abs_error": 1.0547118733938987e-15, + "passed": true + }, + { + "test": "sixth", + "layout": "transpose", + "ranks": 2, + "max_abs_error": 9.992007221626409e-16, + "passed": true + }, + { + "test": "sixth", + "layout": "transpose", + "ranks": 4, + "max_abs_error": 1.0547118733938987e-15, + "passed": true + }, + { + "test": "sixth", + "layout": "halos", + "ranks": 1, + "max_abs_error": 1.0547118733938987e-15, + "passed": true + }, + { + "test": "sixth", + "layout": "halos", + "ranks": 2, + "max_abs_error": 9.992007221626409e-16, + "passed": true + }, + { + "test": "sixth", + "layout": "halos", + "ranks": 4, + "max_abs_error": 1.0547118733938987e-15, + "passed": true + }, + { + "test": "collide", + "layout": "transpose", + "ranks": 1, + "max_abs_error": 1.1102230246251565e-16, + "passed": true + }, + { + "test": "collide", + "layout": "transpose", + "ranks": 2, + "max_abs_error": 1.1102230246251565e-16, + "passed": true + }, + { + "test": "collide", + "layout": "transpose", + "ranks": 4, + "max_abs_error": 1.1102230246251565e-16, + "passed": true + }, + { + "test": "collide", + "layout": "halos", + "ranks": 1, + "max_abs_error": 1.1102230246251565e-16, + "passed": true + }, + { + "test": "collide", + "layout": "halos", + "ranks": 2, + "max_abs_error": 1.1102230246251565e-16, + "passed": true + }, + { + "test": "collide", + "layout": "halos", + "ranks": 4, + "max_abs_error": 1.1102230246251565e-16, + "passed": true + }, + { + "test": "sixth-collide", + "layout": "transpose", + "ranks": 1, + "max_abs_error": 1.0547118733938987e-15, + "passed": true + }, + { + "test": "sixth-collide", + "layout": "transpose", + "ranks": 2, + "max_abs_error": 9.992007221626409e-16, + "passed": true + }, + { + "test": "sixth-collide", + "layout": "transpose", + "ranks": 4, + "max_abs_error": 1.0547118733938987e-15, + "passed": true + }, + { + "test": "sixth-collide", + "layout": "halos", + "ranks": 1, + "max_abs_error": 1.0547118733938987e-15, + "passed": true + }, + { + "test": "sixth-collide", + "layout": "halos", + "ranks": 2, + "max_abs_error": 9.992007221626409e-16, + "passed": true + }, + { + "test": "sixth-collide", + "layout": "halos", + "ranks": 4, + "max_abs_error": 1.0547118733938987e-15, + "passed": true + } +] diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-halos/validation-natural-cpu.json b/benchmarks/vlasov_cuda/results/2026-10-24-halos/validation-natural-cpu.json new file mode 100644 index 00000000..7a9996cb --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-halos/validation-natural-cpu.json @@ -0,0 +1,74 @@ +[ + { + "mode": "transpose", + "ranks": 1, + "max_abs_error": 2.220446049250313e-16, + "passed": true + }, + { + "mode": "transpose", + "ranks": 2, + "max_abs_error": 2.220446049250313e-16, + "passed": true + }, + { + "mode": "transpose", + "ranks": 4, + "max_abs_error": 2.220446049250313e-16, + "passed": true + }, + { + "mode": "stream", + "ranks": 1, + "max_abs_error": 3.3306690738754696e-16, + "passed": true + }, + { + "mode": "stream", + "ranks": 2, + "max_abs_error": 3.3306690738754696e-16, + "passed": true + }, + { + "mode": "stream", + "ranks": 4, + "max_abs_error": 3.3306690738754696e-16, + "passed": true + }, + { + "mode": "accelerate", + "ranks": 1, + "max_abs_error": 5.551115123125783e-16, + "passed": true + }, + { + "mode": "accelerate", + "ranks": 2, + "max_abs_error": 5.551115123125783e-16, + "passed": true + }, + { + "mode": "accelerate", + "ranks": 4, + "max_abs_error": 5.551115123125783e-16, + "passed": true + }, + { + "mode": "landau", + "ranks": 1, + "max_abs_error": 3.1839523243379595e-15, + "passed": true + }, + { + "mode": "landau", + "ranks": 2, + "max_abs_error": 3.1713215548549073e-15, + "passed": true + }, + { + "mode": "landau", + "ranks": 4, + "max_abs_error": 3.1710355546772805e-15, + "passed": true + } +] diff --git a/benchmarks/vlasov_cuda/results/2026-10-24/baseline-source.tar.gz b/benchmarks/vlasov_cuda/results/2026-10-24/baseline-source.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..e54678714ed1484badcc1dad7a8a9fb4fc94f83a GIT binary patch literal 15627 zcmV+mJ@mpKiwFR8humra1MPiXciYI4*d*uVMl`+zFvVvaCw2s&*irw|i|6NqY=gmqdbleh{rk7_{Ja1B@4xxC z-+c4N?u)|{@uQFrrhW4t;9m>={RIE;{eLc3<<9BpvHXc8{yY5pkA*D%CQJWM{cvd6 zqml1gqbQuXfgSW+%y9Wn|L4E{-~TcBU!S3q56`^k$ew=h+KwBod^tXF=lpLr3iE%x zxzYSaOg|?O^Pb=N{9kQ|Vd4$lPIKc~YjbsNeSO1v_N=kK*;-v)HP$x7>w}j&$GhJj z{IG9L?Iek;vgz8{`TLz6`_)G4a^GGz4P|FcRSzhpNQQbf2AF+5KThnP?<$}PTrsr0 z!0L}is%iXPHGl#x+@uRkYUKHDci`H|ICA5B(WB#oA9hanMSnW&#);EyZlnc$FSvsL zdXXJ@^NIn;TU+p7mtgb;I(n*x5Wo`w+OBU07h~+AO3(Eh6Hmj?zw(l_|ClKTAmVTw z^#OS3LEMEA8@W+}&}cmcU=EM=k53O^sMMed)ezPp`=gaE^d&8hrMf4lJ8$=Pj`u`! zgA-__INjepeRm9bHXB%P0lAI~J@Rm1=OC(pmqKx-tvN;i`23I0ks1zO?{`)Iwy>fX zTlKm-O#lt9Q}^v*&#_-geiXHw*^eMQ)%* zR~Oy>rQ5&i4nt=QBV`iBr?h+kqVy9GRi1+W zx4znLTbr%s=CjR4yZNC1{pO!9zxv-b|5uwEtL?)4UtQbScv%0x#`CQg^!>5pii)Cy z%h9M}7-{-*+rJ1SFS#7P&=NsyOfGW?pb~*xswes0&`V-H1@gGlzB@|v#Bt(*I(hQ~ z+Z=#)peq1+10Z>Kqj7-D%E~v5VmKv{ZsCf({T~i?_eB?Ujfn>;QP+UMOT50Agq|Z< z{Ko1`{BJ*s!l;`F7ynkpCj+G)Zy-!c5P#?h%ylyE_uV+If(9Ff*8)1SwIzY+Qbwxl zJ2#=~PLjwAF1Utjb=$Zt0RUh&3G!n%9{Y>h-+c#SeD~zt?(Y7{$pQ!}tWY1dt*uqX3jA5GiFJ@q z)~YppdGX?f3AI68+>rux0Y$vFW2Op|asMsqiyHZoiZ@HXzqU`M!_(MSNPha%)A08# zsfdOV*h4o44c3K9`k-bf?sOChCmi>DSB$WRRjb)3Dg0tc0)LwXcnJqk#8E)(8$d$XHzU^z5;qz}Xd(ba zKSvxrdGf-1G)f{Ep#aGcst*Sy_o9k}z(b&FhMWQk4reG=!damf9GmWtn<}p=rCpe@ zf50~N`Ps9bE*GXo3CEcEmj3@2iI2NF4!wO9xvq(|7hxv=2m04SBeD$|O!@Mt3f*;F z-%Z?YEzcu?Mjtx~qt4>&)U}D*hnx~qFMvd@13$DK=nR+rl!926JtCfGB03T1KBG*zLzxV*z zjiW&3+SbrcQ*}HdlrTa{(C`SYoMShxYS=7!qWoA-{WaVC>VM%tj+z) z$}d4I^;|ZuuzM$B5uv9cqjv#fRkwQXg%@DO@*OZL2eFKPMKnVXpeg})w8U-*jMkk(ePLu@xWbDC zF%{pnC$3{j)`m-Kp07cQ2^o>`;-4O3y_NzD3!I<5^|_6ucw8zOU_Y9U`3OM1wj-y8 zEwM1L*ZRjAH>mTVImJ=1E0q7o)?P_@9icX^7LbP)jd?A=>UM%engaO9UI35Eox$^{ zr)N(GtJ}LtV8l8A#8c5L5)|Uv@rLpHGsy8NxXmh%U_dglMOZxpVBWz^*BK<;sMR-{ zH7Z$CW5r`6iK-@=h@ngzq4`}{UZ`R@n6N&mVJ%dGasq%3KhXf&-9I@!c(Zf5uk}^o zpd8kID=NbJS-~n)Hj9M0aMLoU@ci@7;t#4Li`WyHxUq+#4~?PyUJPu{cb!Vo8{R1D(+N-ioI+T-b~>Dk2B4&-**v-G3ZN6$M?(9IOkjWN_F=kbbBybp zAp!0)wX{1!2(xUNI>x(XZaP2iYGV$W%|!*$aiQ~h26=x}0O!&e$o;Uu{aOMS2={_U z=FT>%z62RMW1Z!4z+Xp^Hvv(9YaTMFTi?Tq24c7zq))fCl#6`2)w`53l=)c+K3pvs zGSUx43^HCnkxDbIY#_OUMi|+_)q;V-jQtR1e701~DnTqA>|!?a5#?5#pEwGqEMOr) zNPBLAfs4h&i#-_QimU_{{S$v24aLyDL46~PT@ivR#o2M~7ubuMfW;pK^0XS@L?{ac zq2pR9G~5{wyPD$BBMk@QMTa|DO(|zSz{4?lNH z+kX}f(2cU@OHq5GP-{A={KW+0*!P zPIV^E_-YYcS){Ui^lpWg0W;g!2kiSd)k;Q>#w`wHs)HH^E4+~lI>ft*sCNseGMVkAYFW$^x`9X+820JEPeC$HZfzv;d{+}(M7uve*x$|3E~AHI3h z{p;cJ>plL|J>Gfy7yODJDFqX-KyDeGc`8S54vIDW(3(^IZBYl%0VY9NNSnw7=wZRc z#7r1V@d1qoprrsfMl|k&ZphZct*rxKeiDrD zA{Z_-&*<_w^A@g5S2x9AQWFDM4`QeWS!aKOzoFmwb_v86j+4dOK@hdY|W|(>PzJHp)Jq#ovtdltweYL6HqY(AaS8l+2AMV!0wscv08z| zGB&V9y*4kZYQ~szaah%QKUscMAeyN;gRh!HG#0P-;F<4E~x zI(UHIiN%_|tk5D8(M)?;>1GBTm&^l1HPNsd+7tvn_xc@3!I(KF78Rh+ z5v&&ITot%mV))*xH_xD(ZTN+M;m^}1>OAXDoYr8)nfz1B(p4WpOE%i*nr!96c10cE z$k+#^{dy^}^%=`rHtoZnMh&Mb*ysr-4l-7FrV0Jn>D?hb1^*OUYajrLf}ha(AMa zt!k@Kz6o+xf;CIFK-Cd%=1v1_qxy@C=}B8@5P~q$!9mv+A;ir`w!(gtk8{>ai3PO9yhp1F1Pb!b;VHXe02LR!o&PJ zgSSXu1(tkOEg=Yf$wDslBu|)26Umh#lBT11{7Mps zlgwo7lNJ!0Q~;#_6y>}ieUGq~!PR?2vUdgycc{JNxSnYgsj@+q2#vUXEXA?sur&TT!Jl_BwmW#*?AKBfPYolNA%DyXQQB0{F^0g-L)EloVTs4N zb!pz(gngo=DW^1GVOVko@Adih5hMcso0;E@#t4%0dRo7-z-Idk#F!5jT_e>IzUu z+SaFLD)ACKbwS=OWv6I4S&nB8kQdHlAM_>1nk6eK>aniIx7-4I%8~A^K>I_Xv>ZiZ zlx~4h=4z`EeH^L;ae-SP z>r2IiTDk=F`#^gU*9u5?i$(_uWaQ3VjFm|5a4t~)}L#uR}` zyo+%-j^U^6yD^#+UAuoN;%msHYP0)M7{|gE*LF0r#Ob9Mi=GXrM`w>q8^6SEgbjd# zKaL_d=)++7A+$l2O}ODQV)2(`{zK-i!yMyge$3q4XZBr~Udr(!0)+{CxoF{Ns)a+T zoF9v`y6B8jvWO5E+hwyEHzL6l>;>k9cw?bC!X($qG;n+=jjH>XGS!;5Mm(JR1rXPS$@IO zX5&arNHh$8#(Cz%iJV;6k8)!2g0Ac&&d7W|tYbnn&UsdN#!j`QKh2QYNKu+Z0bwM! zJWW!)0s+hxsT>&=o~@F4Wa$Z(p6ILHd<|n9{4Em|3N&uNM6pb1NF%$cMs}0WNK|pu z|H?##yp#t^l9!G-1gDe;s6HmftSV9}N@~eSQr0)G5+*8z_soI^|95+-*Jrw+s!eH8 zq(vqf-@XBOs8F6sPEsbxLs`WdW*SC%C~r8Wl(J$=>*%|j;r@K^Dn&vES zjk5ie47AMQRIdr}k$qXZl5T2J(5gx$nqN|3%o`UoL;OqX(u`Si(Vwrkz|CkErwb$P zdJ}g!f7FyT6Au2`-_vhyWzv%q;`osC}ukn*4(OZ~po6tN)$zf3>w%T>l@=e}Ap#+fig+3~dnx=y01q&+WJa{hamC z9fr|O{yaCz=XweY{V(Vk)^23qVAv~>xlqNzOJekX7D;3W@hFTE;fzVhNAKDL$XF9` zD0*Jth=CWyiM6B?@OJ8%O6D0B>C>IxcH`N)+?{Fy>D8o9b_cy+^4`MI)XH1Fn4x8@ zt*--Bt~cAOYtNb+t<>jb(`t}=61yd_5m;^~XI^mASw)3*JqZA1D$cS`O0rLi*>&Yc zX()w4d8-jB8kt8=&dWHP;I%TBnlfDysLtVeF_IcRnFf;7aN z=wZaSZ6bi-7?r~Dh{s#{U`)=%PmF>2nmm$&{k`3ph ztu#6)<;Lp3o>cqh9!9>?caeuOb3>F4$nsSlk>$79BegPp_Dt=Z0z~Dyuc4Nl^K#VF ziwiDSz*^!w`S!(70pb=g5}02 zZTDgBx#qj`bZ7B?tfpbhP(hd#U_QK2}$S#Qbt?+~5AQy4rrQ|9qY2!T$5Relq*dm*WF> z&i}?n(f-reXg}D0zQ)sfR$~8YH$c}}eK7z04xTT+`rjS@uhDE4*Z*egVg3Iq&)xC= zh9eK7lid{&(2gSe=FSj-pknJu16E=6BrhD=fj7!0&1PFNxZnZ{l^*MiR3qpp$L1z~ z%{lS-Ivui9rP27L&mX(vT-uRKR!M@?w*_;AH4nUOtTIt1Cz8n zVRjBhs7LFosp@Lt5toqq*xbManxbZ=Wo1zriI7yTLyn_fesY@}coKJn7P3~OhK|d_ zF&@KP%^ArY_I7R7a>fEmlg^8z^RLaiu1l@sM>s zI?@^>?nN5IOe^b37&1;oj`>KQ?4--Bjgu#B`sg*jk9cTC|(bh@?wg?@5(@GOZ5}szPpo( ztVTATD@W|x(M7+;fefDD%Y^N0DyRt7q6&pvY1vt3tVNz<&iu@D61sBn6b{y(^W;KI z&?lg-dL1dT4wP6S-(5>1OqdkzHk%lOv72+@&DY@iqe{_d8?&$#8A!jXBA$(t8BfLM z&qXB{vj2U(9!$mCGx{-+KX~TxZ#nb$1?Jn{*@vJ~TCczyG?91~{meY`=X7eiKTDf? z>{4yHIlHsCUdgRqQ$ex||G;&m*6$s9R|sZsVG@g12e0?B?z|iN%8JAQP zi({lQj>9pK(N%#cW@4u?Evk4OM2$eRbLlad0xgA>@)PQ$tgaZbgB%hIWhYwM$^FU_ zg;8yZ>%u*XI|&t~wS#!!5@mE!rP77s z3VFxl7$uMKaEx(s9jYV@a33lm0H|^)hY8FQS%w>ohvFo`IFFy;FS5v*IH3)snm7Ux zxF+_euv!HuH@!lGeodT)35JrPQWU~U+^T@LCoq3TGZAlPs+sNrN!hw4H`?Tahr2^b zj4?8L>Xsgi(_(V!Cr0g@Fu>SD$K;O02}yZ+^j~~>R>{T@KI$lNf>+tFJo{IyPtP+5 z^tZWH5u)D{0B3kKSgT9axdok0I6$83c*rSxV~{{*eC!pMlruzb)OPs1P0uQ98I_#5 zVa)JQzJGK^rDzL^cc0S3Ce)A?yG%ByHdI%H4#9Sz`iw@Xi5YPq6E2Fv0zURQ32B21 zn(bMo8$0?_9znN1G0CMkRsl?k#WH;cM+Mf~B!rce3#s6))Y}>j+uq^3m#_CZ%245Z z>5&V)Og?T_88AJoxEF@)6f%|`=UvbhAWMChu(UfpE4N)(-j|Yci@TJO+ikT-8+p=` z`=15k)22fl`V+UCKh(|`E9~EK;+%=NnCOV$vcu{c&6Tu)PZ`O{uuZZ=Io-_+6mPkv zg%n*$zi;)L%0RQjQyi0%cW>?*oRrp7=eRX3S1F;*gFPJxjnI&)%J-6IdysYRzMZ+% z#yz*WtdkF{HRfky`%FeGS%B=oIt0iLYPsR{Qqi__KtO4(z*ti$M5`=SKPr5ZL0bh- z{UT|F)K774G3jCcVQ1WTu(_q@};Fbz=d=3cDB@Q?yaP^#%%pc4pGph z?imPKtswW-L`lbI;V8rDQYido2)n(T4q0X(p=ueKfmjh)7bPdsC9xM=8-}*bITmD+ zmwr+%le@^cBu!y^o_!GjEcNL(nQC0;$qap}At3&AERJhz08UnFqKGz5aK8+rXMTS^rd z$AD%8cgpfn2I8KOxg|5f%(*;bAUXexIYymw=b$Nc?qz;NDXlwq$2yZ4uU+rr5?R;% zLYr~(tm>_3M=tRpR8mMA4-d?93)yFd-=`g+Mf6eu2WImZh5%konN*`9x`53neEB}>}tJJYS${5D zO_Z3Z(KMIiQYz}-ET<}yAMwToQ*s2FzJHzxEy{BhCi9kPl#cdy00Kx^^3bD{9x=EA zr%0JJPfpmb%BwT0MP(ZE2n1J4tq(Ki=umEH;qhS;5muA)JB+usdR|~hy44UeS=SL; z;){{YvjD&^Osxx5*&%7upShDJFU$P_T3C2xlpO?NA_xW|3AXRhNGm3kKaBa)I85@M zjPrLexC93hseGtZw8=nwWqlZDy_ABwkGOqYd5jF6e}DM!n9gxZ)nQ#<+Gx~(cSKW$7}@`(juTaZ`G`CF zh`O0=Q|^YtVHa?qCVgb)Cksy=+W9=`9`B#NJASJP8+nKOv_?#xsppaAiJixA zx@m+F4`{cEq2(@ZuXSMY?$u%U?Yq~n(N_W+%FV0X(T;^65Fe7V@*_Pub-p$)?p|VmxK1wWJ}F{& zokTpZ4NYE^RX{UTVpdSS!#qT5D4P6mu}U3Ij*bt}HRbTQ`*!Ee{#hPsG7#$VEN21y z4mdkZF}&_FjM&RxwE>W773iSk?Jer)joj2QFKV;rYSfco(lA4Z)-A#(Z9rp_oTfXs zWH9LX;kKj!1D%Sf zHkq*a7N02ptg#!qu_ZgRh7tyQ*yLg98AW^E%@&cWKt|V*zigIuE4d3o=Cl!pdf|);!%A;+VWKij;cekT9RaJRypH=)B zMdmSx(IEQL=1OIW0Vkb?`H!kI$GGyQu(>g!=GLF%N!(5s_&JF>s=)1ksZC+Xf)_n8np&5W=B}BT!y-63|BR(rHQQiG9Rr? zxm>(jVahV7jN@+P`amiZx63{sDIH`LQPAD2ZS)?|-|ucm^+p=?yKK%a_DgL?nUz8Nu>OCIr&6iB<-JK=mzgd<}HJhzISx-5vM5 zKD(@1hVd@uLwvhO?>O`u@2~hbc;|XPC>&^%7%xuP6bVpTAy5?)rO#p zxSu~B(wQE45T+(v>%tO`L0xmFZeP>`@rmqUw~w)@$8j_cv_xT9)?>?nej0{1qzH6S zC3y^}_m6QH7_?1-elNb)6Y|3mysAv`rSDGDmvK*Sk*O4fZs7TD+DI@Sj&6hv=o)d| zao-!=SnTB!VnDdX6|qAVf2M0nCr_TV=n6LZWraQ^kv($%IrJQJSQE08R2cE9(3%*& z->QkXVc^n*e@*d}LLV|Hcy}dxvrVhj_n4_Bw$5b7r1$WEYH5t^ik^ zto^il*$4Lb!!Ahh`M$;`-btn7jyp+Z8?v<`=uncsxbn+CyHOZ-eecROBYD4g8FG9# z!0)Q;`{owi@X0?`U_?GrhvBWNoYpjasw?@nGWV#f_jSHpZ!T8c0#+0@GFJlCVQpaX zVkK=52tmClv>o(t>n5QInYEFwX#_)qM3sCi_VK_XP8j%>28Ha0{&*PB{cJyt-RMRo zn8c<;h<<|C9x$b}!B|#@%;@?u^wkv~G)7fRybQ;IV?#|G`3}Dlo5K9c6JPYgDTTen zk+LMVE$)=mf1RH z=(Sz)uqbng*^VPn z%JDF==ns~QO*t~*HGY22msP15gJ*cuf%|3)mSp;m0;sepBOO>)l7w~YqI>{&-}55! zq`omnFd4mRM+T9oOW_@QRIgM~)^JqLj9zi#5jDfyd$0L-)zXPp%T1YDO7mxVqK9=A z>ZZ-~yIDe>h%t~e6hq!Vh$*+r{VG%ncD10qEv{07&F!X5a!?Hi&4z znKATjoB+M%jr&WK(aj-*Zesilu8gE+csHPmFSD#`a_1?!~jU$piIB7iNlk z-2E9;J|Svu>yp+jm_f7gHSnanO{ak0mu{6jA)(VEr#O@Fqh^*hz_=I*ISj-d=|)<9 zwiQO1%km_)a7VvY2H!`|%2LoH8wH$W zcCw-9Q6>=HWAWJf7xRAaiH~?G&hQFOFGPcV$fSuk}N#p5u3%j@KB&P zGOJ9eC|>{{M2T{?I^e%!ay&xe60{;@bTpvIHIT((I{Q^rky24uGU*of>1<^(BA1{7 zCX_@6W0aD6u_+s>r);a}c#IFq>%=TBEIS59p5T^v-w)B5oXXeI;x!Q^q2Fn_^(Mx< z!r!a#d$n`~nc;!{iX5+e=oC-nc4oT<5?y)0C19qy?**`0Ak5MQqFN3KH!l#|OeOD2 zx63PMRrI}vwp%qxPIEz+EB>j6_0K_Lj+9w^P}dzaP}XgrV$=jJwY)2V?ceZ0%H?!h zbDC)?^}M3VD?I6#b9{=H7(xY*Hi|866x;Jhajwu>7&bLzvm5!9GZ-)Rxy&)3ayEA$ zwFZ`^i)RaKQvjq#>WYx%{U0iL`Rrog=gh~TtdTMoi(C_B=(`Imxwck{J)~=Z2{G$) z5rYnv==w)1=N7M=>@pC;EB!^EKlJhG#q50v=`y5XWrVOekbvvaZCT~9S`Fd%EUAqm zJUvGX01KJ5Sh=ASRW`rkSK z*V`NG#r1!!`EdW=S9vOx%JCR?d`2#vltS0(;G!P)ZJ#1@GkKsnT*lM11G2>`Lu9%w zZyCmsZ}(-^OMAi_XK1HnEA1AkC6;&r}e)iT>PDLSn~Pkiw=_RF)k>rPfDoL?z^BI{1L^x9DBq#e%mzMBqXL=?Sqo^ArDq zt^&WsZJJR!=J~yvbcd$W+#)k?bF0OF+Lgt-HDr9A-RfCwZLDuDE;uPIxY>NxT3Alm zvS*978Vd`ihU^k(7w>J+Y%6bbamjQ)O2!dzludc50cR4g)A^&NspMDep0V+?jwk#U zmA+#$EhDjPOAU99()4uqD9f2y8|2Y?b>ZjG9+`yiG_TxVsccm=bDcL(=j23Fq`OqP zmp@6imNSpYKFIT~8srpUgSg@-j6F1iaq$(d40nNGL~Rul9o-rN3iO;-pH-%eGp63R zYToD~Nen^GAk0IjvT*E=3@|mOsi4OEDYlq=x6YKO7*^RI z$Jp_Hgli4P$e9ps?0%oF4<&CgfC@T_YY#3AcI10*q%T8Owo|Gbl0v|US()-SXwl+? z)RLnEAeb^*QF$3NnYV(D(Gp|$uiwVwB~AyiBaL|ZMT}D^J`v{q1zUc(?LEpfv&YA& zlleP(`sN>xJMRGIHuutxoKn?y`qGO&{`NQVRvUi{n|Nd{{yaN)&^M^=J)bGAoMF@$ z=9D;Z@B>?+>BF4e{JFi(IQr0whj@5y3#5?nx~xx%E_jXU;u^z&Ef5BIz?-3<6XtU^ zLrTD?q>c`m*Q%_p9Eu(ptmkiXyg*z2rXU`aFnoNVvFKs35I5=`mNZ_6WO8#61j*KT zgudM-v#t*Rsj11*Nr@fDsq2Rqogy*eW!xD{?_^8DY;(FnzqmJ$8t~^7qMZYCiH(L& zSMJRgp%8wY5wFH;4#>5I2{PRV!5`FLJks<>*Lfc7|G(=ev;TiNK5!5FznuT=jpo{e{r_t`Mf?9|bJc1# zH(DDo9IFrZ|KI%cOCI=UdWnR?wvX138t`X_SO?{4CO{ zVeDWeR#4h~@_Ku9wBD|X=3es$S?YiPm;ZJYdND}qf^Ie(+R>E p}Q<|cNUfRgWy z-o=(UNQ4u*u|c|!42HV1huV=enD6ZEAD!}1K)!Mebe&hPPXDU3FyT(UIEmS)qmC{b znJfIZ@W}TP8?Z4L9WHMUol6|K5gA<2rsbiggSw`46gR&8R=i;|69%q~T~rgc1kmYW zKM8woBwCHux?vm!;>eBs;W$Yk4Lepp`SDG=-fn#NlDb%fR?o>=Ev5pE&*y}$T*y1a zY>Dr@mm(Z+fZM9P39Z%Os7T~OXb-T3RdxHSbgsY1c2;~pyawzVL)Y_P`zH?68HZDC7Xi(t#4Y-@NfH>)o5YbhP8?B z4GZWj{nLa$)3hAsTidiY@Jp+0t+g>n%i7$;`m9xYU9~ngsbF&>D~MlK*R5x3Twrw- z|2C{=m=hMu4fylS+Muu2rYzTz4K`a%hGkBR8|zk+Azsr;;+OR`&WJ6xWQ!YFi>sT~ z8aLTk!-59R7mmoB^aBcw>vFip8Opiv^-fR5td3BBs5y%iC*}WT!*|bw^sZz2*;5{FEThJ z@g5BhZpv{yp8J;U+8AhApF+aE0793Wl4U2A({swK@y8+w4d0D1h_oGu^R`=WuAO5X zG>SB3_u|kWquH9A%Dq6eHsyKfEf@o29|R~T)eKBR(~vweEwK@hBPN2-&V2?f1_4eV zzzdtpcpntzq~Rl%jAU?Dny$4JDrHULdrH`(guLy1%_4{Db81WGHqzV)-mO!Y{-j!| zRSSB(an6h)7x-1QtOoPk7_VCe4rC0dNWFz+n(OCKDYKA-_KDL_h%`PJZx%lEd@sHP zOxr{{*c}xcw!m$16|ns{6!9gVl!6gS2p5PR9BJO7((f)J%Aok(VmS6;Q9)m4-mDgK zvw91wxS=qauq+r%t!vNzgj^OOhstSZcm+wYjufbPt)($IFQ&{iDj054-6?4x$qBp% z)+egpW^6DY1Z0dOIuRwi1rPxqGIH}A7k4)Z#j$JTw$+`G)~XbN(2TxNAp~~h;nGU9 zPV3{!-|hSem_N3!b6fPvUMDVX90(@Gei-4&qyT|oe`20vUSPcElb-Pku5I;Zquy$t znX_ptfY2dsB%mp-%IgY8kj<7|n(Z1ayvHxS;WLXnbpclJga~O26xP^koHOYGPjdZm zgr$v)A;9{FaQjd-p7NqaQC&AeQ|eM(iXHPRp2ag5y2)keFazO>gki07$X(h;6bToP zdPPWDyrHCDD|=qwKSu-t?GD492ctd?JRB7RzoIa55#fnpM)J9{zv4{gIdHlT7g&!u z5YD73d#9!X&?oaYBsJRr3hzgN8vjNI@r^ybc6}d~b$OTRIi5sF!&^SVlL)ORBkzfd z77wi$d%T6rqkGQjoi^lxew<@W&CVwNN^8M$zs++6e*&*fsKwaF>s3$iAjpMaF_5V- z-ZG4)CBC=)0nUVU#!#Ly-H_Zvo4w1Ig)=(_MmnViVl=8FJ01bzVd7kR101jxP*Zx$ zu09Njbdb@eanh-@P5Xi#NAqp&VckLroV2%P8LLoJpo-G{kR19te})g)m^wl(5B0SX z1X^FRtXY@TD$>WZ*=jA()=C&pcQEk!9#XB4QF`%SF*zrM%cs}rmAD+y4hVG)#qzcL zK+qmWV7834*)CvSggM`f8rBm436~57jNlLXF4ML%9ihhURJ<(>25LLg0)C$JYoX&Vqi3 zjJWSQJO^`p8kP=Yf|D?Y0RWH?8dVvq4cmO-1kTe`CBVS|(HRe}sgROhFyZ4o209|( zE;}xvu&_oR#@8kdhA;qv&tc%w(Poy*<7;~)_%d5D3Mn=SOEB@3H!_&B<%FJoIPzT1 zqLT@X`JR>yJ*)b({_?GqaZRUEc*E&4^6fyb5P4w_h`rQBWlW3Q#!F&z&z6p*mHvfQogH0!lP=W*!@C|C@OT?Vh3;W4G{E@nKL%dLaQM$ zCF1c2S2K&?&n5x(NL!nk_F#NqHlb_Nm;y;q4(J0f880y`jbWh32B<>em!fquc6Q(3 z1@$jFAbNu~h&@)$H%wF{=&`P=5J1A_$(v1(sZ?AX?u)##1iJ|Y6p8D|rpp7aaXPDN z@!A!S>Ii*c5ncd{_x$`kzBKwn1f_nGDcSJ~p9SX)(E3ZUk_<;OjT~?49CGIa@TU6S z?wg&z>~~+jJ9xcE*JyN(d0kCE?f&>9#tc0=c)fqz0T}z!scKYKG^hUhN6|Jk=dt#j zj0#-p0Q&Mx1?$S_fI#Xpc2zDydW=UdBgszLAO)F-X49B)#IC87Wq;?&Oi08<5r>3yxf;L1HmTS<=O&hq6g3 zw4>gi3j9p=IYUN`Oa$kJ0Jz4F->gGNA$>90ix8{cWVam zETDhv^6vLtXrX)Z?&#?7__VwKcIV~m{k;yz7Gj@z4gF+&kY~05%;I7+)?7E+f>P>u zvsEVm?3}Z-4SW~Gsv(}G^DfZf5pZv@n}jjy8O}*OUx`K^oSLB3V~`^?ZoA)uy52-+ z$D`~T47I&ZB@T{u1>bBH8#FUnmW?p5ut%-|G#H+bYb>-~96uB&G-0+5Lx`%9A){Z# z>3sf|6$^B<(DC|pV3Cj$x$wXTooFQ-#VfL$6<-RTS9M&5VA}Y7@fapFV!ZB-VL|dn zz<4}7N9+y6dr|r0r<2$3j^EH_Fb8{E^~P-lW+hOKINN5~fn@N;HpuZ%h=-9!nWp>7 zs$FM@k%!3E5v;6&wh4yv*sn2xw9o=Lywt- zAm>H#J*Fr*jy@tnVwU(jtjpfuh6oIV3cd-P5rY0#(l6Mc4|2UsXR%yI$VOEu;P8JUmOmo(&tGAStADVsiEb!ug%4{Gp6 zE~dQvK&K(MQLvya73b|-3rhs7fYD{7N2b$-w??xL2n*0rkXxi8ijeb!#j~C0sf9hF z8GFNuGiMpHU$}$jfhehyfkIj{*gP|kG38)Tfa5tz?&V4Ktrmlhl=dO8hE&`nQ?itw zw(h;4B2&`VQ|+FM&f%+93rKv?Dw^xV&Jdy(>F)JLCz5|GI$Iq;}o<#=7rOVJhYX;tiYV7yX}O~1d*DsU?#5R8 zTofE2WkYihM(Y~QQdAXKiBOk4r4Dv9yFl5C#owJAzI`o25Ag+2HAA}=D0LT;&Zy`? zJ)+r;RiryKx@)=SI-IU_iwP{l!xy42d3t1dD$CA|EJ1D(Ce7Wuxz&V!E%?_KE%@8o z!cv+)yeu*+{tD?|i*|P9PAgVLdT0QCb5R+*F2xtWgV4a%P_oteJQ47O4@sYRTeZAJ zTT(VLNU48I^EHuzMBs>xxWIc^1~M2n5HeV%w#(!kbnBBBqDh-F8zs{SNrE-Zm^54# z;Zg8`g~Z(!+D%^q7Lg>qxgm0i8fZcN3o7{Z8Vitf-H57E?Vn?9(Av4Y>WqwooKKuf zJe}hdmw;iDXorB=H{mHWRCX}d7)jzKZ$uV4an6Xa_Yt}Sx~}vtJXd$F#WM;B>e>NK z8XnM^$jvZi&WsC_iku!T?NT@1^K%`ehd9i)g8eDK)s3%%gH_qTVPNAY4(cSkscl=` z45t_uWYmX&ReRnr0{EoCTFo)!597dNYpp@l!vv**x{?7?2_or+3v*)`(`%4vJiuIm z9R%A3FYppXE@yM7FKj)AIzaSJk#CWvUn6DeQ`s7R>KUjQQ{T^hTCVRE6o zky|LS=Il~++GLYg(hQiTX>tWx5HmRyw?WcQW_voYP9P@i2>XbpN6tL}h*66KLLlp^ z1BpRQn_YA)2e0A<^x{Don3Fx$Qc{Bm=_HnTO&pZ&BjjSnz$2mHaa_C{*8<^V$Q&4Z z%G_PjZ>Vn!D?=T2XLdIj$TwqlsRN1J&?Uxz7n?Gs*y4)&Jz)e&u0aM3vi?DjcrB7v zO)2fxIFi$g%hyujEpnlenj${hlt?9({9IBz%tqY^MxsYS5CO2v z(1F}|sCX=!`EWh~j&i$rbb?9hkp|(Kh;$^klPvKTHy+ptg7#n2eLN(fzHbBOoYc~Z lXP-d711dxItXW0T_WIx~_wYPC56|a#{vT~!Y2N?_0RWZJnw|gv literal 0 HcmV?d00001 diff --git a/benchmarks/vlasov_cuda/results/2026-10-24/scaling-32768x8192.jsonl b/benchmarks/vlasov_cuda/results/2026-10-24/scaling-32768x8192.jsonl new file mode 100644 index 00000000..516da6b8 --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24/scaling-32768x8192.jsonl @@ -0,0 +1,15 @@ +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "nx": 32768, "nv": 8192, "ranks": 1, "nodes": 1, "steps": 20, "warmup": 3, "dt": 0.1, "length": 12.56637061435917, "vmax": 8, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 2147483648, "max_allocated_bytes_per_rank": 21476605968, "stream_ms": 49.01655189999872, "spline_ms": 17.81408299999896, "pack_ms": 0, "mpi_ms": 0, "field_ms": 2.06822614998714, "step_ms": 68.90644464999696, "initial_mass": 12.56637061282561, "final_mass": 12.56637061276199, "mass_relative_change": -5.062453547606825e-12, "initial_kinetic_energy": 6.283185306434273, "final_kinetic_energy": 6.28427053267025, "initial_field_energy": 0.001256637061435577, "final_field_energy": 0.0001735421558638325, "minimum_f": -2.32438533550678e-16, "finite": true, "repeat": 0, "command": ["srun", "--jobid=60204067", "--overlap", "--nodes=1", "--ntasks=1", "--ntasks-per-node=1", "--gpus-per-node=4", "--gpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T095729Z-66315-25333/workdir/cuda-build/vlasov_bench", "--nx", "32768", "--nv", "8192", "--steps", "20", "--warmup", "3", "--mode", "landau"], "rank_log": "rank=0 host=nid001409 local_rank=0 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=937bea67c3242b0cb8313a6de50eef13ee7dcfa6b642142a129b277dd1ac56bb\nsynced_at=2026-10-24T10:06:08Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "nx": 32768, "nv": 8192, "ranks": 1, "nodes": 1, "steps": 20, "warmup": 3, "dt": 0.1, "length": 12.56637061435917, "vmax": 8, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 2147483648, "max_allocated_bytes_per_rank": 21476605968, "stream_ms": 49.02767665001591, "spline_ms": 17.8467182999924, "pack_ms": 0, "mpi_ms": 0, "field_ms": 2.070863450006755, "step_ms": 68.95362119999504, "initial_mass": 12.56637061282561, "final_mass": 12.56637061276199, "mass_relative_change": -5.062453547606825e-12, "initial_kinetic_energy": 6.283185306434273, "final_kinetic_energy": 6.28427053267025, "initial_field_energy": 0.001256637061435577, "final_field_energy": 0.0001735421558638325, "minimum_f": -2.32438533550678e-16, "finite": true, "repeat": 1, "command": ["srun", "--jobid=60204067", "--overlap", "--nodes=1", "--ntasks=1", "--ntasks-per-node=1", "--gpus-per-node=4", "--gpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T095729Z-66315-25333/workdir/cuda-build/vlasov_bench", "--nx", "32768", "--nv", "8192", "--steps", "20", "--warmup", "3", "--mode", "landau"], "rank_log": "rank=0 host=nid001412 local_rank=0 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=937bea67c3242b0cb8313a6de50eef13ee7dcfa6b642142a129b277dd1ac56bb\nsynced_at=2026-10-24T10:06:08Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "nx": 32768, "nv": 8192, "ranks": 1, "nodes": 1, "steps": 20, "warmup": 3, "dt": 0.1, "length": 12.56637061435917, "vmax": 8, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 2147483648, "max_allocated_bytes_per_rank": 21476605968, "stream_ms": 49.01497695000785, "spline_ms": 17.79829555000561, "pack_ms": 0, "mpi_ms": 0, "field_ms": 2.064744599999813, "step_ms": 68.88639965001175, "initial_mass": 12.56637061282561, "final_mass": 12.56637061276199, "mass_relative_change": -5.062453547606825e-12, "initial_kinetic_energy": 6.283185306434273, "final_kinetic_energy": 6.28427053267025, "initial_field_energy": 0.001256637061435577, "final_field_energy": 0.0001735421558638325, "minimum_f": -2.32438533550678e-16, "finite": true, "repeat": 2, "command": ["srun", "--jobid=60204067", "--overlap", "--nodes=1", "--ntasks=1", "--ntasks-per-node=1", "--gpus-per-node=4", "--gpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T095729Z-66315-25333/workdir/cuda-build/vlasov_bench", "--nx", "32768", "--nv", "8192", "--steps", "20", "--warmup", "3", "--mode", "landau"], "rank_log": "rank=0 host=nid001409 local_rank=0 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=937bea67c3242b0cb8313a6de50eef13ee7dcfa6b642142a129b277dd1ac56bb\nsynced_at=2026-10-24T10:06:08Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "nx": 32768, "nv": 8192, "ranks": 2, "nodes": 1, "steps": 20, "warmup": 3, "dt": 0.1, "length": 12.56637061435917, "vmax": 8, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 2147483648, "max_allocated_bytes_per_rank": 10739122192, "stream_ms": 24.53958674999512, "spline_ms": 12.87446425000098, "pack_ms": 6.339934000004632, "mpi_ms": 13.78156439998861, "field_ms": 1.197888349997811, "step_ms": 58.71698414999287, "initial_mass": 12.56637061355123, "final_mass": 12.56637061358818, "mass_relative_change": 2.939963389748054e-12, "initial_kinetic_energy": 6.283185306796558, "final_kinetic_energy": 6.284270533062034, "initial_field_energy": 0.00125663706143567, "final_field_energy": 0.0001735421558640621, "minimum_f": -2.324385335496638e-16, "finite": true, "repeat": 0, "command": ["srun", "--jobid=60204067", "--overlap", "--nodes=1", "--ntasks=2", "--ntasks-per-node=2", "--gpus-per-node=4", "--gpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T095729Z-66315-25333/workdir/cuda-build/vlasov_bench", "--nx", "32768", "--nv", "8192", "--steps", "20", "--warmup", "3", "--mode", "landau"], "rank_log": "rank=0 host=nid001412 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001412 local_rank=1 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=937bea67c3242b0cb8313a6de50eef13ee7dcfa6b642142a129b277dd1ac56bb\nsynced_at=2026-10-24T10:06:08Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "nx": 32768, "nv": 8192, "ranks": 2, "nodes": 1, "steps": 20, "warmup": 3, "dt": 0.1, "length": 12.56637061435917, "vmax": 8, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 2147483648, "max_allocated_bytes_per_rank": 10739122192, "stream_ms": 24.5278908499813, "spline_ms": 12.87684925000008, "pack_ms": 6.339425149968747, "mpi_ms": 13.802522550003, "field_ms": 1.183015450004632, "step_ms": 58.70064285001035, "initial_mass": 12.56637061355123, "final_mass": 12.56637061358818, "mass_relative_change": 2.939963389748054e-12, "initial_kinetic_energy": 6.283185306796558, "final_kinetic_energy": 6.284270533062034, "initial_field_energy": 0.00125663706143567, "final_field_energy": 0.0001735421558640621, "minimum_f": -2.324385335496638e-16, "finite": true, "repeat": 1, "command": ["srun", "--jobid=60204067", "--overlap", "--nodes=1", "--ntasks=2", "--ntasks-per-node=2", "--gpus-per-node=4", "--gpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T095729Z-66315-25333/workdir/cuda-build/vlasov_bench", "--nx", "32768", "--nv", "8192", "--steps", "20", "--warmup", "3", "--mode", "landau"], "rank_log": "rank=1 host=nid001409 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001409 local_rank=0 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=937bea67c3242b0cb8313a6de50eef13ee7dcfa6b642142a129b277dd1ac56bb\nsynced_at=2026-10-24T10:06:08Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "nx": 32768, "nv": 8192, "ranks": 2, "nodes": 1, "steps": 20, "warmup": 3, "dt": 0.1, "length": 12.56637061435917, "vmax": 8, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 2147483648, "max_allocated_bytes_per_rank": 10739122192, "stream_ms": 24.53943520000053, "spline_ms": 12.8782511000054, "pack_ms": 6.340501999989101, "mpi_ms": 13.7808962999884, "field_ms": 1.185581100014588, "step_ms": 58.69956045000322, "initial_mass": 12.56637061355123, "final_mass": 12.56637061358818, "mass_relative_change": 2.939963389748054e-12, "initial_kinetic_energy": 6.283185306796558, "final_kinetic_energy": 6.284270533062034, "initial_field_energy": 0.00125663706143567, "final_field_energy": 0.0001735421558640621, "minimum_f": -2.324385335496638e-16, "finite": true, "repeat": 2, "command": ["srun", "--jobid=60204067", "--overlap", "--nodes=1", "--ntasks=2", "--ntasks-per-node=2", "--gpus-per-node=4", "--gpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T095729Z-66315-25333/workdir/cuda-build/vlasov_bench", "--nx", "32768", "--nv", "8192", "--steps", "20", "--warmup", "3", "--mode", "landau"], "rank_log": "rank=1 host=nid001412 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001412 local_rank=0 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=937bea67c3242b0cb8313a6de50eef13ee7dcfa6b642142a129b277dd1ac56bb\nsynced_at=2026-10-24T10:06:08Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "nx": 32768, "nv": 8192, "ranks": 4, "nodes": 1, "steps": 20, "warmup": 3, "dt": 0.1, "length": 12.56637061435917, "vmax": 8, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 2147483648, "max_allocated_bytes_per_rank": 5370380304, "stream_ms": 12.33204280001132, "spline_ms": 10.97677384998974, "pack_ms": 3.176275849972399, "mpi_ms": 9.777910849993532, "field_ms": 0.7748922500013578, "step_ms": 36.97000044999754, "initial_mass": 12.56637061435844, "final_mass": 12.56637061435783, "mass_relative_change": -4.834443115824827e-14, "initial_kinetic_energy": 6.283185307178614, "final_kinetic_energy": 6.284270533430982, "initial_field_energy": 0.001256637061435906, "final_field_energy": 0.000173542155864126, "minimum_f": -2.324385335491534e-16, "finite": true, "repeat": 0, "command": ["srun", "--jobid=60204067", "--overlap", "--nodes=1", "--ntasks=4", "--ntasks-per-node=4", "--gpus-per-node=4", "--gpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T095729Z-66315-25333/workdir/cuda-build/vlasov_bench", "--nx", "32768", "--nv", "8192", "--steps", "20", "--warmup", "3", "--mode", "landau"], "rank_log": "rank=2 host=nid001409 local_rank=2 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001409 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001409 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001409 local_rank=1 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=937bea67c3242b0cb8313a6de50eef13ee7dcfa6b642142a129b277dd1ac56bb\nsynced_at=2026-10-24T10:06:08Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "nx": 32768, "nv": 8192, "ranks": 4, "nodes": 1, "steps": 20, "warmup": 3, "dt": 0.1, "length": 12.56637061435917, "vmax": 8, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 2147483648, "max_allocated_bytes_per_rank": 5370380304, "stream_ms": 12.3410621499886, "spline_ms": 10.97290559999919, "pack_ms": 3.171595100008062, "mpi_ms": 9.744720099979531, "field_ms": 0.7689962499910052, "step_ms": 36.94513030001758, "initial_mass": 12.56637061435844, "final_mass": 12.56637061435783, "mass_relative_change": -4.834443115824827e-14, "initial_kinetic_energy": 6.283185307178614, "final_kinetic_energy": 6.284270533430982, "initial_field_energy": 0.001256637061435906, "final_field_energy": 0.000173542155864126, "minimum_f": -2.324385335491534e-16, "finite": true, "repeat": 1, "command": ["srun", "--jobid=60204067", "--overlap", "--nodes=1", "--ntasks=4", "--ntasks-per-node=4", "--gpus-per-node=4", "--gpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T095729Z-66315-25333/workdir/cuda-build/vlasov_bench", "--nx", "32768", "--nv", "8192", "--steps", "20", "--warmup", "3", "--mode", "landau"], "rank_log": "rank=3 host=nid001412 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001412 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001412 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001412 local_rank=2 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=937bea67c3242b0cb8313a6de50eef13ee7dcfa6b642142a129b277dd1ac56bb\nsynced_at=2026-10-24T10:06:08Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "nx": 32768, "nv": 8192, "ranks": 4, "nodes": 1, "steps": 20, "warmup": 3, "dt": 0.1, "length": 12.56637061435917, "vmax": 8, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 2147483648, "max_allocated_bytes_per_rank": 5370380304, "stream_ms": 12.33317455000815, "spline_ms": 10.97503700001994, "pack_ms": 3.170171850013048, "mpi_ms": 9.749371850006128, "field_ms": 0.7777887999907307, "step_ms": 36.94672150001566, "initial_mass": 12.56637061435844, "final_mass": 12.56637061435783, "mass_relative_change": -4.834443115824827e-14, "initial_kinetic_energy": 6.283185307178614, "final_kinetic_energy": 6.284270533430982, "initial_field_energy": 0.001256637061435906, "final_field_energy": 0.000173542155864126, "minimum_f": -2.324385335491534e-16, "finite": true, "repeat": 2, "command": ["srun", "--jobid=60204067", "--overlap", "--nodes=1", "--ntasks=4", "--ntasks-per-node=4", "--gpus-per-node=4", "--gpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T095729Z-66315-25333/workdir/cuda-build/vlasov_bench", "--nx", "32768", "--nv", "8192", "--steps", "20", "--warmup", "3", "--mode", "landau"], "rank_log": "rank=3 host=nid001409 local_rank=3 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001409 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001409 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001409 local_rank=2 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=937bea67c3242b0cb8313a6de50eef13ee7dcfa6b642142a129b277dd1ac56bb\nsynced_at=2026-10-24T10:06:08Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "nx": 32768, "nv": 8192, "ranks": 2, "nodes": 2, "steps": 20, "warmup": 3, "dt": 0.1, "length": 12.56637061435917, "vmax": 8, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 2147483648, "max_allocated_bytes_per_rank": 10739122192, "stream_ms": 24.53902204998144, "spline_ms": 12.87798075001092, "pack_ms": 6.338750850022734, "mpi_ms": 76.12148224999373, "field_ms": 1.194125749987052, "step_ms": 121.0326773000133, "initial_mass": 12.56637061355123, "final_mass": 12.56637061358818, "mass_relative_change": 2.939963389748054e-12, "initial_kinetic_energy": 6.283185306796558, "final_kinetic_energy": 6.284270533062034, "initial_field_energy": 0.00125663706143567, "final_field_energy": 0.0001735421558640621, "minimum_f": -2.324385335496638e-16, "finite": true, "repeat": 0, "command": ["srun", "--jobid=60204067", "--overlap", "--nodes=2", "--ntasks=2", "--ntasks-per-node=1", "--gpus-per-node=4", "--gpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T095729Z-66315-25333/workdir/cuda-build/vlasov_bench", "--nx", "32768", "--nv", "8192", "--steps", "20", "--warmup", "3", "--mode", "landau"], "rank_log": "rank=1 host=nid001412 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001409 local_rank=0 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=937bea67c3242b0cb8313a6de50eef13ee7dcfa6b642142a129b277dd1ac56bb\nsynced_at=2026-10-24T10:06:08Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "nx": 32768, "nv": 8192, "ranks": 2, "nodes": 2, "steps": 20, "warmup": 3, "dt": 0.1, "length": 12.56637061435917, "vmax": 8, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 2147483648, "max_allocated_bytes_per_rank": 10739122192, "stream_ms": 24.53835345001494, "spline_ms": 12.87596610001174, "pack_ms": 6.339330200000859, "mpi_ms": 75.99124849999725, "field_ms": 1.182632650005644, "step_ms": 120.8955622999952, "initial_mass": 12.56637061355123, "final_mass": 12.56637061358818, "mass_relative_change": 2.939963389748054e-12, "initial_kinetic_energy": 6.283185306796558, "final_kinetic_energy": 6.284270533062034, "initial_field_energy": 0.00125663706143567, "final_field_energy": 0.0001735421558640621, "minimum_f": -2.324385335496638e-16, "finite": true, "repeat": 1, "command": ["srun", "--jobid=60204067", "--overlap", "--nodes=2", "--ntasks=2", "--ntasks-per-node=1", "--gpus-per-node=4", "--gpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T095729Z-66315-25333/workdir/cuda-build/vlasov_bench", "--nx", "32768", "--nv", "8192", "--steps", "20", "--warmup", "3", "--mode", "landau"], "rank_log": "rank=0 host=nid001409 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001412 local_rank=0 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=937bea67c3242b0cb8313a6de50eef13ee7dcfa6b642142a129b277dd1ac56bb\nsynced_at=2026-10-24T10:06:08Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "nx": 32768, "nv": 8192, "ranks": 2, "nodes": 2, "steps": 20, "warmup": 3, "dt": 0.1, "length": 12.56637061435917, "vmax": 8, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 2147483648, "max_allocated_bytes_per_rank": 10739122192, "stream_ms": 24.54099340000084, "spline_ms": 12.87833215000092, "pack_ms": 6.337992449959984, "mpi_ms": 76.09802300000297, "field_ms": 1.184742850011844, "step_ms": 121.0015570499991, "initial_mass": 12.56637061355123, "final_mass": 12.56637061358818, "mass_relative_change": 2.939963389748054e-12, "initial_kinetic_energy": 6.283185306796558, "final_kinetic_energy": 6.284270533062034, "initial_field_energy": 0.00125663706143567, "final_field_energy": 0.0001735421558640621, "minimum_f": -2.324385335496638e-16, "finite": true, "repeat": 2, "command": ["srun", "--jobid=60204067", "--overlap", "--nodes=2", "--ntasks=2", "--ntasks-per-node=1", "--gpus-per-node=4", "--gpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T095729Z-66315-25333/workdir/cuda-build/vlasov_bench", "--nx", "32768", "--nv", "8192", "--steps", "20", "--warmup", "3", "--mode", "landau"], "rank_log": "rank=0 host=nid001409 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001412 local_rank=0 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=937bea67c3242b0cb8313a6de50eef13ee7dcfa6b642142a129b277dd1ac56bb\nsynced_at=2026-10-24T10:06:08Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "nx": 32768, "nv": 8192, "ranks": 4, "nodes": 2, "steps": 20, "warmup": 3, "dt": 0.1, "length": 12.56637061435917, "vmax": 8, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 2147483648, "max_allocated_bytes_per_rank": 5370380304, "stream_ms": 12.31625059999146, "spline_ms": 10.97326685001008, "pack_ms": 3.172203949969798, "mpi_ms": 38.4375890499939, "field_ms": 0.9794631499858042, "step_ms": 65.72114285000339, "initial_mass": 12.56637061435844, "final_mass": 12.56637061435783, "mass_relative_change": -4.834443115824827e-14, "initial_kinetic_energy": 6.283185307178615, "final_kinetic_energy": 6.284270533430982, "initial_field_energy": 0.001256637061435906, "final_field_energy": 0.000173542155864126, "minimum_f": -2.324385335491534e-16, "finite": true, "repeat": 0, "command": ["srun", "--jobid=60204067", "--overlap", "--nodes=2", "--ntasks=4", "--ntasks-per-node=2", "--gpus-per-node=4", "--gpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T095729Z-66315-25333/workdir/cuda-build/vlasov_bench", "--nx", "32768", "--nv", "8192", "--steps", "20", "--warmup", "3", "--mode", "landau"], "rank_log": "rank=1 host=nid001412 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001409 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001409 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001412 local_rank=1 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=937bea67c3242b0cb8313a6de50eef13ee7dcfa6b642142a129b277dd1ac56bb\nsynced_at=2026-10-24T10:06:08Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "nx": 32768, "nv": 8192, "ranks": 4, "nodes": 2, "steps": 20, "warmup": 3, "dt": 0.1, "length": 12.56637061435917, "vmax": 8, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 2147483648, "max_allocated_bytes_per_rank": 5370380304, "stream_ms": 12.31640194998249, "spline_ms": 10.97109569999475, "pack_ms": 3.171158949965047, "mpi_ms": 38.57280864999666, "field_ms": 0.9742971999912697, "step_ms": 65.93730769999411, "initial_mass": 12.56637061435844, "final_mass": 12.56637061435783, "mass_relative_change": -4.834443115824827e-14, "initial_kinetic_energy": 6.283185307178615, "final_kinetic_energy": 6.284270533430982, "initial_field_energy": 0.001256637061435906, "final_field_energy": 0.000173542155864126, "minimum_f": -2.324385335491534e-16, "finite": true, "repeat": 1, "command": ["srun", "--jobid=60204067", "--overlap", "--nodes=2", "--ntasks=4", "--ntasks-per-node=2", "--gpus-per-node=4", "--gpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T095729Z-66315-25333/workdir/cuda-build/vlasov_bench", "--nx", "32768", "--nv", "8192", "--steps", "20", "--warmup", "3", "--mode", "landau"], "rank_log": "rank=2 host=nid001409 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001409 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=1 host=nid001412 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001412 local_rank=1 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=937bea67c3242b0cb8313a6de50eef13ee7dcfa6b642142a129b277dd1ac56bb\nsynced_at=2026-10-24T10:06:08Z\n"} +{"backend": "cuda", "device": "NVIDIA A100-SXM4-40GB", "precision": "float64", "mode": "landau", "nx": 32768, "nv": 8192, "ranks": 4, "nodes": 2, "steps": 20, "warmup": 3, "dt": 0.1, "length": 12.56637061435917, "vmax": 8, "amplitude": 0.01, "acceleration": 0.2, "host_mpi": false, "state_bytes": 2147483648, "max_allocated_bytes_per_rank": 5370380304, "stream_ms": 12.31745714999875, "spline_ms": 10.97455590000322, "pack_ms": 3.173706649960195, "mpi_ms": 38.54746239999827, "field_ms": 0.9067888499942001, "step_ms": 65.88151769999513, "initial_mass": 12.56637061435844, "final_mass": 12.56637061435783, "mass_relative_change": -4.834443115824827e-14, "initial_kinetic_energy": 6.283185307178615, "final_kinetic_energy": 6.284270533430982, "initial_field_energy": 0.001256637061435906, "final_field_energy": 0.000173542155864126, "minimum_f": -2.324385335491534e-16, "finite": true, "repeat": 2, "command": ["srun", "--jobid=60204067", "--overlap", "--nodes=2", "--ntasks=4", "--ntasks-per-node=2", "--gpus-per-node=4", "--gpu-bind=none", "/pscratch/sd/a/archis/adept-sessions/20261024T095729Z-66315-25333/workdir/cuda-build/vlasov_bench", "--nx", "32768", "--nv", "8192", "--steps", "20", "--warmup", "3", "--mode", "landau"], "rank_log": "rank=1 host=nid001412 local_rank=0 device=NVIDIA A100-SXM4-40GB\nrank=3 host=nid001412 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=2 host=nid001409 local_rank=1 device=NVIDIA A100-SXM4-40GB\nrank=0 host=nid001409 local_rank=0 device=NVIDIA A100-SXM4-40GB\n", "source_state": "version=1\nbase_sha=f9ff13c0b076f707c4b871a3fdb29316229993f4\ndirty=true\nworkspace_fingerprint=937bea67c3242b0cb8313a6de50eef13ee7dcfa6b642142a129b277dd1ac56bb\nsynced_at=2026-10-24T10:06:08Z\n"} diff --git a/benchmarks/vlasov_cuda/results/2026-10-24/summary.json b/benchmarks/vlasov_cuda/results/2026-10-24/summary.json new file mode 100644 index 00000000..8075005f --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24/summary.json @@ -0,0 +1,101 @@ +{ + "date": "2026-10-24", + "system": "NERSC Perlmutter", + "device": "NVIDIA A100-SXM4-40GB", + "software": { + "cuda": "13.2.78", + "gcc": "14.3.0", + "cray_mpich": "9.1.0", + "cmake": "3.30.2" + }, + "allocation_job_id": "60204067", + "nodes": [ + "nid001409", + "nid001412" + ], + "problem": { + "nx": 32768, + "nv": 8192, + "precision": "float64", + "dt": 0.1, + "warmup": 3, + "steps": 20, + "repeats": 3 + }, + "timing": "Median of three runs, each reporting the maximum rank mean over 20 steps; explicit device synchronization; no communication/computation overlap.", + "summary": [ + { + "nodes": 1, + "ranks": 1, + "step_ms": 68.90644464999696, + "stream_ms": 49.01655189999872, + "spline_ms": 17.81408299999896, + "pack_ms": 0, + "mpi_ms": 0, + "field_ms": 2.06822614998714, + "max_allocated_bytes_per_rank": 21476605968, + "speedup": 1.0 + }, + { + "nodes": 1, + "ranks": 2, + "step_ms": 58.70064285001035, + "stream_ms": 24.53943520000053, + "spline_ms": 12.87684925000008, + "pack_ms": 6.339934000004632, + "mpi_ms": 13.78156439998861, + "field_ms": 1.185581100014588, + "max_allocated_bytes_per_rank": 10739122192, + "speedup": 1.1738618404242027 + }, + { + "nodes": 1, + "ranks": 4, + "step_ms": 36.94672150001566, + "stream_ms": 12.33317455000815, + "spline_ms": 10.97503700001994, + "pack_ms": 3.171595100008062, + "mpi_ms": 9.749371850006128, + "field_ms": 0.7748922500013578, + "max_allocated_bytes_per_rank": 5370380304, + "speedup": 1.8650218978148776 + }, + { + "nodes": 2, + "ranks": 2, + "step_ms": 121.0015570499991, + "stream_ms": 24.53902204998144, + "spline_ms": 12.87798075001092, + "pack_ms": 6.338750850022734, + "mpi_ms": 76.09802300000297, + "field_ms": 1.184742850011844, + "max_allocated_bytes_per_rank": 10739122192, + "speedup": 0.5694674211632178 + }, + { + "nodes": 2, + "ranks": 4, + "step_ms": 65.88151769999513, + "stream_ms": 12.31640194998249, + "spline_ms": 10.97326685001008, + "pack_ms": 3.172203949969798, + "mpi_ms": 38.54746239999827, + "field_ms": 0.9742971999912697, + "max_allocated_bytes_per_rank": 5370380304, + "speedup": 1.0459146518721145 + } + ], + "max_abs_mass_relative_change": 5.062453547606825e-12, + "max_abs_energy_relative_change": 3.391486345094563e-07, + "minimum_f": -2.32438533550678e-16, + "source_sha256": { + "CMakeLists.txt": "d5f11680925c336c63ea39e306230371f6dcb3e41ba423c8c09599c7c592664c", + "backend.cpp": "0921b3b030e6ff54cfa5a442ca660d9a51cc1ebb31d94205016c2e06786056de", + "backend.hpp": "d45dbddea158e2c8d78b3c803cd83490354211082b61e988cf07e55d67c0bc47", + "main.cpp": "5c13dff7de6e21aa536d90aa9e46ccbc68e8773d32fe84ea123c0390f80464d5", + "measure.py": "d896e2c6537f650280e3dc7795d255e6d7181241278360b58f098a3a6caef4fb", + "validate.py": "36fd3383185f1a3ddba754d587ca8d76f94c790597a86859200683330907955f" + }, + "completed_slurm_steps": "60204067.8 through .19 (single-node checks), .21 through .24 (two-node checks), .25 through .39 (measurements); COMPLETED with exit 0:0.", + "transport_setup": "MPICH_GPU_SUPPORT_ENABLED=1; --gpus-per-node=4 --gpu-bind=none; cudaSetDevice before MPI_Init." +} diff --git a/benchmarks/vlasov_cuda/results/2026-10-24/validation-cuda-two-nodes.json b/benchmarks/vlasov_cuda/results/2026-10-24/validation-cuda-two-nodes.json new file mode 100644 index 00000000..f8c758a8 --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24/validation-cuda-two-nodes.json @@ -0,0 +1,26 @@ +[ + { + "mode": "transpose", + "ranks": 4, + "max_abs_error": 2.7755575615628914e-16, + "passed": true + }, + { + "mode": "stream", + "ranks": 4, + "max_abs_error": 4.440892098500626e-16, + "passed": true + }, + { + "mode": "accelerate", + "ranks": 4, + "max_abs_error": 6.106226635438361e-16, + "passed": true + }, + { + "mode": "landau", + "ranks": 4, + "max_abs_error": 1.1102230246251565e-15, + "passed": true + } +] diff --git a/benchmarks/vlasov_cuda/results/2026-10-24/validation-cuda.json b/benchmarks/vlasov_cuda/results/2026-10-24/validation-cuda.json new file mode 100644 index 00000000..f2dde4c5 --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24/validation-cuda.json @@ -0,0 +1,74 @@ +[ + { + "mode": "transpose", + "ranks": 1, + "max_abs_error": 2.7755575615628914e-16, + "passed": true + }, + { + "mode": "transpose", + "ranks": 2, + "max_abs_error": 2.7755575615628914e-16, + "passed": true + }, + { + "mode": "transpose", + "ranks": 4, + "max_abs_error": 2.7755575615628914e-16, + "passed": true + }, + { + "mode": "stream", + "ranks": 1, + "max_abs_error": 4.440892098500626e-16, + "passed": true + }, + { + "mode": "stream", + "ranks": 2, + "max_abs_error": 4.440892098500626e-16, + "passed": true + }, + { + "mode": "stream", + "ranks": 4, + "max_abs_error": 4.440892098500626e-16, + "passed": true + }, + { + "mode": "accelerate", + "ranks": 1, + "max_abs_error": 6.106226635438361e-16, + "passed": true + }, + { + "mode": "accelerate", + "ranks": 2, + "max_abs_error": 6.106226635438361e-16, + "passed": true + }, + { + "mode": "accelerate", + "ranks": 4, + "max_abs_error": 6.106226635438361e-16, + "passed": true + }, + { + "mode": "landau", + "ranks": 1, + "max_abs_error": 1.0547118733938987e-15, + "passed": true + }, + { + "mode": "landau", + "ranks": 2, + "max_abs_error": 1.1102230246251565e-15, + "passed": true + }, + { + "mode": "landau", + "ranks": 4, + "max_abs_error": 1.1102230246251565e-15, + "passed": true + } +] diff --git a/benchmarks/vlasov_cuda/results/2026-10-24/validation-fftw.json b/benchmarks/vlasov_cuda/results/2026-10-24/validation-fftw.json new file mode 100644 index 00000000..7a9996cb --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24/validation-fftw.json @@ -0,0 +1,74 @@ +[ + { + "mode": "transpose", + "ranks": 1, + "max_abs_error": 2.220446049250313e-16, + "passed": true + }, + { + "mode": "transpose", + "ranks": 2, + "max_abs_error": 2.220446049250313e-16, + "passed": true + }, + { + "mode": "transpose", + "ranks": 4, + "max_abs_error": 2.220446049250313e-16, + "passed": true + }, + { + "mode": "stream", + "ranks": 1, + "max_abs_error": 3.3306690738754696e-16, + "passed": true + }, + { + "mode": "stream", + "ranks": 2, + "max_abs_error": 3.3306690738754696e-16, + "passed": true + }, + { + "mode": "stream", + "ranks": 4, + "max_abs_error": 3.3306690738754696e-16, + "passed": true + }, + { + "mode": "accelerate", + "ranks": 1, + "max_abs_error": 5.551115123125783e-16, + "passed": true + }, + { + "mode": "accelerate", + "ranks": 2, + "max_abs_error": 5.551115123125783e-16, + "passed": true + }, + { + "mode": "accelerate", + "ranks": 4, + "max_abs_error": 5.551115123125783e-16, + "passed": true + }, + { + "mode": "landau", + "ranks": 1, + "max_abs_error": 3.1839523243379595e-15, + "passed": true + }, + { + "mode": "landau", + "ranks": 2, + "max_abs_error": 3.1713215548549073e-15, + "passed": true + }, + { + "mode": "landau", + "ranks": 4, + "max_abs_error": 3.1710355546772805e-15, + "passed": true + } +] diff --git a/benchmarks/vlasov_cuda/run_adept_comparison.sh b/benchmarks/vlasov_cuda/run_adept_comparison.sh new file mode 100644 index 00000000..2408c3e0 --- /dev/null +++ b/benchmarks/vlasov_cuda/run_adept_comparison.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# Run only inside an owned two-node allocation, after building both CUDA targets. +set -euo pipefail +if [ "${SLURM_PROCID:-0}" -ne 0 ]; then exit 0; fi +: "${EC_SESSION_ROOT:?}" "${SLURM_JOB_ID:?}" +source_root="$EC_SESSION_ROOT/src" +report_root="$EC_SESSION_OUTPUTS/${ADEPT_COMPARISON_OUTPUT:-adept-comparison}" +mkdir "$report_root" +prepared_root=/pscratch/sd/a/archis/adept-sessions/20261024T103224Z-71019-3473/outputs +vp_small=/pscratch/sd/a/archis/vp-turbulence-sessions/20261023T081121Z-nv4096-nx134400-5ce4fe14/src +vp_large=/pscratch/sd/a/archis/vp-turbulence-sessions/20261023T081121Z-nv4096-nx268800-93a418d7/src +export MPICH_GPU_SUPPORT_ENABLED=1 XLA_PYTHON_CLIENT_PREALLOCATE=false JAX_ENABLE_X64=true +export LD_LIBRARY_PATH="/global/common/software/nersc9/nccl/2.29.2-cu13/plugin/lib:/opt/nvidia/hpc_sdk/Linux_x86_64/26.5/cuda/13.2/lib64${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" +export NCCL_SOCKET_IFNAME=hsn NCCL_CROSS_NIC=2 NCCL_NET="AWS Libfabric" NCCL_NET_GDR_LEVEL=LOC FI_CXI_DISABLE_HOST_REGISTER=1 +export XLA_FLAGS="${XLA_FLAGS:+$XLA_FLAGS }--xla_gpu_autotune_level=0" +mapfile -t allocated_nodes < <(scontrol show hostnames "$SLURM_NODELIST") +for case in ${ADEPT_COMPARISON_CASES:-134400:1:4 134400:2:8 268800:2:8}; do + IFS=: read -r nx nodes ranks <<< "$case" + node_list=$(IFS=,; echo "${allocated_nodes[*]:0:nodes}") + prepared="$prepared_root/native-nx$nx" + vp_source="$vp_small" + if [ "$nx" = 268800 ]; then + prepared="$prepared_root/native-nx268800-distributed" + vp_source="$vp_large" + fi + # The pinned environment's own ADEPT must win over this development checkout. + # JAX CUDA wheels supply their runtime; unloading cudatoolkit matches the reference runs. + ( + module unload cudatoolkit 2>/dev/null || true + source /global/common/software/m4490/archis/venvs/vp-turbulence/bin/activate + export PYTHONPATH="$vp_source" + srun --jobid="$SLURM_JOB_ID" --overlap --nodes="$nodes" --nodelist="$node_list" --ntasks="$ranks" \ + --ntasks-per-node=$((ranks / nodes)) --gpus-per-node=4 --gpu-bind=none \ + --cpu-bind=none --mpi=none --kill-on-bad-exit=1 \ + python -u "$source_root/benchmarks/vlasov_cuda/adept_timing.py" \ + --prepared "$prepared" --config "$source_root/benchmarks/vlasov_cuda/reference/latest-nv4096/nx$nx.yaml" \ + --vp-source "$vp_source" --output "$report_root/jax-nx$nx-g$ranks.json" \ + > "$report_root/jax-nx$nx-g$ranks.log" 2>&1 + ) + python -u benchmarks/vlasov_cuda/measure.py "$EC_SESSION_WORKDIR/cuda-build/vlasov_bench" \ + --launcher "srun --jobid=$SLURM_JOB_ID --overlap --nodes={nodes} --nodelist=$node_list --ntasks={ranks} --ntasks-per-node={ranks_per_node} --gpus-per-node=4 --gpu-bind=none --cpu-bind=none" \ + --placements "$nodes:$ranks" --nx "$nx" --nv 4096 --steps 10 --warmup 3 --repeats 3 \ + --expect-cuda --velocity cubic --layout halos --integrator sixth --cells --nu 1e-12 \ + --length 12600 --vmax 15 --dt .02 --input "$prepared/initial.bin" \ + --v-grid "$prepared/v.bin" --background "$prepared/background.bin" --max-mass-change -1 \ + --source-state "$EC_SESSION_ROOT/source-state" --output "$report_root/cuda-nx$nx-g$ranks.jsonl" +done +sha256sum "$EC_SESSION_WORKDIR/cuda-build/vlasov_bench" "$EC_SESSION_WORKDIR/native-build/libadept_vlasov.so" > "$report_root/binaries.sha256" +sha256sum adept/_vlasov1d/native/api.cpp adept/_vlasov1d/native/core/* benchmarks/vlasov_cuda/adept_timing.py > "$report_root/source.sha256" +cp "$EC_SESSION_ROOT/source-state" "$report_root/source-state" +cp benchmarks/vlasov_cuda/run_adept_comparison.sh "$report_root/launch.sh" diff --git a/benchmarks/vlasov_cuda/validate.py b/benchmarks/vlasov_cuda/validate.py new file mode 100644 index 00000000..b72d3cff --- /dev/null +++ b/benchmarks/vlasov_cuda/validate.py @@ -0,0 +1,139 @@ +"""Numerical checks against NumPy FFTs and SciPy's natural CubicSpline. + +Use the CPU build locally or the CUDA build inside a GPU allocation. The launcher +is an argument template, e.g. 'mpiexec -n {ranks}' or 'srun -n {ranks} ...'. +""" + +import argparse +import json +import shlex +import subprocess +import tempfile +from pathlib import Path + +import numpy as np +from scipy.interpolate import CubicSpline + + +def initial(nx, nv): + x = np.arange(nx) * (4 * np.pi / nx) + v = np.linspace(-8, 8, nv) + m = np.exp(-(v**2) / 2) + m /= np.trapezoid(m, v) + return x, v, m[:, None] * (1 + 0.01 * np.cos(0.5 * x))[None, :] + + +def stream(f, v, dt): + k = 0.5 * np.arange(f.shape[1] // 2 + 1) + fk = np.fft.rfft(f, axis=1) * np.exp(-1j * v[:, None] * k[None, :] * dt) + return np.fft.irfft(fk, n=f.shape[1], axis=1) + + +def field(f, v): + rho = 1 - np.trapezoid(f, v, axis=0) + rk = np.fft.rfft(rho) + ek = np.zeros_like(rk) + k = 0.5 * np.arange(len(rk)) + ek[1:] = rk[1:] / (1j * k[1:]) + if f.shape[1] % 2 == 0: + ek[-1] = 0 + return np.fft.irfft(ek, n=f.shape[1]) + + +def accelerate(f, v, a, dt): + q = v[:, None] - np.broadcast_to(a, (f.shape[1],))[None, :] * dt + # Evaluate one spatial column per query column (SciPy otherwise broadcasts + # the whole output axis). Boundary values outside the velocity box are zero. + result = np.empty_like(f) + for x in range(f.shape[1]): + spline = CubicSpline(v, f[:, x], bc_type="natural", extrapolate=False) + result[:, x] = np.nan_to_num(spline(q[:, x]), nan=0.0) + return result + + +def reference(mode, nx, nv, steps, dt): + _, v, f = initial(nx, nv) + for _ in range(steps): + if mode == "stream": + f = stream(f, v, dt) + elif mode == "accelerate": + f = accelerate(f, v, 0.2, dt) + elif mode == "landau": + f = stream(f, v, dt / 2) + e = field(f, v) + f = accelerate(f, v, -e, dt) + f = stream(f, v, dt / 2) + return f + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("binary", type=Path) + parser.add_argument("--launcher", default="mpiexec -n {ranks}") + parser.add_argument("--ranks", nargs="+", type=int, default=[1, 2, 4]) + parser.add_argument("--host-mpi", action="store_true") + parser.add_argument("--output", type=Path) + args = parser.parse_args() + rows = [] + nx, nv, dt, steps = 128, 128, 0.1, 4 + with tempfile.TemporaryDirectory(prefix="vlasov-validation-") as directory: + for mode in ["transpose", "stream", "accelerate", "landau"]: + expected = reference(mode, nx, nv, steps, dt) + first = None + for ranks in args.ranks: + # MPI ranks on different nodes need a shared destination. + # Only rank 0 writes; MPI_Gather collects all velocity slabs. + dump = Path(directory) / f"{mode}-{ranks}.bin" + command = [ + *shlex.split(args.launcher.format(ranks=ranks)), + str(args.binary.resolve()), + "--nx", + str(nx), + "--nv", + str(nv), + "--steps", + str(steps), + "--dt", + str(dt), + "--warmup", + "1", + "--mode", + mode, + "--dump", + str(dump), + ] + if args.host_mpi: + command.append("--host-mpi") + completed = subprocess.run(command, capture_output=True, text=True, timeout=180) + if completed.returncode: + raise RuntimeError(f"{' '.join(command)}\n{completed.stdout}\n{completed.stderr}") + report = json.loads(completed.stdout.strip().splitlines()[-1]) + actual = np.fromfile(dump, dtype=np.float64).reshape(nv, nx) + error = float(np.max(np.abs(actual - expected))) + np.testing.assert_allclose(actual, expected, rtol=2e-11, atol=3e-13) + if first is None: + first = actual + else: + np.testing.assert_allclose(actual, first, rtol=2e-11, atol=3e-13) + if mode in ("transpose", "stream"): + assert abs(report["mass_relative_change"]) < 1e-12, report + if mode == "stream": + x, v, f0 = initial(nx, nv) + m = f0[:, 0] / 1.01 + analytic = m[:, None] * (1 + 0.01 * np.cos(0.5 * (x[None, :] - v[:, None] * steps * dt))) + np.testing.assert_allclose(actual, analytic, rtol=2e-12, atol=3e-14) + if mode == "landau": + assert report["final_field_energy"] < report["initial_field_energy"], report + assert abs(report["mass_relative_change"]) < 1e-9, report + initial_energy = report["initial_kinetic_energy"] + report["initial_field_energy"] + final_energy = report["final_kinetic_energy"] + report["final_field_energy"] + assert abs(final_energy / initial_energy - 1) < 1e-6, report + row = {"mode": mode, "ranks": ranks, "max_abs_error": error, "passed": True} + rows.append(row) + print(json.dumps(row), flush=True) + if args.output: + args.output.write_text(json.dumps(rows, indent=2) + "\n") + + +if __name__ == "__main__": + main() diff --git a/benchmarks/vlasov_cuda/validate_local.py b/benchmarks/vlasov_cuda/validate_local.py new file mode 100644 index 00000000..5f28e947 --- /dev/null +++ b/benchmarks/vlasov_cuda/validate_local.py @@ -0,0 +1,193 @@ +"""Compare C1 cubic halos and sixth-order splitting with interpax/NumPy. + +This is an operator test. Native-checkpoint and collision comparisons are +separate requirements before claiming turbulence-run reproduction. +""" + +import argparse +import json +import os +import shlex +import subprocess +import tempfile +from pathlib import Path + +os.environ.setdefault("JAX_PLATFORMS", "cpu") +import jax + +jax.config.update("jax_enable_x64", True) +import jax.numpy as jnp +import numpy as np +from interpax import interp1d + + +def cubic(f, v, acceleration, dt): + a = np.broadcast_to(acceleration, f.shape[1]) + q = v[:, None] - a[None, :] * dt + return np.asarray( + jax.vmap(lambda query, values: interp1d(query, jnp.asarray(v), values, extrap=1e-30))( + jnp.asarray(q.T), jnp.asarray(f.T) + ) + ).T + + +def stream(f, v, dt, length): + k = 2 * np.pi * np.fft.rfftfreq(f.shape[1], d=length / f.shape[1]) + return np.fft.irfft(np.fft.rfft(f, axis=1) * np.exp(-1j * v[:, None] * k * dt), n=f.shape[1], axis=1) + + +def field(f, dv, length, background): + rho = background - np.sum(f, axis=0) * dv + k = 2 * np.pi * np.fft.rfftfreq(f.shape[1], d=length / f.shape[1]) + ek = np.zeros(len(k), dtype=complex) + ek[1:] = np.fft.rfft(rho)[1:] / (1j * k[1:]) + if f.shape[1] % 2 == 0: + ek[-1] = 0 + return np.fft.irfft(ek, n=f.shape[1]) + + +def sixth(f, v, dt, length, background): + a = [ + 0.168735950563437422448196, + 0.377851589220928303880766, + -0.093175079568731452657924, + 0.377851589220928303880766, + 0.168735950563437422448196, + ] + d1 = 0.049086460976116245491441 - 2 * 0.000069728715055305084099 * dt**2 + d2 = 0.264177609888976700200146 - 2 * 0.000625704827430047189169 * dt**2 - 4 * 2.916600457689847816445691e-6 * dt**4 + d3 = ( + 0.186735929134907054308413 + - 2 * 0.002213085124045325561636 * dt**2 + + 4 * 3.048480261700038788680723e-5 * dt**4 + - 8 * 4.985549387875068121593988e-7 * dt**6 + ) + for i, d in enumerate([d1, d2, d3, d3, d2, d1]): + e = field(f, v[1] - v[0], length, background) + f = cubic(f, v, -e, d * dt) + if i < 5: + f = stream(f, v, a[i] * dt, length) + return f + + +def collide(f, v, dt, nu): + from adept._vlasov1d.solvers.pushers.fokker_planck import Collisions + + cfg = { + "grid": { + "parallel": False, + "species_grids": {"electron": {"v": jnp.asarray(v), "dv": v[1] - v[0]}}, + "species_params": {"electron": {"mass": 1.0}}, + }, + "terms": { + "species": [{"name": "electron"}], + "fokker_planck": {"is_on": True, "type": "chang_cooper_dougherty"}, + "krook": {"is_on": False}, + }, + } + result = Collisions(cfg)(jnp.full(f.shape[1], nu), jnp.zeros(f.shape[1]), jnp.asarray(f.T), dt) + return np.asarray(result).T + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("binary", type=Path) + parser.add_argument("--launcher", default="mpiexec -n {ranks}") + parser.add_argument("--ranks", nargs="+", type=int, default=[1, 2, 4]) + parser.add_argument("--output", type=Path) + args = parser.parse_args() + nx, nv, vmax, length = 128, 128, 8.0, 4 * np.pi + dv = 2 * vmax / nv + v = np.linspace(-vmax + dv / 2, vmax - dv / 2, nv) + x = (np.arange(nx) + 0.5) * length / nx + smooth = np.exp(-(v[:, None] ** 2) / 2) / np.sqrt(2 * np.pi) * (1 + 0.03 * np.cos(0.5 * x)) + signed = np.random.default_rng(4).normal(size=(nv, nx)) + # A shift longer than a rank's velocity block tests multi-neighbour halos. + tests = [ + ("signed-positive", signed, 1.2, 0.1, 0.0), + ("signed-negative", signed, -1.2, 0.1, 0.0), + ("signed-wide", signed, 45.0, 0.1, 0.0), + ("sixth", smooth, 0.0, 0.02, 0.0), + ("collide", smooth, 0.0, 0.1, 1e-3), + ("sixth-collide", smooth, 0.0, 0.02, 1e-12), + ] + records = [] + with tempfile.TemporaryDirectory(prefix="vlasov-local-") as directory: + tmp = Path(directory) + v.tofile(tmp / "v.bin") + background = np.ones(nx) + background.tofile(tmp / "bg.bin") + for name, initial, acceleration, dt, nu in tests: + initial.tofile(tmp / "initial.bin") + if name.startswith("sixth"): + expected = sixth(initial, v, dt, length, background) + elif name == "collide": + expected = initial + else: + expected = cubic(initial, v, acceleration, dt) + if nu: + expected = collide(expected, v, dt, nu) + for layout in ["transpose", "halos"]: + for ranks in args.ranks: + output = tmp / "out.bin" + cmd = [ + *shlex.split(args.launcher.format(ranks=ranks)), + str(args.binary.resolve()), + "--nx", + str(nx), + "--nv", + str(nv), + "--vmax", + str(vmax), + "--length", + str(length), + "--dt", + str(dt), + "--steps", + "1", + "--warmup", + "0", + "--cells", + "--velocity", + "cubic", + "--layout", + layout, + "--halo-capacity", + "64", + "--integrator", + "sixth", + "--mode", + "landau" if name.startswith("sixth") else ("collide" if name == "collide" else "accelerate"), + "--nu", + str(nu), + "--acceleration", + str(acceleration), + "--input", + str(tmp / "initial.bin"), + "--v-grid", + str(tmp / "v.bin"), + "--background", + str(tmp / "bg.bin"), + "--dump", + str(output), + ] + completed = subprocess.run(cmd, capture_output=True, text=True, timeout=90) + if completed.returncode: + raise RuntimeError(f"{' '.join(cmd)}\n{completed.stdout}\n{completed.stderr}") + actual = np.fromfile(output, dtype=np.float64).reshape(nv, nx) + np.testing.assert_allclose(actual, expected, rtol=2e-11, atol=2e-13) + row = dict( + test=name, + layout=layout, + ranks=ranks, + max_abs_error=float(np.max(np.abs(actual - expected))), + passed=True, + ) + records.append(row) + print(json.dumps(row), flush=True) + if args.output: + args.output.write_text(json.dumps(records, indent=2) + "\n") + + +if __name__ == "__main__": + main() diff --git a/docs/source/solvers/vlasov1d/config.md b/docs/source/solvers/vlasov1d/config.md index 27dbd992..cdc2d87d 100644 --- a/docs/source/solvers/vlasov1d/config.md +++ b/docs/source/solvers/vlasov1d/config.md @@ -32,6 +32,73 @@ terms: # Solver configuration ``` +## Execution backend + +`backend: jax` is the default. The optional `backend: native` runs the +electrostatic solver in C++/CUDA/MPI while retaining Python initialization, +save callbacks, xarray/NetCDF postprocessing, and `ergoExo` tracking. It has no +automatic differentiation. Custom `BaseVlasov1D` subclasses passed to +`ergoExo.setup(..., adept_module=MyModule)` are wrapped automatically. + +Build the shared library on the target system (CUDA toolkit, MPI and CMake +are required): + +```sh +cmake -S adept/_vlasov1d/native -B build/native -DCMAKE_BUILD_TYPE=Release +cmake --build build/native -j4 +``` + +On Perlmutter add `-DCMAKE_CXX_COMPILER=CC`, and use the existing allocated +job's `srun` launcher. A two-node example is: + +```yaml +backend: native +native: + library: /absolute/path/build/native/libadept_vlasov.so + ranks: 8 + workdir: /absolute/shared/scratch/path + scalar_stride: 1 + launcher: [srun, --overlap, --nodes=2, '--ntasks={ranks}', + --ntasks-per-node=4, --gpus-per-node=4, --gpu-bind=none, + --cpu-bind=none] +``` + +Run one outer Python driver inside the allocation; the launcher starts its +workers. Set `MPICH_GPU_SUPPORT_ENABLED=1` for CUDA-aware Cray MPI. The work +directory and library must be visible on every worker node. Native execution +uses its own velocity-slab decomposition; `native.ranks` controls it. + +| Native option | Default | Meaning | +|---|---|---| +| `library` | unset | Shared library path; alternatively set `ADEPT_VLASOV_LIBRARY` | +| `device` | `cuda` | `cpu` selects the FFTW validation build (`-DVLASOV_CUDA=OFF`) | +| `ranks` | `1` | One MPI worker per GPU; must divide `nx` and `nv` | +| `launcher` | empty | Argument list, with `{ranks}` substitution; defaults to `mpiexec -n` for multiple ranks | +| `workdir` | `BASE_TEMPDIR` or system temporary directory | Shared staging/output directory, required for multiple ranks | +| `scalar_stride` | `1` | Save scalar diagnostics every this many actual timesteps, plus both endpoints | +| `custom_save_times` | `preserve` | Preserve subclass callback times; `steps` rounds those too | +| `halo_capacity` | `64` | Maximum velocity halo width; exceeding it fails explicitly | +| `host_mpi` | `false` | Stage MPI messages through host memory for MPI without CUDA support | +| `timeout` | unset | Worker timeout in seconds | + +Standard field/distribution save times round to the nearest timestep and +duplicate times are merged. Outputs record actual times in `result.ts` and +`cfg['save'][key]['t']['ax']`; the number of frames can therefore change. +The native backend streams saved outputs into host memory-mapped files instead +of retaining the full time history in GPU memory. Sparse custom callbacks +preserve their requested times by default for existing project postprocessors; +off-step callbacks require a previous-state GPU copy and interpolation. Use +`custom_save_times: steps` when the postprocessor reads actual result times. +Custom postprocessors that rebuild a uniform scalar axis must also use actual +times when the final scalar interval is shorter than `scalar_stride`. + +The initial implementation supports one electron species, a symmetric +cell-centred velocity grid, `time: sixth`, `edfdv: cubic-spline`, spectral +spatial streaming and `field: poisson`. Optional collisions are constant-rate +`chang_cooper_dougherty` with analytic beta. Drivers, Krook relaxation, +Hou–Li filtering, evolved df/dt diagnostics and trainable modules are rejected. +The cubic operator reproduces ADEPT's local C1 Hermite interpolation. + ## units Physical unit normalizations for the simulation. diff --git a/pyproject.toml b/pyproject.toml index ad39ce7a..c29995c3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,6 +37,7 @@ Documentation = "https://adept.readthedocs.io/en/latest/" [project.optional-dependencies] cpu = [] +native = ["cloudpickle"] gpu = [ "jax[cuda12]", "nvidia-ml-py" diff --git a/tests/test_vlasov1d/test_native.py b/tests/test_vlasov1d/test_native.py new file mode 100644 index 00000000..2106ccb8 --- /dev/null +++ b/tests/test_vlasov1d/test_native.py @@ -0,0 +1,182 @@ +"""Native solves retain ADEPT's callbacks and Python outputs at actual save times.""" + +import os +import shutil +import subprocess +from copy import deepcopy +from pathlib import Path + +import jax +import jax.numpy as jnp +import numpy as np +import pytest +import xarray as xr +import yaml +from diffrax import SubSaveAt + +from adept import ergoExo +from adept._vlasov1d.storage import store_f, store_fields +from adept.vlasov1d import BaseVlasov1D, NativeVlasov1D, native_backend + + +@pytest.fixture(scope="module") +def native_library(tmp_path_factory): + configured = os.environ.get("ADEPT_NATIVE_TEST_LIBRARY") + if configured: + return Path(configured) + if not all(shutil.which(name) for name in ["cmake", "mpicxx", "pkg-config"]): + pytest.skip("native integration tests require CMake, MPI and FFTW or ADEPT_NATIVE_TEST_LIBRARY") + if subprocess.run(["pkg-config", "--exists", "fftw3"]).returncode: + pytest.skip("FFTW validation backend is not installed") + build = tmp_path_factory.mktemp("native-build") + source = Path(__file__).resolve().parents[2] / "adept/_vlasov1d/native" + subprocess.run( + ["cmake", "-S", str(source), "-B", str(build), "-DVLASOV_CUDA=OFF", "-DCMAKE_BUILD_TYPE=Release"], + check=True, + capture_output=True, + ) + subprocess.run(["cmake", "--build", str(build), "-j2"], check=True, capture_output=True) + return next(path for path in [build / "libadept_vlasov.so", build / "libadept_vlasov.dylib"] if path.exists()) + + +@pytest.fixture +def config(tmp_path): + cfg = yaml.safe_load(Path("tests/test_base/configs/example.yaml").read_text()) + cfg["grid"].update(nx=32, nv=64, dt=0.02, tmax=0.08, vmax=8.0) + cfg["drivers"] = {"ex": {}, "ey": {}} + cfg["terms"]["time"] = "sixth" + cfg["terms"]["fokker_planck"].update(is_on=True, type="chang_cooper_dougherty") + cfg["terms"]["fokker_planck"]["time"]["baseline"] = 0.001 + cfg["terms"]["fokker_planck"]["space"]["baseline"] = 1.0 + cfg["terms"]["krook"]["is_on"] = False + cfg["save"] = { + "fields": {"t": {"nt": 5}}, + "electron": { + "main": {"t": {"nt": 3}}, + "sampled": { + "t": {"nt": 4}, + "x": {"xmin": 0.0, "xmax": 20.94, "nx": 8}, + "v": {"vmin": -7.0, "vmax": 7.0, "nv": 16}, + }, + }, + } + beam = cfg["density"]["species-background"] + beam.update(v0=3.0, T0=0.2, noise_val=0.001) + cfg["density"]["species-left"] = deepcopy(beam) + cfg["density"]["species-left"].update(v0=-3.0, noise_seed=43) + cfg["native"] = { + "device": os.environ.get("ADEPT_NATIVE_TEST_DEVICE", "cpu"), + "workdir": str(tmp_path), + "timeout": 120, + } + return cfg + + +def setup(cls, config): + module = cls(deepcopy(config)) + module.write_units() + module.get_derived_quantities() + module.get_solver_quantities() + module.init_state_and_args() + module.init_diffeqsolve() + return module + + +def compare_results(original, native): + assert set(original.ts) == set(native.ts) + for key in original.ts: + np.testing.assert_allclose(original.ts[key], native.ts[key], atol=1e-14) + assert jax.tree.structure(original.ys[key]) == jax.tree.structure(native.ys[key]) + for a, b in zip(jax.tree.leaves(original.ys[key]), jax.tree.leaves(native.ys[key]), strict=True): + np.testing.assert_allclose(a, b, rtol=1e-10, atol=1e-11, equal_nan=True, err_msg=key) + + +def reference_at_native_times(original, native): + subs = original.diffeqsolve_quants["saveat"]["subs"] + for key, save in native.diffeqsolve_quants["saveat"]["subs"].items(): + subs[key] = SubSaveAt(ts=save.ts, fn=subs[key].fn) + return original({})["solver result"] + + +@pytest.mark.parametrize("ranks", [1, 2]) +def test_native_dense_saves_and_xarray(config, native_library, tmp_path, ranks): + config["native"]["library"] = str(native_library) + config["native"]["ranks"] = ranks + original, native = (setup(cls, config) for cls in [BaseVlasov1D, NativeVlasov1D]) + a, b = reference_at_native_times(original, native), native({})["solver result"] + compare_results(a, b) + for key, times in b.ts.items(): + np.testing.assert_allclose(np.asarray(times) / 0.02, np.rint(np.asarray(times) / 0.02), atol=1e-14) + assert len(b.ts["default"]) == 6 + output = tmp_path / "output" + (output / "binary").mkdir(parents=True) + fields = store_fields(native.cfg, str(output / "binary"), b.ys["fields"], b.ts["fields"], "fields") + dists = store_f(native.cfg, b.ts, str(output), b.ys) + assert isinstance(fields["electron"], xr.Dataset) + assert dists["electron.sampled"].sizes == {"t": 4, "x": 8, "v_electron": 16} + with xr.open_dataset(output / "binary/dist-electron.sampled.nc") as saved: + xr.testing.assert_allclose(saved, dists["electron.sampled"]) + + +def test_native_step_schedules(config): + config["native"].update(scalar_stride=3, custom_save_times="steps") + native = setup(native_backend(CustomTwoStream), config) + subs = native.diffeqsolve_quants["saveat"]["subs"] + np.testing.assert_allclose(subs["default"].ts, [0.0, 0.06, 0.1]) + np.testing.assert_allclose(subs["f0"].ts, [0.0, 0.04, 0.1]) + + +class CustomTwoStream(BaseVlasov1D): + def init_diffeqsolve(self): + super().init_diffeqsolve() + self.diffeqsolve_quants["saveat"]["subs"]["f0"] = SubSaveAt( + ts=np.array([0.0, 0.037, self.time_quantities["t1"]]), fn=lambda t, y, args: jnp.mean(y["electron"], axis=0) + ) + k = jnp.abs(jnp.asarray(self.cfg["grid"]["kx"])) + self.diffeqsolve_quants["saveat"]["subs"]["peak"] = SubSaveAt( + ts=np.array([0.0, 0.04, self.time_quantities["t1"]]), + fn=lambda t, y, args: k[jnp.argmax(jnp.abs(jnp.fft.fft(y["e"])))], + ) + + def post_process(self, run_output, td): + sol = run_output["solver result"] + result = xr.Dataset( + {"f0": (("t", "v"), sol.ys["f0"])}, + coords={"t": sol.ts["f0"], "v": self.cfg["grid"]["species_grids"]["electron"]["v"]}, + ) + result.to_netcdf(Path(td) / "custom.nc") + return {"custom": result} + + +def test_ergoexo_custom_module_preserves_callbacks(config, native_library, tmp_path, monkeypatch): + from adept import patched_mlflow as mlflow + + mlflow.set_tracking_uri((tmp_path / "mlruns").as_uri()) + monkeypatch.setenv("MLFLOW_ENABLE_SYSTEM_METRICS_LOGGING", "false") + config["native"]["library"] = str(native_library) + original = setup(CustomTwoStream, config) + config["backend"] = "native" + config["mlflow"] = {"experiment": "native-integration-test", "run": "two-stream"} + exo = ergoExo() + modules = exo.setup(config, adept_module=CustomTwoStream) + result, post, run_id = exo(modules) + compare_results(reference_at_native_times(original, exo.adept_module), result["solver result"]) + np.testing.assert_allclose(result["solver result"].ts["f0"], [0.0, 0.037, 0.1]) + assert isinstance(exo.adept_module, CustomTwoStream) + assert not exo.adept_module.jit_compile + assert post["custom"].sizes == {"t": 3, "v": 64} + artifacts = Path(mlflow.get_run(run_id).info.artifact_uri.removeprefix("file://")) + assert (artifacts / "custom.nc").exists() + assert (artifacts / "native/request.json").exists() + assert (artifacts / "native/library.sha256").exists() + + +def test_backend_dispatch_and_guards(config): + assert type(ergoExo()._get_adept_module_(config)) is BaseVlasov1D + config["backend"] = "native" + assert isinstance(ergoExo()._get_adept_module_(config), NativeVlasov1D) + config["terms"]["fokker_planck"]["self_consistent_beta"] = {"enabled": True} + with pytest.raises(NotImplementedError, match="analytic beta"): + NativeVlasov1D(config) + with pytest.raises(TypeError, match="BaseVlasov1D"): + native_backend(object) diff --git a/uv.lock b/uv.lock index de515da8..9bd3147a 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 3 +revision = 5 requires-python = ">=3.11" resolution-markers = [ "python_full_version >= '3.13'", @@ -71,12 +71,16 @@ gpu = [ { name = "jax", extra = ["cuda12"] }, { name = "nvidia-ml-py" }, ] +native = [ + { name = "cloudpickle" }, +] [package.metadata] requires-dist = [ { name = "adept", marker = "extra == 'docs'" }, { name = "adept", extras = ["docs"], marker = "extra == 'dev'" }, { name = "boto3" }, + { name = "cloudpickle", marker = "extra == 'native'" }, { name = "diffrax" }, { name = "h5netcdf" }, { name = "interpax", git = "https://github.com/f0uriest/interpax.git" }, @@ -108,7 +112,7 @@ requires-dist = [ { name = "typeguard", marker = "extra == 'dev'" }, { name = "xarray" }, ] -provides-extras = ["cpu", "gpu", "docs", "dev"] +provides-extras = ["cpu", "native", "gpu", "docs", "dev"] [[package]] name = "alabaster" @@ -2240,20 +2244,20 @@ resolution-markers = [ "python_full_version < '3.12'", ] dependencies = [ - { name = "astropy", marker = "python_full_version < '3.12'" }, - { name = "h5py", marker = "python_full_version < '3.12'" }, - { name = "lmfit", marker = "python_full_version < '3.12'" }, - { name = "matplotlib", marker = "python_full_version < '3.12'" }, - { name = "mpmath", marker = "python_full_version < '3.12'" }, - { name = "numpy", marker = "python_full_version < '3.12'" }, - { name = "packaging", marker = "python_full_version < '3.12'" }, - { name = "pandas", marker = "python_full_version < '3.12'" }, - { name = "requests", marker = "python_full_version < '3.12'" }, - { name = "scipy", marker = "python_full_version < '3.12'" }, - { name = "setuptools", marker = "python_full_version < '3.12'" }, - { name = "tqdm", marker = "python_full_version < '3.12'" }, - { name = "wrapt", marker = "python_full_version < '3.12'" }, - { name = "xarray", marker = "python_full_version < '3.12'" }, + { name = "astropy" }, + { name = "h5py" }, + { name = "lmfit" }, + { name = "matplotlib" }, + { name = "mpmath" }, + { name = "numpy" }, + { name = "packaging" }, + { name = "pandas" }, + { name = "requests" }, + { name = "scipy" }, + { name = "setuptools" }, + { name = "tqdm" }, + { name = "wrapt" }, + { name = "xarray" }, ] sdist = { url = "https://files.pythonhosted.org/packages/2c/f8/941a2f3b1acda062c90ce9c47ad9cb8564df92f77e04db0a5663edc2af7e/plasmapy-2025.8.0.tar.gz", hash = "sha256:b26e6237000d8b69ffb98baa3f3d6043eb6c1828a3d0a1b44ff4053cf61afb90", size = 13148459, upload-time = "2025-08-07T21:16:01.725Z" } wheels = [ @@ -2269,19 +2273,19 @@ resolution-markers = [ "python_full_version == '3.12.*'", ] dependencies = [ - { name = "astropy", marker = "python_full_version >= '3.12'" }, - { name = "h5py", marker = "python_full_version >= '3.12'" }, - { name = "lmfit", marker = "python_full_version >= '3.12'" }, - { name = "matplotlib", marker = "python_full_version >= '3.12'" }, - { name = "mpmath", marker = "python_full_version >= '3.12'" }, - { name = "numpy", marker = "python_full_version >= '3.12'" }, - { name = "packaging", marker = "python_full_version >= '3.12'" }, - { name = "pandas", marker = "python_full_version >= '3.12'" }, - { name = "requests", marker = "python_full_version >= '3.12'" }, - { name = "scipy", marker = "python_full_version >= '3.12'" }, - { name = "tqdm", marker = "python_full_version >= '3.12'" }, - { name = "wrapt", marker = "python_full_version >= '3.12'" }, - { name = "xarray", marker = "python_full_version >= '3.12'" }, + { name = "astropy" }, + { name = "h5py" }, + { name = "lmfit" }, + { name = "matplotlib" }, + { name = "mpmath" }, + { name = "numpy" }, + { name = "packaging" }, + { name = "pandas" }, + { name = "requests" }, + { name = "scipy" }, + { name = "tqdm" }, + { name = "wrapt" }, + { name = "xarray" }, ] sdist = { url = "https://files.pythonhosted.org/packages/84/95/4c7d3f51e5af11286a04643be5596ef8ebcb3cac25386e8bcd5ff846d17a/plasmapy-2025.10.0.tar.gz", hash = "sha256:cfeb6f2785906e080977292e9b7f3bb6e71cdd8ef7273a877959a113d9e56295", size = 13144933, upload-time = "2025-10-25T01:59:00.912Z" } wheels = [ @@ -3006,23 +3010,23 @@ resolution-markers = [ "python_full_version < '3.12'", ] dependencies = [ - { name = "alabaster", marker = "python_full_version < '3.12'" }, - { name = "babel", marker = "python_full_version < '3.12'" }, - { name = "colorama", marker = "python_full_version < '3.12' and sys_platform == 'win32'" }, - { name = "docutils", marker = "python_full_version < '3.12'" }, - { name = "imagesize", marker = "python_full_version < '3.12'" }, - { name = "jinja2", marker = "python_full_version < '3.12'" }, - { name = "packaging", marker = "python_full_version < '3.12'" }, - { name = "pygments", marker = "python_full_version < '3.12'" }, - { name = "requests", marker = "python_full_version < '3.12'" }, - { name = "roman-numerals", marker = "python_full_version < '3.12'" }, - { name = "snowballstemmer", marker = "python_full_version < '3.12'" }, - { name = "sphinxcontrib-applehelp", marker = "python_full_version < '3.12'" }, - { name = "sphinxcontrib-devhelp", marker = "python_full_version < '3.12'" }, - { name = "sphinxcontrib-htmlhelp", marker = "python_full_version < '3.12'" }, - { name = "sphinxcontrib-jsmath", marker = "python_full_version < '3.12'" }, - { name = "sphinxcontrib-qthelp", marker = "python_full_version < '3.12'" }, - { name = "sphinxcontrib-serializinghtml", marker = "python_full_version < '3.12'" }, + { name = "alabaster" }, + { name = "babel" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "docutils" }, + { name = "imagesize" }, + { name = "jinja2" }, + { name = "packaging" }, + { name = "pygments" }, + { name = "requests" }, + { name = "roman-numerals" }, + { name = "snowballstemmer" }, + { name = "sphinxcontrib-applehelp" }, + { name = "sphinxcontrib-devhelp" }, + { name = "sphinxcontrib-htmlhelp" }, + { name = "sphinxcontrib-jsmath" }, + { name = "sphinxcontrib-qthelp" }, + { name = "sphinxcontrib-serializinghtml" }, ] sdist = { url = "https://files.pythonhosted.org/packages/42/50/a8c6ccc36d5eacdfd7913ddccd15a9cee03ecafc5ee2bc40e1f168d85022/sphinx-9.0.4.tar.gz", hash = "sha256:594ef59d042972abbc581d8baa577404abe4e6c3b04ef61bd7fc2acbd51f3fa3", size = 8710502, upload-time = "2025-12-04T07:45:27.343Z" } wheels = [ @@ -3038,23 +3042,23 @@ resolution-markers = [ "python_full_version == '3.12.*'", ] dependencies = [ - { name = "alabaster", marker = "python_full_version >= '3.12'" }, - { name = "babel", marker = "python_full_version >= '3.12'" }, - { name = "colorama", marker = "python_full_version >= '3.12' and sys_platform == 'win32'" }, - { name = "docutils", marker = "python_full_version >= '3.12'" }, - { name = "imagesize", marker = "python_full_version >= '3.12'" }, - { name = "jinja2", marker = "python_full_version >= '3.12'" }, - { name = "packaging", marker = "python_full_version >= '3.12'" }, - { name = "pygments", marker = "python_full_version >= '3.12'" }, - { name = "requests", marker = "python_full_version >= '3.12'" }, - { name = "roman-numerals", marker = "python_full_version >= '3.12'" }, - { name = "snowballstemmer", marker = "python_full_version >= '3.12'" }, - { name = "sphinxcontrib-applehelp", marker = "python_full_version >= '3.12'" }, - { name = "sphinxcontrib-devhelp", marker = "python_full_version >= '3.12'" }, - { name = "sphinxcontrib-htmlhelp", marker = "python_full_version >= '3.12'" }, - { name = "sphinxcontrib-jsmath", marker = "python_full_version >= '3.12'" }, - { name = "sphinxcontrib-qthelp", marker = "python_full_version >= '3.12'" }, - { name = "sphinxcontrib-serializinghtml", marker = "python_full_version >= '3.12'" }, + { name = "alabaster" }, + { name = "babel" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "docutils" }, + { name = "imagesize" }, + { name = "jinja2" }, + { name = "packaging" }, + { name = "pygments" }, + { name = "requests" }, + { name = "roman-numerals" }, + { name = "snowballstemmer" }, + { name = "sphinxcontrib-applehelp" }, + { name = "sphinxcontrib-devhelp" }, + { name = "sphinxcontrib-htmlhelp" }, + { name = "sphinxcontrib-jsmath" }, + { name = "sphinxcontrib-qthelp" }, + { name = "sphinxcontrib-serializinghtml" }, ] sdist = { url = "https://files.pythonhosted.org/packages/cd/bd/f08eb0f4eed5c83f1ba2a3bd18f7745a2b1525fad70660a1c00224ec468a/sphinx-9.1.0.tar.gz", hash = "sha256:7741722357dd75f8190766926071fed3bdc211c74dd2d7d4df5404da95930ddb", size = 8718324, upload-time = "2025-12-31T15:09:27.646Z" } wheels = [ @@ -3069,7 +3073,7 @@ resolution-markers = [ "python_full_version < '3.12'", ] dependencies = [ - { name = "sphinx", version = "9.0.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "sphinx", version = "9.0.4", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/1d/f6/bdd93582b2aaad2cfe9eb5695a44883c8bc44572dd3c351a947acbb13789/sphinx_autodoc_typehints-3.6.1.tar.gz", hash = "sha256:fa0b686ae1b85965116c88260e5e4b82faec3687c2e94d6a10f9b36c3743e2fe", size = 37563, upload-time = "2026-01-02T15:23:46.543Z" } wheels = [ @@ -3085,7 +3089,7 @@ resolution-markers = [ "python_full_version == '3.12.*'", ] dependencies = [ - { name = "sphinx", version = "9.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, + { name = "sphinx", version = "9.1.0", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/97/51/6603ed3786a2d52366c66f49bc8afb31ae5c0e33d4a156afcb38d2bac62c/sphinx_autodoc_typehints-3.6.2.tar.gz", hash = "sha256:3d37709a21b7b765ad6e20a04ecefcb229b9eb0007cb24f6ebaa8a4576ea7f06", size = 37574, upload-time = "2026-01-02T21:25:28.216Z" } wheels = [ From ddc4dd14082a60f9fe8a8f9dfdb80044f5ab91f0 Mon Sep 17 00:00:00 2001 From: archis Date: Mon, 26 Oct 2026 21:37:06 +0000 Subject: [PATCH 3/8] feat(vlasov1d): evolve kinetic ions through the native backend --- .github/workflows/cpu-tests.yaml | 7 ++ adept/_vlasov1d/builder.py | 5 + adept/_vlasov1d/native/README.md | 8 +- adept/_vlasov1d/native/api.cpp | 121 ++++++++++++++++---- adept/_vlasov1d/native/bridge.py | 31 ++++- adept/_vlasov1d/native/core/backend.cpp | 24 +++- adept/_vlasov1d/native/core/backend.hpp | 7 +- adept/_vlasov1d/native/core/solver.hpp | 35 +++--- adept/_vlasov1d/native/module.py | 90 +++++++++------ adept/_vlasov1d/native/worker.py | 17 ++- adept/_vlasov1d/observations.py | 5 +- benchmarks/vlasov_cuda/integration_check.py | 36 +++++- docs/source/solvers/vlasov1d/config.md | 28 +++-- tests/test_vlasov1d/test_native.py | 63 ++++++++++ tests/test_vlasov1d/test_observations.py | 17 +++ 15 files changed, 397 insertions(+), 97 deletions(-) create mode 100644 tests/test_vlasov1d/test_observations.py diff --git a/.github/workflows/cpu-tests.yaml b/.github/workflows/cpu-tests.yaml index 73e661b9..81327fe4 100644 --- a/.github/workflows/cpu-tests.yaml +++ b/.github/workflows/cpu-tests.yaml @@ -153,7 +153,14 @@ jobs: - uses: ./.github/actions/setup_repo with: python-version: '3.11' + - name: Install native validation dependencies + run: | + sudo apt-get update + sudo apt-get install -y cmake pkg-config libfftw3-dev libopenmpi-dev openmpi-bin + uv sync --extra dev --extra native --frozen - name: Test with pytest + env: + OMPI_MCA_rmaps_base_oversubscribe: '1' run: uv run --locked pytest tests/test_vlasov1d --ignore=tests/test_vlasov1d/test_config_regression.py test-farsight1d: diff --git a/adept/_vlasov1d/builder.py b/adept/_vlasov1d/builder.py index bbf678fb..48fcdc34 100644 --- a/adept/_vlasov1d/builder.py +++ b/adept/_vlasov1d/builder.py @@ -83,6 +83,11 @@ def prepare(self, spec: SimulationSpec, *, key: int | jax.Array) -> PreparedSimu raise ValueError(f"Vlasov1DBuilder cannot prepare solver {spec.solver!r}") if spec.schema_version != "1": raise ValueError(f"Unsupported Vlasov1D specification schema version {spec.schema_version!r}") + if spec.config_dict().get("backend", "jax") == "native": + raise NotImplementedError( + "backend: native runs through ergoExo (including custom BaseVlasov1D subclasses); " + "Vlasov1DBuilder prepares a JAX program" + ) if not jax.config.jax_enable_x64: raise RuntimeError("vlasov-1d requires float64; enable jax_enable_x64 before preparation") normalized_key, seed, provenance = normalize_key(key) diff --git a/adept/_vlasov1d/native/README.md b/adept/_vlasov1d/native/README.md index dba0cb25..933c7fc6 100644 --- a/adept/_vlasov1d/native/README.md +++ b/adept/_vlasov1d/native/README.md @@ -38,6 +38,12 @@ last scalar interval is shorter than the requested stride. The validated physics is spectral spatial streaming, ADEPT's local cubic C1 velocity interpolation, sixth-order splitting, Poisson fields, and optional -constant-rate Chang-Cooper Dougherty collisions. Unsupported physics and +constant-rate Chang-Cooper Dougherty collisions. Kinetic ions use the existing +`terms.species` entries, with independent velocity grids, charges and masses. +Each stage sums every species' charge density into one MPI field reduction, +then advances every species under that shared field. Collisions act on the +reference species, as in ADEPT's JAX path; inter-species collisional exchange +is not included. Rebuild the library when updating ADEPT (current ABI: 2). +Unsupported physics and trainable modules fail explicitly. Performance and reproduction evidence is under [the benchmark results](../../../benchmarks/vlasov_cuda/results/2026-10-24-adept). diff --git a/adept/_vlasov1d/native/api.cpp b/adept/_vlasov1d/native/api.cpp index 3db9a19a..0e301c16 100644 --- a/adept/_vlasov1d/native/api.cpp +++ b/adept/_vlasov1d/native/api.cpp @@ -10,20 +10,66 @@ namespace { thread_local std::string last_error; bool own_mpi = false; std::atomic live_engines{0}; -struct Engine { +struct SpeciesState { Options options; std::unique_ptr backend; std::unique_ptr stepper; int local_rank; - Engine(Options opts, int local) : options(std::move(opts)), local_rank(local) { + SpeciesState(Options opts, int local) : options(std::move(opts)), local_rank(local) { backend = std::make_unique(options.grid, local); stepper = std::make_unique(*backend, options); if (!options.coordinates.empty()) backend->set_coordinates(read_binary(options.coordinates, options.grid.nv)); if (!options.background.empty()) backend->set_background(read_binary(options.background, options.grid.nx)); initialize(*backend, options); - ++live_engines; } +}; +struct Engine : SpeciesState { + std::vector> additional; + Engine(Options opts, int local) : SpeciesState(std::move(opts), local) { ++live_engines; } ~Engine() { --live_engines; } + SpeciesState& species(int index) { + if (index < 0 || index > int(additional.size())) throw std::runtime_error("invalid species index"); + return index == 0 ? *this : *additional[index - 1]; + } + template void each(F action) { + action(*this); + for (auto& state : additional) action(*state); + } + void solve_field() { + backend->density(options.grid.charge); + for (auto& state : additional) { + state->backend->density(state->options.grid.charge); + backend->add_density(*state->backend); + } + backend->synchronize(); + if (options.host_mpi) { + auto local = backend->download_density(); + std::vector global(local.size()); + mpi_check(MPI_Allreduce(local.data(), global.data(), options.grid.nx, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD)); + backend->upload_density(global); + } else { + mpi_check(MPI_Allreduce(backend->density_buffer(), backend->reduced_density_buffer(), + options.grid.nx, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD)); + } + backend->field(true); + for (auto& state : additional) state->backend->copy_field(*backend); + } + void step() { + if (additional.empty()) { stepper->step(); return; } + backend->synchronize(); + double started = MPI_Wtime(); + const SixthCoefficients coefficients(options.dt); + for (int k = 0; k < 6; ++k) { + stepper->timed(Field, [&] { solve_field(); }); + each([&](SpeciesState& state) { state.stepper->velocity_step(coefficients.d[k] * options.dt, true); }); + if (k < 5) each([&](SpeciesState& state) { + state.stepper->timed(Stream, [&] { state.backend->stream(coefficients.a[k] * options.dt); }); + }); + } + each([](SpeciesState& state) { state.stepper->collision_step(); }); + backend->synchronize(); + stepper->times[Total] += MPI_Wtime() - started; + } }; using Handle = std::shared_ptr; template int guarded(F action) { @@ -54,8 +100,24 @@ int device_type() { return 1; // kDLCPU #endif } +void validate_options(Options& opts) { + Grid& g = opts.grid; + mpi_check(MPI_Comm_rank(MPI_COMM_WORLD, &g.rank)); + mpi_check(MPI_Comm_size(MPI_COMM_WORLD, &g.ranks)); + g.collisions = opts.nu > 0; + if (g.nx < 8 || g.nv < 4 || g.nx % g.ranks || g.nv % g.ranks || + g.count() > std::size_t(std::numeric_limits::max())) + throw std::runtime_error("native grids must divide across ranks and fit MPI int counts"); + if (!g.local_cubic || !g.halos || !g.cell_centred || opts.dt <= 0 || opts.nu < 0 || + g.mass <= 0 || !std::isfinite(g.mass) || !std::isfinite(g.charge) || + g.length <= 0 || g.vmax <= 0 || g.halo_capacity < 2 || + std::size_t(g.halo_capacity) * g.nx > std::size_t(std::numeric_limits::max()) || + (opts.integrator != "sixth" && opts.integrator != "strang")) + throw std::runtime_error("unsupported native solver options"); +} } extern "C" { +int av_abi_version() { return 2; } const char* av_error() { return last_error.c_str(); } void* av_create(int argc, const char** argv) { void* result = nullptr; @@ -68,18 +130,7 @@ void* av_create(int argc, const char** argv) { int rank; mpi_check(MPI_Comm_rank(MPI_COMM_WORLD, &rank)); Options opts = parse(argc, const_cast(argv), rank); - Grid& g = opts.grid; - g.rank = rank; - mpi_check(MPI_Comm_size(MPI_COMM_WORLD, &g.ranks)); - g.collisions = opts.nu > 0; - if (g.nx < 8 || g.nv < 4 || g.nx % g.ranks || g.nv % g.ranks || - g.count() > std::size_t(std::numeric_limits::max())) - throw std::runtime_error("native grid must divide across ranks and fit MPI int counts"); - if (!g.local_cubic || !g.halos || !g.cell_centred || opts.dt <= 0 || opts.nu < 0 || - g.length <= 0 || g.vmax <= 0 || g.halo_capacity < 2 || - std::size_t(g.halo_capacity) * g.nx > std::size_t(std::numeric_limits::max()) || - (opts.integrator != "sixth" && opts.integrator != "strang")) - throw std::runtime_error("unsupported native solver options"); + validate_options(opts); MPI_Comm local; mpi_check(MPI_Comm_split_type(MPI_COMM_WORLD, MPI_COMM_TYPE_SHARED, rank, MPI_INFO_NULL, &local)); int local_rank; @@ -89,6 +140,25 @@ void* av_create(int argc, const char** argv) { })) return nullptr; return result; } +int av_add_species(void* handle, int argc, const char** argv) { + return guarded([&] { + auto& e = **static_cast(handle); + Options opts = parse(argc, const_cast(argv), e.options.grid.rank); + validate_options(opts); + if (opts.grid.nx != e.options.grid.nx || opts.grid.length != e.options.grid.length || + opts.dt != e.options.dt || opts.integrator != "sixth" || e.options.integrator != "sixth") + throw std::runtime_error("coupled species require the same x grid, timestep and sixth-order integrator"); + e.additional.push_back(std::make_unique(std::move(opts), e.local_rank)); + }); +} +int av_species_info(void* handle, int index, int* output) { + return guarded([&] { + auto& s = (**static_cast(handle)).species(index); + const Grid& g = s.options.grid; + const int values[] = {g.rank, g.ranks, s.local_rank, device_type(), g.nx, g.nv, g.local_v()}; + std::copy_n(values, 7, output); + }); +} int av_info(void* handle, int* output) { return guarded([&] { auto& e = **static_cast(handle); @@ -105,7 +175,7 @@ int av_step(void* handle, int count) { return guarded([&] { if (count < 0) throw std::runtime_error("negative native step count"); auto& e = **static_cast(handle); - for (int i = 0; i < count; ++i) e.stepper->step(); + for (int i = 0; i < count; ++i) e.step(); e.backend->synchronize(); }); } @@ -115,11 +185,11 @@ int av_set_field(void* handle, const char* path) { e.backend->set_field(read_binary(path, e.options.grid.nx)); }); } -void* av_tensor(void* handle, int field) { +void* av_species_tensor(void* handle, int species, int field) { void* result = nullptr; if (guarded([&] { auto owner = *static_cast(handle); - auto& e = *owner; + auto& e = owner->species(species); e.backend->synchronize(); // Producer completion before a consumer stream reads. auto* tensor = new Export{}; tensor->owner = std::move(owner); @@ -134,10 +204,15 @@ void* av_tensor(void* handle, int field) { })) return nullptr; return result; } +void* av_tensor(void* handle, int field) { return av_species_tensor(handle, 0, field); } int av_stats(void* handle, double* output) { return guarded([&] { auto& e = **static_cast(handle); - mpi_check(MPI_Allreduce(e.stepper->times.data(), output, StageCount, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD)); + std::array times{}; + e.each([&](SpeciesState& state) { + for (int i = 0; i < StageCount; ++i) times[i] += state.stepper->times[i]; + }); + mpi_check(MPI_Allreduce(times.data(), output, StageCount, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD)); }); } int av_close(void* handle) { @@ -153,5 +228,11 @@ int av_finalize() { if (own_mpi) { mpi_check(MPI_Finalize()); own_mpi = false; } }); } -void av_abort(int code) { MPI_Abort(MPI_COMM_WORLD, code); } +void av_abort(int code) { + int initialized = 0, finalized = 0; + MPI_Initialized(&initialized); + if (initialized) MPI_Finalized(&finalized); + if (initialized && !finalized) MPI_Abort(MPI_COMM_WORLD, code); + std::_Exit(code); +} } diff --git a/adept/_vlasov1d/native/bridge.py b/adept/_vlasov1d/native/bridge.py index 138b9cf4..01adde90 100644 --- a/adept/_vlasov1d/native/bridge.py +++ b/adept/_vlasov1d/native/bridge.py @@ -1,6 +1,7 @@ """ctypes and DLPack boundary; importing this module does not initialize JAX.""" import ctypes as ct +import sys class NativeCore: @@ -8,10 +9,18 @@ class NativeCore: def __init__(self, library, options): self.lib = ct.CDLL(str(library)) + version = getattr(self.lib, "av_abi_version", None) + if version is None or version() != 2: + raise RuntimeError( + "native library ABI mismatch; rebuild adept/_vlasov1d/native from the installed ADEPT source" + ) signatures = { "av_create": ([ct.c_int, ct.POINTER(ct.c_char_p)], ct.c_void_p), "av_error": ([], ct.c_char_p), "av_info": ([ct.c_void_p, ct.POINTER(ct.c_int)], ct.c_int), + "av_add_species": ([ct.c_void_p, ct.c_int, ct.POINTER(ct.c_char_p)], ct.c_int), + "av_species_info": ([ct.c_void_p, ct.c_int, ct.POINTER(ct.c_int)], ct.c_int), + "av_species_tensor": ([ct.c_void_p, ct.c_int, ct.c_int], ct.c_void_p), "av_step": ([ct.c_void_p, ct.c_int], ct.c_int), "av_tensor": ([ct.c_void_p, ct.c_int], ct.c_void_p), "av_set_field": ([ct.c_void_p, ct.c_char_p], ct.c_int), @@ -28,7 +37,8 @@ def __init__(self, library, options): argv = [b"adept-native", *(str(value).encode() for value in options)] self.handle = self.lib.av_create(len(argv), (ct.c_char_p * len(argv))(*argv)) if not self.handle: - raise RuntimeError(self.lib.av_error().decode()) + print(self.lib.av_error().decode(), file=sys.stderr, flush=True) + self.lib.av_abort(1) values = (ct.c_int * 7)() self.check(self.lib.av_info(self.handle, values)) self.rank, self.size, self.local_rank, self.device_type, self.nx, self.nv, self.local_nv = values @@ -51,8 +61,17 @@ def barrier(self): def set_field(self, path): self.check(self.lib.av_set_field(self.handle, str(path).encode())) - def view(self, field=False): - return _View(self, field) + def add_species(self, options): + argv = [b"adept-native", *(str(value).encode() for value in options)] + self.check(self.lib.av_add_species(self.handle, len(argv), (ct.c_char_p * len(argv))(*argv))) + + def species_info(self, index): + values = (ct.c_int * 7)() + self.check(self.lib.av_species_info(self.handle, index, values)) + return tuple(values) + + def view(self, field=False, species=0): + return _View(self, field, species) def stats(self): values = (ct.c_double * 7)() @@ -70,8 +89,8 @@ def close(self): class _View: """Synchronized, read-only-by-convention view valid until the next native step.""" - def __init__(self, owner, field): - self.owner, self.field = owner, field + def __init__(self, owner, field, species): + self.owner, self.field, self.species = owner, field, species def __dlpack_device__(self): return self.owner.device_type, self.owner.local_rank if self.owner.device_type == 2 else 0 @@ -81,7 +100,7 @@ def __dlpack__(self, stream=None, *, max_version=None, dl_device=None, copy=None raise BufferError("the native exporter provides views; the consumer may copy") if dl_device is not None and tuple(dl_device) != self.__dlpack_device__(): raise BufferError("native DLPack export cannot change devices") - pointer = self.owner.lib.av_tensor(self.owner.handle, int(self.field)) + pointer = self.owner.lib.av_species_tensor(self.owner.handle, self.species, int(self.field)) if not pointer: raise RuntimeError(self.owner.lib.av_error().decode()) # The consumer owns the tensor's C++ deleter after renaming this capsule diff --git a/adept/_vlasov1d/native/core/backend.cpp b/adept/_vlasov1d/native/core/backend.cpp index 94295779..e8b9b9ae 100644 --- a/adept/_vlasov1d/native/core/backend.cpp +++ b/adept/_vlasov1d/native/core/backend.cpp @@ -314,7 +314,7 @@ void Backend::stream(double dt) { p->streaming_fft.c2r(fk, f); parallel_for(g.count(), [=] DEVICE (std::size_t i) { f[i] /= nx; }); } -void Backend::density() { +void Backend::density(double charge) { const Grid g = p->grid; const int nx = g.nx, nv = g.nv, nvl = g.local_v(), v0 = g.rank * nvl; const double dv = g.dv(); @@ -325,10 +325,22 @@ void Backend::density() { double sum = 0.0; for (int j = 0; j < nvl; ++j) sum += f[std::size_t(j) * nx + x] * (!cells && (j + v0 == 0 || j + v0 == nv - 1) ? 0.5 : 1.0); - rho[x] = sum * dv; + rho[x] = charge * (sum * dv); }); } -void Backend::field() { +void Backend::add_density(const Backend& other) { + double* target = p->density_local.ptr; + const double* source = other.p->density_local.ptr; + parallel_for(p->grid.nx, [=] DEVICE (std::size_t x) { target[x] += source[x]; }); +} +void Backend::copy_field(const Backend& other) { +#ifdef VLASOV_CUDA + check_cuda(cudaMemcpy(p->electric.ptr, other.p->electric.ptr, p->grid.nx * sizeof(double), cudaMemcpyDeviceToDevice)); +#else + std::copy_n(other.p->electric.ptr, p->grid.nx, p->electric.ptr); +#endif +} +void Backend::field(bool charge_density) { const Grid g = p->grid; const int nx = g.nx, nk = nx / 2 + 1; const double k0 = 2 * pi / g.length; @@ -336,7 +348,7 @@ void Backend::field() { const double* bg = p->background.ptr; double* e = p->electric.ptr; double* ek = p->field_spectrum.ptr; - parallel_for(nx, [=] DEVICE (std::size_t i) { e[i] = bg[i] - density[i]; }); + parallel_for(nx, [=] DEVICE (std::size_t i) { e[i] = bg[i] + (charge_density ? density[i] : -density[i]); }); p->field_fft.r2c(e, ek); parallel_for(nk, [=] DEVICE (std::size_t i) { if (i == 0 || (nx % 2 == 0 && i == std::size_t(nx / 2))) { @@ -381,7 +393,7 @@ void Backend::velocity(double dt, double acceleration, bool self_consistent) { const double* e = p->electric.ptr; parallel_for(g.count(), [=] DEVICE (std::size_t i) { int x = i % nxl, j = i / nxl; - double a = self_consistent ? -e[x0 + x] : acceleration; + double a = self_consistent ? (g.charge / g.mass) * e[x0 + x] : acceleration; double q = j - a * dt / dv; if (q < 0.0 || q > nv - 1) { out[i] = 0.0; return; } int lo = int(floor(q)); @@ -407,7 +419,7 @@ void Backend::local_velocity(double dt, double acceleration, bool self_consisten double* out = p->output.ptr; parallel_for(g.count(), [=] DEVICE (std::size_t i) { int x = i % width, j = i / width + v0; - double a = self_consistent ? -e[x0 + x] : acceleration; + double a = self_consistent ? (g.charge / g.mass) * e[x0 + x] : acceleration; double query = v[j] - a * dt; int guess = int(floor(j - a * dt / dv)); auto sample = [=] DEVICE (int index) { diff --git a/adept/_vlasov1d/native/core/backend.hpp b/adept/_vlasov1d/native/core/backend.hpp index 933f9bd0..810ff740 100644 --- a/adept/_vlasov1d/native/core/backend.hpp +++ b/adept/_vlasov1d/native/core/backend.hpp @@ -11,6 +11,7 @@ void prepare_device_from_environment(); struct Grid { int nx = 4096, nv = 1024, ranks = 1, rank = 0; double length = 12.566370614359172, vmax = 8.0, amplitude = 0.01; + double charge = -1.0, mass = 1.0; bool cell_centred = false, local_cubic = false, halos = false, collisions = false; int halo_capacity = 64; int local_x() const { return nx / ranks; } @@ -31,8 +32,10 @@ class Backend { void initialize(); void synchronize(); void stream(double dt); - void density(); - void field(); + void density(double charge = 1.0); + void add_density(const Backend& other); + void field(bool charge_density = false); + void copy_field(const Backend& other); void velocity(double dt, double acceleration, bool self_consistent); void local_velocity(double dt, double acceleration, bool self_consistent); void collide(double dt, double nu); diff --git a/adept/_vlasov1d/native/core/solver.hpp b/adept/_vlasov1d/native/core/solver.hpp index 91b7055e..1d970e6c 100644 --- a/adept/_vlasov1d/native/core/solver.hpp +++ b/adept/_vlasov1d/native/core/solver.hpp @@ -69,6 +69,8 @@ Options parse(int argc, char** argv, int rank) { else if (key == "--dt") o.dt = real(value); else if (key == "--acceleration") o.acceleration = real(value); else if (key == "--nu") o.nu = real(value); + else if (key == "--charge") o.grid.charge = real(value); + else if (key == "--mass") o.grid.mass = real(value); else if (key == "--length") o.grid.length = real(value); else if (key == "--vmax") o.grid.vmax = real(value); else if (key == "--amplitude") o.grid.amplitude = real(value); @@ -93,6 +95,19 @@ Options parse(int argc, char** argv, int rank) { // Times include explicit device completion, with no computation/communication // overlap. Reduce each rank's accumulated component times with MPI_MAX later. enum Stage { Stream, Spline, Pack, Exchange, Field, Collision, Total, StageCount }; +struct SixthCoefficients { + double a[5] = {0.168735950563437422448196, 0.377851589220928303880766, + -0.093175079568731452657924, 0.377851589220928303880766, 0.168735950563437422448196}; + double d[6]; + explicit SixthCoefficients(double dt) { + double dt2 = dt * dt, dt4 = dt2 * dt2, dt6 = dt4 * dt2; + d[0] = d[5] = 0.049086460976116245491441 - 2 * 0.000069728715055305084099 * dt2; + d[1] = d[4] = 0.264177609888976700200146 - 2 * 0.000625704827430047189169 * dt2 + - 4 * 2.916600457689847816445691e-6 * dt4; + d[2] = d[3] = 0.186735929134907054308413 - 2 * 0.002213085124045325561636 * dt2 + + 4 * 3.048480261700038788680723e-5 * dt4 - 8 * 4.985549387875068121593988e-7 * dt6; + } +}; class Stepper { public: Stepper(Backend& backend, const Options& opts) : b(backend), o(opts) {} @@ -112,7 +127,7 @@ class Stepper { return; } timed(Exchange, [&] { - double maximum = self_consistent ? b.max_field() : std::abs(o.acceleration); + double maximum = self_consistent ? std::abs(o.grid.charge / o.grid.mass) * b.max_field() : std::abs(o.acceleration); double distance = maximum * std::abs(dt) / o.grid.dv(); if (!std::isfinite(distance) || distance > o.grid.halo_capacity - 2) throw std::runtime_error("departure point exceeds halo capacity; increase --halo-capacity"); @@ -190,7 +205,7 @@ class Stepper { timed(Pack, [&] { b.unpack(to_x); }); } void solve_field() { - b.density(); + b.density(o.grid.charge); b.synchronize(); if (o.host_mpi) { auto density = b.download_density(); @@ -202,7 +217,7 @@ class Stepper { mpi_check(MPI_Allreduce(b.density_buffer(), b.reduced_density_buffer(), o.grid.nx, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD)); } - b.field(); + b.field(true); } void step() { b.synchronize(); @@ -214,19 +229,11 @@ class Stepper { if (o.mode == "landau" && o.integrator == "sixth") { // Same force-gradient Hamiltonian coefficients and ordering as // ADEPT SixthOrderHamIntegrator, including its dt corrections. - const double a[] = {0.168735950563437422448196, 0.377851589220928303880766, - -0.093175079568731452657924, 0.377851589220928303880766, 0.168735950563437422448196}; - double dt2 = o.dt * o.dt, dt4 = dt2 * dt2, dt6 = dt4 * dt2; - double d1 = 0.049086460976116245491441 - 2 * 0.000069728715055305084099 * dt2; - double d2 = 0.264177609888976700200146 - 2 * 0.000625704827430047189169 * dt2 - - 4 * 2.916600457689847816445691e-6 * dt4; - double d3 = 0.186735929134907054308413 - 2 * 0.002213085124045325561636 * dt2 - + 4 * 3.048480261700038788680723e-5 * dt4 - 8 * 4.985549387875068121593988e-7 * dt6; - double d[] = {d1, d2, d3, d3, d2, d1}; + const SixthCoefficients coefficients(o.dt); for (int k = 0; k < 6; ++k) { timed(Field, [&] { solve_field(); }); - velocity_step(d[k] * o.dt, true); - if (k < 5) timed(Stream, [&] { b.stream(a[k] * o.dt); }); + velocity_step(coefficients.d[k] * o.dt, true); + if (k < 5) timed(Stream, [&] { b.stream(coefficients.a[k] * o.dt); }); } collision_step(); b.synchronize(); diff --git a/adept/_vlasov1d/native/module.py b/adept/_vlasov1d/native/module.py index df51550f..16bc492a 100644 --- a/adept/_vlasov1d/native/module.py +++ b/adept/_vlasov1d/native/module.py @@ -10,7 +10,7 @@ import sys import tempfile from contextlib import contextmanager -from functools import cache, lru_cache +from functools import cache from pathlib import Path import jax @@ -106,16 +106,14 @@ def __init__(self, cfg): def _validate_native_physics(self): cfg, sim = self.cfg, self.simulation - if ( - len(sim.species) != 1 - or sim.species[0].name != "electron" - or sim.species[0].charge != -1 - or sim.species[0].mass != 1 - ): - raise NotImplementedError("native Vlasov currently supports one electron species with charge=-1, mass=1") - species = sim.species[0] - if not np.isclose(species.vmin, -species.vmax, rtol=0, atol=1e-12): - raise NotImplementedError("native Vlasov requires a symmetric velocity domain") + names = [species.name for species in sim.species] + if not names or len(set(names)) != len(names): + raise ValueError("native species names must be nonempty and unique") + for species in sim.species: + if not np.isfinite(species.mass) or species.mass <= 0 or not np.isfinite(species.charge): + raise ValueError("native species require positive finite mass and finite charge") + if not np.isclose(species.vmin, -species.vmax, rtol=0, atol=1e-12): + raise NotImplementedError("native Vlasov requires symmetric velocity domains") terms = cfg["terms"] if terms["time"] != "sixth" or terms["edfdv"] != "cubic-spline" or terms["field"] != "poisson": raise NotImplementedError( @@ -125,6 +123,7 @@ def _validate_native_physics(self): raise NotImplementedError("native Vlasov requires exponential spatial streaming") if ( any(cfg["drivers"][key] for key in ["ex", "ey"]) + or cfg["drivers"].get("ex_stochastic") is not None or terms["krook"]["is_on"] or terms.get("hou_li_filter", {}).get("is_on", False) ): @@ -133,7 +132,9 @@ def _validate_native_physics(self): raise NotImplementedError("native Vlasov does not yet evolve df/dt diagnostic state") fp = terms["fokker_planck"] if fp["is_on"]: - if fp["type"] != "chang_cooper_dougherty" or fp.get("self_consistent_beta", {}).get("enabled", False): + if fp["type"].casefold() not in {"chang_cooper_dougherty", "dougherty_chang_cooper"} or fp.get( + "self_consistent_beta", {} + ).get("enabled", False): raise NotImplementedError("native collisions require Chang-Cooper Dougherty with analytic beta") for axis in ["time", "space"]: envelope = fp[axis] @@ -190,6 +191,8 @@ def init_diffeqsolve(self): self.cfg["save"][key]["t"].update(ax=times, nt=len(times), tmin=float(times[0]), tmax=float(times[-1])) def __call__(self, trainable_modules=None, args=None): + if jax.process_count() != 1: + raise RuntimeError("run one outer Python driver; the native backend launches its MPI workers") if trainable_modules: raise NotImplementedError("native Vlasov does not execute trainable modules") if args is not None and args is not self.args: @@ -206,9 +209,11 @@ def __call__(self, trainable_modules=None, args=None): steps = round((t1 - t0) / grid.dt) if steps < 1 or not np.isclose(steps * grid.dt, t1 - t0, rtol=0, atol=1e-9): raise NotImplementedError("native solve endpoints must be separated by an integer number of timesteps") - if grid.nx % options.ranks or self.simulation.species[0].nv % options.ranks: - raise ValueError("native ranks must divide both nx and nv") - known = {"electron", "e", "de", "a", "da", "prev_a"} + species_list = sorted(self.simulation.species, key=lambda species: species.name != "electron") + if grid.nx % options.ranks or any(species.nv % options.ranks for species in species_list): + raise ValueError("native ranks must divide nx and every species nv") + species_names = {species.name for species in species_list} + known = species_names | {"e", "de", "a", "da", "prev_a"} if set(self.state) - known or any(np.any(np.asarray(self.state[key])) for key in ["de", "a", "da", "prev_a"]): raise NotImplementedError( "native Vlasov requires electrostatic state without additional evolved diagnostics" @@ -232,10 +237,7 @@ def __call__(self, trainable_modules=None, args=None): root = Path(workspace.name) # Retain backing files for the NumPy memmaps returned in solver result. self._native_workspaces = getattr(self, "_native_workspaces", []) + [workspace] - _write_vx(root / "initial.bin", self.state["electron"]) - species_grid = self.cfg["grid"]["species_grids"]["electron"] for name, values in [ - ("v.bin", species_grid["v"]), ("background.bin", self.cfg["grid"]["ion_charge"]), ("field.bin", self.state["e"]), ]: @@ -249,7 +251,7 @@ def __call__(self, trainable_modules=None, args=None): dict( callbacks={key: save.fn for key, save in subs.items()}, args=self.args, - state={key: value for key, value in self.state.items() if key not in ["electron", "e"]}, + state={key: value for key, value in self.state.items() if key not in species_names | {"e"}}, ), stream, ) @@ -258,19 +260,13 @@ def __call__(self, trainable_modules=None, args=None): self.cfg["grid"]["species_distributions"] = cache fp = self.cfg["terms"]["fokker_planck"] nu = float(fp["time"]["baseline"]) * float(fp["space"]["baseline"]) if fp["is_on"] else 0.0 - native_args = [ + common_args = [ "--nx", str(grid.nx), - "--nv", - str(species_grid["nv"]), "--length", str(grid.xmax - grid.xmin), - "--vmax", - str(species_grid["vmax"]), "--dt", str(grid.dt), - "--nu", - str(nu), "--cells", "--integrator", "sixth", @@ -280,18 +276,48 @@ def __call__(self, trainable_modules=None, args=None): "halos", "--halo-capacity", str(options.halo_capacity), - "--input", - str(root / "initial.bin"), - "--v-grid", - str(root / "v.bin"), "--background", str(root / "background.bin"), ] if options.host_mpi: - native_args.append("--host-mpi") + common_args.append("--host-mpi") + species_specs = [] + for index, species in enumerate(species_list): + species_grid = self.cfg["grid"]["species_grids"][species.name] + initial, coordinates = root / f"initial-{index}.bin", root / f"v-{index}.bin" + _write_vx(initial, self.state[species.name]) + np.asarray(species_grid["v"], dtype=np.float64).tofile(coordinates) + species_args = [ + *common_args, + "--nv", + str(species.nv), + "--vmax", + str(species.vmax), + "--charge", + str(species.charge), + "--mass", + str(species.mass), + "--nu", + str(nu if index == 0 else 0.0), + "--input", + str(initial), + "--v-grid", + str(coordinates), + ] + species_specs.append( + dict( + name=species.name, + nv=species.nv, + vmax=species.vmax, + charge=species.charge, + mass=species.mass, + options=species_args, + ) + ) request = dict( library=library, - options=native_args, + options=species_specs[0]["options"], + species=species_specs, times=times, t0=t0, dt=grid.dt, diff --git a/adept/_vlasov1d/native/worker.py b/adept/_vlasov1d/native/worker.py index d4c6263d..67a6aad0 100644 --- a/adept/_vlasov1d/native/worker.py +++ b/adept/_vlasov1d/native/worker.py @@ -23,6 +23,8 @@ def run(request): os.environ.setdefault("XLA_PYTHON_CLIENT_PREALLOCATE", "false") core = NativeCore(spec["library"], spec["options"]) try: + for species in spec["species"][1:]: + core.add_species(species["options"]) if core.size != spec["ranks"]: raise RuntimeError("MPI launcher rank count differs from native.ranks") if core.device_type != (2 if spec["device"] == "cuda" else 1): @@ -73,13 +75,12 @@ def snapshot(): # FFTW may align host allocations to 32 bytes; JAX CPU requires 64. # CUDA allocations satisfy JAX's alignment, so GPU views stay strict. copy = False if core.device_type == 2 else None - f = jax.dlpack.from_dlpack(core.view(), copy=copy) e = jax.dlpack.from_dlpack(core.view(field=True), copy=copy) - return dict( - static, - electron=jax.make_array_from_single_device_arrays((core.nx, core.nv), slab, [f]), - e=jax.make_array_from_single_device_arrays((core.nx,), replicated, [e]), - ) + state = dict(static, e=jax.make_array_from_single_device_arrays((core.nx,), replicated, [e])) + for index, species in enumerate(spec["species"]): + f = jax.dlpack.from_dlpack(core.view(species=index), copy=copy) + state[species["name"]] = jax.make_array_from_single_device_arrays((core.nx, species["nv"]), slab, [f]) + return state directory = root / f"rank-{core.rank:05d}" directory.mkdir() @@ -176,6 +177,10 @@ def save(key, frame, t, state): steps=spec["steps"], ranks=core.size, device=spec["device"], + species=[ + {key: value for key, value in species.items() if key != "options"} + for species in spec["species"] + ], libraries={ name: importlib.metadata.version(name) for name in ["jax", "jaxlib", "cloudpickle", "diffrax"] diff --git a/adept/_vlasov1d/observations.py b/adept/_vlasov1d/observations.py index 5285bb82..1dc995de 100644 --- a/adept/_vlasov1d/observations.py +++ b/adept/_vlasov1d/observations.py @@ -4,6 +4,7 @@ import jax from interpax import interp2d from jax import numpy as jnp +from jax.scipy.special import xlogy class FieldsObservation(eqx.Module): @@ -25,7 +26,7 @@ def __call__(self, t, y, args): "v": velocity, "p": jnp.sum(f * centered_v**2.0, axis=1) * dv, "q": jnp.sum(f * centered_v**3.0, axis=1) * dv, - "-flogf": jnp.sum(-jnp.abs(f) * jnp.log(jnp.abs(f)), axis=1) * dv, + "-flogf": jnp.sum(-xlogy(jnp.abs(f), jnp.abs(f)), axis=1) * dv, "f^2": jnp.sum(f * f, axis=1) * dv, } result.update({name: y[name] for name in ("e", "de", "a", "prev_a")}) @@ -80,7 +81,7 @@ def moment(value, spacing=dv): scalars[f"mean_j_{name}"] = moment(f * v) scalars[f"mean_n_{name}"] = moment(f) scalars[f"mean_q_{name}"] = moment(f * v**3.0) - scalars[f"mean_-flogf_{name}"] = moment(-jnp.log(jnp.abs(f)) * jnp.abs(f)) + scalars[f"mean_-flogf_{name}"] = moment(-xlogy(jnp.abs(f), jnp.abs(f))) scalars[f"mean_f2_{name}"] = moment(f * f) mean_kinetic_energy += 0.5 * self.species_params[name]["mass"] * scalars[f"mean_P_{name}"] scalars["mean_de2"] = jnp.mean(y["de"] ** 2.0) diff --git a/benchmarks/vlasov_cuda/integration_check.py b/benchmarks/vlasov_cuda/integration_check.py index 5edba331..9e669ed7 100644 --- a/benchmarks/vlasov_cuda/integration_check.py +++ b/benchmarks/vlasov_cuda/integration_check.py @@ -15,6 +15,8 @@ def main(): parser.add_argument("--ranks", nargs="+", type=int, default=[1, 4, 8]) parser.add_argument("--vp-source", type=Path) parser.add_argument("--module", choices=["base", "vp", "both"], default="both") + parser.add_argument("--ions", action="store_true", help="Validate kinetic protons using ADEPT's species config") + parser.add_argument("--steps", type=int, default=5) args = parser.parse_args() os.environ["JAX_PLATFORMS"] = "cpu" os.environ["JAX_ENABLE_X64"] = "true" @@ -41,7 +43,7 @@ def main(): for cls in classes: for ranks in args.ranks: cfg = yaml.safe_load(Path("tests/test_base/configs/example.yaml").read_text()) - cfg["grid"].update(nx=32, nv=64, dt=0.02, tmax=0.08, vmax=8.0) + cfg["grid"].update(nx=32, nv=64, dt=0.02, tmax=0.02 * (args.steps - 1), vmax=8.0) cfg["drivers"] = {"ex": {}, "ey": {}} cfg["terms"]["time"] = "sixth" cfg["terms"]["fokker_planck"].update(is_on=True, type="chang_cooper_dougherty") @@ -53,6 +55,31 @@ def main(): beam.update(v0=3.0, T0=0.2, noise_val=0.001) cfg["density"]["species-left"] = copy.deepcopy(beam) cfg["density"]["species-left"].update(v0=-3.0, noise_seed=43) + if args.ions: + electron = copy.deepcopy(beam) + electron.update(v0=0.0, T0=1.0, noise_val=0.0) + ion = copy.deepcopy(electron) + ion.update(T0=0.1, basis="sine", baseline=1.0, amplitude=0.03, wavenumber=2 * np.pi / 20.94) + cfg["density"] = {"quasineutrality": True, "species-electrons": electron, "species-ions": ion} + cfg["terms"]["species"] = [ + dict( + name="ion", + charge=1.0, + mass=1836.0, + nv=32, + vmax=6 * np.sqrt(0.1 / 1836.0), + density_components=["species-ions"], + ), + dict( + name="electron", + charge=-1.0, + mass=1.0, + nv=64, + vmax=8.0, + density_components=["species-electrons"], + ), + ] + cfg["save"]["ion"] = {"main": {"t": {"nt": 3}}} cfg["native"] = dict(workdir=str(args.output)) if cls.__name__ == "VPTwoStream1D": cfg["two_stream"] = dict( @@ -129,6 +156,13 @@ def main(): keys=list(sol.ys), ) checks.append(row) + if args.ions: + changes = { + name: float(np.max(np.abs(sol.ys[f"{name}.main"][-1] - sol.ys[f"{name}.main"][0]))) + for name in ["electron", "ion"] + } + assert all(change > 1e-9 for change in changes.values()), changes + row["distribution_changes"] = changes (args.output / "checks.json").write_text(json.dumps(checks, indent=2)) print(json.dumps(row), flush=True) (args.output / "checks.json").write_text(json.dumps(checks, indent=2)) diff --git a/docs/source/solvers/vlasov1d/config.md b/docs/source/solvers/vlasov1d/config.md index 0313350c..e85242e5 100644 --- a/docs/source/solvers/vlasov1d/config.md +++ b/docs/source/solvers/vlasov1d/config.md @@ -41,7 +41,8 @@ automatic differentiation. Custom `BaseVlasov1D` subclasses passed to `ergoExo.setup(..., adept_module=MyModule)` are wrapped automatically. Build the shared library on the target system (CUDA toolkit, MPI and CMake -are required): +are required). Install the Python extra with `pip install 'adept[native]'` +or your environment manager's equivalent: ```sh cmake -S adept/_vlasov1d/native -B build/native -DCMAKE_BUILD_TYPE=Release @@ -72,7 +73,7 @@ uses its own velocity-slab decomposition; `native.ranks` controls it. |---|---|---| | `library` | unset | Shared library path; alternatively set `ADEPT_VLASOV_LIBRARY` | | `device` | `cuda` | `cpu` selects the FFTW validation build (`-DVLASOV_CUDA=OFF`) | -| `ranks` | `1` | One MPI worker per GPU; must divide `nx` and `nv` | +| `ranks` | `1` | One MPI worker per GPU; must divide `nx` and every species' `nv` | | `launcher` | empty | Argument list, with `{ranks}` substitution; defaults to `mpiexec -n` for multiple ranks | | `workdir` | `BASE_TEMPDIR` or system temporary directory | Shared staging/output directory, required for multiple ranks | | `scalar_stride` | `1` | Save scalar diagnostics every this many actual timesteps, plus both endpoints | @@ -92,12 +93,25 @@ off-step callbacks require a previous-state GPU copy and interpolation. Use Custom postprocessors that rebuild a uniform scalar axis must also use actual times when the final scalar interval is shorter than `scalar_stride`. -The initial implementation supports one electron species, a symmetric -cell-centred velocity grid, `time: sixth`, `edfdv: cubic-spline`, spectral -spatial streaming and `field: poisson`. Optional collisions are constant-rate -`chang_cooper_dougherty` with analytic beta. Drivers, Krook relaxation, +Kinetic electrons and ions use the existing `terms.species` and `density` +configuration below. Each species has its own symmetric cell-centred velocity +grid, charge and mass. All species contribute charge to the shared Poisson +solve at every splitting stage and advance with their own charge-to-mass ratio. +With no explicit species list, the usual electron species and static background +are retained. No additional ion parameters or preprocessing are required. + +Supported numerics are `time: sixth`, `edfdv: cubic-spline`, spectral spatial +streaming and `field: poisson`. The cubic operator reproduces ADEPT's local C1 +Hermite interpolation. Optional collisions are constant-rate +`chang_cooper_dougherty` with analytic beta, applied to the reference species +(electron if present, otherwise the first species), matching the JAX path. +Inter-species collisional exchange is not implemented. Drivers, Krook relaxation, Hou–Li filtering, evolved df/dt diagnostics and trainable modules are rejected. -The cubic operator reproduces ADEPT's local C1 Hermite interpolation. + +This backend uses the `ergoExo`/`BaseVlasov1D` lifecycle. The explicit +`Vlasov1DBuilder` prepares JAX programs and rejects `backend: native`; it does +not silently fall back to JAX. Rebuild the shared library after updating ADEPT; +incompatible native library ABIs fail with an actionable error. ## units diff --git a/tests/test_vlasov1d/test_native.py b/tests/test_vlasov1d/test_native.py index 2106ccb8..dd94da9c 100644 --- a/tests/test_vlasov1d/test_native.py +++ b/tests/test_vlasov1d/test_native.py @@ -180,3 +180,66 @@ def test_backend_dispatch_and_guards(config): NativeVlasov1D(config) with pytest.raises(TypeError, match="BaseVlasov1D"): native_backend(object) + from adept import SimulationSpec, solver_registry + + with pytest.raises(NotImplementedError, match="ergoExo"): + solver_registry.prepare(SimulationSpec.from_legacy_config(config), key=0) + + +@pytest.mark.parametrize("ranks,collisions,mass,charge", [(1, False, 25.0, 1.0), (2, True, 1836.0, 2.0)]) +def test_native_kinetic_ions(config, native_library, tmp_path, ranks, collisions, mass, charge): + """Ion charge feeds back on electrons; all species move on independent grids.""" + config["native"].update(library=str(native_library), ranks=ranks) + config["terms"]["fokker_planck"]["is_on"] = collisions + electron = deepcopy(config["density"]["species-background"]) + electron.update(v0=0.0, T0=1.0, noise_val=0.0) + ion = deepcopy(electron) + ion.update(T0=0.1, basis="sine", baseline=0.8 / charge, amplitude=0.03, wavenumber=2 * np.pi / 20.94) + alpha = deepcopy(electron) + alpha.update(T0=0.2, **{"space-profile": {**electron["space-profile"], "baseline": 0.1}}) + config["density"] = { + "quasineutrality": True, + "species-electrons": electron, + "species-ions": ion, + "species-alpha": alpha, + } + # Put the electron last to exercise reference-species selection, rather than + # accidentally applying its collision operator to the first configured ion. + config["terms"]["species"] = [ + dict( + name="ion", + charge=charge, + mass=mass, + nv=32, + vmax=6 * np.sqrt(0.1 / mass), + density_components=["species-ions"], + ), + dict( + name="alpha", + charge=2.0, + mass=4 * mass, + nv=48, + vmax=6 * np.sqrt(0.2 / (4 * mass)), + density_components=["species-alpha"], + ), + dict(name="electron", charge=-1.0, mass=1.0, nv=64, vmax=8.0, density_components=["species-electrons"]), + ] + config["save"].update(ion={"main": {"t": {"nt": 3}}}, alpha={"main": {"t": {"nt": 3}}}) + original, native = (setup(cls, config) for cls in [BaseVlasov1D, NativeVlasov1D]) + a, output = reference_at_native_times(original, native), native({}) + b = output["solver result"] + compare_results(a, b) + for species in ["electron", "ion", "alpha"]: + values = b.ys[f"{species}.main"] + assert np.max(np.abs(values[-1] - values[0])) > 1e-9, species + assert np.max(np.abs(b.ys["fields"]["e"][-1])) > 1e-4 + assert {s["name"] for s in output["native stats"]["species"]} == {"electron", "ion", "alpha"} + directory = tmp_path / "multispecies" + (directory / "binary").mkdir(parents=True) + fields = store_fields(native.cfg, str(directory / "binary"), b.ys["fields"], b.ts["fields"], "fields") + dists = store_f(native.cfg, b.ts, str(directory), b.ys) + assert set(fields) >= {"electron", "ion", "alpha"} + for species, nv in [("ion", 32), ("alpha", 48)]: + with xr.open_dataset(directory / f"binary/dist-{species}.main.nc") as saved: + xr.testing.assert_allclose(saved, dists[f"{species}.main"]) + assert saved.sizes[f"v_{species}"] == nv diff --git a/tests/test_vlasov1d/test_observations.py b/tests/test_vlasov1d/test_observations.py new file mode 100644 index 00000000..4a778807 --- /dev/null +++ b/tests/test_vlasov1d/test_observations.py @@ -0,0 +1,17 @@ +"""Entropy diagnostics use the continuous value 0 log(0) = 0.""" + +import jax.numpy as jnp +import numpy as np + +from adept._vlasov1d.observations import FieldsObservation, ScalarsObservation + + +def test_entropy_with_empty_velocity_cells(): + grids = {"electron": {"v": jnp.array([-1.0, 0.0, 1.0]), "dv": 0.5}} + state = {"electron": jnp.tile(jnp.array([0.0, 1.0, np.exp(-1.0)]), (4, 1))} + state.update({name: jnp.zeros(6 if name in {"a", "prev_a"} else 4) for name in ["e", "de", "a", "prev_a"]}) + expected = 0.5 * np.exp(-1.0) + fields = FieldsObservation(grids, 1.0)(0.0, state, {}) + scalars = ScalarsObservation(grids, {"electron": {"mass": 1.0}}, 1.0)(0.0, state, {}) + np.testing.assert_allclose(fields["electron"]["-flogf"], expected) + np.testing.assert_allclose(scalars["mean_-flogf_electron"], expected) From 15cc3df553361b774d9971d025f75b4706fce0c6 Mon Sep 17 00:00:00 2001 From: archis Date: Mon, 26 Oct 2026 21:42:02 +0000 Subject: [PATCH 4/8] test(vlasov1d): preserve native validation evidence and launch recipe --- adept/_vlasov1d/native/core/solver.hpp | 1 - .../integration-observers-v2.log | 12 +++ .../2026-10-24-adept/integration-v3.log | 74 +++++++++++++++++++ .../2026-10-24-adept/integration-vp.log | 34 +++++++++ .../2026-10-24-adept/jax-nx134400-g4.json | 2 +- .../2026-10-24-adept/jax-nx134400-g4.log | 15 ++++ .../2026-10-24-adept/jax-nx134400-g8.json | 2 +- .../2026-10-24-adept/jax-nx134400-g8.log | 27 +++++++ .../2026-10-24-adept/jax-nx268800-g8.json | 2 +- .../2026-10-24-adept/jax-nx268800-g8.log | 27 +++++++ .../observer-native/request.json | 2 +- .../observer-native/stats.json | 2 +- .../observer-native/worker.log | 24 ++++++ .../pinned-one-node/jax-nx134400-g4.json | 2 +- .../pinned-one-node/jax-nx134400-g4.log | 15 ++++ .../unmatched-node/jax-nx134400-g4.json | 2 +- .../unmatched-node/jax-nx134400-g4.log | 15 ++++ benchmarks/vlasov_cuda/run_integration.sh | 26 +++++++ docs/source/solvers/vlasov1d/config.md | 9 +++ 19 files changed, 285 insertions(+), 8 deletions(-) create mode 100644 benchmarks/vlasov_cuda/results/2026-10-24-adept/integration-observers-v2.log create mode 100644 benchmarks/vlasov_cuda/results/2026-10-24-adept/integration-v3.log create mode 100644 benchmarks/vlasov_cuda/results/2026-10-24-adept/integration-vp.log create mode 100644 benchmarks/vlasov_cuda/results/2026-10-24-adept/jax-nx134400-g4.log create mode 100644 benchmarks/vlasov_cuda/results/2026-10-24-adept/jax-nx134400-g8.log create mode 100644 benchmarks/vlasov_cuda/results/2026-10-24-adept/jax-nx268800-g8.log create mode 100644 benchmarks/vlasov_cuda/results/2026-10-24-adept/observer-native/worker.log create mode 100644 benchmarks/vlasov_cuda/results/2026-10-24-adept/pinned-one-node/jax-nx134400-g4.log create mode 100644 benchmarks/vlasov_cuda/results/2026-10-24-adept/unmatched-node/jax-nx134400-g4.log create mode 100644 benchmarks/vlasov_cuda/run_integration.sh diff --git a/adept/_vlasov1d/native/core/solver.hpp b/adept/_vlasov1d/native/core/solver.hpp index 1d970e6c..d4c51059 100644 --- a/adept/_vlasov1d/native/core/solver.hpp +++ b/adept/_vlasov1d/native/core/solver.hpp @@ -330,4 +330,3 @@ void initialize(Backend& backend, const Options& o) { backend.upload_state(values); } } // namespace adept_native - diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-adept/integration-observers-v2.log b/benchmarks/vlasov_cuda/results/2026-10-24-adept/integration-observers-v2.log new file mode 100644 index 00000000..8e7fe526 --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-adept/integration-observers-v2.log @@ -0,0 +1,12 @@ +2026/10/24 12:27:01 INFO mlflow.tracking.fluent: Experiment with name 'native-adept-integration' does not exist. Creating a new experiment. +Successfully removed /pscratch/sd/a/archis/tmp/tmp2g3h194q +2026/10/24 12:27:04 INFO mlflow.system_metrics.system_metrics_monitor: Started monitoring system metrics. +/pscratch/sd/a/archis/vp-turbulence-sessions/20261023T081121Z-nv4096-nx134400-5ce4fe14/src/vp_turbulence/adeptmodules.py:811: UserWarning: Saving distribution for 'electron.main' at full resolution. + f_result = store_f( +Successfully removed /pscratch/sd/a/archis/tmp/tmp_gtq6flt +2026/10/24 12:27:23 INFO mlflow.system_metrics.system_metrics_monitor: Stopping system metrics monitoring... +2026/10/24 12:27:23 INFO mlflow.system_metrics.system_metrics_monitor: Successfully terminated system metrics monitoring! + 0.00%| | [00:00 + main() + ~~~~^^ + File "/pscratch/sd/a/archis/adept-sessions/20261024T190159Z-99199-22637/src/benchmarks/vlasov_cuda/integration_check.py", line 96, in main + output, post, run_id = exo(modules) + ~~~^^^^^^^^^ + File "/pscratch/sd/a/archis/adept-sessions/20261024T190159Z-99199-22637/src/adept/_base_.py", line 423, in __call__ + run_output = solve(modules, args) + File "/pscratch/sd/a/archis/adept-sessions/20261024T190159Z-99199-22637/src/adept/_vlasov1d/native/module.py", line 317, in __call__ + raise RuntimeError(f"native Vlasov workers failed ({returncode}); log: {log}\n{log.read_text()[-12000:]}") +RuntimeError: native Vlasov workers failed (1); log: /pscratch/sd/a/archis/adept-sessions/20261024T190159Z-99199-22637/outputs/integration-v3/adept-native-5ofo1tiv/worker.log +Traceback (most recent call last): + File "/pscratch/sd/a/archis/adept-sessions/20261024T190159Z-99199-22637/src/adept/_vlasov1d/native/worker.py", line 153, in run + save(key, frame, t, state) + ~~~~^^^^^^^^^^^^^^^^^^^^^^ + File "/pscratch/sd/a/archis/adept-sessions/20261024T190159Z-99199-22637/src/adept/_vlasov1d/native/worker.py", line 83, in save + value = callbacks[key](replicate(np.asarray(t)), state) + File "/pscratch/sd/a/archis/adept-sessions/20261024T190159Z-99199-22637/src/adept/_vlasov1d/native/worker.py", line 66, in + key: jax.jit(lambda t, y, fn=fn: fn(t, y, payload["args"])) for key, fn in payload["callbacks"].items() + ~~^^^^^^^^^^^^^^^^^^^^^^^ + File "/pscratch/sd/a/archis/vp-turbulence-sessions/20261023T081121Z-nv4096-nx134400-5ce4fe14/src/vp_turbulence/adeptmodules.py", line 118, in save + k_peak = k_x[jnp.argmax(e_pow)] + ~~~^^^^^^^^^^^^^^^^^^^ +jax.errors.TracerArrayConversionError: The numpy.ndarray conversion method __array__() was called on traced array with shape int64[] +The error occurred while tracing the function at /pscratch/sd/a/archis/adept-sessions/20261024T190159Z-99199-22637/src/adept/_vlasov1d/native/worker.py:66 for jit. This concrete value was not available in Python because it depends on the value of the argument y['e']. +See https://docs.jax.dev/en/latest/errors.html#jax.errors.TracerArrayConversionError +-------------------- +For simplicity, JAX has removed its internal frames from the traceback of the following exception. Set JAX_TRACEBACK_FILTERING=off to include these. +MPICH ERROR [Rank 0] [job id 60221398.6] [Sat Oct 24 12:12:01 2026] [nid001600] - Abort(1) (rank 0 in comm 0): application called MPI_Abort(MPI_COMM_WORLD, 1) - process 0 + +srun: error: nid001600: task 0: Exited with exit code 1 +srun: Terminating StepId=60221398.6 diff --git a/benchmarks/vlasov_cuda/results/2026-10-24-adept/integration-vp.log b/benchmarks/vlasov_cuda/results/2026-10-24-adept/integration-vp.log new file mode 100644 index 00000000..17de03a6 --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-24-adept/integration-vp.log @@ -0,0 +1,34 @@ +2026/10/24 12:15:01 INFO mlflow.tracking.fluent: Experiment with name 'native-adept-integration' does not exist. Creating a new experiment. +Successfully removed /pscratch/sd/a/archis/tmp/tmp73h5_i9i +2026/10/24 12:15:04 INFO mlflow.system_metrics.system_metrics_monitor: Started monitoring system metrics. +/pscratch/sd/a/archis/vp-turbulence-sessions/20261023T081121Z-nv4096-nx134400-5ce4fe14/src/vp_turbulence/adeptmodules.py:811: UserWarning: Saving distribution for 'electron.main' at full resolution. + f_result = store_f( +Successfully removed /pscratch/sd/a/archis/tmp/tmpu5zif7m1 +2026/10/24 12:15:18 INFO mlflow.system_metrics.system_metrics_monitor: Stopping system metrics monitoring... +2026/10/24 12:15:18 INFO mlflow.system_metrics.system_metrics_monitor: Successfully terminated system metrics monitoring! + 0.00%| | [00:00 "$report_root/build.log" 2>&1 +cmake --build "$EC_SESSION_WORKDIR/native-build" -j4 >> "$report_root/build.log" 2>&1 +library="$EC_SESSION_WORKDIR/native-build/libadept_vlasov.so" +sha256sum "$library" > "$report_root/library.sha256" +sha256sum adept/_vlasov1d/native/api.cpp adept/_vlasov1d/native/core/* > "$report_root/source.sha256" +nvidia-smi --query-gpu=name,driver_version,memory.total --format=csv > "$report_root/gpus.csv" +scontrol show job "$SLURM_JOB_ID" > "$report_root/allocation.txt" +python -u benchmarks/vlasov_cuda/integration_check.py --library "$library" \ + --output "$report_root/coupled" --ranks 1 4 8 --module base --ions --steps 100 \ + > "$report_root/coupled.log" 2>&1 +python -u benchmarks/vlasov_cuda/integration_check.py --library "$library" \ + --output "$report_root/electron" --ranks 4 --module base \ + > "$report_root/electron.log" 2>&1 +echo "Native electron/ion A100 integration checks passed: $report_root" diff --git a/docs/source/solvers/vlasov1d/config.md b/docs/source/solvers/vlasov1d/config.md index e85242e5..6645b5d0 100644 --- a/docs/source/solvers/vlasov1d/config.md +++ b/docs/source/solvers/vlasov1d/config.md @@ -49,6 +49,15 @@ cmake -S adept/_vlasov1d/native -B build/native -DCMAKE_BUILD_TYPE=Release cmake --build build/native -j4 ``` +When ADEPT is installed in another project (such as turbulence), locate the +packaged sources in that project's Python environment: + +```sh +native_source=$(python -c 'from importlib.util import find_spec; print(find_spec("adept._vlasov1d.native").submodule_search_locations[0])') +cmake -S "$native_source" -B build/native -DCMAKE_BUILD_TYPE=Release +cmake --build build/native -j4 +``` + On Perlmutter add `-DCMAKE_CXX_COMPILER=CC`, and use the existing allocated job's `srun` launcher. A two-node example is: From dd58cf9bec6214f49cba99ebeea90d32877f54cc Mon Sep 17 00:00:00 2001 From: archis Date: Mon, 26 Oct 2026 21:54:49 +0000 Subject: [PATCH 5/8] test(vlasov1d): report A100 ion and turbulence compatibility validation --- adept/_base_.py | 9 ++-- adept/_vlasov1d/native/module.py | 8 +-- .../results/2026-10-26-ions/.gitignore | 4 ++ .../results/2026-10-26-ions/README.md | 53 +++++++++++++++++++ tests/test_vlasov1d/test_native.py | 3 ++ 5 files changed, 69 insertions(+), 8 deletions(-) create mode 100644 benchmarks/vlasov_cuda/results/2026-10-26-ions/.gitignore create mode 100644 benchmarks/vlasov_cuda/results/2026-10-26-ions/README.md diff --git a/adept/_base_.py b/adept/_base_.py index 5f4ed9b6..fff3c1f9 100644 --- a/adept/_base_.py +++ b/adept/_base_.py @@ -306,11 +306,6 @@ def _get_adept_module_(self, cfg: dict) -> ADEPTModule: elif cfg["solver"] == "vlasov-1d": from adept.vlasov1d import BaseVlasov1D as this_module - if cfg.get("backend", "jax") == "native": - from adept.vlasov1d import native_backend - - this_module = native_backend(this_module) - # config = ConfigModel(**cfg) elif cfg["solver"] == "vlasov-1d2v": @@ -359,6 +354,10 @@ def _get_adept_module_(self, cfg: dict) -> ADEPTModule: else: raise NotImplementedError("This solver approach has not been implemented yet") + if cfg.get("backend", "jax") == "native": + from adept.vlasov1d import native_backend + + this_module = native_backend(this_module) return this_module(cfg) def _setup_(self, cfg: dict, td: str, adept_module: ADEPTModule = None, log: bool = True) -> dict[str, Module]: diff --git a/adept/_vlasov1d/native/module.py b/adept/_vlasov1d/native/module.py index 16bc492a..a360eb10 100644 --- a/adept/_vlasov1d/native/module.py +++ b/adept/_vlasov1d/native/module.py @@ -99,6 +99,8 @@ class NativeVlasovMixin: jit_compile = False def __init__(self, cfg): + if not jax.config.jax_enable_x64: + raise RuntimeError("native Vlasov requires jax_enable_x64 before preprocessing") with _host_preprocessing(): super().__init__(cfg) self.native_options = NativeSolverConfig.model_validate(cfg.get("native") or {}) @@ -107,7 +109,7 @@ def __init__(self, cfg): def _validate_native_physics(self): cfg, sim = self.cfg, self.simulation names = [species.name for species in sim.species] - if not names or len(set(names)) != len(names): + if not names or any(not name for name in names) or len(set(names)) != len(names): raise ValueError("native species names must be nonempty and unique") for species in sim.species: if not np.isfinite(species.mass) or species.mass <= 0 or not np.isfinite(species.charge): @@ -132,8 +134,8 @@ def _validate_native_physics(self): raise NotImplementedError("native Vlasov does not yet evolve df/dt diagnostic state") fp = terms["fokker_planck"] if fp["is_on"]: - if fp["type"].casefold() not in {"chang_cooper_dougherty", "dougherty_chang_cooper"} or fp.get( - "self_consistent_beta", {} + if fp["type"].casefold() not in {"chang_cooper_dougherty", "dougherty_chang_cooper"} or ( + fp.get("self_consistent_beta") or {} ).get("enabled", False): raise NotImplementedError("native collisions require Chang-Cooper Dougherty with analytic beta") for axis in ["time", "space"]: diff --git a/benchmarks/vlasov_cuda/results/2026-10-26-ions/.gitignore b/benchmarks/vlasov_cuda/results/2026-10-26-ions/.gitignore new file mode 100644 index 00000000..ee3acfa2 --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-26-ions/.gitignore @@ -0,0 +1,4 @@ +# Raw validation evidence remains local and in the isolated NERSC session. +* +!.gitignore +!README.md diff --git a/benchmarks/vlasov_cuda/results/2026-10-26-ions/README.md b/benchmarks/vlasov_cuda/results/2026-10-26-ions/README.md new file mode 100644 index 00000000..ec6493e3 --- /dev/null +++ b/benchmarks/vlasov_cuda/results/2026-10-26-ions/README.md @@ -0,0 +1,53 @@ +# Native kinetic-species integration validation + +All five CUDA integration cases passed on allocation 60336491, nodes +nid003048/nid003744 (A100-SXM4-40GB, driver 580.178.04). The allocation was +released after preserving the outputs; all validation steps completed with +exit code 0. This is correctness evidence, not a production-scale ion timing. +Only this written report is published. Raw source archives, NetCDF outputs, +configs and MLflow metadata remain in the matching local result directory +and the isolated NERSC session recorded below. + +| Case | GPUs | Steps | Maximum absolute saved-value error vs ADEPT/JAX | +|---|---:|---:|---:| +| Kinetic electrons + protons | 1 | 100 | 1.12266e-12 | +| Kinetic electrons + protons | 4 | 100 | 9.30811e-13 | +| Kinetic electrons + protons | 8 / 2 nodes | 100 | 9.66338e-13 | +| Electron two-stream regression | 4 | 5 | 1.49214e-13 | +| Current turbulence custom module | 8 / 2 nodes | 5 | 1.01110e-11 | + +Each saved leaf and time was compared with rtol 1e-10, atol 1e-11, including +full distributions, fields, species moments and scalar diagnostics. The +proton case has nx=32, electron nv=64/vmax=8, ion nv=32, +ion vmax=6*sqrt(0.1/1836), qi=1, mi=1836, Ti=0.1 and dt=0.02. Only the +ions have the initial sine density perturbation. Both species change: max +|f(t=2)-f(t=0)| is approximately 0.02271 for electrons and 0.22539 for ions. +Constant-rate analytic-beta Chang-Cooper Dougherty collisions (nu=0.001) +act on electrons, matching the existing ADEPT reference-species convention. +No inter-species collision operator is claimed. + +The turbulence case uses VPTwoStream1D from vp-turbulence main commit +94df45d0172f861ae42bae5297bacc957294ba32, with no project edits. Its original +f0, velocity-spectrum and cascade callbacks and postprocessor ran through +ergoExo. `vp-artifacts/binary/` contains the actual postprocessed NetCDF +outputs; `vp-artifacts/native/` contains the request, statistics and library +hash logged by the run. `vp-source.tar.gz` preserves its source snapshot. + +ADEPT source: ddc4dd14082a60f9fe8a8f9dfdb80044f5ab91f0 plus the recorded +`run_integration.sh` launcher, archived in source.tar.gz. The compiled C++ +physics is unchanged in the final PR except for trailing blank-line cleanup. +Subsequent Python changes add float64/null-default guards and ensure every +registered Vlasov module honors backend selection. Runtime: JAX/JAXlib +0.10.2, interpax 0.3.14+10.g84aaf30, diffrax 0.7.2, cloudpickle 3.1.2, +CUDA 13.2.78, GNU 14 / Cray MPI 9.1. Existing environments were unchanged. + +Build/launch commands: `../../run_integration.sh` and +`../../integration_check.py`. The turbulence check additionally passes +`--module vp --ranks 8 --vp-source `. Machine paths in hashes, +requests and accounting refer to the isolated session at +/pscratch/sd/a/archis/adept-sessions/20261026T213621Z-5332-10833/. + +Local validation: 8 native/entropy tests, including 1/2-rank MPI and three +kinetic species with independent grids and charges/masses, plus 211 existing +lifecycle/core/builder/configuration regressions. The wheel contains the +CMake/C++ sources needed to build this backend from downstream projects. diff --git a/tests/test_vlasov1d/test_native.py b/tests/test_vlasov1d/test_native.py index dd94da9c..5e5ee654 100644 --- a/tests/test_vlasov1d/test_native.py +++ b/tests/test_vlasov1d/test_native.py @@ -184,6 +184,9 @@ def test_backend_dispatch_and_guards(config): with pytest.raises(NotImplementedError, match="ergoExo"): solver_registry.prepare(SimulationSpec.from_legacy_config(config), key=0) + config["terms"]["fokker_planck"]["self_consistent_beta"] = None + config["solver"] = "vlasov-1d-iaw" + assert not ergoExo()._get_adept_module_(config).jit_compile @pytest.mark.parametrize("ranks,collisions,mass,charge", [(1, False, 25.0, 1.0), (2, True, 1836.0, 2.0)]) From fd6f5b9630edf4be77cacc12eba1beec016630fd Mon Sep 17 00:00:00 2001 From: archis Date: Mon, 26 Oct 2026 22:51:24 +0000 Subject: [PATCH 6/8] fix(vlasov1d): honor resolved collision profiles and CUDA ordinals --- .github/workflows/cpu-tests.yaml | 25 ++++++++++++++++++++- adept/_vlasov1d/native/README.md | 2 +- adept/_vlasov1d/native/api.cpp | 7 ++++-- adept/_vlasov1d/native/bridge.py | 8 +++++-- adept/_vlasov1d/native/core/backend.cpp | 9 ++++++++ adept/_vlasov1d/native/core/backend.hpp | 1 + adept/_vlasov1d/native/module.py | 12 ++++++---- adept/_vlasov1d/native/worker.py | 19 ++++++++++++++-- benchmarks/vlasov_cuda/integration_check.py | 12 +++++++++- docs/source/solvers/vlasov1d/config.md | 6 +++++ tests/test_vlasov1d/test_native.py | 14 ++++++++++++ 11 files changed, 102 insertions(+), 13 deletions(-) diff --git a/.github/workflows/cpu-tests.yaml b/.github/workflows/cpu-tests.yaml index 81327fe4..16847cf9 100644 --- a/.github/workflows/cpu-tests.yaml +++ b/.github/workflows/cpu-tests.yaml @@ -156,13 +156,36 @@ jobs: - name: Install native validation dependencies run: | sudo apt-get update - sudo apt-get install -y cmake pkg-config libfftw3-dev libopenmpi-dev openmpi-bin + packages=(pkg-config libfftw3-dev libopenmpi-dev openmpi-bin) + if ! command -v cmake >/dev/null; then packages+=(cmake); fi + sudo apt-get install -y --no-install-recommends "${packages[@]}" uv sync --extra dev --extra native --frozen - name: Test with pytest env: OMPI_MCA_rmaps_base_oversubscribe: '1' run: uv run --locked pytest tests/test_vlasov1d --ignore=tests/test_vlasov1d/test_config_regression.py + test-native-vlasov1d: + needs: changes + if: needs.changes.outputs.run_all == 'true' || needs.changes.outputs.vlasov1d == 'true' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: ./.github/actions/setup_repo + with: + python-version: '3.11' + - name: Install native validation dependencies + run: | + sudo apt-get update + packages=(pkg-config libfftw3-dev libopenmpi-dev openmpi-bin) + if ! command -v cmake >/dev/null; then packages+=(cmake); fi + sudo apt-get install -y --no-install-recommends "${packages[@]}" + uv sync --extra dev --extra native --frozen + - name: Test native backend + env: + OMPI_MCA_rmaps_base_oversubscribe: '1' + run: uv run --locked pytest tests/test_vlasov1d/test_native.py tests/test_vlasov1d/test_observations.py + test-farsight1d: needs: changes if: needs.changes.outputs.run_all == 'true' || needs.changes.outputs.farsight1d == 'true' diff --git a/adept/_vlasov1d/native/README.md b/adept/_vlasov1d/native/README.md index 933c7fc6..112ca765 100644 --- a/adept/_vlasov1d/native/README.md +++ b/adept/_vlasov1d/native/README.md @@ -43,7 +43,7 @@ constant-rate Chang-Cooper Dougherty collisions. Kinetic ions use the existing Each stage sums every species' charge density into one MPI field reduction, then advances every species under that shared field. Collisions act on the reference species, as in ADEPT's JAX path; inter-species collisional exchange -is not included. Rebuild the library when updating ADEPT (current ABI: 2). +is not included. Rebuild the library when updating ADEPT (current ABI: 3). Unsupported physics and trainable modules fail explicitly. Performance and reproduction evidence is under [the benchmark results](../../../benchmarks/vlasov_cuda/results/2026-10-24-adept). diff --git a/adept/_vlasov1d/native/api.cpp b/adept/_vlasov1d/native/api.cpp index 0e301c16..81ef2de3 100644 --- a/adept/_vlasov1d/native/api.cpp +++ b/adept/_vlasov1d/native/api.cpp @@ -117,7 +117,7 @@ void validate_options(Options& opts) { } } extern "C" { -int av_abi_version() { return 2; } +int av_abi_version() { return 3; } const char* av_error() { return last_error.c_str(); } void* av_create(int argc, const char** argv) { void* result = nullptr; @@ -167,6 +167,9 @@ int av_info(void* handle, int* output) { std::copy_n(values, 7, output); }); } +int av_device_ordinal(void* handle, int* output) { + return guarded([&] { *output = (**static_cast(handle)).backend->device_ordinal(); }); +} int av_broadcast(char* value, int capacity) { return guarded([&] { mpi_check(MPI_Bcast(value, capacity, MPI_CHAR, 0, MPI_COMM_WORLD)); }); } @@ -198,7 +201,7 @@ void* av_species_tensor(void* handle, int species, int field) { tensor->strides[0] = 1; tensor->strides[1] = e.options.grid.nx; tensor->managed = {{field ? e.backend->electric_buffer() : e.backend->velocity_buffer(), - {device_type(), device_type() == 2 ? e.local_rank : 0}, field ? 1 : 2, + {device_type(), e.backend->device_ordinal()}, field ? 1 : 2, {2, 64, 1}, tensor->shape, tensor->strides, 0}, tensor, release_tensor}; result = &tensor->managed; })) return nullptr; diff --git a/adept/_vlasov1d/native/bridge.py b/adept/_vlasov1d/native/bridge.py index 01adde90..19030988 100644 --- a/adept/_vlasov1d/native/bridge.py +++ b/adept/_vlasov1d/native/bridge.py @@ -10,7 +10,7 @@ class NativeCore: def __init__(self, library, options): self.lib = ct.CDLL(str(library)) version = getattr(self.lib, "av_abi_version", None) - if version is None or version() != 2: + if version is None or version() != 3: raise RuntimeError( "native library ABI mismatch; rebuild adept/_vlasov1d/native from the installed ADEPT source" ) @@ -18,6 +18,7 @@ def __init__(self, library, options): "av_create": ([ct.c_int, ct.POINTER(ct.c_char_p)], ct.c_void_p), "av_error": ([], ct.c_char_p), "av_info": ([ct.c_void_p, ct.POINTER(ct.c_int)], ct.c_int), + "av_device_ordinal": ([ct.c_void_p, ct.POINTER(ct.c_int)], ct.c_int), "av_add_species": ([ct.c_void_p, ct.c_int, ct.POINTER(ct.c_char_p)], ct.c_int), "av_species_info": ([ct.c_void_p, ct.c_int, ct.POINTER(ct.c_int)], ct.c_int), "av_species_tensor": ([ct.c_void_p, ct.c_int, ct.c_int], ct.c_void_p), @@ -42,6 +43,9 @@ def __init__(self, library, options): values = (ct.c_int * 7)() self.check(self.lib.av_info(self.handle, values)) self.rank, self.size, self.local_rank, self.device_type, self.nx, self.nv, self.local_nv = values + ordinal = ct.c_int() + self.check(self.lib.av_device_ordinal(self.handle, ct.byref(ordinal))) + self.device_ordinal = ordinal.value def check(self, result): if result: @@ -93,7 +97,7 @@ def __init__(self, owner, field, species): self.owner, self.field, self.species = owner, field, species def __dlpack_device__(self): - return self.owner.device_type, self.owner.local_rank if self.owner.device_type == 2 else 0 + return self.owner.device_type, self.owner.device_ordinal def __dlpack__(self, stream=None, *, max_version=None, dl_device=None, copy=None): if copy: diff --git a/adept/_vlasov1d/native/core/backend.cpp b/adept/_vlasov1d/native/core/backend.cpp index e8b9b9ae..d5a8d8b8 100644 --- a/adept/_vlasov1d/native/core/backend.cpp +++ b/adept/_vlasov1d/native/core/backend.cpp @@ -214,6 +214,15 @@ struct Backend::Impl { Backend::Backend(Grid grid, int local_rank) : p(std::make_unique(grid, local_rank)) {} Backend::~Backend() = default; +int Backend::device_ordinal() const { +#ifdef VLASOV_CUDA + int device = 0; + check_cuda(cudaGetDevice(&device)); + return device; +#else + return 0; +#endif +} void Backend::synchronize() { #ifdef VLASOV_CUDA check_cuda(cudaDeviceSynchronize()); diff --git a/adept/_vlasov1d/native/core/backend.hpp b/adept/_vlasov1d/native/core/backend.hpp index 810ff740..4ae2e694 100644 --- a/adept/_vlasov1d/native/core/backend.hpp +++ b/adept/_vlasov1d/native/core/backend.hpp @@ -62,6 +62,7 @@ class Backend { std::vector download_density(); void upload_density(const std::vector& values); std::string device_name() const; + int device_ordinal() const; std::size_t allocated_bytes() const; private: struct Impl; diff --git a/adept/_vlasov1d/native/module.py b/adept/_vlasov1d/native/module.py index a360eb10..d75ebf7f 100644 --- a/adept/_vlasov1d/native/module.py +++ b/adept/_vlasov1d/native/module.py @@ -133,15 +133,20 @@ def _validate_native_physics(self): if any(cfg.get("diagnostics", {}).values()): raise NotImplementedError("native Vlasov does not yet evolve df/dt diagnostic state") fp = terms["fokker_planck"] + self.native_collision_frequency = 0.0 if fp["is_on"]: if fp["type"].casefold() not in {"chang_cooper_dougherty", "dougherty_chang_cooper"} or ( fp.get("self_consistent_beta") or {} ).get("enabled", False): raise NotImplementedError("native collisions require Chang-Cooper Dougherty with analytic beta") for axis in ["time", "space"]: - envelope = fp[axis] - if envelope.get("bump_height", 0) != 0 or envelope.get("slope", 0) != 0: + envelope = getattr(sim.nu_fp_prof, f"{axis}_envelope") + resolved_config = getattr(self.config_model.terms.fokker_planck, axis) + if envelope.bump_height != 0 or float(resolved_config.slope) != 0: raise NotImplementedError("native collisions currently require constant frequency profiles") + self.native_collision_frequency = float( + sim.nu_fp_prof.time_envelope.baseline * sim.nu_fp_prof.space_envelope.baseline + ) def write_units(self): with _host_preprocessing(): @@ -260,8 +265,7 @@ def __call__(self, trainable_modules=None, args=None): finally: if cache is not None: self.cfg["grid"]["species_distributions"] = cache - fp = self.cfg["terms"]["fokker_planck"] - nu = float(fp["time"]["baseline"]) * float(fp["space"]["baseline"]) if fp["is_on"] else 0.0 + nu = self.native_collision_frequency common_args = [ "--nx", str(grid.nx), diff --git a/adept/_vlasov1d/native/worker.py b/adept/_vlasov1d/native/worker.py index 67a6aad0..4d85a9ae 100644 --- a/adept/_vlasov1d/native/worker.py +++ b/adept/_vlasov1d/native/worker.py @@ -44,10 +44,10 @@ def run(request): coordinator_address=coordinator, num_processes=core.size, process_id=core.rank, - local_device_ids=[core.local_rank] if core.device_type == 2 else [0], + local_device_ids=[core.device_ordinal], ) elif core.device_type == 2: - jax.config.update("jax_cuda_visible_devices", str(core.local_rank)) + jax.config.update("jax_cuda_visible_devices", str(core.device_ordinal)) sys.path[:0] = spec.get("pythonpath", []) import cloudpickle import jax.numpy as jnp @@ -84,6 +84,17 @@ def snapshot(): directory = root / f"rank-{core.rank:05d}" directory.mkdir() + (directory / "placement.json").write_text( + json.dumps( + dict( + rank=core.rank, + local_rank=core.local_rank, + device_ordinal=core.device_ordinal, + cuda_visible_devices=os.environ.get("CUDA_VISIBLE_DEVICES"), + host=socket.gethostname(), + ) + ) + ) writers, metadata, layout = {}, {}, {} def save(key, frame, t, state): @@ -177,6 +188,10 @@ def save(key, frame, t, state): steps=spec["steps"], ranks=core.size, device=spec["device"], + placement=[ + json.loads((root / f"rank-{rank:05d}/placement.json").read_text()) + for rank in range(core.size) + ], species=[ {key: value for key, value in species.items() if key != "options"} for species in spec["species"] diff --git a/benchmarks/vlasov_cuda/integration_check.py b/benchmarks/vlasov_cuda/integration_check.py index 9e669ed7..13ad7b68 100644 --- a/benchmarks/vlasov_cuda/integration_check.py +++ b/benchmarks/vlasov_cuda/integration_check.py @@ -17,7 +17,12 @@ def main(): parser.add_argument("--module", choices=["base", "vp", "both"], default="both") parser.add_argument("--ions", action="store_true", help="Validate kinetic protons using ADEPT's species config") parser.add_argument("--steps", type=int, default=5) + parser.add_argument( + "--per-rank-gpu", action="store_true", help="Expose one GPU per worker; validate host-staged MPI" + ) args = parser.parse_args() + if args.per_rank_gpu: + os.environ["MPICH_GPU_SUPPORT_ENABLED"] = "0" os.environ["JAX_PLATFORMS"] = "cpu" os.environ["JAX_ENABLE_X64"] = "true" os.environ["MLFLOW_ALLOW_FILE_STORE"] = "true" @@ -112,6 +117,7 @@ def main(): library=str(args.library), ranks=ranks, timeout=180, + host_mpi=args.per_rank_gpu, launcher=[ "srun", f"--jobid={os.environ['SLURM_JOB_ID']}", @@ -120,7 +126,7 @@ def main(): "--ntasks={ranks}", f"--ntasks-per-node={ranks // nodes}", "--gpus-per-node=4", - "--gpu-bind=none", + "--gpu-bind=single:1" if args.per_rank_gpu else "--gpu-bind=none", "--cpu-bind=none", ], ) @@ -155,6 +161,10 @@ def main(): stats=output["native stats"], keys=list(sol.ys), ) + if args.per_rank_gpu: + placement = output["native stats"]["placement"] + assert all(worker["device_ordinal"] == 0 for worker in placement), placement + assert len({(worker["host"], worker["cuda_visible_devices"]) for worker in placement}) == ranks checks.append(row) if args.ions: changes = { diff --git a/docs/source/solvers/vlasov1d/config.md b/docs/source/solvers/vlasov1d/config.md index 6645b5d0..e1c46783 100644 --- a/docs/source/solvers/vlasov1d/config.md +++ b/docs/source/solvers/vlasov1d/config.md @@ -77,6 +77,12 @@ Run one outer Python driver inside the allocation; the launcher starts its workers. Set `MPICH_GPU_SUPPORT_ENABLED=1` for CUDA-aware Cray MPI. The work directory and library must be visible on every worker node. Native execution uses its own velocity-slab decomposition; `native.ranks` controls it. +CUDA device ordinals follow each worker's visible device list, independently +of its MPI local rank. The example exposes all four GPUs per node for +CUDA-aware Cray MPI. A launcher that exposes one GPU per worker uses ordinal +zero within each worker. On Cray systems requiring all GPUs visible for GPU +transport, use `host_mpi: true` and `MPICH_GPU_SUPPORT_ENABLED=0` to test that +visibility mode with host-staged communication. | Native option | Default | Meaning | |---|---|---| diff --git a/tests/test_vlasov1d/test_native.py b/tests/test_vlasov1d/test_native.py index 5e5ee654..50a80d49 100644 --- a/tests/test_vlasov1d/test_native.py +++ b/tests/test_vlasov1d/test_native.py @@ -126,6 +126,20 @@ def test_native_step_schedules(config): np.testing.assert_allclose(subs["f0"].ts, [0.0, 0.04, 0.1]) +@pytest.mark.parametrize("axis", ["time", "space"]) +def test_native_rejects_omitted_collision_pulse_default(config, axis): + config["terms"]["fokker_planck"][axis].pop("bump_height") + with pytest.raises(NotImplementedError, match="constant frequency"): + NativeVlasov1D(config) + + +def test_native_constant_collision_defaults(config): + native = NativeVlasov1D(deepcopy(config)) + assert native.native_collision_frequency == 0.001 + config["terms"]["fokker_planck"]["time"].pop("baseline") + assert NativeVlasov1D(config).native_collision_frequency == 0.0 + + class CustomTwoStream(BaseVlasov1D): def init_diffeqsolve(self): super().init_diffeqsolve() From 1441e510cb49b5712a14a1f01243dca5895e3338 Mon Sep 17 00:00:00 2001 From: archis Date: Mon, 26 Oct 2026 22:58:53 +0000 Subject: [PATCH 7/8] fix(vlasov1d): retain allocation device identity across callbacks --- adept/_vlasov1d/native/core/backend.cpp | 21 ++++++++------ benchmarks/vlasov_cuda/integration_check.py | 10 ++++--- .../vlasov_cuda/run_visibility_check.sh | 28 +++++++++++++++++++ 3 files changed, 47 insertions(+), 12 deletions(-) create mode 100644 benchmarks/vlasov_cuda/run_visibility_check.sh diff --git a/adept/_vlasov1d/native/core/backend.cpp b/adept/_vlasov1d/native/core/backend.cpp index d5a8d8b8..7e7023bc 100644 --- a/adept/_vlasov1d/native/core/backend.cpp +++ b/adept/_vlasov1d/native/core/backend.cpp @@ -163,6 +163,15 @@ std::string select_device(int local_rank) { return "CPU/FFTW (validation only)"; #endif } +int current_device_ordinal() { +#ifdef VLASOV_CUDA + int device = 0; + check_cuda(cudaGetDevice(&device)); + return device; +#else + return 0; +#endif +} } // namespace void prepare_device_from_environment() { @@ -180,12 +189,13 @@ void prepare_device_from_environment() { struct Backend::Impl { Grid grid; std::string device; + int ordinal; Buffer fv, fx, send, recv, second, rhs, output, inverse; Buffer spectrum, density_local, density_global, electric, field_spectrum; Buffer coordinates, background, ghosts, field_maximum; FFT streaming_fft, field_fft; Impl(Grid g, int local_rank) - : grid(g), device(select_device(local_rank)), + : grid(g), device(select_device(local_rank)), ordinal(current_device_ordinal()), fv(g.count()), fx(!g.halos || g.collisions ? g.count() : 0), send((!g.halos || g.collisions) && g.ranks > 1 ? g.count() : 0), recv((!g.halos || g.collisions) && g.ranks > 1 ? g.count() : 0), @@ -215,16 +225,11 @@ struct Backend::Impl { Backend::Backend(Grid grid, int local_rank) : p(std::make_unique(grid, local_rank)) {} Backend::~Backend() = default; int Backend::device_ordinal() const { -#ifdef VLASOV_CUDA - int device = 0; - check_cuda(cudaGetDevice(&device)); - return device; -#else - return 0; -#endif + return p->ordinal; } void Backend::synchronize() { #ifdef VLASOV_CUDA + check_cuda(cudaSetDevice(p->ordinal)); check_cuda(cudaDeviceSynchronize()); #endif } diff --git a/benchmarks/vlasov_cuda/integration_check.py b/benchmarks/vlasov_cuda/integration_check.py index 13ad7b68..c8c5a7ac 100644 --- a/benchmarks/vlasov_cuda/integration_check.py +++ b/benchmarks/vlasov_cuda/integration_check.py @@ -17,11 +17,13 @@ def main(): parser.add_argument("--module", choices=["base", "vp", "both"], default="both") parser.add_argument("--ions", action="store_true", help="Validate kinetic protons using ADEPT's species config") parser.add_argument("--steps", type=int, default=5) + parser.add_argument("--gpus-per-node", type=int, default=4) + parser.add_argument("--host-mpi", action="store_true") parser.add_argument( "--per-rank-gpu", action="store_true", help="Expose one GPU per worker; validate host-staged MPI" ) args = parser.parse_args() - if args.per_rank_gpu: + if args.per_rank_gpu or args.host_mpi: os.environ["MPICH_GPU_SUPPORT_ENABLED"] = "0" os.environ["JAX_PLATFORMS"] = "cpu" os.environ["JAX_ENABLE_X64"] = "true" @@ -110,14 +112,14 @@ def main(): original.get_solver_quantities() original.init_state_and_args() original.init_diffeqsolve() - nodes = 2 if ranks > 4 else 1 + nodes = (ranks + args.gpus_per_node - 1) // args.gpus_per_node cfg["backend"] = "native" cfg["native"].update( device="cuda", library=str(args.library), ranks=ranks, timeout=180, - host_mpi=args.per_rank_gpu, + host_mpi=args.per_rank_gpu or args.host_mpi, launcher=[ "srun", f"--jobid={os.environ['SLURM_JOB_ID']}", @@ -125,7 +127,7 @@ def main(): f"--nodes={nodes}", "--ntasks={ranks}", f"--ntasks-per-node={ranks // nodes}", - "--gpus-per-node=4", + f"--gpus-per-node={args.gpus_per_node}", "--gpu-bind=single:1" if args.per_rank_gpu else "--gpu-bind=none", "--cpu-bind=none", ], diff --git a/benchmarks/vlasov_cuda/run_visibility_check.sh b/benchmarks/vlasov_cuda/run_visibility_check.sh new file mode 100644 index 00000000..344bb4f4 --- /dev/null +++ b/benchmarks/vlasov_cuda/run_visibility_check.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# ABI3 regression on an owned two-GPU shared allocation. +set -euo pipefail +if [ "${SLURM_PROCID:-0}" -ne 0 ]; then exit 0; fi +: "${EC_SESSION_WORKDIR:?}" "${EC_SESSION_OUTPUTS:?}" "${SLURM_JOB_ID:?}" +report_root="$EC_SESSION_OUTPUTS/native-visibility" +mkdir "$report_root" +export MPICH_GPU_SUPPORT_ENABLED=0 XLA_PYTHON_CLIENT_PREALLOCATE=false JAX_ENABLE_X64=true +export LD_LIBRARY_PATH="/global/common/software/nersc9/nccl/2.29.2-cu13/plugin/lib:/opt/nvidia/hpc_sdk/Linux_x86_64/26.5/cuda/13.2/lib64${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" +export NCCL_SOCKET_IFNAME=hsn NCCL_CROSS_NIC=2 NCCL_NET="AWS Libfabric" NCCL_NET_GDR_LEVEL=LOC FI_CXI_DISABLE_HOST_REGISTER=1 +export XLA_FLAGS="${XLA_FLAGS:+$XLA_FLAGS }--xla_gpu_autotune_level=0" +cmake -S adept/_vlasov1d/native -B "$EC_SESSION_WORKDIR/native-build" \ + -DCMAKE_CXX_COMPILER=CC -DCMAKE_BUILD_TYPE=Release > "$report_root/build.log" 2>&1 +cmake --build "$EC_SESSION_WORKDIR/native-build" -j4 >> "$report_root/build.log" 2>&1 +library="$EC_SESSION_WORKDIR/native-build/libadept_vlasov.so" +sha256sum "$library" > "$report_root/library.sha256" +sha256sum adept/_vlasov1d/native/api.cpp adept/_vlasov1d/native/core/* > "$report_root/source.sha256" +python -u benchmarks/vlasov_cuda/integration_check.py --library "$library" \ + --output "$report_root/shared" --ranks 1 2 --module base --ions --steps 100 \ + --gpus-per-node 2 --host-mpi > "$report_root/shared.log" 2>&1 +python -u benchmarks/vlasov_cuda/integration_check.py --library "$library" \ + --output "$report_root/bound" --ranks 2 --module base --ions --steps 100 \ + --gpus-per-node 2 --per-rank-gpu > "$report_root/bound.log" 2>&1 +python -u benchmarks/vlasov_cuda/integration_check.py --library "$library" \ + --output "$report_root/vp-main" --ranks 2 --module vp \ + --vp-source "$EC_SESSION_OUTPUTS/vp-source" --gpus-per-node 2 --host-mpi \ + > "$report_root/vp-main.log" 2>&1 +echo "Shared and per-worker GPU visibility checks passed: $report_root" From e767f672fa101672363739b3933c3ae142f59087 Mon Sep 17 00:00:00 2001 From: archis Date: Mon, 26 Oct 2026 23:20:01 +0000 Subject: [PATCH 8/8] docs(vlasov1d): record validated masked-GPU launch configuration --- benchmarks/vlasov_cuda/integration_check.py | 5 ++- .../results/2026-10-26-ions/README.md | 32 +++++++++++++++++++ .../vlasov_cuda/run_visibility_check.sh | 9 +++--- docs/source/solvers/vlasov1d/config.md | 9 ++++++ 4 files changed, 50 insertions(+), 5 deletions(-) diff --git a/benchmarks/vlasov_cuda/integration_check.py b/benchmarks/vlasov_cuda/integration_check.py index c8c5a7ac..53e01f34 100644 --- a/benchmarks/vlasov_cuda/integration_check.py +++ b/benchmarks/vlasov_cuda/integration_check.py @@ -121,6 +121,9 @@ def main(): timeout=180, host_mpi=args.per_rank_gpu or args.host_mpi, launcher=[ + "env", + "-u", + "CUDA_VISIBLE_DEVICES", "srun", f"--jobid={os.environ['SLURM_JOB_ID']}", "--overlap", @@ -166,7 +169,7 @@ def main(): if args.per_rank_gpu: placement = output["native stats"]["placement"] assert all(worker["device_ordinal"] == 0 for worker in placement), placement - assert len({(worker["host"], worker["cuda_visible_devices"]) for worker in placement}) == ranks + assert all(len(worker["cuda_visible_devices"].split(",")) == 1 for worker in placement), placement checks.append(row) if args.ions: changes = { diff --git a/benchmarks/vlasov_cuda/results/2026-10-26-ions/README.md b/benchmarks/vlasov_cuda/results/2026-10-26-ions/README.md index ec6493e3..53dc7de4 100644 --- a/benchmarks/vlasov_cuda/results/2026-10-26-ions/README.md +++ b/benchmarks/vlasov_cuda/results/2026-10-26-ions/README.md @@ -51,3 +51,35 @@ Local validation: 8 native/entropy tests, including 1/2-rank MPI and three kinetic species with independent grids and charges/masses, plus 211 existing lifecycle/core/builder/configuration regressions. The wheel contains the CMake/C++ sources needed to build this backend from downstream projects. + +## Review corrections and masked-GPU regression + +ABI3 uses the CUDA ordinal selected at allocation time for both JAX and +DLPack, separately from the MPI local rank. Native synchronization restores +that allocation's device. Collision guards and the constant frequency use +the resolved ADEPT envelopes, so an omitted bump height defaults to one and +is rejected as a nonconstant profile; an omitted constant baseline retains +ADEPT's zero default. Eleven focused CPU tests pass, including both omitted +time/space pulse defaults, constant defaults and MPI/NetCDF parity. + +Supplemental ABI3 checks on shared allocation 60339880, nid200301: + +| Case | GPUs | Steps | Maximum absolute saved-value error | +|---|---:|---:|---:| +| Kinetic electrons + protons | 1 | 100 | 1.12266e-12 | +| Same, one GPU visible per worker | 2 | 100 | 8.31335e-13 | +| Current turbulence callbacks, one GPU visible per worker | 2 | 5 | 5.74119e-12 | + +Both MPI local ranks 0/1 use CUDA ordinal zero and report +CUDA_VISIBLE_DEVICES=0. A separate CUDA probe confirmed distinct physical PCI +devices 0000:41:00.0 and 0000:C1:00.0. Comparison tolerances remain unchanged. +The earlier 1/4/8-GPU tests above use the normal all-visible production path. + +Two launcher/transport failures were retained in the local evidence: shared +Slurm steps with gpu-bind=none exposed the same GPU to both workers, causing +NCCL to reject duplicate devices; explicit single-GPU binding exposed distinct +GPUs, but NCCL SHM imports failed with an invalid device ordinal. The masked +regression passed with host-staged MPI and socket NCCL, disabling P2P/SHM and +GPU-direct registration. These transport settings are for the regression, +not a new throughput benchmark. Reproduction: ../../run_visibility_check.sh. +The shared lease was released after preserving reports and source. diff --git a/benchmarks/vlasov_cuda/run_visibility_check.sh b/benchmarks/vlasov_cuda/run_visibility_check.sh index 344bb4f4..b125997f 100644 --- a/benchmarks/vlasov_cuda/run_visibility_check.sh +++ b/benchmarks/vlasov_cuda/run_visibility_check.sh @@ -6,8 +6,9 @@ if [ "${SLURM_PROCID:-0}" -ne 0 ]; then exit 0; fi report_root="$EC_SESSION_OUTPUTS/native-visibility" mkdir "$report_root" export MPICH_GPU_SUPPORT_ENABLED=0 XLA_PYTHON_CLIENT_PREALLOCATE=false JAX_ENABLE_X64=true +export NCCL_P2P_DISABLE=1 NCCL_SHM_DISABLE=1 export LD_LIBRARY_PATH="/global/common/software/nersc9/nccl/2.29.2-cu13/plugin/lib:/opt/nvidia/hpc_sdk/Linux_x86_64/26.5/cuda/13.2/lib64${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" -export NCCL_SOCKET_IFNAME=hsn NCCL_CROSS_NIC=2 NCCL_NET="AWS Libfabric" NCCL_NET_GDR_LEVEL=LOC FI_CXI_DISABLE_HOST_REGISTER=1 +export NCCL_SOCKET_IFNAME=hsn NCCL_CROSS_NIC=2 NCCL_NET=Socket NCCL_NET_GDR_LEVEL=0 FI_CXI_DISABLE_HOST_REGISTER=1 export XLA_FLAGS="${XLA_FLAGS:+$XLA_FLAGS }--xla_gpu_autotune_level=0" cmake -S adept/_vlasov1d/native -B "$EC_SESSION_WORKDIR/native-build" \ -DCMAKE_CXX_COMPILER=CC -DCMAKE_BUILD_TYPE=Release > "$report_root/build.log" 2>&1 @@ -16,13 +17,13 @@ library="$EC_SESSION_WORKDIR/native-build/libadept_vlasov.so" sha256sum "$library" > "$report_root/library.sha256" sha256sum adept/_vlasov1d/native/api.cpp adept/_vlasov1d/native/core/* > "$report_root/source.sha256" python -u benchmarks/vlasov_cuda/integration_check.py --library "$library" \ - --output "$report_root/shared" --ranks 1 2 --module base --ions --steps 100 \ + --output "$report_root/shared" --ranks 1 --module base --ions --steps 100 \ --gpus-per-node 2 --host-mpi > "$report_root/shared.log" 2>&1 python -u benchmarks/vlasov_cuda/integration_check.py --library "$library" \ --output "$report_root/bound" --ranks 2 --module base --ions --steps 100 \ --gpus-per-node 2 --per-rank-gpu > "$report_root/bound.log" 2>&1 python -u benchmarks/vlasov_cuda/integration_check.py --library "$library" \ --output "$report_root/vp-main" --ranks 2 --module vp \ - --vp-source "$EC_SESSION_OUTPUTS/vp-source" --gpus-per-node 2 --host-mpi \ + --vp-source "$EC_SESSION_OUTPUTS/vp-source" --gpus-per-node 2 --per-rank-gpu \ > "$report_root/vp-main.log" 2>&1 -echo "Shared and per-worker GPU visibility checks passed: $report_root" +echo "Single-worker and per-worker GPU visibility checks passed: $report_root" diff --git a/docs/source/solvers/vlasov1d/config.md b/docs/source/solvers/vlasov1d/config.md index e1c46783..bebe3531 100644 --- a/docs/source/solvers/vlasov1d/config.md +++ b/docs/source/solvers/vlasov1d/config.md @@ -83,6 +83,15 @@ CUDA-aware Cray MPI. A launcher that exposes one GPU per worker uses ordinal zero within each worker. On Cray systems requiring all GPUs visible for GPU transport, use `host_mpi: true` and `MPICH_GPU_SUPPORT_ENABLED=0` to test that visibility mode with host-staged communication. +On the tested shared A100 node, masked-worker NCCL shared-memory imports also +reported an invalid device ordinal. The validated host-transport regression +uses `NCCL_P2P_DISABLE=1`, `NCCL_SHM_DISABLE=1`, `NCCL_NET=Socket` and +`NCCL_NET_GDR_LEVEL=0`. Keep the all-visible CUDA-aware example above for +the production throughput path. +If an outer Slurm step already restricted `CUDA_VISIBLE_DEVICES`, prefix the +launcher with `[env, -u, CUDA_VISIBLE_DEVICES, srun, ...]` so Slurm can apply +the new worker step's GPU binding. Separate GPU cgroups may each report +ordinal zero and `CUDA_VISIBLE_DEVICES=0`; physical GPUs still differ. | Native option | Default | Meaning | |---|---|---|