diff --git a/.github/workflows/libs-apps.yml b/.github/workflows/libs-apps.yml new file mode 100644 index 000000000..4689a72dc --- /dev/null +++ b/.github/workflows/libs-apps.yml @@ -0,0 +1,122 @@ +name: Run app tests (/run-app-tests) + +# chipStar plus every downstream library with its tests, and the zeroRK and +# OpenSn checks, on an Intel discrete GPU under Level Zero and OpenCL. Run it +# from the Actions tab, or comment /run-app-tests on a pull request to get an +# app-tests status on its head commit. +# +# Runner needs: modules oneapi/2025.0.4 and llvm/23.0, kernel journal access +# (adm group), fuser, curl, sha256sum, git over ssh to github.com, OpenMPI, gfortran, and on Ubuntu libgtest-dev +# libgmock-dev libblas-dev liblapack-dev libboost-dev pybind11-dev. +on: + workflow_dispatch: + inputs: + pr: + description: Pull request number to test (overrides ref) + required: false + ref: + description: Branch, tag or sha to test, by default the dispatched branch + required: false + issue_comment: + types: [created] + +permissions: {} + +jobs: + # Runs on GitHub's hosted runners: the job below executes the tested tree's code, so it never holds the write token. + resolve: + if: >- + github.event_name == 'workflow_dispatch' || + (github.event.issue.pull_request && + (github.event.comment.body == '/run-app-tests' || + startsWith(github.event.comment.body, '/run-app-tests ') || + startsWith(github.event.comment.body, fromJSON('"/run-app-tests\n"')) || + startsWith(github.event.comment.body, fromJSON('"/run-app-tests\r"'))) && + contains(fromJSON('["colleeneb", "pvelesko"]'), github.event.comment.user.login)) + runs-on: ubuntu-latest + permissions: + statuses: write + pull-requests: write + outputs: + sha: ${{ steps.target.outputs.sha }} + env: + PR: ${{ github.event.issue.number || inputs.pr }} + REF: ${{ inputs.ref }} + steps: + - name: Resolve the commit to test + id: target + uses: actions/github-script@v7 + with: + script: | + const pr = process.env.PR; + if (!pr) return core.setOutput('sha', process.env.REF || context.sha); + const { data } = await github.rest.pulls.get({ ...context.repo, pull_number: Number(pr) }); + core.setOutput('sha', data.head.sha); + await github.rest.repos.createCommitStatus({ + ...context.repo, sha: data.head.sha, state: 'pending', context: 'app-tests', + description: 'libraries and apps running', + target_url: `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`, + }); + if (context.eventName === 'issue_comment') + await github.rest.reactions.createForIssueComment({ + ...context.repo, comment_id: context.payload.comment.id, content: 'rocket', + }); + + run: + needs: resolve + runs-on: [self-hosted, intel-dgpu] + timeout-minutes: 1440 + permissions: + contents: read + concurrency: + group: libs-apps + queue: max + steps: + # The script comes from the default or dispatched branch, so it can test PRs older than it. + - uses: actions/checkout@v4 + with: + path: ci + persist-credentials: false + - uses: actions/checkout@v4 + with: + ref: ${{ needs.resolve.outputs.sha }} + path: src + persist-credentials: false + fetch-depth: 0 + submodules: recursive + # The job owns the machine; the script's load-based default would shrink by the previous job's load. + - name: Build and test + run: | + jobs=$(awk -v n="$(nproc)" '/^MemTotal/ { j = int($2 / 3145728); if (n - 1 < j) j = n - 1; print (j < 1 ? 1 : j) }' /proc/meminfo) + ci/scripts/ci/libs-apps.sh -s src -w "${{ runner.temp }}/libs-apps" -j "$jobs" + - uses: actions/upload-artifact@v4 + if: always() + with: + name: libs-apps-logs + path: | + ${{ runner.temp }}/libs-apps/summary.txt + ${{ runner.temp }}/libs-apps/logs + + report: + needs: [resolve, run] + if: always() && needs.resolve.outputs.sha != '' && (github.event.issue.number || inputs.pr) + runs-on: ubuntu-latest + permissions: + statuses: write + steps: + - uses: actions/github-script@v7 + env: + RESULT: ${{ needs.run.result }} + SHA: ${{ needs.resolve.outputs.sha }} + with: + script: | + const ok = process.env.RESULT === 'success'; + const url = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; + // A newer request on this commit owns the status. + const { data } = await github.rest.repos.getCombinedStatusForRef({ ...context.repo, ref: process.env.SHA }); + if (data.statuses.find(s => s.context === 'app-tests')?.target_url !== url) return; + await github.rest.repos.createCommitStatus({ + ...context.repo, sha: process.env.SHA, state: ok ? 'success' : 'failure', + context: 'app-tests', description: ok ? 'libraries and apps pass' : 'see summary.txt in the run artifacts', + target_url: url, + }); diff --git a/scripts/ci/libs-apps.sh b/scripts/ci/libs-apps.sh new file mode 100755 index 000000000..4f05e88a4 --- /dev/null +++ b/scripts/ci/libs-apps.sh @@ -0,0 +1,195 @@ +#!/bin/bash +# Build chipStar plus every downstream library with its tests, then run each +# library's ctest suite and the zeroRK and OpenSn checks on one Intel GPU per +# backend. Waits while another process holds the GPU and stops all GPU work only +# when the kernel declares it wedged. Results go to summary.txt. +# +# scripts/ci/libs-apps.sh -w WORK_DIR [-s CHIPSTAR_SRC] [-j JOBS] [-b "level0 opencl"] [-p build,libs,apps] +set -o pipefail + +HERE=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) +SRC=$HERE WRK="" JOBS="" BACKENDS="level0 opencl" PHASES=build,libs,apps +INTEL_GPUS=$(grep -l 0x8086 /sys/bus/pci/devices/*/vendor | xargs -n1 dirname | + while read -r d; do grep -q '^0x03' "$d/class" && echo "${d##*/}"; done | sort) +# First Intel display device off bus 00, where the integrated GPU sits. +GPU_PCI=${GPU_PCI:-$(grep -v '^0000:00:' <<< "$INTEL_GPUS" | head -1)} +DEPS_CACHE=${DEPS_CACHE:-$HOME/.cache/chipstar-libs-apps} +OPENSN_COMMIT=d0644cd9c633c6ae5e7110d3f8721641e5ef2982 +while getopts "s:w:j:b:p:" o; do + case $o in + s) SRC=$(cd "$OPTARG" && pwd) || exit 2 ;; w) WRK=$OPTARG ;; j) JOBS=$OPTARG ;; + b) BACKENDS=$OPTARG ;; p) PHASES=$OPTARG ;; *) sed -n '2,7p' "$0"; exit 2 ;; + esac +done +[ -n "$WRK" ] && mkdir -p "$WRK/logs" && WRK=$(cd "$WRK" && pwd) || { sed -n '2,7p' "$0"; exit 2; } +LOGS=$WRK/logs SUMMARY=$WRK/summary.txt FAILED=0 GPU_DEAD=0 +# idle cores minus one, at most one job per 3 GB of available RAM +: "${JOBS:=$(awk -v n="$(nproc)" -v m="$(free -g | awk '/^Mem:/ {print $7}')" \ + '{ j = int(n - $1 - 1); if (m / 3 < j) j = int(m / 3); print (j < 1 ? 1 : j) }' /proc/loadavg)}" +RENDER=/dev/dri/$(basename "$(ls -d /sys/bus/pci/devices/"$GPU_PCI"/drm/renderD*)") +DRV=$(basename "$(readlink /sys/bus/pci/devices/"$GPU_PCI"/driver)") +# Level Zero and the Intel OpenCL platform both hide every GPU but this one, indexed in PCI order. +ZE_IDX=$(($(grep -nx "$GPU_PCI" <<< "$INTEL_GPUS" | cut -d: -f1) - 1)) +[ "$ZE_IDX" -ge 0 ] || { echo "$GPU_PCI is not an Intel display device"; exit 2; } +export ZE_ENABLE_PCI_ID_DEVICE_ORDER=1 ZE_AFFINITY_MASK=$ZE_IDX CHIP_LOGLEVEL=err +unset CHIP_PLATFORM CHIP_DEVICE CHIP_DEVICE_TYPE +# Arc A-series GPUs have no FP64 hardware; the apps and most libraries need double. +export OverrideDefaultFP64Settings=1 IGC_EnableDPEmulation=1 +source "$HERE/scripts/module-env.sh" +module load oneapi/2025.0.4 llvm/23.0 || exit 1 +# zeroRK and OpenSn bundle projects that declare a CMake minimum below 3.5. +export CMAKE_POLICY_VERSION_MINIMUM=3.5 + +trap '[ -n "$STEP" ] && { kill -KILL "$STEP"; pkill -KILL -s "$STEP"; } 2>/dev/null; exit 143' INT TERM +wants() { case ,$PHASES, in *,$1,*) return 0 ;; esac; return 1; } +note() { echo "$*" | tee -a "$SUMMARY"; } +result() { [ "$2" -eq 0 ] && note "PASS $1" || { note "FAIL $1 (rc=$2, $3)"; FAILED=$((FAILED + 1)); }; } +cursor() { journalctl -k -b -n0 --show-cursor -q --no-pager | sed -n 's/^-- cursor: //p'; } +# i915 wedges without saying so after these (intel_reset.c); the word reset also appears in recoverable lines. +FATAL_MSG="wedged|GPU reset not supported|Failed to reset chip|Failed to initialise HW following reset|timed out, cancelling all in-flight rendering" +# Fatal: the driver gave up on the GPU (FATAL_MSG) or the journal is unreadable so the GPU cannot be watched. Sticky. +gpu_faults() { + local j p="$DRV $GPU_PCI.*($FATAL_MSG)|^Failed|^No journal" + [ -e "$LOGS/fault" ] && { echo 1; return; } + j=$(journalctl -k -b -q --no-pager --after-cursor="$CURSOR" 2>&1) || j="Failed: journalctl exit $? +$j" + j=$(grep -E "$DRV $GPU_PCI|^Failed|^No journal" <<< "$j") + grep -qiE "$p" <<< "$j" || { echo 0; return; } + { grep -iE "$p" <<< "$j" | head -100; head -900 <<< "$j"; } > "$LOGS/fault"; echo 1 +} + +# gpu_step NAME LOG CMD...; CMD gets its own session so a fault kills all of it. +gpu_step() { + local name=$1 log=$2 pid="" f rc sc resets; shift 2 + [ $GPU_DEAD -eq 0 ] || { note "SKIP $name"; return; } + while f=$(gpu_faults) && [ "$f" -eq 0 ] && fuser "$RENDER" > /dev/null 2>&1; do echo "[gpu] $RENDER busy, waiting"; sleep 60; done + if [ "$f" -eq 0 ]; then + sc=$(cursor) + setsid "$@" > "$log" 2>&1 < /dev/null & + pid=$! STEP=$! + while kill -0 $pid 2>/dev/null && f=$(gpu_faults) && [ "$f" -eq 0 ]; do sleep 1; done + [ "$f" -gt 0 ] || f=$(gpu_faults) + [ "$f" -gt 0 ] || { + wait $pid; rc=$? + pkill -KILL -s $pid 2>/dev/null + # journald ingests kernel messages asynchronously: let a reset logged at the end of the step reach the journal. + sleep 2 + # A reset the driver recovered from: the hung test fails by itself, the rest keep running. + if [ -z "$sc" ] || ! resets=$(journalctl -k -b -q --no-pager --after-cursor="$sc" 2>&1); then + note "NOTE journal unreadable after $name, cannot rule out a GPU reset"; resets=""; [ $rc -ne 0 ] || rc=1 + fi + resets=$(grep -iE "$DRV $GPU_PCI.*(reset|timedout|hang)" <<< "$resets" | grep -viE "$DRV $GPU_PCI.*($FATAL_MSG)") + [ -z "$resets" ] || { echo "$resets" > "$log.gpu-faults"; note "NOTE GPU reset during $name, see $log.gpu-faults"; [ $rc -ne 0 ] || rc=1; } + # A fatal line logged after the last poll is still fatal. + f=$(gpu_faults) + } + fi + if [ "$f" -gt 0 ]; then + [ -n "$pid" ] && { kill -KILL $pid; pkill -KILL -s $pid; } 2>/dev/null; GPU_DEAD=1 + note "STOP GPU wedged or journal error at $name $(grep -so 'Start *[0-9]*: [^ ]*' "$log" | tail -1), see logs/fault" + result "$name" 137 "$log"; STEP=""; return + fi + result "$name" $rc "$log"; STEP="" +} + +# Same .ci/exclude.txt and .ci/ctest-labels.txt convention as libraries-presubmit.yml. +run_libs() { + local lib s ex lb + for s in "$WRK"/staging/*/; do + lib=$(basename "$s") + [ -f "$s/build/CTestTestfile.cmake" ] || { note "SKIP $1 $lib (no CTest tests)"; continue; } + ex=$(grep -Evhs '^\s*(#|$)' "$s/.ci/exclude.txt" | paste -sd '|' -) + lb=$(grep -Evhs '^\s*(#|$)' "$s/.ci/ctest-labels.txt" | paste -sd '|' -) + rm -f "$LOGS/$1/$lib.log" + gpu_step "$1 $lib" "$LOGS/$1/$lib.log" ctest --test-dir "$s/build" -j1 --output-on-failure \ + --timeout "${CTEST_TIMEOUT:-900}" ${lb:+-L "$lb"} ${ex:+-E "$ex"} + [ $GPU_DEAD -eq 0 ] && grep -hs 'tests passed' "$LOGS/$1/$lib.log" | sed 's/^/ /' | tee -a "$SUMMARY" + done +} + +# Built trees are cached per runner by app commit; the libraries they load come from this run's PREFIX. +lock() { mkdir -p "$DEPS_CACHE" && exec 9> "$DEPS_CACHE/lock" && flock 9; } + +# zeroRK's scripts/build_aurora.sh without its module loads and unbounded make -j. +build_zerork() { + lock || return 1 + [ -e "$ZRK/.built" ] && { echo "reusing $ZRK"; return 0; } + rm -rf "$ZRK" && git clone -q --depth 1 -b hip-chipStar git@github.com:CHIP-SPV/zero-rk.git "$ZRK" && + mkdir "$ZRK/build_aurora" && cd "$ZRK/build_aurora" && + cmake -DCMAKE_EXE_LINKER_FLAGS=-lintlc -DENABLE_GPU=ON -DENABLE_MAGMA=OFF -DCMAKE_CXX_COMPILER=hipcc \ + -DCMAKE_C_COMPILER=hipcc -DZERORK_TESTS=OFF -DZERORK_HAVE_MKL=OFF .. && make -j"$JOBS" && make install && + touch "$ZRK/.built" +} + +# OpenSn's dependencies do not use chipStar, so each runner builds them once. +build_opensn() { + local deps=$DEPS_CACHE/opensn-deps-$OPENSN_COMMIT + lock || return 1 + [ -e "$OSN/.built" ] && { echo "reusing $OSN"; return 0; } + rm -rf "$OSN" && git clone -q https://github.com/Open-Sn/opensn.git "$OSN" && cd "$OSN" && + git checkout -q $OPENSN_COMMIT || return 1 + [ -s "$deps/bin/set_opensn_env.sh" ] || { + rm -rf "$deps" && cmake -S tools/dependencies -B build_deps -DCMAKE_INSTALL_PREFIX="$deps" -DJOBS="$JOBS" \ + -DCMAKE_DISABLE_FIND_PACKAGE_Boost=ON && + cmake --build build_deps -j1; } || { rm -rf "$deps"; return 1; } + # CMAKE_HIP_PLATFORM=spirv needs CMake 4.3. + local cm=$DEPS_CACHE/cmake-4.3.2-linux-x86_64 tgz=$DEPS_CACHE/cmake-4.3.2.tar.gz + [ -x "$cm/bin/cmake" ] || { + curl -sSfL -o "$tgz" https://github.com/Kitware/CMake/releases/download/v4.3.2/cmake-4.3.2-linux-x86_64.tar.gz && + echo "791ae3604841ca03cb3889a3ad89165346e4b180ae3448efd4b0caa9ef46d245 $tgz" | sha256sum -c - && + tar xzf "$tgz" -C "$DEPS_CACHE" && rm -f "$tgz"; } || return 1 + source "$deps/bin/set_opensn_env.sh" && + CC=hipcc CXX=hipcc "$cm/bin/cmake" -S . -B build -DOPENSN_WITH_HIP=ON -DCMAKE_HIP_ARCHITECTURES=spirv \ + -DCMAKE_HIP_PLATFORM=spirv -DCMAKE_BUILD_TYPE=RelWithDebInfo && + "$cm/bin/cmake" --build build -j"$JOBS" && touch "$OSN/.built" +} + +run_apps() { + note "SKIP $1 zeroRK box_reactor (zerork_correctness_box_reactor.sh needs Aurora's PBS and MPI launcher)" + rm -rf "$LOGS/$1/zerork-out" + gpu_step "$1 zeroRK GPU paths" "$LOGS/$1/zerork-paths.log" timeout -k 60 22000 bash -c 'cd "$0" && bash "$1" "$2"' \ + "$ZRK" "$HERE/tests/apps/ZeroRK/zerork_gpu_paths.sh" "$WRK/apps/zerork-out" + [ $GPU_DEAD -eq 0 ] && grep -hsE '^(PASS|FAIL) ' "$LOGS/$1/zerork-paths.log" | sed "s/^/ $1 /" | tee -a "$SUMMARY" + cp -r "$WRK/apps/zerork-out" "$LOGS/$1/" 2>/dev/null + rm -rf "$WRK/apps/opensn-run" && mkdir "$WRK/apps/opensn-run" && cp "$SRC"/tests/apps/OpenSn/* "$WRK/apps/opensn-run" + gpu_step "$1 OpenSn strong_scaling" "$LOGS/$1/opensn.log" timeout -k 60 1800 bash -c \ + 'cd "$0" && source "$1" && CHIP_LOGLEVEL=off mpiexec -n 1 "$2" -i strong_scaling.py' "$WRK/apps/opensn-run" \ + "$DEPS_CACHE/opensn-deps-$OPENSN_COMMIT/bin/set_opensn_env.sh" "$OSN/build/python/opensn" +} + +note "=== $(date -u +%FT%TZ) $(hostname) GPU $GPU_PCI $DRV ($RENDER) chipStar $(git -C "$SRC" log -1 --format='%h %s') -j$JOBS" +if wants build; then + rm -rf "$WRK/install" "$WRK/staging" + (cd "$SRC" && ./install_chipstar.py --all --with-tests -j "$JOBS" --install-dir "$WRK/install" \ + --staging-dir "$WRK/staging" --module-dir "$WRK/modulefiles") > "$LOGS/build.log" 2>&1 + result "build: install_chipstar.py --all --with-tests" $? "$LOGS/build.log" + sed 's/\x1b\[[0-9;]*m//g' "$LOGS/build.log" | grep -E '^\[(SUCCESS|FAILED)\]' | cut -c1-150 | sed 's/^/ /' | tee -a "$SUMMARY" + for s in "$WRK"/staging/*/; do note " $(basename "$s") $(git -C "$s" rev-parse --short HEAD)"; done +fi +PREFIX=$(ls -d "$WRK"/install/chipStar/*/ | tail -1) && PREFIX=${PREFIX%/} +[ -x "$PREFIX/bin/hipcc" ] || { note "FAIL no chipStar install under $WRK/install"; exit 1; } +export HIP_PATH=$PREFIX HIP_PLATFORM=spirv PATH=$PREFIX/bin:$PATH CMAKE_PREFIX_PATH=$PREFIX:$CMAKE_PREFIX_PATH +export LD_LIBRARY_PATH=$PREFIX/lib:$LD_LIBRARY_PATH LIBRARY_PATH=$PREFIX/lib:$LIBRARY_PATH +if wants apps; then + rm -rf "$WRK/apps" && mkdir "$WRK/apps" + # hipcc embeds headers, device bitcode and pass plugins of the install into what it builds. + INST=$(find "$PREFIX/include" "$PREFIX/lib/hip-device-lib" "$PREFIX/lib/llvm" "$PREFIX/bin" -type f | sort | xargs cat | cksum | cut -d' ' -f1) + ZRK=$DEPS_CACHE/zerork-$(git ls-remote git@github.com:CHIP-SPV/zero-rk.git hip-chipStar | cut -c1-40)-$INST + OSN=$DEPS_CACHE/opensn-$OPENSN_COMMIT-$INST + (cd "$WRK/apps" && build_zerork) > "$LOGS/zerork-build.log" 2>&1; result "build zeroRK" $? "$LOGS/zerork-build.log" + (cd "$WRK/apps" && build_opensn) > "$LOGS/opensn-build.log" 2>&1; result "build OpenSn" $? "$LOGS/opensn-build.log" +fi +wants libs || wants apps || BACKENDS="" +rm -f "$LOGS/fault" +CURSOR=$(cursor) +[ -z "$BACKENDS" ] || [ -n "$CURSOR" ] || { note "FAIL kernel journal unreadable, cannot watch the GPU"; exit 1; } +[ -z "$BACKENDS" ] || { n=$(journalctl -k -b -q --no-pager | grep -ciE "$DRV $GPU_PCI.*($FATAL_MSG)"; echo "${PIPESTATUS[0]}"); [ "$(echo $n)" = "0 0" ]; } || + { note "FAIL $GPU_PCI was declared wedged or failed to reset earlier in this boot, or the kernel journal is unreadable"; exit 1; } +[ -z "$BACKENDS" ] || command -v fuser > /dev/null || { note "FAIL fuser not found, cannot see other GPU users"; exit 1; } +for be in $BACKENDS; do + mkdir -p "$LOGS/$be" && export CHIP_BE=$be + wants libs && run_libs "$be" + wants apps && run_apps "$be" +done +note "=== $FAILED failed" +[ "$FAILED" -eq 0 ] && [ "$GPU_DEAD" -eq 0 ] diff --git a/tests/apps/ZeroRK/zerork_gpu_paths.sh b/tests/apps/ZeroRK/zerork_gpu_paths.sh new file mode 100755 index 000000000..fb17362dc --- /dev/null +++ b/tests/apps/ZeroRK/zerork_gpu_paths.sh @@ -0,0 +1,107 @@ +#!/bin/bash +# ZeroRK GPU code path check: run the h2 mechanism on the GPU through five +# integrator and reactor configurations and compare each with a CPU reference. +# +# Usage: zerork_gpu_paths.sh +# CPU references already in are reused, so the CPU runs happen +# once for any number of invocations (one per backend). +# +# Must be invoked from the zero-rk source root with build_aurora/ built and installed. + +[ $# -eq 1 ] || { echo "Usage: $0 "; exit 1; } +out=$(mkdir -p "$1" && cd "$1" && pwd) +export CHIP_JIT_FLAGS_OVERRIDE="-ze-opt-enable-auto-large-GRF-mode" CHIP_LOGLEVEL=off +TOLERANCE=1e-4 # max relative difference in temperature and pressure + +cd build_aurora/inst_dir || exit 1 +exe=$PWD/bin/zerork_cfd_plugin_tester_gpu.x +export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$PWD/lib:$PWD/lib64 +cd share/zerork/examples/cfd_plugin_tester || exit 1 + +# Replace KEY in FILE, or append it: the base ymls lack some keys. +setYML() { + grep -q "^ *$2 *:" "$1" && sed -i "s|^ *$2 *:.*$|$2: $3|" "$1" || echo "$2: $3" >> "$1" +} + +# run NAME GPU CV INTEGRATOR ANALYTIC ALWAYS_SOLVE_T N_REACTORS N_REACTORS_MAX LU +run() { + local name=$1 in=$out/$1_input.yml zrk=$out/$1_zerork.yml + local fuel oxid + fuel=$(grep -v '^#' inputs/h2_fracs.log | grep -Eiv '^(O2|N2)\b' | sed 's/$/,/' | tr '\n' ' ') + oxid=$(grep -v '^#' inputs/h2_fracs.log | grep -Ei '^(O2|N2)\b' | sed 's/$/,/' | tr '\n' ' ') + cp inputs/base_tester.yml "$in"; cp inputs/base_plugin.yml "$zrk" + setYML "$in" mechanism_file "$PWD/../../mechanisms/hydrogen/h2_v1b_mech.txt" + setYML "$in" thermo_file "$PWD/../../mechanisms/hydrogen/h2_v1a_therm.txt" + setYML "$in" solution_time "${SOLT:-0.00000165131996}" + setYML "$in" fuel_composition "{ $fuel }" + setYML "$in" oxidizer_composition "{ $oxid }" + setYML "$in" zerork_cfd_plugin_input "$zrk" + setYML "$in" n_reactors "$7" + setYML "$in" log_species "[h, h2, o, o2, oh, h2o, n2, ho2, h2o2, ar]" + setYML "$in" n_steps "${STEPS:-4}" + [ -n "$STATE" ] && setYML "$in" state_files "[$STATE]" + setYML "$in" constant_volume "$3" + setYML "$in" reactor_history_file_prefix "$out/$name" + setYML "$zrk" mechanism_parsing_log "$out/$name.cklog" + setYML "$zrk" reactor_timing_log "$out/$name.log" + setYML "$zrk" preconditioner_threshold 3.20e-5 + setYML "$zrk" gpu "$2" + setYML "$zrk" always_solve_temperature "$6" + setYML "$zrk" dense 1 + setYML "$zrk" iterative 1 + setYML "$zrk" integrator "$4" + setYML "$zrk" analytic "$5" + setYML "$zrk" n_reactors_max "$8" + rm -f "$out/$name"_*.hist "$out/$name.log" "$out/$name.stdout" + ZERORK_REACTOR_USE_LU=$9 timeout -k 30 1800 "$exe" "$in" > "$out/$name.stdout" 2>&1 +} + +# Temperature (col 3) and pressure (col 4) row by row against the CPU reference; +# $3 = leading rows of the GPU history to skip. +compare() { + local gpu + gpu=$(awk -v s="${3:-0}" '/^#/ || ++n > s' "$2") + [ -s "$1" ] && [ -n "$gpu" ] && [ "$(wc -l < "$1")" -eq "$(echo "$gpu" | wc -l)" ] || return 1 + paste "$1" <(echo "$gpu") | LC_ALL=C awk -v tol=$TOLERANCE '/^#/ { next } { + h = NF / 2; dt = ($(h+3) - $3) / $3; dp = ($(h+4) - $4) / $4 + if (dt < 0) dt = -dt; if (dp < 0) dp = -dp + if (!(dt <= tol && dp <= tol) || $3 $4 $(h+3) $(h+4) ~ /[nN][aA][nN]|[iI][nN][fF]/) { printf "step %s: T %g vs %g, P %g vs %g\n", $1, $3, $(h+3), $4, $(h+4); bad = 1 } } + END { exit bad }' +} + +# Every timing log row must show all reactors on the GPU (zeroRK retries failed GPU reactors on the CPU); +# with $2 = 1 also some row with n_gpu_notemp > 0 (zeroRK counts them only on some solves). +# Columns: 3 n_cpu, 5 n_gpu, 6 n_gpu_notemp. +onGpu() { + awk -v nt="$2" '/^#/ { next } { n++; if ($6 > 0) any = 1 + if ($3 != 0 || $5 <= 0) { print "row " $1 ": n_cpu " $3 ", n_gpu " $5; bad = 1 } } + END { if (nt && !any) { print "n_gpu_notemp is 0 on every row"; bad = 1 } exit (bad || !n) }' "$1" +} + +# n_reactors must stay >= the plugin's n_reactors_min (256) or the GPU is skipped silently. +N=512 +# One CPU reference per constant_volume and always_solve_temperature pair. +for ref in cv1_t1 cv0_t1; do + [ -s "$out/ref_${ref}_000.hist" ] || run ref_$ref 0 ${ref:2:1} 1 1 ${ref:5:1} 32 $N 0 || rm -f "$out/ref_${ref}_000.hist" +done +# zeroRK solves temperature on the first GPU call whatever always_solve_temperature says, so the +# no-temperature reference restarts from step 0 of the temperature reference without it. +# One reactor: the tester reads one state file per reactor (species, T, P). +if [ ! -s "$out/ref_cv1_t0_000.hist" ]; then + awk '!/^#/ { print $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $3, $4; exit }' "$out/ref_cv1_t1_000.hist" > "$out/ref_s0.state" && + STEPS=3 SOLT=0.00000123848997 STATE=$out/ref_s0.state run ref_cv1_t0 0 1 1 1 0 1 $N 0 || rm -f "$out/ref_cv1_t0_000.hist" +fi +rc=0 +# name constant_volume integrator analytic always_solve_temperature n_reactors_max ZERORK_REACTOR_USE_LU +while read -r name cv integ analytic ast nmax lu; do + if run "$name" 1 "$cv" "$integ" "$analytic" "$ast" $N "$nmax" "$lu" && + diff=$(onGpu "$out/$name.log" $((1 - ast)) && compare "$out/ref_cv${cv}_t${ast}_000.hist" "$out/${name}_000.hist" $((1 - ast))); then echo "PASS zeroRK $name" + else echo "FAIL zeroRK $name ($out/$name.stdout)"; echo "$diff" | head -5; rc=1; fi +done << 'CASES' +cvode_analytic_cv 1 0 1 1 512 1 +seulex_fd_cv 1 1 0 1 512 0 +radau_analytic_cv 1 3 1 1 512 1 +cvode_analytic_cp 0 0 1 1 512 0 +seulex_fd_notemp_batch 1 1 0 0 256 1 +CASES +exit $rc