diff --git a/.github/workflows/test-wheel-linux.yml b/.github/workflows/test-wheel-linux.yml index 4971d56f83..ea7e59f2d9 100644 --- a/.github/workflows/test-wheel-linux.yml +++ b/.github/workflows/test-wheel-linux.yml @@ -137,9 +137,10 @@ jobs: with: # for artifact fetching, graphics libs, g++ required for cffi in # example; util-linux for `nsenter` (custom-DRIVER rows re-exec - # install_gpu_driver.sh onto the host through nsenter) - dependencies: "jq wget libgl1 libegl1 g++ util-linux" - dependent_exes: "jq wget" + # install_gpu_driver.sh onto the host through nsenter); gdb for + # crash backtraces in run-tests + dependencies: "jq wget libgl1 libegl1 g++ util-linux gdb" + dependent_exes: "jq wget gdb" - name: Install GPU driver if: ${{ matrix.DRIVER != 'latest' && matrix.DRIVER != 'earliest' }} diff --git a/ci/tools/gdb_pytest_cmds b/ci/tools/gdb_pytest_cmds new file mode 100644 index 0000000000..a21da3ba1d --- /dev/null +++ b/ci/tools/gdb_pytest_cmds @@ -0,0 +1,23 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +# GDB batch commands: run pytest and print a full backtrace only when the +# process terminates due to a signal (e.g. SIGSEGV). Exit with the +# inferior's own exit code so the caller sees the right failure status. +set pagination off +set print thread-events off +handle SIGTERM nostop pass +run +# $_exitcode is void when the inferior crashed (exited via signal) or was +# stopped mid-run by a signal (e.g. SIGSEGV during import). In both cases +# the backtrace is meaningful and we want to exit non-zero. +if $_isvoid($_exitcode) + echo \n=== GDB: process crashed or was stopped by signal, backtrace follows ===\n + bt full + echo \n=== Thread backtraces ===\n + thread apply all bt full + echo \n============================================================\n + quit 1 +end +quit $_exitcode diff --git a/ci/tools/run-tests b/ci/tools/run-tests index c093ed9e4d..6bdd72c306 100755 --- a/ci/tools/run-tests +++ b/ci/tools/run-tests @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # SPDX-License-Identifier: Apache-2.0 @@ -8,6 +8,28 @@ set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +_GDB_CMDS="${SCRIPT_DIR}/gdb_pytest_cmds" + +# Run pytest, wrapping with gdb on Linux to capture backtraces on crashes. +# When compute-sanitizer is active (SANITIZER_CMD non-empty) gdb is skipped +# because the two tools conflict. +run_pytest() { + local san="${SANITIZER_CMD:-}" + if [[ -n "${san}" ]]; then + ${san} pytest "$@" + elif [[ "$(uname -s)" == "Linux" ]] && command -v gdb &>/dev/null; then + # python is the ELF binary gdb needs; the pytest shebang script cannot be + # used as gdb's inferior directly. Passing the script path as python's + # argument makes Python set sys.path[0] to the script's directory rather + # than '' (CWD), matching a bare "pytest" call and preventing the source + # tree from shadowing installed namespace packages such as cuda.pathfinder. + gdb -batch -x "${_GDB_CMDS}" --args python "$(command -v pytest)" "$@" + else + pytest "$@" + fi +} + # Check if the script was called with exactly 1 argument if [[ ${#} -ne 1 ]]; then echo "Error: This script requires exactly 1 argument. You provided ${#}" @@ -36,7 +58,7 @@ if [[ "${test_module}" == "pathfinder" ]]; then "LD:${CUDA_PATHFINDER_TEST_LOAD_NVIDIA_DYNAMIC_LIB_STRICTNESS} " \ "FH:${CUDA_PATHFINDER_TEST_FIND_NVIDIA_HEADERS_STRICTNESS} " \ "BC:${CUDA_PATHFINDER_TEST_FIND_NVIDIA_BITCODE_LIB_STRICTNESS}" - pytest -ra -s -v --durations=0 tests/ |& tee /tmp/pathfinder_test_log.txt + run_pytest -ra -s -v --durations=0 tests/ |& tee /tmp/pathfinder_test_log.txt # Report the number of "INFO test_" lines (including zero) # to support quick validations based on GHA log archives. line_count=$(awk '/^INFO test_/ {count++} END {print count+0}' /tmp/pathfinder_test_log.txt) @@ -51,9 +73,9 @@ elif [[ "${test_module}" == "bindings" ]]; then pip install $(ls "${CUDA_BINDINGS_ARTIFACTS_DIR}"/*.whl)[all] --group test fi echo "Running bindings tests" - ${SANITIZER_CMD} pytest -rxXs -v --durations=0 --randomly-dont-reorganize tests/ + run_pytest -rxXs -v --durations=0 --randomly-dont-reorganize tests/ if [[ "${SKIP_CYTHON_TEST}" == 0 ]]; then - ${SANITIZER_CMD} pytest -rxXs -v --durations=0 --randomly-dont-reorganize tests/cython + run_pytest -rxXs -v --durations=0 --randomly-dont-reorganize tests/cython fi popd elif [[ "${test_module}" == "core" || "${test_module}" == nightly-* ]]; then @@ -105,11 +127,11 @@ elif [[ "${test_module}" == "core" || "${test_module}" == nightly-* ]]; then echo "Installed packages before core tests:" pip list echo "Running core tests" - ${SANITIZER_CMD} pytest -rxXs -v --durations=0 --randomly-dont-reorganize tests/ + run_pytest -rxXs -v --durations=0 --randomly-dont-reorganize tests/ # Currently our CI always installs the latest bindings (from either major version). # This is not compatible with the test requirements. if [[ "${SKIP_CYTHON_TEST}" == 0 ]]; then - ${SANITIZER_CMD} pytest -rxXs -v --durations=0 --randomly-dont-reorganize tests/cython + run_pytest -rxXs -v --durations=0 --randomly-dont-reorganize tests/cython fi popd elif [[ "${test_module}" == "nightly-cuda-core" ]]; then