Skip to content

DAOS-18626 ci: Add run-unit_tests.sh sanitizer test runner script #19104

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
knard38 wants to merge 1 commit into
base: master
Choose a base branch
from
+454 −0
Draft

DAOS-18626 ci: Add run-unit_tests.sh sanitizer test runner script #19104

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
291 changes: 291 additions & 0 deletions utils/ci/run-unit_tests.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,291 @@
#!/bin/bash
# SPDX-License-Identifier: BSD-2-Clause-Patent
# Copyright 2025-2026 Hewlett Packard Enterprise Development LP.
# Run the DAOS unit-test suite, optionally under a sanitizer, and collect
# all reports. Shared by the standard, ASan, UBSan, and TSan GHA jobs (and by
# utils/docker/run-unit_tests-docker.sh for local reproduction).
#
# Usage:
# run-unit_tests.sh --standard|--asan|--ubsan|--tsan \
# [--list] [--suite_filter=REGEX] [--test_filter=REGEX]
#
# --list List the suites/tests that would run (honoring the
# mode and any --suite_filter/--test_filter) without
# actually running them, then exit.
# --suite_filter=REGEX Only run suites whose name matches REGEX.
# --test_filter=REGEX Only run tests whose command line matches REGEX.
#
# Sanitizer options (ASAN_OPTIONS/UBSAN_OPTIONS/TSAN_OPTIONS/LSAN_OPTIONS) can
# be fully overridden by pre-setting them in the environment before invoking
# this script (e.g. to add fast_unwind_on_malloc=1 for a deeper leak trace --
# see docs/dev/development.md). A separate *_EXTRA_OPTIONS variable per
# sanitizer (e.g. ASAN_EXTRA_OPTIONS) can instead be set to append extra
# options to whichever value (default or overridden) ends up in effect,
# without having to restate the whole options string.
#
# Exit codes written to /home/daos/test-results/exit_status:
# sanitizer_detected=1 Set when sanitizer log files are found in sanitizer-logs/
# (always 0 for --standard)
# functional_failure=1 Set when any test recorded a failure in its JUnit/
# cmocka XML output (may be 1 at the same time as
# sanitizer_detected=1, when a sanitizer finding and an
# unrelated plain test failure both occur in one run)
# runner_error=1 Set when run_utest.py itself crashed/errored out
# before finishing (e.g. malformed utest.yaml, missing
# build vars) -- an infrastructure-level problem,
# distinct from functional_failure, since it isn't
# necessarily also caught by another build's run
#
# Outputs mounted back to the host runner:
# /home/daos/sanitizer-logs/ – per-PID log files (asan.<pid>, ubsan.<pid>, tsan.<pid>)
# /home/daos/test-results/ – JUnit XML files + exit_status

set -uo pipefail

# Join non-empty ':'-delimited sanitizer-option fragments, skipping any empty
# ones (e.g. an unset *_EXTRA_OPTIONS) to avoid stray/leading colons.
join_options() {
local out=""
for frag in "$@"; do
[ -z "${frag}" ] && continue
if [ -z "${out}" ]; then out="${frag}"; else out="${out}:${frag}"; fi
done
printf '%s' "${out}"
}

# ── Parse mode argument + optional pass-through args ─────────────────────────
MODE="${1:-}"
if [ "${MODE}" != "--standard" ] && [ "${MODE}" != "--asan" ] \
&& [ "${MODE}" != "--ubsan" ] && [ "${MODE}" != "--tsan" ]; then
echo "Usage: $(basename "$0") --standard|--asan|--ubsan|--tsan" \
"[--list] [--suite_filter=REGEX] [--test_filter=REGEX]" >&2
exit 1
fi
shift
EXTRA_ARGS=()
LIST_MODE=0
for arg in "$@"; do
case "${arg}" in
--list)
EXTRA_ARGS+=("${arg}")
LIST_MODE=1
;;
--suite_filter=*|--test_filter=*)
EXTRA_ARGS+=("${arg}")
;;
*)
echo "Unknown argument: ${arg}" >&2
exit 1
;;
esac
done

# ── Mode-specific configuration ───────────────────────────────────────────────
RESULTS_DIR=/home/daos/test-results
LOG_DIR=/home/daos/sanitizer-logs
mkdir -p "${RESULTS_DIR}" "${LOG_DIR}"

if [ "${MODE}" = "--standard" ]; then
# No sanitizer runtime: run every suite not explicitly excluded from GHA
# (asan:/tsan: tags in utest.yaml only apply when --asan/--ubsan/--tsan
# is passed to run_utest.py, so passing nothing here selects them all).
PYTHON_FLAG=""

elif [ "${MODE}" = "--asan" ]; then
# ASan: detect_odr_violation=0 suppresses the DAOS dual-library false positives.
# UBSAN_OPTIONS is intentionally NOT set: the UBSan workflow uses a separate
# image (SANITIZERS=undefined_behavior) so log_path is never shared between
# runtimes.
if [ -z "${ASAN_OPTIONS+set}" ]; then
ASAN_OPTIONS=$(join_options "log_path=${LOG_DIR}/asan" "exitcode=42" \
"print_summary=1" "symbolize=1" "detect_odr_violation=0")
fi
ASAN_OPTIONS=$(join_options "${ASAN_OPTIONS}" "${ASAN_EXTRA_OPTIONS:-}")
export ASAN_OPTIONS

# LSan (leak detection, part of the ASan runtime): use a suppressions file
# if it exists, for third-party/vendored leaks that aren't DAOS bugs (see
# utils/test_lsan.supp).
LSAN_SUPP_FILE=""
if [ -f "$(pwd)/daos/utils/test_lsan.supp" ]; then
LSAN_SUPP_FILE="$(pwd)/daos/utils/test_lsan.supp"
fi
if [ -z "${LSAN_OPTIONS+set}" ]; then
LSAN_OPTIONS=$(join_options "${LSAN_SUPP_FILE:+suppressions=${LSAN_SUPP_FILE}}")
fi
LSAN_OPTIONS=$(join_options "${LSAN_OPTIONS}" "${LSAN_EXTRA_OPTIONS:-}")
export LSAN_OPTIONS

PYTHON_FLAG="--asan"

elif [ "${MODE}" = "--ubsan" ]; then
# UBSan: halt_on_error=1 makes UBSan stop at the first violation, giving
# one report per process (consistent with ASan and TSan behavior) and a
# simpler test-name attribution via the snapshot mechanism.
# flush=1 ensures the report is fully written before _exit() is called.
# ASAN_OPTIONS is intentionally NOT set: this image is built with
# SANITIZERS=undefined_behavior only, so there is no ASan runtime present.
# Note: UBSan has no runtime suppressions= mechanism (unlike ASan/LSan/
# TSan) in either GCC or Clang -- see docs/dev/development.md for the
# source-level __attribute__((no_sanitize("undefined"))) alternative.
if [ -z "${UBSAN_OPTIONS+set}" ]; then
UBSAN_OPTIONS=$(join_options "log_path=${LOG_DIR}/ubsan" "exitcode=42" \
"halt_on_error=1" "print_summary=1" "print_stacktrace=1" "flush=1")
fi
UBSAN_OPTIONS=$(join_options "${UBSAN_OPTIONS}" "${UBSAN_EXTRA_OPTIONS:-}")
export UBSAN_OPTIONS

PYTHON_FLAG="--ubsan"

else
# TSan: use suppressions file if it exists (for Argobots ULT false positives
# and other confirmed-benign races, e.g. the lazy log-facility cache --
# see utils/test_tsan.supp).
TSAN_SUPP_FILE=""
if [ -f "$(pwd)/daos/utils/test_tsan.supp" ]; then
TSAN_SUPP_FILE="$(pwd)/daos/utils/test_tsan.supp"
fi
if [ -z "${TSAN_OPTIONS+set}" ]; then
TSAN_OPTIONS=$(join_options "log_path=${LOG_DIR}/tsan" "exitcode=42" \
"second_deadlock_stack=1" "print_summary=1" \
"${TSAN_SUPP_FILE:+suppressions=${TSAN_SUPP_FILE}}")
fi
TSAN_OPTIONS=$(join_options "${TSAN_OPTIONS}" "${TSAN_EXTRA_OPTIONS:-}")
export TSAN_OPTIONS

# UCX's libucm memory-hook layer (madvise/mmap interception for RDMA
# registration tracking) is not TSan-safe: its interceptor can take a
# lock via a TSan-instrumented pthread_rwlock_rdlock() call while a
# thread is exiting, which crashes with a plain SIGSEGV (no TSan report
# at all, confirmed via gdb -- __tsan::MutexPreReadLock() inside
# ucm_madvise()/ucm_event_enter()). This is a known third-party UCX/TSan
# interaction, not a DAOS bug. Since the crash emits no sanitizer report,
# it would otherwise not set sanitizer_detected and could let a real
# crash silently pass a --tsan job. Disabling UCX's memory-event hooks
# avoids the crash entirely without affecting TSan's ability to detect
# and report genuine data races (confirmed: the same test's real race is
# still reported with this set).
if [ -z "${UCX_MEM_EVENTS+set}" ]; then
export UCX_MEM_EVENTS=n
fi

PYTHON_FLAG="--tsan"
fi

# ── Mount tmpfs for PMDK / VOS tests ─────────────────────────────────────────
# Applies to all modes, including --standard: VOS/storage-engine suites are
# I/O heavy and are dramatically slower against a disk-backed directory.
mkdir -p /mnt/daos
mount -t tmpfs \
-o rw,noatime,inode64,huge=always,mpol=prefer:0,uid="$(id -u)",gid="$(id -g)" \
tmpfs /mnt/daos

cd daos
# shellcheck source=/dev/null
source utils/sl/setup_local.sh

# ── Run the unit-test suite ───────────────────────────────────────────────────
# --{asan,tsan} : select suites via asan:/tsan: flags in utest.yaml
# --sudo no : container already runs as root; no nested sudo needed
# --no-fail-on-error: collect the exit code ourselves so every suite runs
export CMOCKA_XML_FILE="${RESULTS_DIR}/cmocka-%g.xml"
# CMOCKA_XML_FILE alone is not enough: cmocka only writes XML when "xml" is
# one of the active CMOCKA_MESSAGE_OUTPUT formats (defaults to stdout only).
export CMOCKA_MESSAGE_OUTPUT=xml
export PMEMOBJ_CONF="sds.at_create=0"

set +e # do not abort on test failure; we capture the exit code
# shellcheck disable=SC2086 # PYTHON_FLAG is intentionally empty for --standard
python3 utils/run_utest.py \
${PYTHON_FLAG} \
--gha \
--sudo no \
--no-fail-on-error \
--log_dir "${RESULTS_DIR}/logs" \
"${EXTRA_ARGS[@]}"
RUNNER_RC=$?
set -e

# --list only prints the selected suites/tests and exits; no tests ran, so
# there is nothing to report -- skip the exit_status/summary machinery below
# entirely rather than print an all-zero trailer that could be misread as "0
# tests failed" instead of "no tests were run".
if [ "${LIST_MODE}" -eq 1 ]; then
exit "${RUNNER_RC}"
fi

# ── Detect sanitizer findings and functional-test failures ───────────────────
FUNCTIONAL_FAILURE=0
SANITIZER_DETECTED=0
RUNNER_ERROR=0

if [ "${MODE}" != "--standard" ]; then
# Derive the log-file prefix from the mode flag to check for findings.
case "${MODE}" in --asan) PREFIX=asan ;; --ubsan) PREFIX=ubsan ;; *) PREFIX=tsan ;; esac
if [ "${MODE}" = "--ubsan" ]; then
# UBSan has no runtime suppressions mechanism (see utils/ci/run-unit_tests.sh's
# --ubsan branch above), so any log file it writes is always a genuine,
# unsuppressable finding: file existence alone is a correct check here.
ls "${LOG_DIR}/${PREFIX}".* >/dev/null 2>&1 && SANITIZER_DETECTED=1
else
# ASan/LSan and TSan both support suppressions=<file>, and a fully
# suppressed report can still leave a log file behind (confirmed:
# LSan writes a "Suppressions used:" footer even when every leak it
# found was suppressed) -- so file *existence* alone would wrongly
# still flag a run whose only finding(s) were all intentionally
# suppressed. Check the file's content for each tool's own marker
# line for a genuine, still-unsuppressed report instead:
# ASan/LSan : "==<pid>==ERROR: ..." (absent when fully suppressed)
# TSan : "WARNING: ThreadSanitizer: ..." (in practice TSan
# does not write a file at all when fully suppressed,
# but this content check is kept for consistency and
# as a safety net against relying on that).
case "${MODE}" in
--asan) MARKER='^==[0-9]+==ERROR:' ;;
*) MARKER='^WARNING: ThreadSanitizer:' ;;
esac
if ls "${LOG_DIR}/${PREFIX}".* >/dev/null 2>&1 \
&& grep -E -q "${MARKER}" "${LOG_DIR}/${PREFIX}".* 2>/dev/null; then
SANITIZER_DETECTED=1
fi
fi
fi

# run_utest.py is always run with --no-fail-on-error so that every suite still
# runs to completion after a failure; this means RUNNER_RC is 0 whenever
# run_utest.py itself ran to completion, even when individual tests failed --
# it never propagates a test's own exit code (e.g. a sanitizer's exitcode=42)
# as its own (subprocess.run() is called with check=False). So RUNNER_RC can
# never actually be 42; a nonzero RUNNER_RC here only ever means run_utest.py
# crashed/errored out before finishing (e.g. malformed utest.yaml, missing
# build vars). This is an infrastructure-level problem, not a test-content
# failure, so it is tracked separately from FUNCTIONAL_FAILURE below and
# always fails the job regardless of mode: unlike a plain test failure (which
# the standard build's own, separate run would independently catch too), a
# crash specific to one build's container/environment might not be.
if [ "${RUNNER_RC}" -ne 0 ]; then
RUNNER_ERROR=1
fi

# Individual test failures are recorded in the JUnit/cmocka XML files but (per
# above) never affect RUNNER_RC, so check them directly. This is the same
# check summarize_functional_failures.py uses to build the job summary, kept
# in sync so the job's pass/fail status always matches what the summary shows.
if ! python3 utils/ci/summarize_functional_failures.py --results-dir "${RESULTS_DIR}" >/dev/null; then
FUNCTIONAL_FAILURE=1
fi

echo " sanitizer_detected=${SANITIZER_DETECTED}"
echo " runner_error=${RUNNER_ERROR}"

# ── Write a structured status file readable by the workflow ──────────────────
printf 'sanitizer_detected=%s\nfunctional_failure=%s\nrunner_error=%s\n' \
"${SANITIZER_DETECTED}" "${FUNCTIONAL_FAILURE}" "${RUNNER_ERROR}" > "${RESULTS_DIR}/exit_status"

echo "=== Test run complete ==="
echo " mode=${MODE}"
echo " functional_failure=${FUNCTIONAL_FAILURE}"

# Always exit 0 here so Docker returns control to the workflow, which
# will inspect exit_status and fail the job with a meaningful message.
exit 0
Loading
Loading