diff --git a/utils/ci/run-unit_tests.sh b/utils/ci/run-unit_tests.sh new file mode 100755 index 00000000000..72e7d3ec0af --- /dev/null +++ b/utils/ci/run-unit_tests.sh @@ -0,0 +1,291 @@ +#!/bin/bash +# SPDX-License-Identifier: BSD-2-Clause-Patent +# Copyright 2025-2026 Hewlett Packard Enterprise Development LP. +# Run the DAOS unit-test suite, optionally under a sanitizer, and collect +# all reports. Shared by the standard, ASan, UBSan, and TSan GHA jobs (and by +# utils/docker/run-unit_tests-docker.sh for local reproduction). +# +# Usage: +# run-unit_tests.sh --standard|--asan|--ubsan|--tsan \ +# [--list] [--suite_filter=REGEX] [--test_filter=REGEX] +# +# --list List the suites/tests that would run (honoring the +# mode and any --suite_filter/--test_filter) without +# actually running them, then exit. +# --suite_filter=REGEX Only run suites whose name matches REGEX. +# --test_filter=REGEX Only run tests whose command line matches REGEX. +# +# Sanitizer options (ASAN_OPTIONS/UBSAN_OPTIONS/TSAN_OPTIONS/LSAN_OPTIONS) can +# be fully overridden by pre-setting them in the environment before invoking +# this script (e.g. to add fast_unwind_on_malloc=1 for a deeper leak trace -- +# see docs/dev/development.md). A separate *_EXTRA_OPTIONS variable per +# sanitizer (e.g. ASAN_EXTRA_OPTIONS) can instead be set to append extra +# options to whichever value (default or overridden) ends up in effect, +# without having to restate the whole options string. +# +# Exit codes written to /home/daos/test-results/exit_status: +# sanitizer_detected=1 Set when sanitizer log files are found in sanitizer-logs/ +# (always 0 for --standard) +# functional_failure=1 Set when any test recorded a failure in its JUnit/ +# cmocka XML output (may be 1 at the same time as +# sanitizer_detected=1, when a sanitizer finding and an +# unrelated plain test failure both occur in one run) +# runner_error=1 Set when run_utest.py itself crashed/errored out +# before finishing (e.g. malformed utest.yaml, missing +# build vars) -- an infrastructure-level problem, +# distinct from functional_failure, since it isn't +# necessarily also caught by another build's run +# +# Outputs mounted back to the host runner: +# /home/daos/sanitizer-logs/ – per-PID log files (asan., ubsan., tsan.) +# /home/daos/test-results/ – JUnit XML files + exit_status + +set -uo pipefail + +# Join non-empty ':'-delimited sanitizer-option fragments, skipping any empty +# ones (e.g. an unset *_EXTRA_OPTIONS) to avoid stray/leading colons. +join_options() { + local out="" + for frag in "$@"; do + [ -z "${frag}" ] && continue + if [ -z "${out}" ]; then out="${frag}"; else out="${out}:${frag}"; fi + done + printf '%s' "${out}" +} + +# ── Parse mode argument + optional pass-through args ───────────────────────── +MODE="${1:-}" +if [ "${MODE}" != "--standard" ] && [ "${MODE}" != "--asan" ] \ + && [ "${MODE}" != "--ubsan" ] && [ "${MODE}" != "--tsan" ]; then + echo "Usage: $(basename "$0") --standard|--asan|--ubsan|--tsan" \ + "[--list] [--suite_filter=REGEX] [--test_filter=REGEX]" >&2 + exit 1 +fi +shift +EXTRA_ARGS=() +LIST_MODE=0 +for arg in "$@"; do + case "${arg}" in + --list) + EXTRA_ARGS+=("${arg}") + LIST_MODE=1 + ;; + --suite_filter=*|--test_filter=*) + EXTRA_ARGS+=("${arg}") + ;; + *) + echo "Unknown argument: ${arg}" >&2 + exit 1 + ;; + esac +done + +# ── Mode-specific configuration ─────────────────────────────────────────────── +RESULTS_DIR=/home/daos/test-results +LOG_DIR=/home/daos/sanitizer-logs +mkdir -p "${RESULTS_DIR}" "${LOG_DIR}" + +if [ "${MODE}" = "--standard" ]; then + # No sanitizer runtime: run every suite not explicitly excluded from GHA + # (asan:/tsan: tags in utest.yaml only apply when --asan/--ubsan/--tsan + # is passed to run_utest.py, so passing nothing here selects them all). + PYTHON_FLAG="" + +elif [ "${MODE}" = "--asan" ]; then + # ASan: detect_odr_violation=0 suppresses the DAOS dual-library false positives. + # UBSAN_OPTIONS is intentionally NOT set: the UBSan workflow uses a separate + # image (SANITIZERS=undefined_behavior) so log_path is never shared between + # runtimes. + if [ -z "${ASAN_OPTIONS+set}" ]; then + ASAN_OPTIONS=$(join_options "log_path=${LOG_DIR}/asan" "exitcode=42" \ + "print_summary=1" "symbolize=1" "detect_odr_violation=0") + fi + ASAN_OPTIONS=$(join_options "${ASAN_OPTIONS}" "${ASAN_EXTRA_OPTIONS:-}") + export ASAN_OPTIONS + + # LSan (leak detection, part of the ASan runtime): use a suppressions file + # if it exists, for third-party/vendored leaks that aren't DAOS bugs (see + # utils/test_lsan.supp). + LSAN_SUPP_FILE="" + if [ -f "$(pwd)/daos/utils/test_lsan.supp" ]; then + LSAN_SUPP_FILE="$(pwd)/daos/utils/test_lsan.supp" + fi + if [ -z "${LSAN_OPTIONS+set}" ]; then + LSAN_OPTIONS=$(join_options "${LSAN_SUPP_FILE:+suppressions=${LSAN_SUPP_FILE}}") + fi + LSAN_OPTIONS=$(join_options "${LSAN_OPTIONS}" "${LSAN_EXTRA_OPTIONS:-}") + export LSAN_OPTIONS + + PYTHON_FLAG="--asan" + +elif [ "${MODE}" = "--ubsan" ]; then + # UBSan: halt_on_error=1 makes UBSan stop at the first violation, giving + # one report per process (consistent with ASan and TSan behavior) and a + # simpler test-name attribution via the snapshot mechanism. + # flush=1 ensures the report is fully written before _exit() is called. + # ASAN_OPTIONS is intentionally NOT set: this image is built with + # SANITIZERS=undefined_behavior only, so there is no ASan runtime present. + # Note: UBSan has no runtime suppressions= mechanism (unlike ASan/LSan/ + # TSan) in either GCC or Clang -- see docs/dev/development.md for the + # source-level __attribute__((no_sanitize("undefined"))) alternative. + if [ -z "${UBSAN_OPTIONS+set}" ]; then + UBSAN_OPTIONS=$(join_options "log_path=${LOG_DIR}/ubsan" "exitcode=42" \ + "halt_on_error=1" "print_summary=1" "print_stacktrace=1" "flush=1") + fi + UBSAN_OPTIONS=$(join_options "${UBSAN_OPTIONS}" "${UBSAN_EXTRA_OPTIONS:-}") + export UBSAN_OPTIONS + + PYTHON_FLAG="--ubsan" + +else + # TSan: use suppressions file if it exists (for Argobots ULT false positives + # and other confirmed-benign races, e.g. the lazy log-facility cache -- + # see utils/test_tsan.supp). + TSAN_SUPP_FILE="" + if [ -f "$(pwd)/daos/utils/test_tsan.supp" ]; then + TSAN_SUPP_FILE="$(pwd)/daos/utils/test_tsan.supp" + fi + if [ -z "${TSAN_OPTIONS+set}" ]; then + TSAN_OPTIONS=$(join_options "log_path=${LOG_DIR}/tsan" "exitcode=42" \ + "second_deadlock_stack=1" "print_summary=1" \ + "${TSAN_SUPP_FILE:+suppressions=${TSAN_SUPP_FILE}}") + fi + TSAN_OPTIONS=$(join_options "${TSAN_OPTIONS}" "${TSAN_EXTRA_OPTIONS:-}") + export TSAN_OPTIONS + + # UCX's libucm memory-hook layer (madvise/mmap interception for RDMA + # registration tracking) is not TSan-safe: its interceptor can take a + # lock via a TSan-instrumented pthread_rwlock_rdlock() call while a + # thread is exiting, which crashes with a plain SIGSEGV (no TSan report + # at all, confirmed via gdb -- __tsan::MutexPreReadLock() inside + # ucm_madvise()/ucm_event_enter()). This is a known third-party UCX/TSan + # interaction, not a DAOS bug. Since the crash emits no sanitizer report, + # it would otherwise not set sanitizer_detected and could let a real + # crash silently pass a --tsan job. Disabling UCX's memory-event hooks + # avoids the crash entirely without affecting TSan's ability to detect + # and report genuine data races (confirmed: the same test's real race is + # still reported with this set). + if [ -z "${UCX_MEM_EVENTS+set}" ]; then + export UCX_MEM_EVENTS=n + fi + + PYTHON_FLAG="--tsan" +fi + +# ── Mount tmpfs for PMDK / VOS tests ───────────────────────────────────────── +# Applies to all modes, including --standard: VOS/storage-engine suites are +# I/O heavy and are dramatically slower against a disk-backed directory. +mkdir -p /mnt/daos +mount -t tmpfs \ + -o rw,noatime,inode64,huge=always,mpol=prefer:0,uid="$(id -u)",gid="$(id -g)" \ + tmpfs /mnt/daos + +cd daos +# shellcheck source=/dev/null +source utils/sl/setup_local.sh + +# ── Run the unit-test suite ─────────────────────────────────────────────────── +# --{asan,tsan} : select suites via asan:/tsan: flags in utest.yaml +# --sudo no : container already runs as root; no nested sudo needed +# --no-fail-on-error: collect the exit code ourselves so every suite runs +export CMOCKA_XML_FILE="${RESULTS_DIR}/cmocka-%g.xml" +# CMOCKA_XML_FILE alone is not enough: cmocka only writes XML when "xml" is +# one of the active CMOCKA_MESSAGE_OUTPUT formats (defaults to stdout only). +export CMOCKA_MESSAGE_OUTPUT=xml +export PMEMOBJ_CONF="sds.at_create=0" + +set +e # do not abort on test failure; we capture the exit code +# shellcheck disable=SC2086 # PYTHON_FLAG is intentionally empty for --standard +python3 utils/run_utest.py \ + ${PYTHON_FLAG} \ + --gha \ + --sudo no \ + --no-fail-on-error \ + --log_dir "${RESULTS_DIR}/logs" \ + "${EXTRA_ARGS[@]}" +RUNNER_RC=$? +set -e + +# --list only prints the selected suites/tests and exits; no tests ran, so +# there is nothing to report -- skip the exit_status/summary machinery below +# entirely rather than print an all-zero trailer that could be misread as "0 +# tests failed" instead of "no tests were run". +if [ "${LIST_MODE}" -eq 1 ]; then + exit "${RUNNER_RC}" +fi + +# ── Detect sanitizer findings and functional-test failures ─────────────────── +FUNCTIONAL_FAILURE=0 +SANITIZER_DETECTED=0 +RUNNER_ERROR=0 + +if [ "${MODE}" != "--standard" ]; then + # Derive the log-file prefix from the mode flag to check for findings. + case "${MODE}" in --asan) PREFIX=asan ;; --ubsan) PREFIX=ubsan ;; *) PREFIX=tsan ;; esac + if [ "${MODE}" = "--ubsan" ]; then + # UBSan has no runtime suppressions mechanism (see utils/ci/run-unit_tests.sh's + # --ubsan branch above), so any log file it writes is always a genuine, + # unsuppressable finding: file existence alone is a correct check here. + ls "${LOG_DIR}/${PREFIX}".* >/dev/null 2>&1 && SANITIZER_DETECTED=1 + else + # ASan/LSan and TSan both support suppressions=, and a fully + # suppressed report can still leave a log file behind (confirmed: + # LSan writes a "Suppressions used:" footer even when every leak it + # found was suppressed) -- so file *existence* alone would wrongly + # still flag a run whose only finding(s) were all intentionally + # suppressed. Check the file's content for each tool's own marker + # line for a genuine, still-unsuppressed report instead: + # ASan/LSan : "====ERROR: ..." (absent when fully suppressed) + # TSan : "WARNING: ThreadSanitizer: ..." (in practice TSan + # does not write a file at all when fully suppressed, + # but this content check is kept for consistency and + # as a safety net against relying on that). + case "${MODE}" in + --asan) MARKER='^==[0-9]+==ERROR:' ;; + *) MARKER='^WARNING: ThreadSanitizer:' ;; + esac + if ls "${LOG_DIR}/${PREFIX}".* >/dev/null 2>&1 \ + && grep -E -q "${MARKER}" "${LOG_DIR}/${PREFIX}".* 2>/dev/null; then + SANITIZER_DETECTED=1 + fi + fi +fi + +# run_utest.py is always run with --no-fail-on-error so that every suite still +# runs to completion after a failure; this means RUNNER_RC is 0 whenever +# run_utest.py itself ran to completion, even when individual tests failed -- +# it never propagates a test's own exit code (e.g. a sanitizer's exitcode=42) +# as its own (subprocess.run() is called with check=False). So RUNNER_RC can +# never actually be 42; a nonzero RUNNER_RC here only ever means run_utest.py +# crashed/errored out before finishing (e.g. malformed utest.yaml, missing +# build vars). This is an infrastructure-level problem, not a test-content +# failure, so it is tracked separately from FUNCTIONAL_FAILURE below and +# always fails the job regardless of mode: unlike a plain test failure (which +# the standard build's own, separate run would independently catch too), a +# crash specific to one build's container/environment might not be. +if [ "${RUNNER_RC}" -ne 0 ]; then + RUNNER_ERROR=1 +fi + +# Individual test failures are recorded in the JUnit/cmocka XML files but (per +# above) never affect RUNNER_RC, so check them directly. This is the same +# check summarize_functional_failures.py uses to build the job summary, kept +# in sync so the job's pass/fail status always matches what the summary shows. +if ! python3 utils/ci/summarize_functional_failures.py --results-dir "${RESULTS_DIR}" >/dev/null; then + FUNCTIONAL_FAILURE=1 +fi + +echo " sanitizer_detected=${SANITIZER_DETECTED}" +echo " runner_error=${RUNNER_ERROR}" + +# ── Write a structured status file readable by the workflow ────────────────── +printf 'sanitizer_detected=%s\nfunctional_failure=%s\nrunner_error=%s\n' \ + "${SANITIZER_DETECTED}" "${FUNCTIONAL_FAILURE}" "${RUNNER_ERROR}" > "${RESULTS_DIR}/exit_status" + +echo "=== Test run complete ===" +echo " mode=${MODE}" +echo " functional_failure=${FUNCTIONAL_FAILURE}" + +# Always exit 0 here so Docker returns control to the workflow, which +# will inspect exit_status and fail the job with a meaningful message. +exit 0 diff --git a/utils/ci/summarize_functional_failures.py b/utils/ci/summarize_functional_failures.py new file mode 100644 index 00000000000..516dde2cc2c --- /dev/null +++ b/utils/ci/summarize_functional_failures.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python3 +""" + Copyright 2025-2026 Hewlett Packard Enterprise Development LP + All rights reserved. + + SPDX-License-Identifier: BSD-2-Clause-Patent + + Summarize functional test failures from JUnit/cmocka XML files. + + Scans a test-results/ directory and builds a GitHub-flavoured Markdown + section for GITHUB_STEP_SUMMARY that lists every failing test case with + its suite and name. + + Two granularity levels are handled automatically: + - cmocka-level (UTEST_*.xml): individual test case names + e.g. test_d_rank_list_to_str + - binary-level (test_*.xml): test binary basename + e.g. test_gurt + + cmocka-level entries take precedence when available (they are more + specific): a suite's binary-level entry is dropped as redundant when that + suite already has a cmocka-level entry AND exactly one binary failed in + that suite (with more than one failing binary in the same suite, it's not + possible to tell which one is already covered, so none are dropped). The + aggregate summary file test_run_utest.py.native.xml is skipped. + + Usage (from .github/workflows/unit-test-template.yml, called by unit-testing.yml): + + python3 utils/ci/summarize_functional_failures.py \\ + --results-dir test-results + + Exits 1 when at least one failure is found (so the caller can gate on it), + 0 when all tests passed. +""" + +import argparse +import os +import re +import sys +import xml.etree.ElementTree as ET +from pathlib import Path + + +def _suite_from_classname(classname: str) -> str: + """Extract suite name from a JUnit classname attribute. + + Examples: + "UTEST_gurt.gurt_tests" → "gurt" + "gurt" → "gurt" + "UTEST_cart.cart" → "cart" + """ + m = re.match(r'UTEST_([^.]+)', classname) + return m.group(1) if m else classname.split('.')[0] + + +def collect_failures(results_dir: Path) -> list: + """Scan *results_dir* for JUnit XML files and return a list of failure dicts. + + Each dict has: + suite str — test suite name (e.g. "gurt") + name str — test case name: cmocka function OR binary path + is_binary bool — True when name is a binary path (contains '/') + """ + failures = [] + for xml_path in sorted(results_dir.glob('*.xml')): + fname = xml_path.name + if 'run_utest.py' in fname: # skip aggregate summary + continue + try: + root = ET.parse(xml_path).getroot() + except ET.ParseError: + continue + for tc in root.iter('testcase'): + fail_el = tc.find('failure') + err_el = tc.find('error') + if fail_el is None and err_el is None: + continue + tc_name = tc.get('name', '') + classname = tc.get('classname', '') + failures.append({ + 'suite': _suite_from_classname(classname), + 'name': tc_name, + 'is_binary': '/' in tc_name, + }) + return failures + + +def build_summary_md(failures: list) -> str: + """Return a Markdown summary section for the given failure list.""" + if not failures: + return '#### ✅ All functional unit tests passed\n' + + # cmocka-level entries (specific test function names) take precedence over + # binary-level entries for the same suite. + cmocka_entries = [f for f in failures if not f['is_binary']] + binary_entries = [f for f in failures if f['is_binary']] + + # A suite's binary-level entry is only known to be fully redundant with its + # cmocka-level entry/entries when exactly one binary failed in that suite -- + # with more than one, we can't tell which is already covered by a cmocka-level + # entry, so nothing is suppressed for that suite in that (rarer) case. + cmocka_suites = {f['suite'] for f in cmocka_entries} + binary_names_by_suite: dict = {} + for f in binary_entries: + binary_names_by_suite.setdefault(f['suite'], set()).add(f['name']) + binary_entries = [ + f for f in binary_entries + if not (f['suite'] in cmocka_suites and len(binary_names_by_suite[f['suite']]) == 1) + ] + + # Deduplicate while preserving order (e.g. accidental double-reporting of the + # exact same entry). + seen: set = set() + items: list = [] + for f in cmocka_entries + binary_entries: + key = (f['suite'], f['name']) + if key not in seen: + seen.add(key) + items.append(f) + + lines = [f'#### ❌ Functional test failures — {len(items)} failure(s)', ''] + lines += [ + '| Suite | Test |', + '|-------|------|', + ] + for item in items: + if item['is_binary']: + display = f'`{os.path.basename(item["name"])}`' + else: + display = f'`{item["name"]}`' + lines.append(f'| {item["suite"]} | {display} |') + + return '\n'.join(lines) + '\n' + + +def main() -> int: + """Entry point.""" + parser = argparse.ArgumentParser( + description='Summarize functional test failures for GITHUB_STEP_SUMMARY') + parser.add_argument('--results-dir', required=True, + help='Directory containing test result XML files') + parser.add_argument('--summary-out', default=None, + help='Output path for the Markdown file (default: stdout)') + args = parser.parse_args() + + results_dir = Path(args.results_dir) + if not results_dir.is_dir(): + print(f'Results directory not found: {results_dir}', file=sys.stderr) + return 0 + + failures = collect_failures(results_dir) + summary = build_summary_md(failures) + + if args.summary_out: + Path(args.summary_out).write_text(summary, encoding='utf-8') + else: + print(summary, end='') + + return 1 if failures else 0 + + +if __name__ == '__main__': + sys.exit(main())