diff --git a/.github/workflows/slurm.yml b/.github/workflows/slurm.yml index dffe7cee69..e658835ae9 100644 --- a/.github/workflows/slurm.yml +++ b/.github/workflows/slurm.yml @@ -88,7 +88,7 @@ jobs: sudo apt-get install --no-install-recommends -y clang-tidy-23 libslurm-dev cmake -S spank -B build/spank -DCMAKE_BUILD_TYPE=Release -DCMAKE_EXPORT_COMPILE_COMMANDS=ON cmake --build build/spank - clang-tidy-23 -p build/spank spank/spank.cpp --warnings-as-errors='*' + clang-tidy-23 -p build/spank spank/spank.cpp --header-filter='(^|/)spank/' --warnings-as-errors='*' - name: Build the MQT Core wheel timeout-minutes: 15 diff --git a/docs/qdmi/slurm.md b/docs/qdmi/slurm.md index 6b79a9f300..f91353f8ee 100644 --- a/docs/qdmi/slurm.md +++ b/docs/qdmi/slurm.md @@ -218,6 +218,51 @@ Provider-specific installation and credentials remain in the [Braket guide](https://github.com/munich-quantum-software/amazon-braket-qdmi-device) and [IQM guide](https://github.com/iqm-finland/QDMI-on-IQM). +## Optionally validate before task launch + +Administrators can enable a bounded readiness check by adding the absolute +checker path to the shared module's plugstack line: + +```ini +required /usr/local/lib/slurm/mqt-core-qdmi-spank.so licenses=mqt.sc.default validate=/usr/local/bin/mqt-core-qdmi-check validation_timeout=30 +``` + +Validation is disabled when `validate` is absent. `validation_timeout` defaults +to 30 seconds and accepts whole seconds from 1 to 3600. The selected license +must name one configured device with an implicit count or `:1`. + +The module runs the checker as the job user with the job environment after +configuration injection. It does not load a provider in a Slurm daemon. An +unavailable device, checker failure, or timeout prevents tasks using that check +from starting and does not drain the node. The checker accepts both `IDLE` and +`BUSY`; it does not reserve the device or authorize a later submission. + +Each node runs one check per job step and shares the result across its tasks. +Tasks on another node may already have started when a check fails. All tasks +must use the same site-established provider runtime, catalogue, and credentials +on that node, configured before this hook. Place the module after plugins that +set these inputs. Task prologs run later and must not change them. + +The configured checker must use the workload's provider runtime and have its +[device catalogue](driver.md#check-a-device-before-launch) available. It cannot +validate a virtual environment, module, credentials, or container activated +later in a batch script. For such jobs, administrators must leave automatic +validation disabled. Job authors can run the checker inside the chosen +environment after setup, for example: + +```bash +#!/bin/bash +#SBATCH --licenses=mqt.sc.default +set -eu +source /path/to/venv/bin/activate +export MQT_CORE_QDMI_CONFIG_FILE=/path/to/qdmi.json +mqt-core-qdmi-check --device mqt.sc.default --timeout 30 +python workload.py +``` + +Both forms provide a readiness snapshot. The application must still handle +changes in device availability and credentials. + ## Submit a DDSIM job Save this program as `bell.py` in a location that all compute nodes can read: diff --git a/spank/Validation.hpp b/spank/Validation.hpp new file mode 100644 index 0000000000..4639b28245 --- /dev/null +++ b/spank/Validation.hpp @@ -0,0 +1,179 @@ +/* + * Copyright (c) 2025 - 2026 Munich Quantum Software Company GmbH + * All rights reserved. + * + * SPDX-License-Identifier: GPL-3.0-or-later + * + * This program is free software: you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation, either version 3 of the License, or (at your + * option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General + * Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program. If not, see . + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include // NOLINT(modernize-deprecated-headers) +#include +#include +#include +#include +#include +#include +#include +#include + +namespace mqt::spank { + +class Validation final { + enum class State : uint8_t { Empty, Checking, Available, Failed }; + struct Shared { + std::atomic state{State::Empty}; + }; + static_assert(std::atomic::is_always_lock_free); + +public: + Validation() = default; + Validation(const Validation&) = delete; + Validation& operator=(const Validation&) = delete; + Validation(Validation&&) = delete; + Validation& operator=(Validation&&) = delete; + + ~Validation() { + if (shared_ != nullptr) { + munmap(shared_, sizeof(Shared)); + } + } + + void prepare() { + auto* memory = mmap(nullptr, sizeof(Shared), PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_ANONYMOUS, -1, 0); + if (memory == MAP_FAILED) { + throw std::runtime_error("could not prepare launch validation"); + } + shared_ = std::construct_at(static_cast(memory)); + } + + void check(const std::string& executable, const int timeout, + const std::string& device, char* const* environment) { + if (shared_ == nullptr) { + throw std::runtime_error("launch validation was not prepared"); + } + const auto deadline = + std::chrono::steady_clock::now() + std::chrono::seconds(timeout + 2); + auto expected = State::Empty; + if (shared_->state.compare_exchange_strong(expected, State::Checking)) { + const bool available = execute(executable, timeout, device, environment); + shared_->state.store(available ? State::Available : State::Failed, + std::memory_order_release); + } + + while (true) { + const auto state = shared_->state.load(std::memory_order_acquire); + if (state == State::Available) { + return; + } + if (state == State::Failed) { + throw std::runtime_error("QDMI launch validation failed"); + } + if (std::chrono::steady_clock::now() >= deadline) { + throw std::runtime_error("QDMI launch validation timed out"); + } + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + } + +private: + static bool execute(const std::string& executable, const int timeout, + const std::string& device, char* const* environment) { + const auto seconds = std::to_string(timeout); + // execve does not modify the strings despite its mutable argument types. + // NOLINTBEGIN(cppcoreguidelines-pro-type-const-cast) + std::array arguments{ + const_cast(executable.c_str()), const_cast("--device"), + const_cast(device.c_str()), const_cast("--timeout"), + const_cast(seconds.c_str()), static_cast(nullptr), + }; + // NOLINTEND(cppcoreguidelines-pro-type-const-cast) + const auto parent = getpid(); + const auto child = fork(); + if (child < 0) { + return false; + } + if (child == 0) { + // Slurm's caught handlers survive fork until execve resets them. + struct sigaction action{}; + action.sa_handler = SIG_DFL; + sigemptyset(&action.sa_mask); + sigset_t mask{}; + sigemptyset(&mask); + sigaddset(&mask, SIGTERM); + if (sigaction(SIGTERM, &action, nullptr) != 0 || + sigprocmask(SIG_UNBLOCK, &mask, nullptr) != 0) { + _exit(1); + } + // prctl uses the Linux variadic C ABI. + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg) + if (prctl(PR_SET_PDEATHSIG, SIGTERM) != 0 || getppid() != parent || + setpgid(0, 0) != 0) { + _exit(1); + } + // Never send provider or launcher diagnostics to Slurm's log stream. + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg) + const auto sink = open("/dev/null", O_RDWR); + if (sink < 0 || dup2(sink, STDIN_FILENO) < 0 || + dup2(sink, STDOUT_FILENO) < 0 || dup2(sink, STDERR_FILENO) < 0) { + _exit(1); + } + if (sink > STDERR_FILENO) { + close(sink); + } + execve(executable.c_str(), arguments.data(), environment); + _exit(1); + } + static_cast(setpgid(child, child)); + const auto deadline = + std::chrono::steady_clock::now() + std::chrono::seconds(timeout + 1); + bool available = false; + while (true) { + siginfo_t status{}; + const auto result = waitid(P_PID, static_cast(child), &status, + WEXITED | WNOHANG | WNOWAIT); + if (result == 0 && status.si_pid == child) { + available = status.si_code == CLD_EXITED && status.si_status == 0; + break; + } + if ((result < 0 && errno != EINTR) || + std::chrono::steady_clock::now() >= deadline) { + static_cast(kill(child, SIGTERM)); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + break; + } + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + // Reap only after cleanup, so the checker PID cannot be reused. + static_cast(kill(-child, SIGKILL)); + static_cast(kill(child, SIGKILL)); + while (waitpid(child, nullptr, 0) < 0 && errno == EINTR) { + } + return available; + } + + Shared* shared_ = nullptr; +}; + +} // namespace mqt::spank diff --git a/spank/spank.cpp b/spank/spank.cpp index 83630812a7..2ad97e1fb6 100644 --- a/spank/spank.cpp +++ b/spank/spank.cpp @@ -22,7 +22,10 @@ /// @brief Transport administrator-declared configuration references to QDMI /// jobs. +#include "Validation.hpp" + #include +#include #include #include #include @@ -30,6 +33,7 @@ #include #include #include +#include #include extern "C" { @@ -159,6 +163,25 @@ class Configuration final { throw std::runtime_error("invalid catalogue reference"); } catalogueDefault_ = value; + } else if (key == "validate" && checker_.empty()) { + if (!validValue(value) || !value.starts_with('/')) { + throw std::runtime_error( + "validate requires an absolute checker path"); + } + checker_ = value; + } else if (key == "validation_timeout" && !hasTimeout_) { + // from_chars accepts the bounds of this string view as pointers. + // NOLINTBEGIN(cppcoreguidelines-pro-bounds-pointer-arithmetic) + const auto parsed = std::from_chars( + value.data(), value.data() + value.size(), timeout_); + if (parsed.ec != std::errc{} || + parsed.ptr != value.data() + value.size() || timeout_ < 1 || + timeout_ > 3600) { + throw std::runtime_error( + "validation_timeout must be 1 to 3600 seconds"); + } + // NOLINTEND(cppcoreguidelines-pro-bounds-pointer-arithmetic) + hasTimeout_ = true; } else if (key == "reference") { const auto nameEnd = value.find(':'); const auto idsEnd = @@ -199,6 +222,9 @@ class Configuration final { if (licenses_.empty()) { throw std::runtime_error("configure concrete license IDs with licenses="); } + if (hasTimeout_ && checker_.empty()) { + throw std::runtime_error("validation_timeout requires validate"); + } for (const auto& reference : references_) { for (const auto& id : reference.licenses) { if (std::ranges::find(licenses_, id) == licenses_.end()) { @@ -246,7 +272,23 @@ class Configuration final { references_[static_cast(option) - 1].option = argument; } - void inject(spank_t spank) const { + [[nodiscard]] bool validationEnabled() const { return !checker_.empty(); } + + void validate(spank_t spank, mqt::spank::Validation& validation, + const std::string& device) const { + // S_JOB_ENV requires a mutable char*** output parameter. + char** values = nullptr; // NOLINT(misc-const-correctness) + // Slurm exposes item lookup through a variadic C ABI. + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg) + if (spank_get_item(spank, S_JOB_ENV, &values) != ESPANK_SUCCESS || + values == nullptr) { + throw std::runtime_error("could not read the job environment"); + } + // S_JOB_ENV is a borrowed, null-terminated array owned by Slurm. + validation.check(checker_, timeout_, device, values); + } + + auto inject(spank_t spank) const -> std::optional { const auto expression = jobEnvironment(spank, LICENSE_ENVIRONMENT); const auto selected = std::ranges::find_if(licenses_, [&](const auto& id) { return expression && (*expression == id || *expression == id + ":1"); @@ -259,7 +301,7 @@ class Configuration final { throw std::runtime_error( "QDMI options require a configured device license"); } - return; + return std::nullopt; } apply(spank, CATALOGUE_ENVIRONMENT, catalogueDefault_, catalogueOption_); @@ -275,6 +317,10 @@ class Configuration final { apply(spank, reference.environment.c_str(), reference.defaultValue, reference.option); } + if (!validationEnabled()) { + return std::nullopt; + } + return *selected; } private: @@ -307,12 +353,19 @@ class Configuration final { std::vector references_; std::optional catalogueDefault_; std::optional catalogueOption_; + std::string checker_; + int timeout_ = 30; + bool hasTimeout_ = false; }; Configuration configuration; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) bool licenseEnvironmentReady = true; +mqt::spank::Validation + validation; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) +bool validationReady = true; int optionCallback(const int value, const char* argument, int /*remote*/) { try { @@ -336,6 +389,7 @@ int slurm_spank_init(spank_t spank, const int count, char* arguments[]) { try { configuration = Configuration{}; licenseEnvironmentReady = true; + validationReady = true; configuration.parse(count, arguments); configuration.registerOptions(spank); return ESPANK_SUCCESS; @@ -357,6 +411,14 @@ int slurm_spank_user_init(spank_t spank, int /*count*/, char* /*arguments*/[]) { licenseEnvironmentReady = false; fail("could not clear inherited license selection"); } + if (configuration.validationEnabled()) { + try { + validation.prepare(); + } catch (...) { + validationReady = false; + fail("could not prepare launch validation"); + } + } } return ESPANK_SUCCESS; } @@ -369,7 +431,13 @@ int slurm_spank_task_init(spank_t spank, int /*count*/, char* /*arguments*/[]) { if (!licenseEnvironmentReady) { throw std::runtime_error("could not clear inherited license selection"); } - configuration.inject(spank); + const auto device = configuration.inject(spank); + if (device) { + if (!validationReady) { + throw std::runtime_error("could not prepare launch validation"); + } + configuration.validate(spank, validation, *device); + } return ESPANK_SUCCESS; } catch (const std::exception& error) { fail(error.what()); diff --git a/test/python/test_slurm_integration.py b/test/python/test_slurm_integration.py index 51152a543e..391e8cdc81 100644 --- a/test/python/test_slurm_integration.py +++ b/test/python/test_slurm_integration.py @@ -230,8 +230,10 @@ def controller(*command: str, **_kwargs: object) -> subprocess.CompletedProcess[ runner.test_provider(options) assert commands[0][:3] == ("env", "PROVIDER_PROFILE=profile", "MQT_CORE_QDMI_CONFIG_FILE=/opt/provider.qdmi.json") assert commands[1][0] == "srun" - assert commands[0][-2:] == commands[1][-2:] == ("python3", "/workload/probe.py") + assert commands[0][-2:] == commands[1][-2:] == commands[2][-2:] == ("python3", "/workload/probe.py") assert not configurations[0] assert "licenses=provider.device" in configurations[1] assert "reference=PROVIDER_PROFILE:provider.device:profile" in configurations[1] + assert "validate=" not in configurations[1] + assert configurations[2] == configurations[1].rstrip() + " validate=/usr/local/bin/mqt-core-qdmi-check\n" assert not (tmp_path / "plugstack.conf").read_text() diff --git a/test/slurm/checker_probe.py b/test/slurm/checker_probe.py new file mode 100755 index 0000000000..08aed183e8 --- /dev/null +++ b/test/slurm/checker_probe.py @@ -0,0 +1,52 @@ +#!/usr/bin/python3 +# Copyright (c) 2023 - 2026 Chair for Design Automation, TUM +# Copyright (c) 2025 - 2026 Munich Quantum Software Company GmbH +# All rights reserved. +# +# SPDX-License-Identifier: MIT +# +# Licensed under the MIT License + +"""Observe checker execution inside real Slurm tasks using non-secret test data.""" + +from __future__ import annotations + +import json +import os +import pathlib +import signal +import subprocess +import sys +import time + +mode = os.environ.get("MQT_SLURM_CHECKER_MODE", "check") +child = None +if mode == "hang": + signal.signal(signal.SIGTERM, signal.SIG_IGN) + child = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(300)"]) + +record = { + "uid": os.getuid(), + "euid": os.geteuid(), + "gid": os.getgid(), + "groups": os.getgroups(), + "job": os.environ["SLURM_JOB_ID"], + "step": os.environ.get("SLURM_STEP_ID", "batch"), + "node": os.environ["SLURMD_NODENAME"], + "pid": os.getpid(), + "child": child.pid if child else None, + "reference": os.environ.get("MQT_SLURM_TEST_REFERENCE"), + "catalogue": os.environ.get("MQT_CORE_QDMI_CONFIG_FILE"), + "arguments": sys.argv[1:], +} +with pathlib.Path(os.environ["MQT_SLURM_CHECKER_LOG"]).open("a", encoding="utf-8") as output: + output.write(json.dumps(record) + "\n") + +if mode == "hang": + time.sleep(300) +elif mode == "fail": + sys.stderr.write("test provider credential must stay private\n") + sys.exit(1) +else: + executable = "/usr/local/bin/mqt-core-qdmi-check" + os.execv(executable, [executable, *sys.argv[1:]]) # ruff: ignore[start-process-with-no-shell] diff --git a/test/slurm/run_integration.py b/test/slurm/run_integration.py index 24bd2f2b08..7a3d859ae3 100644 --- a/test/slurm/run_integration.py +++ b/test/slurm/run_integration.py @@ -464,7 +464,7 @@ def parse_arguments(arguments: Sequence[str]) -> argparse.Namespace: def test_provider(options: argparse.Namespace) -> None: - """Run the same provider workload with direct configuration and SPANK.""" + """Run provider workloads with direct configuration, injection, and validation.""" environment = list(options.reference) if options.qdmi_config_file: environment.append(f"MQT_CORE_QDMI_CONFIG_FILE={options.qdmi_config_file}") @@ -480,8 +480,9 @@ def test_provider(options: argparse.Namespace) -> None: for reference in options.reference: name, _, value = reference.partition("=") configuration.append(f"reference={name}:{options.device_license}:{value}") - (RUNTIME / "plugstack.conf").write_text(" ".join(configuration) + "\n", encoding="utf-8") - job(*allocation, *options.command, timeout=300) + for validation in ("", " validate=/usr/local/bin/mqt-core-qdmi-check"): + (RUNTIME / "plugstack.conf").write_text(" ".join(configuration) + validation + "\n", encoding="utf-8") + job(*allocation, *options.command, timeout=300) (RUNTIME / "plugstack.conf").write_text("", encoding="utf-8") @@ -595,6 +596,149 @@ def values(*arguments: str) -> list[str | None]: (RUNTIME / "plugstack.conf").write_text("", encoding="utf-8") +def test_spank_validation() -> None: + """Prove bounded readiness checks at the real, unprivileged task hook.""" + selected = "mqt.sc.default" + log = RUNTIME / "jobs" / "checker.jsonl" + body = RUNTIME / "jobs" / "validated-body" + catalogue = "/jobs/validation.qdmi.json" + (RUNTIME / "jobs" / "validation.qdmi.json").write_text( + '{"schema-version":1,"qdmi":{"devices":[]}}', encoding="utf-8" + ) + configuration = ( + "required /usr/local/lib/slurm/mqt-core-qdmi-spank.so " + f"licenses={selected},mqt.ddsim.default qdmi_config_file={catalogue} " + f"reference=MQT_SLURM_TEST_REFERENCE:{selected}:site-value" + ) + probe = "/workspace/test/slurm/checker_probe.py" + environment = ( + "env", + "MQT_SLURM_CHECKER_LOG=/jobs/checker.jsonl", + "MQT_SLURM_TEST_REFERENCE=job-value", + ) + allocation = ("srun", "--immediate=5", "--time=1", "--nodes=2", "--ntasks=4", f"--licenses={selected}") + program = "from pathlib import Path; Path('/jobs/validated-body').touch()" + + def calls() -> list[dict[str, Any]]: + return [json.loads(line) for line in log.read_text(encoding="utf-8").splitlines()] if log.exists() else [] + + def configure(checker: str = probe, timeout: int = 3) -> None: + validation = f" validate={checker} validation_timeout={timeout}" + (RUNTIME / "plugstack.conf").write_text(configuration + validation + "\n", encoding="utf-8") + log.unlink(missing_ok=True) + body.unlink(missing_ok=True) + + def assert_released() -> None: + for node in ("node1", "node2"): + wait_for(f"{node} to return to IDLE after validation", lambda node=node: node_is_idle(node)) + assert "DRAIN" not in node_record(node) + assert_license(selected, total=1, used=0, free=1) + + def assert_no_processes(records: list[dict[str, Any]]) -> None: + for record in records: + for pid in (record["pid"], record["child"]): + if pid is not None: + compute( + record["node"], + "python3", + "-c", + "from pathlib import Path; " + f"p=Path('/proc/{pid}/stat'); " + "assert not p.exists() or p.read_text().split(') ', 1)[1].startswith('Z ')", + ) + + configure() + job(*environment, *allocation, "--mpi=pmi2", "python3", "-c", program, timeout=60) + records = calls() + assert body.exists() + assert len(records) == 2, records + assert {record["node"] for record in records} == {"node1", "node2"} + for record in records: + assert record["uid"] == record["euid"] == record["gid"] == 10000, record + assert record["reference"] == "job-value", record + assert record["catalogue"] == catalogue, record + assert record["arguments"] == ["--device", selected, "--timeout", "3"], record + assert_released() + + configure() + job( + *environment, + "sbatch", + "--wait", + "--time=1", + "--nodes=2", + "--ntasks=4", + f"--licenses={selected}", + "--output=/jobs/validator-batch.out", + "--wrap", + shlex.join(("srun", "--ntasks=4", "python3", "-c", program)), + timeout=120, + ) + records = calls() + assert body.exists() + assert len(records) == 3, records + assert len({(record["node"], record["step"]) for record in records}) == 3, records + assert_released() + + for mode in ("fail", "hang"): + configure() + started = time.monotonic() + result = job( + *environment, + f"MQT_SLURM_CHECKER_MODE={mode}", + *allocation, + "--mpi=pmi2", + "python3", + "-c", + program, + check=False, + timeout=60, + ) + assert result.returncode != 0 + assert not body.exists() + assert "test provider credential" not in result.stdout + result.stderr + assert time.monotonic() - started < 15 + records = calls() + assert len(records) == 2, records + assert_released() + assert_no_processes(records) + + for checker, extra in (("/missing/mqt-core-qdmi-check", ()), (probe, ("MQT_CORE_QDMI_CONFIG_JSON=invalid",))): + configure(checker) + result = job(*environment, *extra, *allocation, "python3", "-c", program, check=False, timeout=60) + assert result.returncode != 0 + assert not body.exists() + assert_released() + + configure() + job(*environment, "SLURM_JOB_LICENSES=mqt.sc.default", "srun", "--time=1", "--ntasks=1", "/bin/true") + assert not calls() + + configure(timeout=300) + submitted = ( + job( + *environment, + "MQT_SLURM_CHECKER_MODE=hang", + "sbatch", + "--parsable", + "--time=5", + f"--licenses={selected}", + "--output=/jobs/validator-cancel.out", + "--wrap", + shlex.join(("python3", "-c", program)), + ) + .stdout.strip() + .split(";", maxsplit=1)[0] + ) + wait_for("the cancellable checker to start", lambda: bool(calls())) + job("scancel", submitted) + wait_for("the validation job to be cancelled", lambda: job_finished(submitted, expected_state="CANCELLED")) + assert not body.exists() + assert_released() + assert_no_processes(calls()) + (RUNTIME / "plugstack.conf").write_text("", encoding="utf-8") + + def main(arguments: Sequence[str] = ()) -> None: """Build the shared cluster and run its Core or provider workload.""" options = parse_arguments(arguments) @@ -670,6 +814,7 @@ def main(arguments: Sequence[str] = ()) -> None: else: test_core() test_spank_transport() + test_spank_validation() success = True LOGGER.info("Slurm admission and execution checks: %.2fs", time.monotonic() - testing_at)