From 53525f96e9751c63c752bedeef93cb52fe8db744 Mon Sep 17 00:00:00 2001 From: flowerthrower Date: Tue, 22 Sep 2026 10:38:33 +0200 Subject: [PATCH 1/6] =?UTF-8?q?=E2=9C=A8=20Add=20optional=20Slurm=20launch?= =?UTF-8?q?=20validation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Assisted-by: GPT-6 via Codex Signed-off-by: flowerthrower --- .agent/plans/slurm-launch-validation.md | 47 +++++ .github/workflows/ci.yml | 2 +- .github/workflows/slurm.yml | 3 +- .license-tools-config.json | 1 + docs/qdmi/slurm.md | 35 ++++ noxfile.py | 2 +- spank/Validation.hpp | 220 ++++++++++++++++++++++++ spank/spank.cpp | 92 +++++++++- test/python/test_slurm_integration.py | 4 +- test/slurm/Dockerfile | 6 +- test/slurm/checker_probe.py | 52 ++++++ test/slurm/run_integration.py | 189 +++++++++++++++++++- test/slurm/validation_environment.cpp | 51 ++++++ 13 files changed, 693 insertions(+), 11 deletions(-) create mode 100644 .agent/plans/slurm-launch-validation.md create mode 100644 spank/Validation.hpp create mode 100755 test/slurm/checker_probe.py create mode 100644 test/slurm/validation_environment.cpp diff --git a/.agent/plans/slurm-launch-validation.md b/.agent/plans/slurm-launch-validation.md new file mode 100644 index 0000000000..f051aefb40 --- /dev/null +++ b/.agent/plans/slurm-launch-validation.md @@ -0,0 +1,47 @@ +# Optional Slurm launch validation + +Status: implemented and validated on real Slurm; final provider image checks +remain in progress. + +## Goal and scope + +Add opt-in readiness validation to the shared SPANK component. Execute the MIT +QDMI checker as the job user before task launch. Keep provider code outside +Slurm daemons and preserve static injection when validation is disabled. + +## Decisions + +- Validate in remote `task_init`, after Slurm restores allocated licenses and + permanently drops privileges. Return a task failure, never a node failure. +- Copy the final `S_JOB_ENV` for `execve`; do not inherit daemon credentials. +- Check once per node and step through an anonymous shared gate created before + task forks. Compare later tasks' effective configuration with the first task's + snapshot. Reject a mismatch without claiming that earlier tasks did not start. +- Provider configuration must be step-wide. Slurm rank metadata is excluded from + snapshot comparison and is unsupported as provider configuration input. +- Bound environment storage, checker execution, and waiting for another task's + result. Do not add controller RPCs, a persistent cache, or an all-task + barrier. +- Reset and unblock the checker's parent-death signal before exec. Slurm's + caught signal handlers remain installed in a fork child until exec resets + them. + +## Work remaining + +- Complete both provider workloads with direct configuration, shared injection, + and optional validation against the smaller native and wheel images. + +## Validation + +The existing runner unit tests pass (22 tests), as do full repository lint, +standalone Linux compilation, and full-file clang-tidy 23 checks. The installed +Linux checker passed its forced-parent-death case. Native configuration +correctly rejects the macOS host before compiling this Linux-only component. The +full strict documentation build passed with MLIR 23.1.0. + +The full real Slurm fixture passed on Linux aarch64 with cgroup v2 and Slurm +25.11.2. It verifies hook ordering, task launch, license release, node state, +task environment, and process cleanup. Cases cover disabled validation, multiple +nodes/tasks, batch and nested steps, failures, timeouts, cancellation, differing +task inputs, and bounded storage. Static injection and provider migrations do +not depend on this optional PR. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 875029b5ad..aca9ad7a54 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -139,7 +139,7 @@ jobs: cpp-linter-extra-args: "-std=c++20 -Wunused-template" # Vendored headers are checked upstream; standalone SPANK is checked in # the Slurm workflow against its own compilation database and headers. - cpp-linter-ignore-extra: "vendor/**|spank/**" + cpp-linter-ignore-extra: "vendor/**|spank/**|test/slurm/validation_environment.cpp" setup-mlir: true llvm-version: 23.1.1 permissions: diff --git a/.github/workflows/slurm.yml b/.github/workflows/slurm.yml index cc587b5dd5..1377569faa 100644 --- a/.github/workflows/slurm.yml +++ b/.github/workflows/slurm.yml @@ -88,7 +88,8 @@ jobs: sudo apt-get install --no-install-recommends -y clang-tidy-23 libslurm-dev cmake -S spank -B build/spank -DCMAKE_BUILD_TYPE=Release -DCMAKE_EXPORT_COMPILE_COMMANDS=ON cmake --build build/spank - clang-tidy-23 -p build/spank spank/spank.cpp --warnings-as-errors='*' + clang-tidy-23 -p build/spank spank/spank.cpp --header-filter='(^|/)spank/' --warnings-as-errors='*' + clang-tidy-23 test/slurm/validation_environment.cpp --warnings-as-errors='*' -- -std=c++20 - name: Build the MQT Core wheel timeout-minutes: 15 diff --git a/.license-tools-config.json b/.license-tools-config.json index 150ed120d7..1817b9829f 100644 --- a/.license-tools-config.json +++ b/.license-tools-config.json @@ -20,6 +20,7 @@ "exclude": [ "^spank/", "^test/slurm/Dockerfile$", + "^test/slurm/validation_environment\\.cpp$", "^\\.[^/]+", "/\\.[^/]+", ".*\\.qasm", diff --git a/docs/qdmi/slurm.md b/docs/qdmi/slurm.md index e441431ba8..2a7129e71c 100644 --- a/docs/qdmi/slurm.md +++ b/docs/qdmi/slurm.md @@ -199,6 +199,41 @@ and [IQM guide](https://github.com/iqm-finland/QDMI-on-IQM). Provider migrations replace the legacy SPANK module and options, including IQM's embedded early validation and alias-derived license names. Use one shared module per cluster. +## Optionally validate before task launch + +Administrators can enable a bounded readiness check by adding the absolute +checker path to the shared module's plugstack line: + +```ini +required /usr/local/lib/slurm/mqt-core-qdmi-spank.so licenses=mqt.sc.default validate=/usr/local/bin/mqt-core-qdmi-check validation_timeout=30 +``` + +Validation is disabled when `validate` is absent. `validation_timeout` defaults +to 30 seconds and accepts whole seconds from 1 to 3600. The selected license +must name one configured device with an implicit count or `:1`. + +The module runs the checker as the job user with a copy of the job environment +after configuration injection. It does not load a provider in a Slurm daemon. An +unavailable device, checker failure, or timeout prevents matching tasks on that +node from starting and does not drain the node. The checker still accepts both +`IDLE` and `BUSY`; it does not reserve the device or authorize a later +submission. + +Each node runs at most one check per job step. Tasks on another node may already +have started when a check fails. Later tasks on the same node compare their +configuration with the first task's snapshot before they reuse its result. +Different configuration rejects that task; earlier tasks may already have +started. The snapshot includes the device ID and all environment entries except +`SLURM_` and `SLURMD_` metadata. Provider configuration must be the same across +tasks and must not depend on those metadata variables. Environments larger than +128 KiB are rejected when validation is enabled. MPI and task plugins that add +different non-Slurm environment values can also cause a mismatch. + +Place the module after other plugins that set provider configuration. Slurm runs +task prologs after the validation hook, so task prologs must not change provider +configuration. This is a readiness snapshot of the inputs at the hook; the +application must still handle changes in device availability and credentials. + ## Submit a DDSIM job Save this program as `bell.py` in a location that all compute nodes can read: diff --git a/noxfile.py b/noxfile.py index 7705c730a5..bdf873c190 100755 --- a/noxfile.py +++ b/noxfile.py @@ -111,7 +111,7 @@ def cpp_lint(session: nox.Session) -> None: "--style=", "--tidy-checks=", f"--version={llvm_bin}", - "--ignore=build|!build/mlir/**|**/include|include|vendor/**|spank/**", + "--ignore=build|!build/mlir/**|**/include|include|vendor/**|spank/**|test/slurm/validation_environment.cpp", "--thread-comments=false", "--step-summary=false", "--database=build/cpp-lint", diff --git a/spank/Validation.hpp b/spank/Validation.hpp new file mode 100644 index 0000000000..95c03583d3 --- /dev/null +++ b/spank/Validation.hpp @@ -0,0 +1,220 @@ +/* + * Copyright (c) 2025 - 2026 Munich Quantum Software Company GmbH + * All rights reserved. + * + * SPDX-License-Identifier: GPL-3.0-or-later + * + * This program is free software: you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation, either version 3 of the License, or (at your + * option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General + * Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program. If not, see . + */ + +#pragma once + +#include +#include // NOLINT(modernize-deprecated-headers) +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace mqt::spank { + +class Validation final { + static constexpr size_t MAX_ENVIRONMENT_BYTES = 128UL * 1024; + enum class State : uint8_t { Empty, Copying, Checking, Available, Failed }; + struct Shared { + std::atomic state{State::Empty}; + size_t size = 0; + std::array snapshot{}; + }; + static_assert(std::atomic::is_always_lock_free); + +public: + Validation() = default; + Validation(const Validation&) = delete; + Validation& operator=(const Validation&) = delete; + Validation(Validation&&) = delete; + Validation& operator=(Validation&&) = delete; + + ~Validation() { + if (shared_ != nullptr) { + munmap(shared_, sizeof(Shared)); + } + } + + void prepare() { + auto* memory = mmap(nullptr, sizeof(Shared), PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_ANONYMOUS, -1, 0); + if (memory == MAP_FAILED) { + throw std::runtime_error("could not prepare launch validation"); + } + shared_ = std::construct_at(static_cast(memory)); + } + + void check(const std::string& executable, const int timeout, + const std::string& device, std::vector environment) { + if (shared_ == nullptr) { + throw std::runtime_error("launch validation was not prepared"); + } + std::vector ordered(environment.begin(), + environment.end()); + std::ranges::sort(ordered); + std::string snapshot = device + '\0'; + for (const auto entry : ordered) { + if (!entry.starts_with("SLURM_") && !entry.starts_with("SLURMD_")) { + snapshot += entry; + snapshot += '\0'; + } + if (snapshot.size() > MAX_ENVIRONMENT_BYTES) { + throw std::runtime_error("validation environment exceeds 128 KiB"); + } + } + + const auto deadline = + std::chrono::steady_clock::now() + std::chrono::seconds(timeout + 2); + auto expected = State::Empty; + if (shared_->state.compare_exchange_strong(expected, State::Copying)) { + std::memcpy(shared_->snapshot.data(), snapshot.data(), snapshot.size()); + shared_->size = snapshot.size(); + shared_->state.store(State::Checking, std::memory_order_release); + const bool available = execute(executable, timeout, device, environment); + shared_->state.store(available ? State::Available : State::Failed, + std::memory_order_release); + } + + while (true) { + const auto state = shared_->state.load(std::memory_order_acquire); + if (state >= State::Checking && + (shared_->size != snapshot.size() || + std::memcmp(shared_->snapshot.data(), snapshot.data(), + snapshot.size()) != 0)) { + throw std::runtime_error( + "provider configuration differs between tasks"); + } + if (state == State::Available) { + return; + } + if (state == State::Failed) { + throw std::runtime_error("QDMI launch validation failed"); + } + if (std::chrono::steady_clock::now() >= deadline) { + throw std::runtime_error("QDMI launch validation timed out"); + } + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + } + +private: + static bool execute(const std::string& executable, const int timeout, + const std::string& device, + std::vector& environment) { + const auto seconds = std::to_string(timeout); + // execve does not modify the strings despite its mutable argument types. + // NOLINTBEGIN(cppcoreguidelines-pro-type-const-cast) + std::array arguments{ + const_cast(executable.c_str()), const_cast("--device"), + const_cast(device.c_str()), const_cast("--timeout"), + const_cast(seconds.c_str()), static_cast(nullptr), + }; + // NOLINTEND(cppcoreguidelines-pro-type-const-cast) + std::vector envp; + envp.reserve(environment.size() + 1); + for (auto& entry : environment) { + envp.push_back(entry.data()); + } + envp.push_back(nullptr); + + const auto parent = getpid(); + const auto child = fork(); + if (child < 0) { + return false; + } + if (child == 0) { + // Slurm's caught handlers survive fork until execve resets them. + struct sigaction action{}; + action.sa_handler = SIG_DFL; + sigemptyset(&action.sa_mask); + sigset_t mask{}; + sigemptyset(&mask); + sigaddset(&mask, SIGTERM); + if (sigaction(SIGTERM, &action, nullptr) != 0 || + sigprocmask(SIG_UNBLOCK, &mask, nullptr) != 0) { + _exit(1); + } + // prctl uses the Linux variadic C ABI. + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg) + if (prctl(PR_SET_PDEATHSIG, SIGTERM) != 0 || getppid() != parent || + setpgid(0, 0) != 0) { + _exit(1); + } + // Never send provider or launcher diagnostics to Slurm's log stream. + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg) + const auto sink = open("/dev/null", O_RDWR); + if (sink < 0 || dup2(sink, STDIN_FILENO) < 0 || + dup2(sink, STDOUT_FILENO) < 0 || dup2(sink, STDERR_FILENO) < 0) { + _exit(1); + } + if (sink > STDERR_FILENO) { + close(sink); + } + execve(executable.c_str(), arguments.data(), envp.data()); + _exit(1); + } + static_cast(setpgid(child, child)); + const auto deadline = + std::chrono::steady_clock::now() + std::chrono::seconds(timeout + 1); + bool available = false; + while (true) { + siginfo_t status{}; + const auto result = waitid(P_PID, static_cast(child), &status, + WEXITED | WNOHANG | WNOWAIT); + if (result == 0 && status.si_pid == child) { + available = status.si_code == CLD_EXITED && status.si_status == 0; + break; + } + if ((result < 0 && errno != EINTR) || + std::chrono::steady_clock::now() >= deadline) { + static_cast(kill(child, SIGTERM)); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + break; + } + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + // Reap only after cleanup, so the checker PID cannot be reused. + static_cast(kill(-child, SIGKILL)); + static_cast(kill(child, SIGKILL)); + while (waitpid(child, nullptr, 0) < 0 && errno == EINTR) { + } + return available; + } + + Shared* shared_ = nullptr; +}; + +} // namespace mqt::spank diff --git a/spank/spank.cpp b/spank/spank.cpp index e398568564..a8b3dd0266 100644 --- a/spank/spank.cpp +++ b/spank/spank.cpp @@ -22,7 +22,10 @@ /// @brief Transport administrator-declared configuration references to QDMI /// jobs. +#include "Validation.hpp" + #include +#include #include #include #include @@ -30,6 +33,8 @@ #include #include #include +#include +#include #include extern "C" { @@ -159,6 +164,25 @@ class Configuration final { throw std::runtime_error("invalid catalogue reference"); } catalogueDefault_ = value; + } else if (key == "validate" && checker_.empty()) { + if (!validValue(value) || !value.starts_with('/')) { + throw std::runtime_error( + "validate requires an absolute checker path"); + } + checker_ = value; + } else if (key == "validation_timeout" && !hasTimeout_) { + // from_chars accepts the bounds of this string view as pointers. + // NOLINTBEGIN(cppcoreguidelines-pro-bounds-pointer-arithmetic) + const auto parsed = std::from_chars( + value.data(), value.data() + value.size(), timeout_); + if (parsed.ec != std::errc{} || + parsed.ptr != value.data() + value.size() || timeout_ < 1 || + timeout_ > 3600) { + throw std::runtime_error( + "validation_timeout must be 1 to 3600 seconds"); + } + // NOLINTEND(cppcoreguidelines-pro-bounds-pointer-arithmetic) + hasTimeout_ = true; } else if (key == "reference") { const auto nameEnd = value.find(':'); const auto idsEnd = @@ -199,6 +223,9 @@ class Configuration final { if (licenses_.empty()) { throw std::runtime_error("configure concrete license IDs with licenses="); } + if (hasTimeout_ && checker_.empty()) { + throw std::runtime_error("validation_timeout requires validate"); + } for (const auto& reference : references_) { for (const auto& id : reference.licenses) { if (std::ranges::find(licenses_, id) == licenses_.end()) { @@ -246,7 +273,35 @@ class Configuration final { references_[static_cast(option) - 1].option = argument; } - void inject(spank_t spank) const { + [[nodiscard]] bool validationEnabled() const { return !checker_.empty(); } + + void validate(spank_t spank, mqt::spank::Validation& validation, + const std::string& device) const { + // S_JOB_ENV requires a mutable char*** output parameter. + char** values = nullptr; // NOLINT(misc-const-correctness) + // Slurm exposes item lookup through a variadic C ABI. + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg) + if (spank_get_item(spank, S_JOB_ENV, &values) != ESPANK_SUCCESS || + values == nullptr) { + throw std::runtime_error("could not read the job environment"); + } + std::vector environment; + size_t bytes = 0; + // S_JOB_ENV is a borrowed, null-terminated array owned by Slurm. + // NOLINTBEGIN(cppcoreguidelines-pro-bounds-pointer-arithmetic) + for (size_t index = 0; values[index] != nullptr; ++index) { + std::string entry{values[index]}; + bytes += entry.size() + 1; + if (bytes > 128UL * 1024) { + throw std::runtime_error("validation environment exceeds 128 KiB"); + } + environment.push_back(std::move(entry)); + } + // NOLINTEND(cppcoreguidelines-pro-bounds-pointer-arithmetic) + validation.check(checker_, timeout_, device, std::move(environment)); + } + + auto inject(spank_t spank) const -> std::optional { const auto expression = jobEnvironment(spank, LICENSE_ENVIRONMENT); std::vector selected; if (expression) { @@ -275,7 +330,7 @@ class Configuration final { throw std::runtime_error( "QDMI options require a configured device license"); } - return; + return std::nullopt; } apply(spank, CATALOGUE_ENVIRONMENT, catalogueDefault_, catalogueOption_); @@ -294,6 +349,15 @@ class Configuration final { apply(spank, reference.environment.c_str(), reference.defaultValue, reference.option); } + if (!validationEnabled()) { + return std::nullopt; + } + if (selected.size() != 1 || + (*expression != selected.front() && + *expression != std::string{selected.front()} + ":1")) { + throw std::runtime_error("validation requires one unit device license"); + } + return std::string{selected.front()}; } private: @@ -326,12 +390,19 @@ class Configuration final { std::vector references_; std::optional catalogueDefault_; std::optional catalogueOption_; + std::string checker_; + int timeout_ = 30; + bool hasTimeout_ = false; }; Configuration configuration; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) bool licenseEnvironmentReady = true; +mqt::spank::Validation + validation; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) +bool validationReady = true; int optionCallback(const int value, const char* argument, int /*remote*/) { try { @@ -355,6 +426,7 @@ int slurm_spank_init(spank_t spank, const int count, char* arguments[]) { try { configuration = Configuration{}; licenseEnvironmentReady = true; + validationReady = true; configuration.parse(count, arguments); configuration.registerOptions(spank); return ESPANK_SUCCESS; @@ -376,6 +448,14 @@ int slurm_spank_user_init(spank_t spank, int /*count*/, char* /*arguments*/[]) { licenseEnvironmentReady = false; fail("could not clear inherited license selection"); } + if (configuration.validationEnabled()) { + try { + validation.prepare(); + } catch (...) { + validationReady = false; + fail("could not prepare launch validation"); + } + } } return ESPANK_SUCCESS; } @@ -388,7 +468,13 @@ int slurm_spank_task_init(spank_t spank, int /*count*/, char* /*arguments*/[]) { if (!licenseEnvironmentReady) { throw std::runtime_error("could not clear inherited license selection"); } - configuration.inject(spank); + const auto device = configuration.inject(spank); + if (device) { + if (!validationReady) { + throw std::runtime_error("could not prepare launch validation"); + } + configuration.validate(spank, validation, *device); + } return ESPANK_SUCCESS; } catch (const std::exception& error) { fail(error.what()); diff --git a/test/python/test_slurm_integration.py b/test/python/test_slurm_integration.py index 51152a543e..391e8cdc81 100644 --- a/test/python/test_slurm_integration.py +++ b/test/python/test_slurm_integration.py @@ -230,8 +230,10 @@ def controller(*command: str, **_kwargs: object) -> subprocess.CompletedProcess[ runner.test_provider(options) assert commands[0][:3] == ("env", "PROVIDER_PROFILE=profile", "MQT_CORE_QDMI_CONFIG_FILE=/opt/provider.qdmi.json") assert commands[1][0] == "srun" - assert commands[0][-2:] == commands[1][-2:] == ("python3", "/workload/probe.py") + assert commands[0][-2:] == commands[1][-2:] == commands[2][-2:] == ("python3", "/workload/probe.py") assert not configurations[0] assert "licenses=provider.device" in configurations[1] assert "reference=PROVIDER_PROFILE:provider.device:profile" in configurations[1] + assert "validate=" not in configurations[1] + assert configurations[2] == configurations[1].rstrip() + " validate=/usr/local/bin/mqt-core-qdmi-check\n" assert not (tmp_path / "plugstack.conf").read_text() diff --git a/test/slurm/Dockerfile b/test/slurm/Dockerfile index 5385875dba..2949aad70f 100644 --- a/test/slurm/Dockerfile +++ b/test/slurm/Dockerfile @@ -16,9 +16,12 @@ RUN apt-get update \ && DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ cmake g++ make libslurm-dev COPY --from=core spank /src/spank +COPY --from=core test/slurm/validation_environment.cpp /src/validation_environment.cpp RUN cmake -S /src/spank -B /build -DCMAKE_BUILD_TYPE=Release \ && cmake --build /build --parallel 2 \ - && cmake --install /build --prefix /usr/local + && cmake --install /build --prefix /usr/local \ + && g++ -std=c++20 -fPIC -shared /src/validation_environment.cpp \ + -o /usr/local/lib/slurm/mqt-test-validation-environment.so FROM ubuntu:26.04 AS base @@ -37,6 +40,7 @@ RUN apt-get update \ COPY --from=uv /uv /uvx /bin/ COPY --from=spank /usr/local/lib/slurm/mqt-core-qdmi-spank.so /usr/local/lib/slurm/ +COPY --from=spank /usr/local/lib/slurm/mqt-test-validation-environment.so /usr/local/lib/slurm/ COPY --from=spank /usr/local/share/licenses/mqt-core-qdmi-spank /usr/local/share/licenses/mqt-core-qdmi-spank COPY mqt-slurm-prepare /usr/local/sbin/mqt-slurm-prepare COPY mqt-slurm-install-provider /usr/local/sbin/mqt-slurm-install-provider diff --git a/test/slurm/checker_probe.py b/test/slurm/checker_probe.py new file mode 100755 index 0000000000..08aed183e8 --- /dev/null +++ b/test/slurm/checker_probe.py @@ -0,0 +1,52 @@ +#!/usr/bin/python3 +# Copyright (c) 2023 - 2026 Chair for Design Automation, TUM +# Copyright (c) 2025 - 2026 Munich Quantum Software Company GmbH +# All rights reserved. +# +# SPDX-License-Identifier: MIT +# +# Licensed under the MIT License + +"""Observe checker execution inside real Slurm tasks using non-secret test data.""" + +from __future__ import annotations + +import json +import os +import pathlib +import signal +import subprocess +import sys +import time + +mode = os.environ.get("MQT_SLURM_CHECKER_MODE", "check") +child = None +if mode == "hang": + signal.signal(signal.SIGTERM, signal.SIG_IGN) + child = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(300)"]) + +record = { + "uid": os.getuid(), + "euid": os.geteuid(), + "gid": os.getgid(), + "groups": os.getgroups(), + "job": os.environ["SLURM_JOB_ID"], + "step": os.environ.get("SLURM_STEP_ID", "batch"), + "node": os.environ["SLURMD_NODENAME"], + "pid": os.getpid(), + "child": child.pid if child else None, + "reference": os.environ.get("MQT_SLURM_TEST_REFERENCE"), + "catalogue": os.environ.get("MQT_CORE_QDMI_CONFIG_FILE"), + "arguments": sys.argv[1:], +} +with pathlib.Path(os.environ["MQT_SLURM_CHECKER_LOG"]).open("a", encoding="utf-8") as output: + output.write(json.dumps(record) + "\n") + +if mode == "hang": + time.sleep(300) +elif mode == "fail": + sys.stderr.write("test provider credential must stay private\n") + sys.exit(1) +else: + executable = "/usr/local/bin/mqt-core-qdmi-check" + os.execv(executable, [executable, *sys.argv[1:]]) # ruff: ignore[start-process-with-no-shell] diff --git a/test/slurm/run_integration.py b/test/slurm/run_integration.py index 5cf05a1490..ead408672d 100644 --- a/test/slurm/run_integration.py +++ b/test/slurm/run_integration.py @@ -464,7 +464,7 @@ def parse_arguments(arguments: Sequence[str]) -> argparse.Namespace: def test_provider(options: argparse.Namespace) -> None: - """Run the same provider workload with direct configuration and SPANK.""" + """Run provider workloads with direct configuration, injection, and validation.""" environment = list(options.reference) if options.qdmi_config_file: environment.append(f"MQT_CORE_QDMI_CONFIG_FILE={options.qdmi_config_file}") @@ -480,8 +480,9 @@ def test_provider(options: argparse.Namespace) -> None: for reference in options.reference: name, _, value = reference.partition("=") configuration.append(f"reference={name}:{options.device_license}:{value}") - (RUNTIME / "plugstack.conf").write_text(" ".join(configuration) + "\n", encoding="utf-8") - job(*allocation, *options.command, timeout=300) + for validation in ("", " validate=/usr/local/bin/mqt-core-qdmi-check"): + (RUNTIME / "plugstack.conf").write_text(" ".join(configuration) + validation + "\n", encoding="utf-8") + job(*allocation, *options.command, timeout=300) (RUNTIME / "plugstack.conf").write_text("", encoding="utf-8") @@ -625,6 +626,186 @@ def values(*arguments: str) -> list[str | None]: (RUNTIME / "plugstack.conf").write_text("", encoding="utf-8") +def test_spank_validation() -> None: + """Prove bounded readiness checks at the real, unprivileged task hook.""" + selected = "mqt.sc.default" + log = RUNTIME / "jobs" / "checker.jsonl" + body = RUNTIME / "jobs" / "validated-body" + catalogue = "/jobs/validation.qdmi.json" + (RUNTIME / "jobs" / "validation.qdmi.json").write_text( + '{"schema-version":1,"qdmi":{"devices":[]}}', encoding="utf-8" + ) + configuration = ( + "required /usr/local/lib/slurm/mqt-core-qdmi-spank.so " + f"licenses={selected},mqt.ddsim.default qdmi_config_file={catalogue} " + f"reference=MQT_SLURM_TEST_REFERENCE:{selected}:site-value" + ) + probe = "/workspace/test/slurm/checker_probe.py" + environment = ( + "env", + "MQT_SLURM_CHECKER_LOG=/jobs/checker.jsonl", + "MQT_SLURM_TEST_REFERENCE=job-value", + ) + allocation = ("srun", "--immediate=5", "--time=1", "--nodes=2", "--ntasks=4", f"--licenses={selected}") + program = "from pathlib import Path; Path('/jobs/validated-body').touch()" + + def calls() -> list[dict[str, Any]]: + return [json.loads(line) for line in log.read_text(encoding="utf-8").splitlines()] if log.exists() else [] + + def configure(checker: str | None = probe) -> None: + validation = f" validate={checker} validation_timeout=3" if checker else "" + (RUNTIME / "plugstack.conf").write_text(configuration + validation + "\n", encoding="utf-8") + log.unlink(missing_ok=True) + body.unlink(missing_ok=True) + + def assert_released() -> None: + for node in ("node1", "node2"): + wait_for(f"{node} to return to IDLE after validation", lambda node=node: node_is_idle(node)) + assert "DRAIN" not in node_record(node) + assert_license(selected, total=1, used=0, free=1) + + def assert_no_processes(records: list[dict[str, Any]]) -> None: + for record in records: + for pid in (record["pid"], record["child"]): + if pid is not None: + compute( + record["node"], + "python3", + "-c", + "from pathlib import Path; " + f"p=Path('/proc/{pid}/stat'); " + "assert not p.exists() or p.read_text().split(') ', 1)[1].startswith('Z ')", + ) + + configure(None) + job(*environment, *allocation, "python3", "-c", program, timeout=60) + assert body.exists() + assert not calls() + assert_released() + + configure() + job(*environment, *allocation, "python3", "-c", program, timeout=60) + records = calls() + assert body.exists() + assert len(records) == 2, records + assert {record["node"] for record in records} == {"node1", "node2"} + for record in records: + assert record["uid"] == record["euid"] == record["gid"] == 10000, record + assert record["reference"] == "job-value", record + assert record["catalogue"] == catalogue, record + assert record["arguments"] == ["--device", selected, "--timeout", "3"], record + assert_released() + + configure() + plugstack = RUNTIME / "plugstack.conf" + plugstack.write_text( + "required /usr/local/lib/slurm/mqt-test-validation-environment.so\n" + plugstack.read_text(encoding="utf-8"), + encoding="utf-8", + ) + result = job(*environment, *allocation, "/bin/true", check=False, timeout=60) + assert result.returncode != 0 + assert "provider configuration differs between tasks" in result.stdout + result.stderr + assert len(calls()) <= 2 + assert_released() + + configure() + job( + *environment, + "sbatch", + "--wait", + "--time=1", + "--nodes=2", + "--ntasks=4", + f"--licenses={selected}", + "--output=/jobs/validator-batch.out", + "--wrap", + shlex.join(("srun", "--ntasks=4", "python3", "-c", program)), + timeout=120, + ) + records = calls() + assert body.exists() + assert len(records) == 3, records + assert len({(record["node"], record["step"]) for record in records}) == 3, records + assert_released() + + for mode in ("fail", "hang"): + configure() + started = time.monotonic() + result = job( + *environment, + f"MQT_SLURM_CHECKER_MODE={mode}", + *allocation, + "python3", + "-c", + program, + check=False, + timeout=60, + ) + assert result.returncode != 0 + assert not body.exists() + assert "test provider credential" not in result.stdout + result.stderr + assert time.monotonic() - started < 15 + records = calls() + assert len(records) == 2, records + assert_released() + assert_no_processes(records) + + for checker, extra in (("/missing/mqt-core-qdmi-check", ()), (probe, ("MQT_CORE_QDMI_CONFIG_JSON=invalid",))): + configure(checker) + result = job(*environment, *extra, *allocation, "python3", "-c", program, check=False, timeout=60) + assert result.returncode != 0 + assert not body.exists() + assert_released() + + configure() + job(*environment, "SLURM_JOB_LICENSES=mqt.sc.default", "srun", "--time=1", "--ntasks=1", "/bin/true") + assert not calls() + + configure() + result = job( + *environment, + "python3", + "-c", + "import os, sys; os.environ['LARGE_ONE']='x'*65500; os.environ['LARGE_TWO']='x'*65500; " + "os.execvp(sys.argv[1], sys.argv[1:])", + *allocation, + "python3", + "-c", + program, + check=False, + timeout=60, + ) + assert result.returncode != 0 + assert "validation environment exceeds 128 KiB" in result.stdout + result.stderr + assert not calls() + assert not body.exists() + assert_released() + + configure() + submitted = ( + job( + *environment, + "MQT_SLURM_CHECKER_MODE=hang", + "sbatch", + "--parsable", + "--time=1", + f"--licenses={selected}", + "--output=/jobs/validator-cancel.out", + "--wrap", + shlex.join(("python3", "-c", program)), + ) + .stdout.strip() + .split(";", maxsplit=1)[0] + ) + wait_for("the cancellable checker to start", lambda: bool(calls())) + job("scancel", submitted) + wait_for("the validation job to be cancelled", lambda: job_finished(submitted, expected_state="CANCELLED")) + assert not body.exists() + assert_released() + assert_no_processes(calls()) + (RUNTIME / "plugstack.conf").write_text("", encoding="utf-8") + + def main(arguments: Sequence[str] = ()) -> None: """Build the shared cluster and run its Core or provider workload.""" options = parse_arguments(arguments) @@ -700,6 +881,8 @@ def main(arguments: Sequence[str] = ()) -> None: else: test_core() test_spank_transport() + if not options.command: + test_spank_validation() success = True LOGGER.info("Slurm admission and execution checks: %.2fs", time.monotonic() - testing_at) diff --git a/test/slurm/validation_environment.cpp b/test/slurm/validation_environment.cpp new file mode 100644 index 0000000000..f6af0951b3 --- /dev/null +++ b/test/slurm/validation_environment.cpp @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2025 - 2026 Munich Quantum Software Company GmbH + * All rights reserved. + * + * SPDX-License-Identifier: GPL-3.0-or-later + * + * This program is free software: you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation, either version 3 of the License, or (at your + * option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General + * Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program. If not, see . + */ + +extern "C" { +#include +#include +} + +// This fixture creates different provider inputs before the production hook. +// Names and signatures are fixed by Slurm's plugin ABI. +// NOLINTBEGIN(readability-identifier-naming, misc-use-internal-linkage, +// cppcoreguidelines-avoid-c-arrays) +extern "C" { +extern const char plugin_name[] = "mqt_test_validation_environment"; +extern const char plugin_type[] = "spank"; +extern const unsigned int plugin_version = SLURM_VERSION_NUMBER; +extern const unsigned int spank_plugin_version = 1; + +int slurm_spank_task_init(spank_t spank, int /*count*/, char* /*arguments*/[]) { + int task = 0; + // Slurm exposes item lookup through a variadic C ABI. + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg) + if (spank_get_item(spank, S_TASK_ID, &task) != ESPANK_SUCCESS) { + return -1; + } + if (task == 1 && spank_setenv(spank, "MQT_SLURM_TEST_REFERENCE", + "task-specific", 1) != ESPANK_SUCCESS) { + return -1; + } + return ESPANK_SUCCESS; +} +} +// NOLINTEND(readability-identifier-naming, misc-use-internal-linkage, +// cppcoreguidelines-avoid-c-arrays) From 7d06dd2b8a963d2a5652ef4996ebd82fd58fa5e6 Mon Sep 17 00:00:00 2001 From: flowerthrower Date: Tue, 22 Sep 2026 12:48:57 +0200 Subject: [PATCH 2/6] =?UTF-8?q?=F0=9F=93=9D=20Record=20completed=20provide?= =?UTF-8?q?r=20validation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Assisted-by: GPT-6 via Codex Signed-off-by: flowerthrower --- .agent/plans/slurm-launch-validation.md | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/.agent/plans/slurm-launch-validation.md b/.agent/plans/slurm-launch-validation.md index f051aefb40..69a46825b2 100644 --- a/.agent/plans/slurm-launch-validation.md +++ b/.agent/plans/slurm-launch-validation.md @@ -1,7 +1,6 @@ # Optional Slurm launch validation -Status: implemented and validated on real Slurm; final provider image checks -remain in progress. +Status: complete. ## Goal and scope @@ -26,11 +25,6 @@ Slurm daemons and preserve static injection when validation is disabled. caught signal handlers remain installed in a fork child until exec resets them. -## Work remaining - -- Complete both provider workloads with direct configuration, shared injection, - and optional validation against the smaller native and wheel images. - ## Validation The existing runner unit tests pass (22 tests), as do full repository lint, @@ -45,3 +39,8 @@ task environment, and process cleanup. Cases cover disabled validation, multiple nodes/tasks, batch and nested steps, failures, timeouts, cancellation, differing task inputs, and bounded storage. Static injection and provider migrations do not depend on this optional PR. + +Braket and IQM passed direct configuration, shared injection, and enabled +validation with both native and wheel installations. These runs used released +Core 4.0.0, the installed native checker, local mock services, and a 4 GiB +Docker RAM cap. No live provider jobs were submitted. From 4f5edee3953e283a0297241d43f918c4e83b3170 Mon Sep 17 00:00:00 2001 From: flowerthrower Date: Tue, 22 Sep 2026 18:09:33 +0200 Subject: [PATCH 3/6] =?UTF-8?q?=F0=9F=94=A7=20Apply=20current=20validation?= =?UTF-8?q?=20formatting?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Qualify earlier validation evidence after rebasing onto main. Assisted-by: GPT-6 via Codex Signed-off-by: flowerthrower --- .agent/plans/slurm-launch-validation.md | 8 +++++++- spank/Validation.hpp | 15 +++++++-------- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/.agent/plans/slurm-launch-validation.md b/.agent/plans/slurm-launch-validation.md index 69a46825b2..0063947565 100644 --- a/.agent/plans/slurm-launch-validation.md +++ b/.agent/plans/slurm-launch-validation.md @@ -25,7 +25,7 @@ Slurm daemons and preserve static injection when validation is disabled. caught signal handlers remain installed in a fork child until exec resets them. -## Validation +## Validation before the main update The existing runner unit tests pass (22 tests), as do full repository lint, standalone Linux compilation, and full-file clang-tidy 23 checks. The installed @@ -44,3 +44,9 @@ Braket and IQM passed direct configuration, shared injection, and enabled validation with both native and wheel installations. These runs used released Core 4.0.0, the installed native checker, local mock services, and a 4 GiB Docker RAM cap. No live provider jobs were submitted. + +## Validation after the main update + +The shared runner's 22 tests and full repository lint pass. The new formatting +rules reorder the validation header's includes without changing its behavior. +Refreshed Linux and provider integration results are recorded in the PR. diff --git a/spank/Validation.hpp b/spank/Validation.hpp index 95c03583d3..690dc7d691 100644 --- a/spank/Validation.hpp +++ b/spank/Validation.hpp @@ -20,14 +20,6 @@ #pragma once -#include -#include // NOLINT(modernize-deprecated-headers) -#include -#include -#include -#include -#include - #include #include #include @@ -36,11 +28,18 @@ #include #include #include +#include #include +#include // NOLINT(modernize-deprecated-headers) #include #include #include +#include +#include +#include +#include #include +#include #include namespace mqt::spank { From 4899499ad800962c7d5ce85d8238f61d8ff55ad1 Mon Sep 17 00:00:00 2001 From: flowerthrower Date: Wed, 23 Sep 2026 13:08:03 +0200 Subject: [PATCH 4/6] =?UTF-8?q?=F0=9F=A7=B9=20Remove=20Slurm=20validation?= =?UTF-8?q?=20implementation=20plan?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Assisted-by: GPT-6 via Codex Signed-off-by: flowerthrower Assisted-by: GPT 5.6 via Codex --- .agent/plans/slurm-launch-validation.md | 52 ------------------------- 1 file changed, 52 deletions(-) delete mode 100644 .agent/plans/slurm-launch-validation.md diff --git a/.agent/plans/slurm-launch-validation.md b/.agent/plans/slurm-launch-validation.md deleted file mode 100644 index 0063947565..0000000000 --- a/.agent/plans/slurm-launch-validation.md +++ /dev/null @@ -1,52 +0,0 @@ -# Optional Slurm launch validation - -Status: complete. - -## Goal and scope - -Add opt-in readiness validation to the shared SPANK component. Execute the MIT -QDMI checker as the job user before task launch. Keep provider code outside -Slurm daemons and preserve static injection when validation is disabled. - -## Decisions - -- Validate in remote `task_init`, after Slurm restores allocated licenses and - permanently drops privileges. Return a task failure, never a node failure. -- Copy the final `S_JOB_ENV` for `execve`; do not inherit daemon credentials. -- Check once per node and step through an anonymous shared gate created before - task forks. Compare later tasks' effective configuration with the first task's - snapshot. Reject a mismatch without claiming that earlier tasks did not start. -- Provider configuration must be step-wide. Slurm rank metadata is excluded from - snapshot comparison and is unsupported as provider configuration input. -- Bound environment storage, checker execution, and waiting for another task's - result. Do not add controller RPCs, a persistent cache, or an all-task - barrier. -- Reset and unblock the checker's parent-death signal before exec. Slurm's - caught signal handlers remain installed in a fork child until exec resets - them. - -## Validation before the main update - -The existing runner unit tests pass (22 tests), as do full repository lint, -standalone Linux compilation, and full-file clang-tidy 23 checks. The installed -Linux checker passed its forced-parent-death case. Native configuration -correctly rejects the macOS host before compiling this Linux-only component. The -full strict documentation build passed with MLIR 23.1.0. - -The full real Slurm fixture passed on Linux aarch64 with cgroup v2 and Slurm -25.11.2. It verifies hook ordering, task launch, license release, node state, -task environment, and process cleanup. Cases cover disabled validation, multiple -nodes/tasks, batch and nested steps, failures, timeouts, cancellation, differing -task inputs, and bounded storage. Static injection and provider migrations do -not depend on this optional PR. - -Braket and IQM passed direct configuration, shared injection, and enabled -validation with both native and wheel installations. These runs used released -Core 4.0.0, the installed native checker, local mock services, and a 4 GiB -Docker RAM cap. No live provider jobs were submitted. - -## Validation after the main update - -The shared runner's 22 tests and full repository lint pass. The new formatting -rules reorder the validation header's includes without changing its behavior. -Refreshed Linux and provider integration results are recorded in the PR. From 5bee90b68cd580428fc37af5627d2bc2e7015227 Mon Sep 17 00:00:00 2001 From: flowerthrower Date: Wed, 23 Sep 2026 15:08:25 +0200 Subject: [PATCH 5/6] =?UTF-8?q?=F0=9F=90=9B=20Validate=20tasks=20when=20la?= =?UTF-8?q?unch=20environments=20differ?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 🤖 *AI text below* 🤖 Treat a changed task environment as a cache miss. Reuse the existing bounded checker so ordinary MPI ranks and task-specific provider inputs can launch independently. Keep cached results for matching inputs. Extend the real Slurm fixture with PMI2 success, failure, and timeout coverage, and update the existing configuration-variation assertion. Assisted-by: GPT-6 via Codex Signed-off-by: flowerthrower Assisted-by: GPT 5.6 via Codex --- docs/qdmi/slurm.md | 24 ++++++++++-------------- spank/Validation.hpp | 7 ++++--- test/slurm/run_integration.py | 18 +++++++++++++----- 3 files changed, 27 insertions(+), 22 deletions(-) diff --git a/docs/qdmi/slurm.md b/docs/qdmi/slurm.md index 2a7129e71c..9c572254b2 100644 --- a/docs/qdmi/slurm.md +++ b/docs/qdmi/slurm.md @@ -214,20 +214,16 @@ must name one configured device with an implicit count or `:1`. The module runs the checker as the job user with a copy of the job environment after configuration injection. It does not load a provider in a Slurm daemon. An -unavailable device, checker failure, or timeout prevents matching tasks on that -node from starting and does not drain the node. The checker still accepts both -`IDLE` and `BUSY`; it does not reserve the device or authorize a later -submission. - -Each node runs at most one check per job step. Tasks on another node may already -have started when a check fails. Later tasks on the same node compare their -configuration with the first task's snapshot before they reuse its result. -Different configuration rejects that task; earlier tasks may already have -started. The snapshot includes the device ID and all environment entries except -`SLURM_` and `SLURMD_` metadata. Provider configuration must be the same across -tasks and must not depend on those metadata variables. Environments larger than -128 KiB are rejected when validation is enabled. MPI and task plugins that add -different non-Slurm environment values can also cause a mismatch. +unavailable device, checker failure, or timeout prevents tasks using that check +from starting and does not drain the node. The checker still accepts both `IDLE` +and `BUSY`; it does not reserve the device or authorize a later submission. + +Each node caches the first check per job step. Tasks with matching configuration +reuse its result; tasks with different configuration run their own bounded +check. Other tasks may already have started when a check fails. The comparison +includes the device ID and all environment entries except `SLURM_` and `SLURMD_` +metadata. Provider configuration must not depend on those metadata variables. +Environments larger than 128 KiB are rejected when validation is enabled. Place the module after other plugins that set provider configuration. Slurm runs task prologs after the validation hook, so task prologs must not change provider diff --git a/spank/Validation.hpp b/spank/Validation.hpp index 690dc7d691..1c8e89e4c1 100644 --- a/spank/Validation.hpp +++ b/spank/Validation.hpp @@ -108,13 +108,14 @@ class Validation final { } while (true) { - const auto state = shared_->state.load(std::memory_order_acquire); + auto state = shared_->state.load(std::memory_order_acquire); if (state >= State::Checking && (shared_->size != snapshot.size() || std::memcmp(shared_->snapshot.data(), snapshot.data(), snapshot.size()) != 0)) { - throw std::runtime_error( - "provider configuration differs between tasks"); + state = execute(executable, timeout, device, environment) + ? State::Available + : State::Failed; } if (state == State::Available) { return; diff --git a/test/slurm/run_integration.py b/test/slurm/run_integration.py index ead408672d..18196444a9 100644 --- a/test/slurm/run_integration.py +++ b/test/slurm/run_integration.py @@ -696,16 +696,23 @@ def assert_no_processes(records: list[dict[str, Any]]) -> None: assert record["arguments"] == ["--device", selected, "--timeout", "3"], record assert_released() + configure() + job(*environment, *allocation, "--mpi=pmi2", "python3", "-c", program, timeout=60) + assert body.exists() + assert len(calls()) == 4, calls() + assert_released() + configure() plugstack = RUNTIME / "plugstack.conf" plugstack.write_text( "required /usr/local/lib/slurm/mqt-test-validation-environment.so\n" + plugstack.read_text(encoding="utf-8"), encoding="utf-8", ) - result = job(*environment, *allocation, "/bin/true", check=False, timeout=60) - assert result.returncode != 0 - assert "provider configuration differs between tasks" in result.stdout + result.stderr - assert len(calls()) <= 2 + job(*environment, *allocation, "python3", "-c", program, timeout=60) + assert body.exists() + records = calls() + assert len(records) == 4, records + assert {record["reference"] for record in records} == {"job-value", "task-specific"} assert_released() configure() @@ -735,6 +742,7 @@ def assert_no_processes(records: list[dict[str, Any]]) -> None: *environment, f"MQT_SLURM_CHECKER_MODE={mode}", *allocation, + "--mpi=pmi2", "python3", "-c", program, @@ -746,7 +754,7 @@ def assert_no_processes(records: list[dict[str, Any]]) -> None: assert "test provider credential" not in result.stdout + result.stderr assert time.monotonic() - started < 15 records = calls() - assert len(records) == 2, records + assert len(records) == 4, records assert_released() assert_no_processes(records) From 4bbfa3063c79605dd429427235c0751550437e0c Mon Sep 17 00:00:00 2001 From: flowerthrower Date: Tue, 6 Oct 2026 09:48:36 +0200 Subject: [PATCH 6/6] =?UTF-8?q?=F0=9F=A7=AA=20Distinguish=20Slurm=20cancel?= =?UTF-8?q?lation=20from=20timeout?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 🤖 *AI text below* 🤖 Keep the checker and batch time limits beyond the cancellation observation window. Fold MPI success into the existing identity and environment assertions, and clarify who disables site validation. Assisted-by: GPT-6 via Codex Signed-off-by: flowerthrower Assisted-by: GPT 5.6 via Codex --- docs/qdmi/slurm.md | 9 +++++---- test/slurm/run_integration.py | 16 +++++----------- 2 files changed, 10 insertions(+), 15 deletions(-) diff --git a/docs/qdmi/slurm.md b/docs/qdmi/slurm.md index 97a7862f0e..7be4323f20 100644 --- a/docs/qdmi/slurm.md +++ b/docs/qdmi/slurm.md @@ -234,8 +234,8 @@ must name one configured device with an implicit count or `:1`. The module runs the checker as the job user with the job environment after configuration injection. It does not load a provider in a Slurm daemon. An unavailable device, checker failure, or timeout prevents tasks using that check -from starting and does not drain the node. The checker still accepts both `IDLE` -and `BUSY`; it does not reserve the device or authorize a later submission. +from starting and does not drain the node. The checker accepts both `IDLE` and +`BUSY`; it does not reserve the device or authorize a later submission. Each node runs one check per job step and shares the result across its tasks. Tasks on another node may already have started when a check fails. All tasks @@ -245,8 +245,9 @@ set these inputs. Task prologs run later and must not change them. The configured checker must use the workload's provider runtime. It cannot validate a virtual environment, module, credentials, or container activated -later in a batch script. For such jobs, leave automatic validation disabled and -run the checker inside the chosen environment after setup, for example: +later in a batch script. For such jobs, administrators must leave automatic +validation disabled. Job authors can run the checker inside the chosen +environment after setup, for example: ```bash #!/bin/bash diff --git a/test/slurm/run_integration.py b/test/slurm/run_integration.py index e192cd9fbc..eaf4b12173 100644 --- a/test/slurm/run_integration.py +++ b/test/slurm/run_integration.py @@ -622,8 +622,8 @@ def test_spank_validation() -> None: def calls() -> list[dict[str, Any]]: return [json.loads(line) for line in log.read_text(encoding="utf-8").splitlines()] if log.exists() else [] - def configure(checker: str | None = probe) -> None: - validation = f" validate={checker} validation_timeout=3" if checker else "" + def configure(checker: str | None = probe, timeout: int = 3) -> None: + validation = f" validate={checker} validation_timeout={timeout}" if checker else "" (RUNTIME / "plugstack.conf").write_text(configuration + validation + "\n", encoding="utf-8") log.unlink(missing_ok=True) body.unlink(missing_ok=True) @@ -654,7 +654,7 @@ def assert_no_processes(records: list[dict[str, Any]]) -> None: assert_released() configure() - job(*environment, *allocation, "python3", "-c", program, timeout=60) + job(*environment, *allocation, "--mpi=pmi2", "python3", "-c", program, timeout=60) records = calls() assert body.exists() assert len(records) == 2, records @@ -666,12 +666,6 @@ def assert_no_processes(records: list[dict[str, Any]]) -> None: assert record["arguments"] == ["--device", selected, "--timeout", "3"], record assert_released() - configure() - job(*environment, *allocation, "--mpi=pmi2", "python3", "-c", program, timeout=60) - assert body.exists() - assert len(calls()) == 2, calls() - assert_released() - configure() job( *environment, @@ -726,14 +720,14 @@ def assert_no_processes(records: list[dict[str, Any]]) -> None: job(*environment, "SLURM_JOB_LICENSES=mqt.sc.default", "srun", "--time=1", "--ntasks=1", "/bin/true") assert not calls() - configure() + configure(timeout=300) submitted = ( job( *environment, "MQT_SLURM_CHECKER_MODE=hang", "sbatch", "--parsable", - "--time=1", + "--time=5", f"--licenses={selected}", "--output=/jobs/validator-cancel.out", "--wrap",