diff --git a/.containerignore b/.containerignore new file mode 100644 index 00000000..8443b16e --- /dev/null +++ b/.containerignore @@ -0,0 +1,26 @@ +# Podman/Docker ignore file +# Excludes files from being copied into container images during build + +# Exclude the saved images tarball (6+ GB) +spindle-podman-images.tar + +# Exclude build artifacts +workspace-Spindle/build/ +workspace-Spindle/install/ +*.o +*.a +*.so + +# Exclude git metadata +.git/ +.gitignore + +# Exclude test output +out.* +*.log + +# Exclude investigation directory +podman-concurrency-investigation/ + +# Exclude debugging artifacts +debugging/ diff --git a/.dockerignore b/.dockerignore new file mode 120000 index 00000000..092a75da --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +.containerignore \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index be44ec15..bbe6725f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,10 +25,10 @@ jobs: timeout-minutes: 20 steps: - name: Check out Spindle - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 - name: Setup Docker Compose - uses: docker/setup-compose-action@16feee727cbdc83b6a014e6cc26fec4a79bcf30c + uses: docker/setup-compose-action@4eb059ff7f16592f9c84d5ca339c53cb7c5064e2 with: version: latest @@ -78,10 +78,10 @@ jobs: timeout-minutes: 30 steps: - name: Check out Spindle - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 - name: Setup Docker Compose - uses: docker/setup-compose-action@16feee727cbdc83b6a014e6cc26fec4a79bcf30c + uses: docker/setup-compose-action@4eb059ff7f16592f9c84d5ca339c53cb7c5064e2 with: version: latest @@ -131,10 +131,10 @@ jobs: timeout-minutes: 20 steps: - name: Check out Spindle - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 - name: Setup Docker Compose - uses: docker/setup-compose-action@16feee727cbdc83b6a014e6cc26fec4a79bcf30c + uses: docker/setup-compose-action@4eb059ff7f16592f9c84d5ca339c53cb7c5064e2 with: version: latest @@ -184,16 +184,16 @@ jobs: timeout-minutes: 20 steps: - name: Check out Spindle - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 - name: Setup Docker Compose - uses: docker/setup-compose-action@16feee727cbdc83b6a014e6cc26fec4a79bcf30c + uses: docker/setup-compose-action@4eb059ff7f16592f9c84d5ca339c53cb7c5064e2 with: version: latest - name: Login to GitHub Container Registry if: ${{ !env.ACT }} - uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f with: registry: ghcr.io username: ${{ github.actor }} @@ -251,16 +251,16 @@ jobs: timeout-minutes: 20 steps: - name: Check out Spindle - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 - name: Setup Docker Compose - uses: docker/setup-compose-action@16feee727cbdc83b6a014e6cc26fec4a79bcf30c + uses: docker/setup-compose-action@4eb059ff7f16592f9c84d5ca339c53cb7c5064e2 with: version: latest - name: Login to GitHub Container Registry if: ${{ !env.ACT }} - uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f with: registry: ghcr.io username: ${{ github.actor }} @@ -309,16 +309,16 @@ jobs: timeout-minutes: 20 steps: - name: Check out Spindle - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 - name: Setup Docker Compose - uses: docker/setup-compose-action@16feee727cbdc83b6a014e6cc26fec4a79bcf30c + uses: docker/setup-compose-action@4eb059ff7f16592f9c84d5ca339c53cb7c5064e2 with: version: latest - name: Login to GitHub Container Registry if: ${{ !env.ACT }} - uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f with: registry: ghcr.io username: ${{ github.actor }} diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml index 41cef5c7..3bb473c6 100644 --- a/.github/workflows/container.yml +++ b/.github/workflows/container.yml @@ -11,20 +11,20 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 + uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c - name: Login to GitHub Container Registry - uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build & Push Slurm Base Image - uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a with: context: ./containers/spindle-slurm-ubuntu/base platforms: linux/amd64 diff --git a/.gitignore b/.gitignore index 626ce7fa..a2ac4612 100644 --- a/.gitignore +++ b/.gitignore @@ -35,3 +35,16 @@ run_driver run_driver_rm preload_file_list build + +# Generated by generate_config.sh +containers/spindle-slurm-ubuntu/testing/conf/slurmdbd.conf +containers/spindle-slurm-ubuntu/testing/mariadb.env +containers/spindle-slurm-ubuntu/testing-srun/conf/slurmdbd.conf +containers/spindle-slurm-ubuntu/testing-srun/mariadb.env + +# spindle-podman +.podman-state.json +.podman-metrics.log +.podman-logs + + diff --git a/DEBUGGING.txt b/DEBUGGING.txt new file mode 100644 index 00000000..27628203 --- /dev/null +++ b/DEBUGGING.txt @@ -0,0 +1,21 @@ +# Build +podman rm -f spindlenode-interactive +./scripts/podman/build-spindle-serial-interactive.sh +# Start +./scripts/podman/run-spindle-serial-interactive.sh +# Connect +podman exec -it spindlenode-interactive bash +# Enable core dumps +ulimit -c unlimited +# Set up spindle config file +mkdir -p /home/spindleuser/Spindle-inst/etc/spindle +cat > /home/spindleuser/Spindle-inst/etc/spindle/spindle.conf <<'EOF' +noclean = true +EOF +# cd +cd Spindle-build/testsuite +# Run the problematic test +SPINDLE_DEBUG=3 SPINDLE_FLAGS="--noclean" ./run_driver --thrdopen --push +# Core file analysis +RELOCATED=$(find /tmp/commpath/cachepath -name "*test_driver" -type f | grep -v "\.so" | head -1) +gdb ${RELOCATED} /tmp/core.test_driver.$(ls -t *.core | head -1 ) diff --git a/containers/spindle-flux-ubuntu/Dockerfile.podman b/containers/spindle-flux-ubuntu/Dockerfile.podman new file mode 100644 index 00000000..56862c53 --- /dev/null +++ b/containers/spindle-flux-ubuntu/Dockerfile.podman @@ -0,0 +1,97 @@ +# This is based on the Flux Container Tutorial +# See https://flux-framework.readthedocs.io/en/latest/tutorials/containers +ARG flux_sched_version=noble-v0.48.0-amd64 +FROM fluxrm/flux-sched:${flux_sched_version} AS builder +ARG replicas=4 +ENV workers=${replicas} +USER root +ENV TMPDIR=/tmp +RUN echo 'TMPDIR="/tmp"' >> /etc/environment +ENV SPINDLE_TEST_CONTAINER=1 + +# LC-specific fix for podman setgroups issue +ARG PODMAN_BUILD=false +RUN if [ "$PODMAN_BUILD" = "true" ]; then \ + echo 'APT::Sandbox::User root;' > /etc/apt/apt.conf.d/00-apt-sandbox; \ + fi + +RUN DEBIAN_FRONTEND="noninteractive" apt-get update \ + && apt-get -qq install -y --no-install-recommends \ + autotools-dev \ + autoconf \ + automake \ + cmake \ + git \ + python3 \ + openssh-server \ + openssh-client \ + libdb-dev \ + apt-utils \ + dnsutils \ + iputils-ping \ + python3-pip \ + libgcrypt20 \ + libgcrypt20-dev \ + gdb \ + libc6-dbg \ + software-properties-common + +ARG USER=fluxuser +ARG CONFIG_ROOT=containers/spindle-flux-ubuntu + +# Allow fluxuser to run as other users so it can start munged +RUN sh -c "printf \"${USER} ALL=(ALL) NOPASSWD: ALL\\n\" >> /etc/sudoers" + +# Configure flux +ENV STATE_DIR=/var/lib/flux +RUN mkdir -p ${STATE_DIR} /etc/flux/system /etc/flux/system/cron.d /etc/flux/config /run/flux /etc/flux/imp/conf.d +COPY ${CONFIG_ROOT}/flux/imp.toml /etc/flux/imp/conf.d/ +COPY ${CONFIG_ROOT}/flux/broker.toml /etc/flux/config/ +RUN mkdir -p /etc/flux/system/cron.d && \ + mkdir -p /mnt/curve && \ + flux keygen /mnt/curve/curve.cert && \ + flux R encode --hosts="node-[1-${workers}]" > /etc/flux/system/R && \ + chmod -R a+rX /etc/flux && \ + chown -R ${USER}:${USER} /run/flux ${STATE_DIR} /mnt/curve/curve.cert + +# Build Spindle +WORKDIR /home/${USER} +# Copy the whole git repo into the container +COPY . /home/${USER}/Spindle + +# Fix permissions on Spindle source (configure needs +x) +RUN chmod -R u+rwX /home/${USER}/Spindle && \ + chown -R ${USER}:${USER} /home/${USER}/Spindle + +# Copy and prepare build script as root +COPY ${CONFIG_ROOT}/scripts/build_spindle.sh /home/${USER}/build_spindle.sh +RUN chmod +rx /home/${USER}/build_spindle.sh && \ + chown ${USER}:${USER} /home/${USER}/build_spindle.sh + +# Switch to user to run the build +USER ${USER} +RUN bash ./build_spindle.sh + +# Copy scripts as root +USER root +RUN chown -R ${USER}:${USER} /home/fluxuser && \ + chown -R ${USER}:${USER} /run/flux + +COPY ${CONFIG_ROOT}/scripts/flux_healthcheck.sh /home/${USER}/flux_healthcheck.sh +COPY ${CONFIG_ROOT}/scripts/entrypoint.sh.podman /home/${USER}/entrypoint.sh +RUN chmod +rx /home/${USER}/flux_healthcheck.sh /home/${USER}/entrypoint.sh && \ + chown ${USER}:${USER} /home/${USER}/flux_healthcheck.sh /home/${USER}/entrypoint.sh + +USER ${USER} +WORKDIR /home/${USER} +ENV PATH /home/${USER}/Spindle-inst/bin:${PATH} +# Make libfabric work with fork. +ENV RDMAV_FORK_SAFE 1 +# Silence warning from hwloc about unsupported PCI device +# on GitHub-hosted runners. +ENV HWLOC_HIDE_ERRORS 2 +# Suppress UCX warnings about /proc/sys tuning parameters +# These are expected in rootless podman containers +ENV UCX_LOG_LEVEL=error + +ENTRYPOINT /bin/bash ./entrypoint.sh diff --git a/containers/spindle-flux-ubuntu/scripts/entrypoint.sh.podman b/containers/spindle-flux-ubuntu/scripts/entrypoint.sh.podman new file mode 100644 index 00000000..3fb7f9ac --- /dev/null +++ b/containers/spindle-flux-ubuntu/scripts/entrypoint.sh.podman @@ -0,0 +1,51 @@ +#!/bin/bash +# +# Podman-compatible entrypoint for Flux containers +# Starts munged and the flux broker. +# +# For documentation on running Flux in containers, see +# https://flux-framework.readthedocs.io/en/latest/tutorials/containers + +set -x # Debug output + +brokerOptions="-Scron.directory=/etc/flux/system/cron.d \ + -Stbon.fanout=256 \ + -Srundir=/run/flux \ + -Sstatedir=${STATE_DIRECTORY:-/var/lib/flux} \ + -Slocal-uri=local:///run/flux/local \ + -Slog-stderr-level=6 \ + -Slog-stderr-mode=local" + +# Get the hostname that will resolve for the Docker bridge network. +address=$(echo $( nslookup "$( hostname -i )" | head -n 1 )) +parts=(${address//=/ }) +hostName=${parts[2]} +thisHost=(${hostName//./ }) +thisHost=${thisHost[0]} +echo "This host: $thisHost" +echo "Main host: $mainHost" +export FLUX_FAKE_HOSTNAME=$thisHost + +if [ -d /shared ]; then + sudo chown -R "$(id -un):$(id -gn)" /shared + sudo chmod 755 /shared +fi + +# Start munged +echo "Starting munged..." +sudo -u munge /usr/sbin/munged + +# Give munge time to start +sleep 2 + +if [ "${thisHost}" != "${mainHost}" ]; then + # Worker node -- wait for head node before connecting + echo "Worker node: waiting for head node..." + sleep 15 + echo "Starting flux broker (worker)..." + exec flux start -o --config /etc/flux/config ${brokerOptions} sleep inf +else + # Head node + echo "Head node: starting flux broker..." + exec flux start -o --config /etc/flux/config ${brokerOptions} sleep inf +fi diff --git a/containers/spindle-hello-podman/01-basic/Dockerfile b/containers/spindle-hello-podman/01-basic/Dockerfile new file mode 100644 index 00000000..41fc7907 --- /dev/null +++ b/containers/spindle-hello-podman/01-basic/Dockerfile @@ -0,0 +1,28 @@ +FROM ubuntu:noble + +# LC-specific fix for podman setgroups issue +# This is a no-op in Docker, but required for podman on LC systems +ARG PODMAN_BUILD=false +RUN if [ "$PODMAN_BUILD" = "true" ]; then \ + echo 'APT::Sandbox::User root;' > /etc/apt/apt.conf.d/00-apt-sandbox; \ + fi + +# Install curl to test both apt-get and SSL connectivity +RUN apt-get update && apt-get install -y --no-install-recommends \ + curl \ + ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +# Test script that validates: +# 1. Container runs +# 2. Network access works +# 3. SSL certificates work +CMD echo "=== Hello from Podman! ===" && \ + echo "Container is running successfully." && \ + echo "" && \ + echo "Testing network and SSL connectivity..." && \ + curl -I https://www.google.com 2>&1 | head -5 && \ + echo "" && \ + echo "If you see HTTP headers above, SSL is working!" && \ + echo "" && \ + echo "=== 01-basic test complete ===" diff --git a/containers/spindle-hello-podman/02-user-switch/Dockerfile b/containers/spindle-hello-podman/02-user-switch/Dockerfile new file mode 100644 index 00000000..e05c7018 --- /dev/null +++ b/containers/spindle-hello-podman/02-user-switch/Dockerfile @@ -0,0 +1,78 @@ +FROM ubuntu:noble + +# LC-specific fix for podman setgroups issue +ARG PODMAN_BUILD=false +RUN if [ "$PODMAN_BUILD" = "true" ]; then \ + echo 'APT::Sandbox::User root;' > /etc/apt/apt.conf.d/00-apt-sandbox; \ + fi + +# Install basic tools +RUN apt-get update && apt-get install -y --no-install-recommends \ + sudo \ + && rm -rf /var/lib/apt/lists/* + +# Create a non-root user similar to how Spindle containers work +# This pattern is used in all Spindle containers +ARG USER=testuser +ARG UID=1001 +ARG GID=1001 + +RUN groupadd -g ${GID} ${USER} && \ + useradd -m -u ${UID} -g ${GID} -s /bin/bash ${USER} && \ + echo "${USER} ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/${USER} && \ + chmod 0440 /etc/sudoers.d/${USER} + +# Create a test directory owned by root to verify permission handling +RUN mkdir -p /root-owned && \ + echo "This is owned by root" > /root-owned/root-file.txt && \ + chmod 644 /root-owned/root-file.txt + +# Create a test directory that will be owned by the user +RUN mkdir -p /user-owned && \ + chown ${UID}:${GID} /user-owned + +# Switch to non-root user +USER ${USER} +WORKDIR /home/${USER} + +# Create a file as the user to verify we're running as that user +RUN echo "This is owned by ${USER}" > /home/${USER}/user-file.txt + +# Test script that validates: +# 1. Running as the correct user +# 2. Can read files owned by root +# 3. Can write to user-owned directories +# 4. Cannot write to root-owned directories +# 5. Can use sudo if needed +CMD echo "=== User Switch Test ===" && \ + echo "" && \ + echo "Current user information:" && \ + id && \ + echo "" && \ + echo "Home directory:" && \ + pwd && \ + ls -la /home/${USER} && \ + echo "" && \ + echo "Testing file permissions:" && \ + echo " Reading root-owned file..." && \ + cat /root-owned/root-file.txt && \ + echo " ✓ Can read root-owned files" && \ + echo "" && \ + echo " Writing to user-owned directory..." && \ + echo "test content" > /user-owned/test.txt && \ + cat /user-owned/test.txt && \ + echo " ✓ Can write to user-owned directories" && \ + echo "" && \ + echo " Testing root-owned directory (should fail)..." && \ + (echo "test" > /root-owned/fail.txt 2>&1 && echo " ✗ UNEXPECTED: Could write to root directory" || echo " ✓ Correctly denied write to root directory") && \ + echo "" && \ + echo " Testing sudo access..." && \ + sudo -n echo " ✓ Sudo works (passwordless for container convenience)" && \ + echo "" && \ + echo "=== 02-user-switch test complete ===" && \ + echo "" && \ + echo "Key patterns validated:" && \ + echo " ✓ Non-root user created (uid=${UID})" && \ + echo " ✓ User has home directory" && \ + echo " ✓ Correct permission handling" && \ + echo " ✓ Sudo available when needed" diff --git a/containers/spindle-hello-podman/03-filesystem/Dockerfile b/containers/spindle-hello-podman/03-filesystem/Dockerfile new file mode 100644 index 00000000..b1fb59c3 --- /dev/null +++ b/containers/spindle-hello-podman/03-filesystem/Dockerfile @@ -0,0 +1,85 @@ +FROM ubuntu:noble + +# LC-specific fix for podman setgroups issue +ARG PODMAN_BUILD=false +RUN if [ "$PODMAN_BUILD" = "true" ]; then \ + echo 'APT::Sandbox::User root;' > /etc/apt/apt.conf.d/00-apt-sandbox; \ + fi + +# Install tools for filesystem operations +RUN apt-get update && apt-get install -y --no-install-recommends \ + file \ + && rm -rf /var/lib/apt/lists/* + +# Create a non-root user (matching pattern from 02-user-switch) +ARG USER=testuser +ARG UID=1001 +ARG GID=1001 + +# Make USER available at runtime (ARG only available at build time) +ENV USER=${USER} + +RUN groupadd -g ${GID} ${USER} && \ + useradd -m -u ${UID} -g ${GID} -s /bin/bash ${USER} + +# Switch to non-root user +USER ${USER} +WORKDIR /home/${USER} + +# Create mount points for testing different volume scenarios +# These will be populated via volume mounts at runtime +RUN mkdir -p /home/${USER}/host-data && \ + mkdir -p /home/${USER}/output + +# Test script that validates volume mounts and filesystem operations +# Note: The actual volume content will be mounted at runtime +CMD echo "=== Filesystem Mount Test ===" && \ + echo "" && \ + echo "Testing volume mounts and file operations..." && \ + echo "" && \ + echo "1. Checking mounted host directory:" && \ + if [ -d "/home/${USER}/host-data" ]; then \ + echo " Mount point exists: /home/${USER}/host-data" && \ + if [ "$(ls -A /home/${USER}/host-data 2>/dev/null)" ]; then \ + echo " ✓ Host directory mounted with content:" && \ + ls -lh /home/${USER}/host-data | head -10 && \ + echo "" && \ + echo " Reading a file from host..." && \ + if [ -f "/home/${USER}/host-data/test-input.txt" ]; then \ + cat /home/${USER}/host-data/test-input.txt && \ + echo " ✓ Can read files from host mount" ; \ + else \ + echo " ℹ No test-input.txt found (expected for manual testing)" ; \ + fi ; \ + else \ + echo " ⚠ Mount point empty - volume may not be mounted" ; \ + fi ; \ + else \ + echo " ✗ Mount point missing" ; \ + fi && \ + echo "" && \ + echo "2. Testing write to output directory:" && \ + echo "Test data written from container at $(date)" > /home/${USER}/output/container-output.txt && \ + echo " ✓ Created file: container-output.txt" && \ + cat /home/${USER}/output/container-output.txt && \ + echo "" && \ + echo "3. Checking file ownership:" && \ + ls -ln /home/${USER}/output/container-output.txt && \ + echo " ℹ File owned by UID=$(stat -c %u /home/${USER}/output/container-output.txt) GID=$(stat -c %g /home/${USER}/output/container-output.txt)" && \ + echo "" && \ + echo "4. Testing file type detection:" && \ + file /home/${USER}/output/container-output.txt && \ + echo "" && \ + echo "=== 03-filesystem test complete ===" && \ + echo "" && \ + echo "Key patterns validated:" && \ + echo " ✓ Volume mounts work" && \ + echo " ✓ Can read files from host" && \ + echo " ✓ Can write files to mounted volumes" && \ + echo " ✓ File ownership preserved" && \ + echo "" && \ + echo "This demonstrates the pattern used for Spindle:" && \ + echo " - Mount source code from host (read-only)" && \ + echo " - Mount build directory (read-write)" && \ + echo " - Build artifacts persist on host" && \ + echo " - Logs accessible after container exits" diff --git a/containers/spindle-hello-podman/04-networking/Dockerfile b/containers/spindle-hello-podman/04-networking/Dockerfile new file mode 100644 index 00000000..c2be0154 --- /dev/null +++ b/containers/spindle-hello-podman/04-networking/Dockerfile @@ -0,0 +1,41 @@ +FROM ubuntu:noble + +# LC-specific fix for podman setgroups issue +ARG PODMAN_BUILD=false +RUN if [ "$PODMAN_BUILD" = "true" ]; then \ + echo 'APT::Sandbox::User root;' > /etc/apt/apt.conf.d/00-apt-sandbox; \ + fi + +# Install networking tools +RUN apt-get update && apt-get install -y --no-install-recommends \ + iputils-ping \ + iproute2 \ + curl \ + netcat-openbsd \ + && rm -rf /var/lib/apt/lists/* + +# Create a non-root user +ARG USER=testuser +ARG UID=1001 +ARG GID=1001 + +# Make USER available at runtime +ENV USER=${USER} + +RUN groupadd -g ${GID} ${USER} && \ + useradd -m -u ${UID} -g ${GID} -s /bin/bash ${USER} + +# Switch to non-root user +USER ${USER} +WORKDIR /home/${USER} + +# Default command: run as server listening on port 8080 +# This can be overridden when running as a client +CMD echo "=== Network Container ===" && \ + echo "Container: $(hostname)" && \ + echo "IP Address: $(hostname -I)" && \ + echo "" && \ + echo "Starting simple HTTP server on port 8080..." && \ + while true; do \ + echo "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\n\r\nHello from $(hostname) at $(date)" | nc -l -p 8080 -q 1; \ + done diff --git a/containers/spindle-hello-podman/05-flux-multi/Dockerfile b/containers/spindle-hello-podman/05-flux-multi/Dockerfile new file mode 100644 index 00000000..8b90d595 --- /dev/null +++ b/containers/spindle-hello-podman/05-flux-multi/Dockerfile @@ -0,0 +1,59 @@ +# Simplified Flux multi-node example +# Based on https://flux-framework.readthedocs.io/en/latest/tutorials/containers +# Uses the official Flux Framework base image + +ARG flux_sched_version=noble-v0.48.0-amd64 +FROM fluxrm/flux-sched:${flux_sched_version} + +USER root + +# LC-specific fix for podman setgroups issue +ARG PODMAN_BUILD=false +RUN if [ "$PODMAN_BUILD" = "true" ]; then \ + echo 'APT::Sandbox::User root;' > /etc/apt/apt.conf.d/00-apt-sandbox; \ + fi + +# Install basic tools for testing +RUN apt-get update && apt-get install -y --no-install-recommends \ + iputils-ping \ + iproute2 \ + && rm -rf /var/lib/apt/lists/* + +# Configure Flux for multi-node cluster +ARG workers=3 +ENV workers=${workers} +ENV STATE_DIR=/var/lib/flux + +# Set up Flux directories and configuration +RUN mkdir -p ${STATE_DIR} /etc/flux/system /etc/flux/config /run/flux /mnt/curve && \ + flux keygen /mnt/curve/curve.cert && \ + flux R encode --hosts="flux-node-[1-${workers}]" > /etc/flux/system/R + +# Create broker configuration +RUN echo '[bootstrap]' > /etc/flux/config/broker.toml && \ + echo 'default_port = 8050' >> /etc/flux/config/broker.toml && \ + echo 'default_bind = "tcp://eth0:%p"' >> /etc/flux/config/broker.toml && \ + echo 'default_connect = "tcp://%h.local:%p"' >> /etc/flux/config/broker.toml && \ + echo 'hosts = [' >> /etc/flux/config/broker.toml && \ + echo ' { host = "flux-node-1" },' >> /etc/flux/config/broker.toml && \ + echo ' { host = "flux-node-[2-'${workers}']" },' >> /etc/flux/config/broker.toml && \ + echo ']' >> /etc/flux/config/broker.toml + +# Set proper ownership +ARG USER=fluxuser +RUN chown -R ${USER}:${USER} /run/flux ${STATE_DIR} /mnt/curve /etc/flux + +# Allow fluxuser to run munge +RUN sh -c "printf \"${USER} ALL=(ALL) NOPASSWD: /usr/sbin/munged\\n\" >> /etc/sudoers" + +# Copy entrypoint as root, make executable, then set ownership +COPY containers/spindle-hello-podman/05-flux-multi/entrypoint.sh /home/${USER}/entrypoint.sh +RUN chmod +x /home/${USER}/entrypoint.sh && \ + chown ${USER}:${USER} /home/${USER}/entrypoint.sh + +USER ${USER} +WORKDIR /home/${USER} + +ENV HWLOC_HIDE_ERRORS=2 + +ENTRYPOINT ["./entrypoint.sh"] diff --git a/containers/spindle-hello-podman/05-flux-multi/entrypoint.sh b/containers/spindle-hello-podman/05-flux-multi/entrypoint.sh new file mode 100755 index 00000000..bac5a870 --- /dev/null +++ b/containers/spindle-hello-podman/05-flux-multi/entrypoint.sh @@ -0,0 +1,67 @@ +#!/bin/bash +# +# Flux multi-node entrypoint +# Based on Spindle's Flux container setup +# +# All nodes run the same image but behave differently based on hostname. +# flux-node-1 is the head node, others are workers. + +set -e + +echo "=== Flux Container Startup ===" +echo "Hostname: $(hostname)" +echo "IP: $(hostname -I)" + +# Start munge for authentication +echo "Starting munge..." +sudo /usr/sbin/munged +sleep 1 + +# Determine if this is the head node or a worker +MAIN_HOST="flux-node-1" +THIS_HOST=$(hostname) + +brokerOptions="-Stbon.fanout=256 \ + -Srundir=/run/flux \ + -Sstatedir=${STATE_DIR} \ + -Slog-stderr-level=6 \ + -Slog-stderr-mode=local" + +if [ "${THIS_HOST}" != "${MAIN_HOST}" ]; then + # Worker node - wait for head node to be ready + echo "Worker node: connecting to ${MAIN_HOST}..." + sleep 5 + + # Start flux broker and connect to head node + flux start -o --config /etc/flux/config ${brokerOptions} sleep inf +else + # Head node + echo "Head node: starting Flux broker..." + + # Start flux broker + flux start -o --config /etc/flux/config ${brokerOptions} bash -c ' + echo "" + echo "=== Flux Instance Started ===" + echo "" + + # Wait for workers to connect + echo "Waiting for workers to connect..." + sleep 10 + + echo "" + echo "Flux instance status:" + flux resource list + + echo "" + echo "Running test job across cluster..." + flux run -N 3 hostname + + echo "" + echo "=== Flux cluster is operational ===" + echo "You can now run flux commands." + echo "" + + # Keep running + sleep inf + ' +fi diff --git a/containers/spindle-hello-podman/06-slurm-multi/Dockerfile b/containers/spindle-hello-podman/06-slurm-multi/Dockerfile new file mode 100644 index 00000000..2385e36e --- /dev/null +++ b/containers/spindle-hello-podman/06-slurm-multi/Dockerfile @@ -0,0 +1,90 @@ +FROM ubuntu:noble + +USER root + +# LC-specific fix for podman setgroups issue +ARG PODMAN_BUILD=false +RUN if [ "$PODMAN_BUILD" = "true" ]; then \ + echo 'APT::Sandbox::User root;' > /etc/apt/apt.conf.d/00-apt-sandbox; \ + fi + +# Install build dependencies and runtime tools +# Build Slurm from source instead of using Ubuntu packages +# to avoid dpkg post-install chown failures in rootless podman +RUN apt-get update && \ + DEBIAN_FRONTEND="noninteractive" apt-get install -y --no-install-recommends \ + ca-certificates \ + build-essential \ + git \ + autoconf \ + automake \ + libtool \ + pkg-config \ + libmunge-dev \ + libmariadb-dev \ + libhwloc-dev \ + libjson-c-dev \ + libhttp-parser-dev \ + python3-dev \ + munge \ + sudo \ + iputils-ping \ + && rm -rf /var/lib/apt/lists/* + +# Set up munge +RUN mkdir -p /run/munge /etc/munge && \ + chown munge:munge /run/munge && \ + chmod 0755 /run/munge + +# Create munge key (same key on all nodes for simplicity) +RUN dd if=/dev/urandom bs=1 count=1024 > /etc/munge/munge.key && \ + chown munge:munge /etc/munge/munge.key && \ + chmod 400 /etc/munge/munge.key + +# Create slurm user BEFORE building Slurm +RUN groupadd -r -g 900 slurm && \ + useradd -r -u 900 -g 900 -s /bin/false -d /var/lib/slurmd slurm + +# Build Slurm from source +# Based on Spindle's build_slurm.sh +ARG SLURM_VERSION=slurm-25-05-3-1 +RUN git clone -b ${SLURM_VERSION} --single-branch --depth=1 \ + https://github.com/SchedMD/slurm.git /tmp/slurm && \ + cd /tmp/slurm && \ + ./configure \ + --prefix=/usr \ + --sysconfdir=/etc/slurm \ + --with-mysql_config=/usr/bin \ + --libdir=/usr/lib && \ + make -j$(nproc) && \ + make install && \ + cd / && \ + rm -rf /tmp/slurm + +# Create Slurm directories with correct ownership +# Based on Spindle's setup_slurm.sh +RUN mkdir -p \ + /etc/slurm \ + /var/spool/slurmd \ + /var/spool/slurmctld \ + /var/run/slurmd \ + /var/lib/slurmd \ + /var/log/slurm && \ + chown -R slurm:slurm \ + /etc/slurm \ + /var/spool/slurmd \ + /var/spool/slurmctld \ + /var/run/slurmd \ + /var/lib/slurmd \ + /var/log/slurm + +# Copy Slurm configuration +COPY containers/spindle-hello-podman/06-slurm-multi/slurm.conf /etc/slurm/slurm.conf +COPY containers/spindle-hello-podman/06-slurm-multi/cgroup.conf /etc/slurm/cgroup.conf +RUN chown slurm:slurm /etc/slurm/slurm.conf /etc/slurm/cgroup.conf + +# Copy entrypoint +COPY containers/spindle-hello-podman/06-slurm-multi/entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh + +ENTRYPOINT ["/entrypoint.sh"] diff --git a/containers/spindle-hello-podman/06-slurm-multi/cgroup.conf b/containers/spindle-hello-podman/06-slurm-multi/cgroup.conf new file mode 100644 index 00000000..e59e9aee --- /dev/null +++ b/containers/spindle-hello-podman/06-slurm-multi/cgroup.conf @@ -0,0 +1 @@ +CgroupPlugin=cgroup/v1 diff --git a/containers/spindle-hello-podman/06-slurm-multi/entrypoint.sh b/containers/spindle-hello-podman/06-slurm-multi/entrypoint.sh new file mode 100755 index 00000000..77741aa5 --- /dev/null +++ b/containers/spindle-hello-podman/06-slurm-multi/entrypoint.sh @@ -0,0 +1,50 @@ +#!/bin/bash +# +# Slurm multi-node entrypoint +# Simplified version for hello-world demonstration +# Based on Spindle's working entrypoint +# +# Determines role based on SLURM_ROLE environment variable: +# - ctl: runs slurmctld (head node) +# - worker: runs slurmd (compute node) + +set -e + +echo "=== Slurm Container Startup ===" +echo "Hostname: $(hostname)" +echo "Role: ${SLURM_ROLE}" + +# Start munge for authentication +echo "Starting munge..." +sudo -u munge /usr/sbin/munged +sleep 2 + +case "${SLURM_ROLE}" in + ctl) + echo "Starting slurmctld (controller daemon)..." + # Run as slurm user in foreground with high verbosity + sudo -u slurm /usr/sbin/slurmctld -i -Dvvv + ;; + + worker) + echo "Waiting for controller to be ready..." + sleep 5 + + echo "Testing connectivity to controller..." + ping -c 3 slurm-head || echo "WARNING: Cannot ping slurm-head" + + echo "Testing munge authentication..." + munge -n | unmunge || echo "WARNING: Munge test failed" + + echo "Starting slurmd (compute daemon)..." + # Run as root (via sudo) in foreground - slurmd needs root for cgroup management + # Based on Spindle's pattern: sudo bash -c 'exec /usr/sbin/slurmd -Dvvv' + sudo /usr/sbin/slurmd -Dvvv + ;; + + *) + echo "ERROR: SLURM_ROLE must be 'ctl' or 'worker'" + echo "Got: ${SLURM_ROLE}" + exit 1 + ;; +esac diff --git a/containers/spindle-hello-podman/06-slurm-multi/slurm.conf b/containers/spindle-hello-podman/06-slurm-multi/slurm.conf new file mode 100644 index 00000000..1c3f756f --- /dev/null +++ b/containers/spindle-hello-podman/06-slurm-multi/slurm.conf @@ -0,0 +1,46 @@ +# Minimal Slurm configuration for hello-world demo +# Based on Spindle's working configuration +ClusterName=hello-slurm +ControlMachine=slurm-head +ControlAddr=slurm-head +SlurmUser=slurm +SlurmctldPort=6817 +SlurmdPort=6818 + +# Authentication +AuthType=auth/munge + +# Logging +SlurmctldLogFile=/var/log/slurm/slurmctld.log +SlurmdLogFile=/var/log/slurm/slurmd.log +SlurmctldDebug=3 +SlurmdDebug=3 + +# Process tracking +ProctrackType=proctrack/linuxproc +TaskPlugin=task/none + +# Scheduling +SchedulerType=sched/builtin +SelectType=select/linear + +# State preservation +StateSaveLocation=/var/lib/slurmd +SlurmdSpoolDir=/var/spool/slurmd +SlurmctldPidFile=/var/run/slurmd/slurmctld.pid +SlurmdPidFile=/var/run/slurmd/slurmd.pid + +# Accounting (minimal - no database) +AccountingStorageType=accounting_storage/none +JobAcctGatherType=jobacct_gather/none + +# Return to service +ReturnToService=2 + +# MPI +MpiDefault=none + +# Node definitions - explicitly set NodeAddr +NodeName=slurm-node-1 NodeAddr=slurm-node-1 CPUs=1 State=UNKNOWN +NodeName=slurm-node-2 NodeAddr=slurm-node-2 CPUs=1 State=UNKNOWN +PartitionName=debug Nodes=slurm-node-[1-2] Default=YES MaxTime=INFINITE State=UP diff --git a/containers/spindle-hello-podman/README.md b/containers/spindle-hello-podman/README.md new file mode 100644 index 00000000..60999e8f --- /dev/null +++ b/containers/spindle-hello-podman/README.md @@ -0,0 +1,115 @@ +# Spindle Hello World - Podman Tutorial + +This directory contains a series of progressive test containers that demonstrate and validate the patterns needed to run Spindle containers with podman on LC systems. + +## Purpose + +These containers serve as: +1. **Validation** - Test that podman is properly configured +2. **Tutorial** - Show common container patterns (user switching, volumes, networking) +3. **Documentation** - Demonstrate LC-specific workarounds + +## Tests + +### 01-basic (✓ Available) +**What it tests:** +- Container builds with apt-get (tests setgroups fix) +- Container runs successfully +- Network connectivity works +- SSL certificates are properly configured + +**Run from outside the sandbox:** +```bash +cd /path/to/workspace-Spindle/Spindle/podman-port +./scripts/podman/run-hello-01-basic.sh +``` + +### 02-user-switch (✓ Available) +**What it tests:** +- Creating a non-root user (uid=1001, gid=1001) +- Switching to that user with USER directive +- File permission handling (read root files, write user files) +- Sudo access (passwordless for container convenience) + +**Run from outside the sandbox:** +```bash +cd /path/to/workspace-Spindle/Spindle/podman-port +./scripts/podman/run-hello-02-user.sh +``` + +### 03-filesystem (✓ Available) +**What it tests:** +- Volume mounts with -v flag +- Reading files from host directories +- Writing files to mounted volumes +- File persistence after container exit +- SELinux context handling (:Z flag) + +**Run from outside the sandbox:** +```bash +cd /path/to/workspace-Spindle/Spindle/podman-port +./scripts/podman/run-hello-03-filesystem.sh +``` + +This creates temporary test directories, mounts them into the container, +and verifies files can be read/written. Artifacts persist after the +container exits, demonstrating how Spindle build/test output is preserved. + +### 04-networking (✓ Available) +**What it tests:** +- Custom network creation +- Multiple containers on same network +- DNS resolution (hostname lookups) +- Container-to-container HTTP communication +- Service discovery patterns + +**Run from outside the sandbox:** +```bash +cd /path/to/workspace-Spindle/Spindle/podman-port +./scripts/podman/run-hello-04-networking.sh +``` + +This creates a server container and two client containers, all on a custom +network. Clients can resolve the server by hostname and communicate via HTTP. +This demonstrates the pattern used for Slurm/Flux multi-node clusters. + +## LC-Specific Issues Addressed + +### Issue 1: setgroups errors with apt-get +**Problem:** Ubuntu/Debian apt fails with "setgroups 65534 failed" on LC systems + +**Solution:** Add to Dockerfile: +```dockerfile +ARG PODMAN_BUILD=false +RUN if [ "$PODMAN_BUILD" = "true" ]; then \ + echo 'APT::Sandbox::User root;' > /etc/apt/apt.conf.d/00-apt-sandbox; \ + fi +``` + +This is controlled via build arg (automatically set by scripts). + +### Issue 2: SSL certificate errors +**Problem:** HTTPS fails with "certificate signed by unknown authority" + +**Solution:** Mount LC certificates into container: +- Build: `-v /etc/pki/ca-trust/source/anchors/PAN-cspca.llnl.gov.crt.pem:/usr/local/share/ca-certificates/cspca.crt:ro` +- Run: `-v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/ssl/certs/ca-certificates.crt:ro` + +This is handled automatically by `scripts/podman/common.sh`. + +## Usage + +All scripts are designed to be run from **outside the sandbox** where podman is available: + +```bash +# From your normal home directory or workspace +cd /path/to/workspace-Spindle/Spindle/podman-port +./scripts/podman/run-hello-01-basic.sh +``` + +## Next Steps + +After completing these hello-world tests, proceed to: +- `scripts/podman/run-serial.sh` - Single container Spindle tests +- `scripts/podman/run-flux.sh` - Flux resource manager tests +- `scripts/podman/run-slurm-srun.sh` - Multi-container Slurm cluster tests diff --git a/containers/spindle-serial-ubuntu/Dockerfile.podman b/containers/spindle-serial-ubuntu/Dockerfile.podman new file mode 100644 index 00000000..6151e276 --- /dev/null +++ b/containers/spindle-serial-ubuntu/Dockerfile.podman @@ -0,0 +1,93 @@ +ARG ubuntu_version=noble +FROM ubuntu:${ubuntu_version} +USER root +ENV TMPDIR=/tmp +RUN echo 'TMPDIR="/tmp"' >> /etc/environment +ENV SPINDLE_TEST_CONTAINER=1 + +# LC-specific fix for podman setgroups issue +ARG PODMAN_BUILD=false +RUN if [ "$PODMAN_BUILD" = "true" ]; then \ + echo 'APT::Sandbox::User root;' > /etc/apt/apt.conf.d/00-apt-sandbox; \ + fi + +RUN DEBIAN_FRONTEND="noninteractive" apt-get update \ +# install latest pkg utils: + && apt-get -qq install -y --no-install-recommends \ + apt-utils + +RUN DEBIAN_FRONTEND="noninteractive" apt-get -qq install -y --no-install-recommends \ + locales \ + ca-certificates \ + wget \ + git \ + ssh \ + sudo \ + build-essential \ + pkg-config \ + autotools-dev \ + libtool \ + autoconf \ + automake \ + make \ + gfortran-13 \ + gcc-13 \ + g++-13 \ + gdb \ + libc6-dbg \ + munge \ + libmunge-dev \ + libhwloc-dev \ + mpich \ + libmpich-dev + +# Prevent hwloc from trying to use graphics cards +# as this fails when X is not running. +ENV HWLOC_COMPONENTS=-gl + +# Suppress UCX warnings about /proc/sys tuning parameters +# These are expected in rootless podman containers +ENV UCX_LOG_LEVEL=error + +# Set up munge +RUN mkdir -p /run/munge && \ + chown munge:munge /run/munge && \ + chmod 0755 /run/munge + +ARG USER=spindleuser +ARG UID=1001 +ARG BUILD_ROOT=./containers/spindle-serial-ubuntu +COPY ${BUILD_ROOT}/scripts/add_docker_user.sh /add_docker_user.sh +RUN /add_docker_user.sh + +USER ${USER} +WORKDIR /home/${USER} +RUN mkdir -p /home/${USER}/Spindle +# Copy the Spindle repo into the container +COPY . /home/${USER}/Spindle + +# Fix permissions on copied files (configure, scripts, etc.) as root +USER root +RUN chmod -R u+rwX /home/${USER}/Spindle && \ + chown -R ${USER}:${USER} /home/${USER}/Spindle + +# Copy and prepare build script as root, then chown to user +COPY ${BUILD_ROOT}/scripts/build_spindle.sh /home/${USER}/build_spindle.sh +RUN chmod +rx /home/${USER}/build_spindle.sh && \ + chown ${USER}:${USER} /home/${USER}/build_spindle.sh + +# Switch back to user to run the build +USER ${USER} +RUN bash ./build_spindle.sh + +# Copy entrypoint as root and set permissions +USER root +COPY ${BUILD_ROOT}/scripts/entrypoint.sh.podman /home/${USER}/entrypoint.sh +RUN chmod +rx /home/${USER}/entrypoint.sh && \ + chown ${USER}:${USER} /home/${USER}/entrypoint.sh + +# Final switch to user for runtime +USER ${USER} +ENV PATH /home/${USER}/Spindle-inst/bin:$PATH + +ENTRYPOINT /bin/bash ./entrypoint.sh diff --git a/containers/spindle-serial-ubuntu/scripts/entrypoint.sh.podman b/containers/spindle-serial-ubuntu/scripts/entrypoint.sh.podman new file mode 100644 index 00000000..b4e7a05d --- /dev/null +++ b/containers/spindle-serial-ubuntu/scripts/entrypoint.sh.podman @@ -0,0 +1,10 @@ +#!/bin/bash +# Podman-compatible entrypoint for Spindle serial container +# Start munged in background and keep container alive + +printf "\nStarting munged\n" + +sudo -u munge /usr/sbin/munged + +# Keep container alive +sleep inf diff --git a/containers/spindle-slurm-ubuntu/base/Dockerfile.podman b/containers/spindle-slurm-ubuntu/base/Dockerfile.podman new file mode 100644 index 00000000..0e44ee6e --- /dev/null +++ b/containers/spindle-slurm-ubuntu/base/Dockerfile.podman @@ -0,0 +1,90 @@ +ARG UBUNTU_VERSION=noble +FROM ubuntu:${UBUNTU_VERSION} +USER root +ENV TMPDIR=/tmp +RUN echo 'TMPDIR="/tmp"' >> /etc/environment + +# LC-specific fix for podman setgroups issue +ARG PODMAN_BUILD=false +RUN if [ "$PODMAN_BUILD" = "true" ]; then \ + echo 'APT::Sandbox::User root;' > /etc/apt/apt.conf.d/00-apt-sandbox; \ + fi + +RUN apt-get update \ + && DEBIAN_FRONTEND="noninteractive" apt-get -qq install -y --no-install-recommends \ + apt-utils + +RUN apt-get update \ + && DEBIAN_FRONTEND="noninteractive" apt-get -qq install -y --no-install-recommends \ + locales \ + ca-certificates \ + wget \ + git \ + ssh \ + sudo \ + psmisc \ + build-essential \ + pkg-config \ + autotools-dev \ + libtool \ + autoconf \ + automake \ + make \ + gfortran-13 \ + gcc-13 \ + g++-13 \ + gdb \ + munge \ + libmunge-dev \ + libhwloc-dev \ + python3-dev \ + python3-pip \ + python3-setuptools \ + python3-wheel \ + python-is-python3 \ + openssh-server \ + openssh-client \ + mariadb-client \ + libmariadb-dev \ + libhttp-parser-dev \ + libjson-c-dev + + +# Prevent hwloc from trying to use graphics cards +# as this fails when X is not running. +ENV HWLOC_COMPONENTS=-gl + +# Suppress UCX warnings about /proc/sys tuning parameters +ENV UCX_LOG_LEVEL=error + +# Set up munge +RUN mkdir -p /run/munge && \ + chown munge:munge /run/munge && \ + chmod 0755 /run/munge + +ARG BUILD_ROOT=. +COPY ${BUILD_ROOT}/scripts/add_docker_user.sh /add_docker_user.sh + +# Slurm daemons run as $SLURM_USER +ARG SLURM_USER=slurm +ARG USER=${SLURM_USER} +ARG UID=1002 +RUN /add_docker_user.sh + +# Applications run as $USER +ARG USER=slurmuser +ARG UID=1001 +RUN /add_docker_user.sh + +ARG SLURM_VERSION=slurm-25-05-3-1 +COPY ${BUILD_ROOT}/scripts/build_slurm.sh /build_slurm.sh +RUN chmod +x /build_slurm.sh && /build_slurm.sh + +ARG MPICH_VERSION=4.2.2 +COPY ${BUILD_ROOT}/scripts/build_mpich.sh.podman /build_mpich.sh +RUN chmod +x /build_mpich.sh && /build_mpich.sh + +COPY ${BUILD_ROOT}/conf/mpich.conf /etc/ld.so.conf.d/mpich.conf +RUN ldconfig + +ENV SLURM_MPI_TYPE pmi2 diff --git a/containers/spindle-slurm-ubuntu/base/scripts/build_mpich.sh.podman b/containers/spindle-slurm-ubuntu/base/scripts/build_mpich.sh.podman new file mode 100644 index 00000000..b35a7e48 --- /dev/null +++ b/containers/spindle-slurm-ubuntu/base/scripts/build_mpich.sh.podman @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -euxo pipefail + +mkdir mpich +pushd mpich +# Add --no-same-owner for rootless podman compatibility +wget -O - https://www.mpich.org/static/downloads/${MPICH_VERSION}/mpich-${MPICH_VERSION}.tar.gz | tar xvz --no-same-owner --strip-components 1 +mkdir -p build +pushd build +../configure --prefix=/usr --disable-fortran --with-slurm=/usr/include/slurm +make -j$(nproc) install +popd +popd +rm -rf mpich diff --git a/containers/spindle-slurm-ubuntu/testing-plugin/conf/slurm.conf b/containers/spindle-slurm-ubuntu/testing-plugin/conf/slurm.conf index abf060d5..e4298157 100644 --- a/containers/spindle-slurm-ubuntu/testing-plugin/conf/slurm.conf +++ b/containers/spindle-slurm-ubuntu/testing-plugin/conf/slurm.conf @@ -34,6 +34,7 @@ JobAcctGatherFrequency=30 AccountingStorageType=accounting_storage/slurmdbd AccountingStorageHost=slurm-db AccountingStoragePort=6819 +PrologFlags=Contain NodeName=slurm-node-1 NodeAddr=slurm-node-1 CPUs=3 RealMemory=1000 State=UNKNOWN NodeName=slurm-node-2 NodeAddr=slurm-node-2 CPUs=3 RealMemory=1000 State=UNKNOWN NodeName=slurm-node-3 NodeAddr=slurm-node-3 CPUs=3 RealMemory=1000 State=UNKNOWN diff --git a/containers/spindle-slurm-ubuntu/testing-plugin/scripts/build_spindle.sh b/containers/spindle-slurm-ubuntu/testing-plugin/scripts/build_spindle.sh index 8b759415..a52917ba 100755 --- a/containers/spindle-slurm-ubuntu/testing-plugin/scripts/build_spindle.sh +++ b/containers/spindle-slurm-ubuntu/testing-plugin/scripts/build_spindle.sh @@ -3,7 +3,7 @@ set -euxo pipefail mkdir -p /home/${USER}/Spindle-build cd /home/${USER}/Spindle-build -/home/${USER}/Spindle/configure --prefix=/home/${USER}/Spindle-inst --enable-sec-munge --with-rm=slurm-plugin --enable-slurm-plugin --with-cachepaths=/tmp/commpath/cachepath --with-commpath=/tmp/commpath --enable-crash-dedup CFLAGS="-O2 -g" CXXFLAGS="-O2 -g" +/home/${USER}/Spindle/configure --prefix=/home/${USER}/Spindle-inst --enable-sec-munge --with-rm=slurm-plugin --enable-slurm-plugin --with-cachepaths=/tmp/commpath/cachepath --with-commpath='$TMPDIR/commpath' --enable-crash-dedup CFLAGS="-O2 -g" CXXFLAGS="-O2 -g" make -j$(nproc) make install diff --git a/containers/spindle-slurm-ubuntu/testing-srun/Dockerfile.podman b/containers/spindle-slurm-ubuntu/testing-srun/Dockerfile.podman new file mode 100644 index 00000000..681eb582 --- /dev/null +++ b/containers/spindle-slurm-ubuntu/testing-srun/Dockerfile.podman @@ -0,0 +1,55 @@ +FROM spindle-slurm-base:latest +ARG replicas=4 +ENV workers=${replicas} +ENV TMPDIR=/tmp +RUN echo 'TMPDIR="/tmp"' >> /etc/environment +ENV SPINDLE_TEST_CONTAINER=1 + +ARG BUILD_ROOT=containers/spindle-slurm-ubuntu/testing-srun + +# Slurm daemons run as $SLURM_USER +ARG SLURM_USER=slurm + +# Applications run as $USER +ARG USER=slurmuser +ARG UID=1001 + +# Set up the Slurm install already present in the base image +USER root +COPY ${BUILD_ROOT}/scripts/setup_slurm.sh.podman /setup_slurm.sh +COPY ${BUILD_ROOT}/conf/slurm.conf.template /home/${SLURM_USER}/slurm.conf.template +COPY ${BUILD_ROOT}/conf/slurmdbd.conf.template /home/${SLURM_USER}/slurmdbd.conf.template +COPY ${BUILD_ROOT}/conf/cgroup.conf /home/${SLURM_USER}/cgroup.conf +RUN chmod +x /setup_slurm.sh && /setup_slurm.sh + +USER ${USER} +WORKDIR /home/${USER} + +# Copy the Spindle repo into the container and build it +RUN mkdir -p /home/${USER}/Spindle +COPY . /home/${USER}/Spindle + +# Fix permissions on Spindle source +USER root +RUN chmod -R u+rwX /home/${USER}/Spindle && \ + chown -R ${USER}:${USER} /home/${USER}/Spindle + +# Copy and prepare build script +COPY ${BUILD_ROOT}/scripts/build_spindle.sh /home/${USER}/build_spindle.sh +RUN chmod +rx /home/${USER}/build_spindle.sh && \ + chown ${USER}:${USER} /home/${USER}/build_spindle.sh + +# Build Spindle as user +USER ${USER} +RUN bash ./build_spindle.sh + +# Copy entrypoint as root +USER root +COPY ${BUILD_ROOT}/scripts/entrypoint.sh.podman /home/${USER}/entrypoint.sh +RUN chmod +rx /home/${USER}/entrypoint.sh && \ + chown ${USER}:${USER} /home/${USER}/entrypoint.sh + +USER ${USER} +ENV PATH /home/${USER}/Spindle-inst/bin:$PATH + +ENTRYPOINT /bin/bash ./entrypoint.sh diff --git a/containers/spindle-slurm-ubuntu/testing-srun/conf/slurm.conf.template b/containers/spindle-slurm-ubuntu/testing-srun/conf/slurm.conf.template new file mode 100644 index 00000000..22de2c23 --- /dev/null +++ b/containers/spindle-slurm-ubuntu/testing-srun/conf/slurm.conf.template @@ -0,0 +1,40 @@ +ClusterName=linux +ControlMachine=${SLURM_HEAD_NODE} +ControlAddr=${SLURM_HEAD_NODE} +SlurmUser=slurm +SlurmctldPort=6817 +SlurmdPort=6818 +AuthType=auth/munge +StateSaveLocation=/var/lib/slurmd +SlurmdSpoolDir=/var/spool/slurmd +SwitchType=switch/none +MpiDefault=none +SlurmctldPidFile=/var/run/slurmd/slurmctld.pid +SlurmdPidFile=/var/run/slurmd/slurmd.pid +ProctrackType=proctrack/linuxproc +# TaskPlugin=task/affinity disabled - causes "Operation not permitted" in rootless podman +TaskPlugin=task/none +ReturnToService=2 +SlurmctldTimeout=300 +SlurmdTimeout=300 +InactiveLimit=0 +MinJobAge=300 +KillWait=30 +Waittime=0 +SchedulerType=sched/backfill +SelectType=select/cons_tres +SelectTypeParameters=CR_Core_Memory +SlurmctldDebug=3 +SlurmctldLogFile=/var/log/slurm/slurmctld.log +SlurmdDebug=3 +SlurmdLogFile=/var/log/slurm/slurmd.log +JobCompType=jobcomp/filetxt +JobCompLoc=/var/log/slurm/jobcomp.log +JobAcctGatherType=jobacct_gather/linux +JobAcctGatherFrequency=30 +AccountingStorageType=accounting_storage/slurmdbd +AccountingStorageHost=${SLURM_DB_HOST} +AccountingStoragePort=6819 +# NodeName entries are generated at runtime by entrypoint based on $workers and $SLURM_NODE_PREFIX +PartitionName=debug Nodes=ALL Default=YES MaxTime=INFINITE State=UP + diff --git a/containers/spindle-slurm-ubuntu/testing-srun/conf/slurmdbd.conf b/containers/spindle-slurm-ubuntu/testing-srun/conf/slurmdbd.conf new file mode 100644 index 00000000..01c05970 --- /dev/null +++ b/containers/spindle-slurm-ubuntu/testing-srun/conf/slurmdbd.conf @@ -0,0 +1,11 @@ +AuthType=auth/munge +DbdAddr=slurm-db +DbdHost=slurm-db +SlurmUser=slurm +DebugLevel=4 +LogFile=/var/log/slurm/slurmdbd.log +PidFile=/var/run/slurmdbd/slurmdbd.pid +StorageType=accounting_storage/mysql +StorageHost=slurm-mariadb +StorageUser=slurm +StoragePass=LUJWpLWqKaowyZHEOaQIXQ diff --git a/containers/spindle-slurm-ubuntu/testing-srun/conf/slurmdbd.conf.template b/containers/spindle-slurm-ubuntu/testing-srun/conf/slurmdbd.conf.template index 0e274118..52907024 100644 --- a/containers/spindle-slurm-ubuntu/testing-srun/conf/slurmdbd.conf.template +++ b/containers/spindle-slurm-ubuntu/testing-srun/conf/slurmdbd.conf.template @@ -1,10 +1,10 @@ AuthType=auth/munge -DbdAddr=slurm-db -DbdHost=slurm-db +DbdAddr=${SLURM_DB_HOST} +DbdHost=${SLURM_DB_HOST} SlurmUser=slurm DebugLevel=4 LogFile=/var/log/slurm/slurmdbd.log PidFile=/var/run/slurmdbd/slurmdbd.pid StorageType=accounting_storage/mysql -StorageHost=slurm-mariadb +StorageHost=${SLURM_MARIADB_HOST} StorageUser=slurm diff --git a/containers/spindle-slurm-ubuntu/testing-srun/mariadb.env b/containers/spindle-slurm-ubuntu/testing-srun/mariadb.env new file mode 100644 index 00000000..d61fe3eb --- /dev/null +++ b/containers/spindle-slurm-ubuntu/testing-srun/mariadb.env @@ -0,0 +1 @@ +MARIADB_PASSWORD: "LUJWpLWqKaowyZHEOaQIXQ" diff --git a/containers/spindle-slurm-ubuntu/testing-srun/scripts/entrypoint.sh.podman b/containers/spindle-slurm-ubuntu/testing-srun/scripts/entrypoint.sh.podman new file mode 100644 index 00000000..c9333e84 --- /dev/null +++ b/containers/spindle-slurm-ubuntu/testing-srun/scripts/entrypoint.sh.podman @@ -0,0 +1,85 @@ +#!/usr/bin/env bash +# +# Podman-compatible entrypoint for Slurm srun containers + +set -x # Debug output + +echo "SLURM_ROLE: ${SLURM_ROLE}" + +echo "Starting munged..." +sudo -u munge /usr/sbin/munged +sleep 2 + +# Generate slurm.conf from template with runtime hostname substitution +echo "Generating slurm.conf from template..." +export SLURM_HEAD_NODE=${SLURM_HEAD_NODE:-slurm-head} +export SLURM_DB_HOST=${SLURM_DB_HOST:-slurm-db} +export SLURM_NODE_PREFIX=${SLURM_NODE_PREFIX:-slurm-node} +echo "DEBUG: SLURM_HEAD_NODE=$SLURM_HEAD_NODE" +echo "DEBUG: SLURM_DB_HOST=$SLURM_DB_HOST" +echo "DEBUG: SLURM_NODE_PREFIX=$SLURM_NODE_PREFIX" +echo "DEBUG: Template first 3 lines:" +sudo head -3 /etc/slurm/slurm.conf.template +echo "DEBUG: Running sed to substitute variables..." +sudo sed -e "s/\${SLURM_HEAD_NODE}/$SLURM_HEAD_NODE/g" \ + -e "s/\${SLURM_DB_HOST}/$SLURM_DB_HOST/g" \ + -e "s/\${SLURM_NODE_PREFIX}/$SLURM_NODE_PREFIX/g" \ + /etc/slurm/slurm.conf.template | sudo tee /etc/slurm/slurm.conf > /dev/null + +# Add NodeName entries dynamically based on $workers +workers=${workers:-4} +for i in $(seq 1 $workers); do + echo "NodeName=${SLURM_NODE_PREFIX}-${i} NodeAddr=${SLURM_NODE_PREFIX}-${i} CPUs=3 RealMemory=1000 State=UNKNOWN" | sudo tee -a /etc/slurm/slurm.conf > /dev/null +done +sudo chmod 644 /etc/slurm/slurm.conf + +echo "Generated slurm.conf (first 10 lines):" +sudo head -10 /etc/slurm/slurm.conf +echo "..." +echo "Last 5 lines:" +sudo tail -5 /etc/slurm/slurm.conf +echo "" + +if [ "${SLURM_ROLE}" = "db" ]; then + echo "=== ENTRYPOINT DEBUG (v2) ===" + echo "MARIADB_PASSWORD is set: $([ -n "$MARIADB_PASSWORD" ] && echo YES || echo NO)" + echo "MARIADB_PASSWORD length: ${#MARIADB_PASSWORD}" + echo "Template exists: $([ -f /etc/slurm/slurmdbd.conf.template ] && echo YES || echo NO)" + + # Generate slurmdbd.conf from template with runtime hostname and password substitution + if [ -n "$MARIADB_PASSWORD" ]; then + echo "Generating slurmdbd.conf from template..." + export SLURM_MARIADB_HOST=${SLURM_MARIADB_HOST:-slurm-mariadb} + echo "DEBUG: SLURM_DB_HOST=$SLURM_DB_HOST" + echo "DEBUG: SLURM_MARIADB_HOST=$SLURM_MARIADB_HOST" + echo "DEBUG: Template first 3 lines:" + sudo head -3 /etc/slurm/slurmdbd.conf.template + echo "DEBUG: Running sed to substitute variables..." + sudo sed -e "s/\${SLURM_DB_HOST}/$SLURM_DB_HOST/g" \ + -e "s/\${SLURM_MARIADB_HOST}/$SLURM_MARIADB_HOST/g" \ + /etc/slurm/slurmdbd.conf.template | sudo tee /etc/slurm/slurmdbd.conf > /dev/null + echo "StoragePass=${MARIADB_PASSWORD}" | sudo tee -a /etc/slurm/slurmdbd.conf > /dev/null + sudo chmod 600 /etc/slurm/slurmdbd.conf + sudo chown slurm:slurm /etc/slurm/slurmdbd.conf + echo "Generated slurmdbd.conf (complete file):" + sudo cat /etc/slurm/slurmdbd.conf + else + echo "ERROR: MARIADB_PASSWORD not set!" + echo "Old config exists: $([ -f /etc/slurm/slurmdbd.conf ] && echo YES || echo NO)" + fi + echo "=== END DEBUG ===" + echo "" + + echo "Starting slurmdbd..." + exec sudo -u slurm /usr/sbin/slurmdbd -Dvvv +elif [ "${SLURM_ROLE}" = "ctl" ] ; then + echo "Starting slurmctld..." + exec sudo -u slurm /usr/sbin/slurmctld -i -Dvvv +elif [ "${SLURM_ROLE}" = "worker" ] ; then + echo "Starting slurmd..." + exec sudo /usr/sbin/slurmd -Dvvv +else + echo "Unknown SLURM_ROLE: ${SLURM_ROLE}" + echo "Valid roles: db, ctl, worker" + exit 1 +fi diff --git a/containers/spindle-slurm-ubuntu/testing-srun/scripts/setup_slurm.sh.podman b/containers/spindle-slurm-ubuntu/testing-srun/scripts/setup_slurm.sh.podman new file mode 100644 index 00000000..7d73ccac --- /dev/null +++ b/containers/spindle-slurm-ubuntu/testing-srun/scripts/setup_slurm.sh.podman @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +set -euxo pipefail + +mkdir -p /etc/slurm /etc/sysconfig/slurm /var/spool/slurmd /var/spool/slurmctld /var/run/slurmd /var/run/slurmdbd /var/lib/slurmd /var/log/slurm +touch /var/lib/slurmd/node_state /var/lib/slurmd/front_end_state /var/lib/slurmd/job_state /var/lib/slurmd/resv_state /var/lib/slurmd/trigger_state /var/lib/slurmd/assoc_mgr_state /var/lib/slurmd/assoc_usage /var/lib/slurmd/qos_usage /var/lib/slurmd/fed_mgr_state +cp /home/${SLURM_USER}/slurm.conf.template /etc/slurm/slurm.conf.template +cp /home/${SLURM_USER}/slurmdbd.conf.template /etc/slurm/slurmdbd.conf.template +cp /home/${SLURM_USER}/cgroup.conf /etc/slurm/cgroup.conf +chown -R slurm:slurm /etc/slurm /etc/sysconfig/slurm /var/spool/slurmd /var/spool/slurmctld /var/run/slurmd /var/run/slurmdbd /var/lib/slurmd /var/log/slurm +# Templates should be readable by slurm (will be used to generate configs at runtime) +chmod 644 /etc/slurm/slurm.conf.template /etc/slurm/slurmdbd.conf.template +# cgroup.conf should be world-readable (needed by slurmuser for salloc/srun) +chmod 644 /etc/slurm/cgroup.conf diff --git a/containers/spindle-slurm-ubuntu/testing/Dockerfile.podman b/containers/spindle-slurm-ubuntu/testing/Dockerfile.podman new file mode 100644 index 00000000..96ab2219 --- /dev/null +++ b/containers/spindle-slurm-ubuntu/testing/Dockerfile.podman @@ -0,0 +1,68 @@ +FROM spindle-slurm-base:latest +ARG replicas=4 +ENV workers=${replicas} +ENV TMPDIR=/tmp +RUN echo 'TMPDIR="/tmp"' >> /etc/environment +ENV SPINDLE_TEST_CONTAINER=1 + +ARG BUILD_ROOT=containers/spindle-slurm-ubuntu/testing + +# Slurm daemons run as $SLURM_USER +ARG SLURM_USER=slurm + +# Applications run as $USER +ARG USER=slurmuser +ARG UID=1001 + +RUN apt-get update \ + && DEBIAN_FRONTEND="noninteractive" apt-get -qq install -y --no-install-recommends \ + gdb \ + libc6-dbg + +# Set up the Slurm install already present in the base image +USER root +COPY ${BUILD_ROOT}/scripts/setup_slurm.sh.podman /setup_slurm.sh +COPY ${BUILD_ROOT}/conf/slurm.conf.podman /home/${SLURM_USER}/slurm.conf +COPY ${BUILD_ROOT}/conf/slurmdbd.conf /home/${SLURM_USER}/slurmdbd.conf +COPY ${BUILD_ROOT}/conf/cgroup.conf /home/${SLURM_USER}/cgroup.conf +RUN chmod +x /setup_slurm.sh && /setup_slurm.sh + +# Slurm without Spank plugin needs passwordless ssh +USER root +COPY ${BUILD_ROOT}/conf/ssh_config /home/${USER}/ +COPY ${BUILD_ROOT}/scripts/setup_ssh.sh /home/${USER}/ +RUN chmod +x /home/${USER}/setup_ssh.sh && \ + chown ${USER}:${USER} /home/${USER}/ssh_config /home/${USER}/setup_ssh.sh + +USER ${USER} +WORKDIR /home/${USER} +RUN bash ./setup_ssh.sh + +# Copy the Spindle repo into the container and build it +RUN mkdir -p /home/${USER}/Spindle +COPY . /home/${USER}/Spindle + +# Fix permissions on Spindle source +USER root +RUN chmod -R u+rwX /home/${USER}/Spindle && \ + chown -R ${USER}:${USER} /home/${USER}/Spindle + +# Copy and prepare build script +COPY ${BUILD_ROOT}/scripts/build_spindle.sh /home/${USER}/build_spindle.sh +RUN chmod +rx /home/${USER}/build_spindle.sh && \ + chown ${USER}:${USER} /home/${USER}/build_spindle.sh + +# Build Spindle as user +USER ${USER} +RUN bash ./build_spindle.sh + +# Copy entrypoint as root +USER root +COPY ${BUILD_ROOT}/scripts/entrypoint.sh.podman /home/${USER}/entrypoint.sh +RUN chmod +rx /home/${USER}/entrypoint.sh && \ + chown ${USER}:${USER} /home/${USER}/entrypoint.sh + +USER ${USER} +ENV PATH /home/${USER}/Spindle-inst/bin:$PATH + +ENTRYPOINT /bin/bash ./entrypoint.sh diff --git a/containers/spindle-slurm-ubuntu/testing/conf/slurm.conf.podman b/containers/spindle-slurm-ubuntu/testing/conf/slurm.conf.podman new file mode 100644 index 00000000..1a308beb --- /dev/null +++ b/containers/spindle-slurm-ubuntu/testing/conf/slurm.conf.podman @@ -0,0 +1,43 @@ +ClusterName=linux +ControlMachine=slurm-head +ControlAddr=slurm-head +SlurmUser=slurm +SlurmctldPort=6817 +SlurmdPort=6818 +AuthType=auth/munge +StateSaveLocation=/var/lib/slurmd +SlurmdSpoolDir=/var/spool/slurmd +SwitchType=switch/none +MpiDefault=none +SlurmctldPidFile=/var/run/slurmd/slurmctld.pid +SlurmdPidFile=/var/run/slurmd/slurmd.pid +ProctrackType=proctrack/linuxproc +# TaskPlugin=task/affinity disabled - causes "Operation not permitted" in rootless podman +TaskPlugin=task/none +ReturnToService=2 +SlurmctldTimeout=300 +SlurmdTimeout=300 +InactiveLimit=0 +MinJobAge=300 +KillWait=30 +Waittime=0 +SchedulerType=sched/backfill +SelectType=select/cons_tres +SelectTypeParameters=CR_Core_Memory +SlurmctldDebug=3 +SlurmctldLogFile=/var/log/slurm/slurmctld.log +SlurmdDebug=3 +SlurmdLogFile=/var/log/slurm/slurmd.log +JobCompType=jobcomp/filetxt +JobCompLoc=/var/log/slurm/jobcomp.log +JobAcctGatherType=jobacct_gather/linux +JobAcctGatherFrequency=30 +AccountingStorageType=accounting_storage/slurmdbd +AccountingStorageHost=slurm-db +AccountingStoragePort=6819 +NodeName=slurm-node-1 NodeAddr=slurm-node-1 CPUs=3 RealMemory=1000 State=UNKNOWN +NodeName=slurm-node-2 NodeAddr=slurm-node-2 CPUs=3 RealMemory=1000 State=UNKNOWN +NodeName=slurm-node-3 NodeAddr=slurm-node-3 CPUs=3 RealMemory=1000 State=UNKNOWN +NodeName=slurm-node-4 NodeAddr=slurm-node-4 CPUs=3 RealMemory=1000 State=UNKNOWN +PartitionName=debug Nodes=ALL Default=YES MaxTime=INFINITE State=UP + diff --git a/containers/spindle-slurm-ubuntu/testing/scripts/entrypoint.sh.podman b/containers/spindle-slurm-ubuntu/testing/scripts/entrypoint.sh.podman new file mode 100644 index 00000000..72d2d681 --- /dev/null +++ b/containers/spindle-slurm-ubuntu/testing/scripts/entrypoint.sh.podman @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# +# Podman-compatible entrypoint for Slurm rshlaunch containers + +set -x # Debug output + +echo "SLURM_ROLE: ${SLURM_ROLE}" + +echo "Configuring SSH..." +# Create privilege separation directory +sudo mkdir -p /run/sshd +sudo chmod 755 /run/sshd + +# Regenerate SSH host keys if they don't exist or are problematic +if [ ! -f /etc/ssh/ssh_host_rsa_key ]; then + sudo ssh-keygen -A +fi +# Fix permissions +sudo chmod 600 /etc/ssh/ssh_host_*_key +sudo chmod 644 /etc/ssh/ssh_host_*_key.pub + +echo "Starting sshd..." +# Disable privilege separation - setgroups fails in rootless podman on LC systems +sudo bash -c 'ulimit -c unlimited; /usr/sbin/sshd -o UsePrivilegeSeparation=no' + +echo "Starting munged..." +sudo -u munge /usr/sbin/munged +sleep 2 + +if [ -d /shared ]; then + sudo chown -R "$(id -un):$(id -gn)" /shared + sudo chmod 755 /shared +fi + +if [ "${SLURM_ROLE}" = "db" ]; then + echo "Starting slurmdbd..." + exec sudo -u slurm /usr/sbin/slurmdbd -Dvvv +elif [ "${SLURM_ROLE}" = "ctl" ] ; then + echo "Starting slurmctld..." + exec sudo -u slurm /usr/sbin/slurmctld -i -Dvvv +elif [ "${SLURM_ROLE}" = "worker" ] ; then + echo "Starting slurmd..." + exec sudo bash -c 'ulimit -c unlimited; exec /usr/sbin/slurmd -Dvvv' +else + echo "Unknown SLURM_ROLE: ${SLURM_ROLE}" + echo "Valid roles: db, ctl, worker" + exit 1 +fi diff --git a/containers/spindle-slurm-ubuntu/testing/scripts/setup_slurm.sh.podman b/containers/spindle-slurm-ubuntu/testing/scripts/setup_slurm.sh.podman new file mode 100644 index 00000000..1adb6e42 --- /dev/null +++ b/containers/spindle-slurm-ubuntu/testing/scripts/setup_slurm.sh.podman @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +set -euxo pipefail + +mkdir -p /etc/slurm /etc/sysconfig/slurm /var/spool/slurmd /var/spool/slurmctld /var/run/slurmd /var/run/slurmdbd /var/lib/slurmd /var/log/slurm +touch /var/lib/slurmd/node_state /var/lib/slurmd/front_end_state /var/lib/slurmd/job_state /var/lib/slurmd/resv_state /var/lib/slurmd/trigger_state /var/lib/slurmd/assoc_mgr_state /var/lib/slurmd/assoc_usage /var/lib/slurmd/qos_usage /var/lib/slurmd/fed_mgr_state +cp /home/${SLURM_USER}/slurm.conf /etc/slurm/slurm.conf +cp /home/${SLURM_USER}/slurmdbd.conf /etc/slurm/slurmdbd.conf +cp /home/${SLURM_USER}/cgroup.conf /etc/slurm/cgroup.conf +chown -R slurm:slurm /etc/slurm /etc/sysconfig/slurm /var/spool/slurmd /var/spool/slurmctld /var/run/slurmd /var/run/slurmdbd /var/lib/slurmd /var/log/slurm +# slurmdbd.conf should be readable only by slurm (has password) +chmod 600 /etc/slurm/slurmdbd.conf +# slurm.conf and cgroup.conf should be world-readable (needed by slurmuser for salloc/srun) +chmod 644 /etc/slurm/slurm.conf /etc/slurm/cgroup.conf diff --git a/doc/slurm_plugin.md b/doc/slurm_plugin.md new file mode 100644 index 00000000..f6ac1982 --- /dev/null +++ b/doc/slurm_plugin.md @@ -0,0 +1,125 @@ +Spindle Slurm plugin +==================== + +The Spindle Slurm plugin integrates Spindle into Slurm through the +SPANK interface as an alternative launch mechanism to the srun wrapper. +It adds the ability to launch job steps using `srun --spindle`. + +## Building and configuring the plugin + +Configure Spindle with `--enable-slurm-plugin`: + +```bash +./configure --with-rm=slurm-plugin --enable-slurm-plugin [--with-slurm-dir=/path/to/slurm] ... +make +make install +``` + +Refer to `INSTALL` for more details on configuring Spindle. + +After installation of Spindle, the plugin is installed at +`$PREFIX/lib/libspindleslurm.so`. It is registered with Slurm by adding the +following line to `/etc/slurm/plugstack.conf`: + +``` +required /path/to/spindle/lib/libspindleslurm.so +``` + +### Site-wide default options + +Any arguments listed after the library path in `plugstack.conf` are +passed to Spindle as site-wide default options, applied to every +Spindle launch on the cluster. For example: + +``` +required /path/to/spindle/lib/libspindleslurm.so --level=low +``` + +## Session launch modes + +The manner in which Spindle sessions are started varies depending on +the configuration of Spindle and of Slurm. + +When starting a session, the plugin must arrange for Spindle to start +on each compute node before any step runs within the allocation. +The most straightforward way to do this is to configure the cluster +to run job prologs at allocation time. If your `slurm.conf` includes +`PrologFlags=Alloc` (or another flag that implies it: `Contain`, +`RunInJob`, `X11`, or `ForceRequeueOnFail`), then sessions +will be started on each node of the allocation at the time the allocation +is made. + +If `PrologFlags=Alloc` or a related setting is *not* used, one of two +mechanisms is used to start the job on every node: + +**RSH launch**: Spindle can use RSH/SSH to launch daemons from the +frontend (FE) process. To use the RSH launch mode, the cluster must be configured +such that passwordless ssh can be used to run commands on every compute +node within the allocation without any interactive user input. +This mode is enabled by configuring Spindle with: + +```bash +./configure --with-rm=slurm-plugin --enable-slurm-plugin --with-rsh-launch [--with-rsh-cmd=/usr/bin/ssh] ... +``` + +**Dummy srun fallback**: If neither `PrologFlags=Alloc` nor RSH launch is available, +Spindle will fall back on using a dummy `srun` invocation to force the prolog +to run on every compute node of the allocation. Note that this has the side-effect +of consuming step 0, so that the user's first step will instead be numbered 1. + +## Using Spindle through the Slurm plugin + +### Per-step mode: `--spindle` + +Add `--spindle` to any `srun` command to use Spindle for that step. +Spindle daemons start before the application runs and shut down when +the step finishes. + +```bash +srun --spindle ./my_application +``` + +Additional arguments can be passed to Spindle as an optional value of the argument `--spindle`: + +```bash +srun --spindle="--level=low" ./my_application +``` + +### Session mode: `--spindle-session` + +Session mode shares a Spindle session across multiple steps. +The use of sessions in the Slurm plugin differs from its use with +the other launchers. Unlike the other launchers, sessions are *not* +started with `spindle --start-session`. Rather, an additional argument +`--spindle-session` is added to `salloc` and `sbatch`. + +To use a session, include `--spindle-session` when creating the allocation: + +```bash +salloc --spindle-session ... +``` + +Then run steps with `--spindle`: + +```bash +srun --spindle ./app1 +srun --spindle ./app2 +srun --spindle ./app3 +``` + +All steps within the allocation will run in the same Spindle session. +When the allocation exits, the session will terminate automatically. + +Sessions can be used with an `sbatch` script as shown below: + +```bash +#!/bin/bash +#SBATCH --spindle-session +#SBATCH -N 4 +#SBATCH -n 4 + +srun --spindle ./app1 +srun --spindle ./app2 +srun --spindle ./app3 +``` + diff --git a/scripts/podman/README.md b/scripts/podman/README.md new file mode 100644 index 00000000..2c87e895 --- /dev/null +++ b/scripts/podman/README.md @@ -0,0 +1,219 @@ +# Spindle Podman Scripts + +Scripts for running Spindle containers locally with podman on LC systems. + +## Prerequisites + +- Podman installed and working +- **LC systems: Run `enable-podman` before using these scripts** +- Subuid/subgid configured for your user + +## Quick Start + +### Hello World Tests (verify environment) + +```bash +./run-hello-01-basic.sh # Basic container execution +./run-hello-02-user.sh # User switching +./run-hello-03-filesystem.sh # Volume mounts +./run-hello-04-networking.sh # Multi-container networking +./run-hello-05-flux.sh # Flux cluster +./run-hello-06-slurm.sh # Slurm cluster +``` + +### Spindle Tests + +```bash +# Build images +./build-spindle-serial.sh +./build-spindle-slurm-base.sh && ./build-spindle-slurm-srun.sh +./build-spindle-flux.sh + +# Run tests +./test-spindle-serial.sh +./test-spindle-slurm-srun.sh +./test-spindle-flux.sh + +# Debug variants (keeps containers running) +./test-spindle-serial-debug.sh + +# Crash tests +./test-spindle-serial-crash.sh +``` + +### Parallel Testing on Compute Nodes + +```bash +# On login node: save images to tarball +./save-images.sh + +# On compute node: load images +./load-images.sh /path/to/spindle-podman-images.tar + +# Run multiple tests in parallel +for i in $(seq 1 10); do + ./test-spindle-slurm-srun-parallel.sh $i > out.$i 2>&1 & +done +wait + +# Clean up +./cleanup.sh +``` + +## Script Reference + +### Infrastructure + +- **common.sh** - Shared functions and LC-specific configurations + - SSL certificate mounts for LC systems + - `podman_build()` and `podman_run()` wrappers with PODMAN_BUILD arg + - Handles setgroups workaround + +- **cleanup.sh** - Remove all or specific test containers/networks + - Usage: `./cleanup.sh` (all) or `./cleanup.sh ` (specific) + - Parallel cleanup for speed + +- **save-images.sh** - Save Spindle images to tarball for compute node deployment + - Creates `spindle-podman-images.tar` in repo root + - Bundles `mariadb.env` for portable Slurm testing + +- **load-images.sh** - Load images from tarball on compute nodes + - Usage: `./load-images.sh ` + - Extracts and loads each image separately + +### Hello World Tests + +- **run-hello-01-basic.sh** - Basic execution (validates environment) +- **run-hello-02-user.sh** - User switching (validates non-root patterns) +- **run-hello-03-filesystem.sh** - Volume mounts (validates host/container filesystem) +- **run-hello-04-networking.sh** - Multi-container networking (validates cluster patterns) +- **run-hello-05-flux.sh** - Flux cluster (validates Flux setup) +- **run-hello-06-slurm.sh** - Slurm cluster (validates Slurm setup) + +### Build Scripts + +- **build-spindle-serial.sh** - Build serial Spindle container +- **build-spindle-slurm-base.sh** - Build Slurm base image (Slurm + MPICH from source, ~6 min) +- **build-spindle-slurm-srun.sh** - Build Slurm srun test layer (requires base) +- **build-spindle-slurm-rshlaunch.sh** - Build Slurm rshlaunch test layer (BLOCKED: SSH issues) +- **build-spindle-flux.sh** - Build Flux Spindle container (~7 min) + +### Test Scripts + +- **test-spindle-serial.sh** - Serial launcher tests +- **test-spindle-serial-debug.sh** - Serial tests with containers kept running for debugging +- **test-spindle-serial-crash.sh** - Serial crash dump tests + +- **test-spindle-slurm-srun.sh** - Slurm srun tests (7 containers: MariaDB, slurmdbd, slurmctld, 4 workers) +- **test-spindle-slurm-srun-parallel.sh** - Parallel-safe variant with unique container names + - Usage: `./test-spindle-slurm-srun-parallel.sh ` + - Enables multiple concurrent test runs without conflicts + +- **test-spindle-slurm-rshlaunch.sh** - Slurm rshlaunch tests (BLOCKED: SSH setgroups issue) + +- **test-spindle-flux.sh** - Flux launcher tests (PARTIALLY WORKING: cluster starts but Spindle tests fail) + +## Deployment Workflow + +### Local Development (Login Node) + +1. Build images: `./build-spindle-*.sh` +2. Test locally: `./test-spindle-*.sh` +3. Iterate and debug + +### Compute Node Deployment + +1. **On login node:** + ```bash + ./save-images.sh + # Creates spindle-podman-images.tar + mariadb.env + ``` + +2. **On compute node:** + ```bash + enable-podman # REQUIRED on LC systems + ./load-images.sh /path/to/spindle-podman-images.tar + ``` + +3. **Run parallel tests:** + ```bash + for i in $(seq 1 50); do + ./test-spindle-slurm-srun-parallel.sh $i > out.$i 2>&1 & + done + wait + + # Check results + grep -l "ALL TESTS PASSED" out.* + grep -l "SOME TESTS FAILED" out.* + ``` + +4. **Cleanup:** + ```bash + ./cleanup.sh + ``` + +## Known Issues + +### On LC Systems: Must Run enable-podman First + +LC systems require running `enable-podman` before using podman. This configures storage and clears any stale state. If you see "Access denied" or image loading issues, run `enable-podman` and retry. + +### Slurm Tests Require mariadb.env + +Slurm tests need the MariaDB password from `mariadb.env`. The `save-images.sh` script copies this to the repo root for portability. If you get "Access denied for user 'slurm'@...", check that `mariadb.env` exists in the repo root. + +### Parallel Test Limits + +Rootless podman has resource limits. On compute nodes, 50-100 parallel tests work well. Beyond that, you may hit: +- File descriptor limits +- Network namespace limits +- Memory pressure + +### Flux Tests Partially Working + +Flux cluster starts correctly, but Spindle tests fail with "spindleRunBE failed!". Needs debugging. See PODMAN.md for details. + +### Slurm rshlaunch Blocked + +SSH privilege separation calls `setgroups()`, which fails in rootless podman on LC systems. Tried `UsePrivilegeSeparation=no` but still fails. May need sshd rebuild or alternative SSH implementation. + +## Troubleshooting + +### "newuidmap failed: Operation not permitted" +Your user doesn't have subuid/subgid mappings. Check: +```bash +grep $(whoami) /etc/subuid /etc/subgid +``` + +If empty, contact your sysadmin to add entries. + +### "certificate signed by unknown authority" +The SSL certificate mounts may be incorrect for your system. Check if these files exist: +```bash +ls -l /etc/pki/ca-trust/source/anchors/PAN-cspca.llnl.gov.crt.pem +ls -l /etc/pki/tls/certs/ca-bundle.trust.crt +``` + +### "setgroups 65534 failed" +The Dockerfile needs the apt sandbox fix. Verify the Dockerfile has: +```dockerfile +ARG PODMAN_BUILD=false +RUN if [ "$PODMAN_BUILD" = "true" ]; then \ + echo 'APT::Sandbox::User root;' > /etc/apt/apt.conf.d/00-apt-sandbox; \ + fi +``` + +And you're using `podman_build()` from common.sh (not raw `podman build`). + +### Image IDs All the Same After Load + +If `podman images | grep spindle` shows the same image ID for all tags, the tarball wasn't created correctly. Re-run `save-images.sh` which saves each image separately then combines them. + +### Containers Exit Immediately + +Check that you ran `enable-podman` before loading images. Also verify images loaded correctly with `podman images | grep spindle` - each should have a different ID. + +## LC System Documentation + +For more details on podman on LC systems: +https://hpc.llnl.gov/documentation/user-guides/using-containers-lc-hpc-systems/containers-how-build-container diff --git a/scripts/podman/build-spindle-flux.sh b/scripts/podman/build-spindle-flux.sh new file mode 100755 index 00000000..17b6397a --- /dev/null +++ b/scripts/podman/build-spindle-flux.sh @@ -0,0 +1,44 @@ +#!/bin/bash +# +# Build Spindle Flux container for podman +# +# This builds the Spindle Flux multi-node container. +# Run this from outside the sandbox where podman is available. + +set -e + +# Get the directory containing this script +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +# Source common functions +source "$SCRIPT_DIR/common.sh" + +# Configuration +IMAGE_NAME="spindle-flux-ubuntu" +DOCKERFILE="$REPO_ROOT/containers/spindle-flux-ubuntu/Dockerfile.podman" + +echo "==========================================" +echo "Building Spindle Flux Container" +echo "==========================================" +echo "" +echo "This builds the Spindle Flux multi-node test container." +echo "Image: $IMAGE_NAME" +echo "Dockerfile: $DOCKERFILE" +echo "" + +# Build the image +# Pass replicas=4 to match docker-compose configuration +podman_build "$IMAGE_NAME" "$DOCKERFILE" "$REPO_ROOT" "--build-arg replicas=4" + +echo "" +echo "==========================================" +echo "Build complete!" +echo "==========================================" +echo "" +echo "Image: $IMAGE_NAME" +echo "" +echo "Next steps:" +echo " 1. Run regular tests: ./scripts/podman/test-spindle-flux.sh" +echo " 2. Run crash tests: ./scripts/podman/test-spindle-flux-crash.sh" +echo "" diff --git a/scripts/podman/build-spindle-serial-interactive.sh b/scripts/podman/build-spindle-serial-interactive.sh new file mode 100755 index 00000000..0bda1b93 --- /dev/null +++ b/scripts/podman/build-spindle-serial-interactive.sh @@ -0,0 +1,44 @@ +#!/bin/bash +# +# Build Spindle serial-interactive container for podman +# +# This builds a serial container that stays running for interactive debugging. +# Run this from outside the sandbox where podman is available. + +set -e + +# Get the directory containing this script +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +# Source common functions +source "$SCRIPT_DIR/common.sh" + +# Configuration +IMAGE_NAME="spindle-serial-interactive-ubuntu" +DOCKERFILE="$REPO_ROOT/containers/spindle-serial-interactive-ubuntu/Dockerfile.podman" + +echo "==========================================" +echo "Building Spindle Serial Interactive Container" +echo "==========================================" +echo "" +echo "This builds a serial container for interactive debugging." +echo "Image: $IMAGE_NAME" +echo "Dockerfile: $DOCKERFILE" +echo "" + +# Build the image +podman_build "$IMAGE_NAME" "$DOCKERFILE" "$REPO_ROOT" + +echo "" +echo "==========================================" +echo "Build complete!" +echo "==========================================" +echo "" +echo "Image: $IMAGE_NAME" +echo "" +echo "Next steps:" +echo " 1. Start container: ./scripts/podman/run-spindle-serial-interactive.sh" +echo " 2. Connect to it: podman exec -it spindlenode-interactive bash" +echo " 3. Run tests manually: cd Spindle-build/testsuite && SPINDLE_DEBUG=3 ./runTests" +echo "" diff --git a/scripts/podman/build-spindle-serial.sh b/scripts/podman/build-spindle-serial.sh new file mode 100755 index 00000000..b7ae2428 --- /dev/null +++ b/scripts/podman/build-spindle-serial.sh @@ -0,0 +1,44 @@ +#!/bin/bash +# +# Build Spindle serial container for podman +# +# This builds the actual Spindle serial container (not a hello-world demo). +# Run this from outside the sandbox where podman is available. + +set -e + +# Get the directory containing this script +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +# Source common functions +source "$SCRIPT_DIR/common.sh" + +# Configuration +IMAGE_NAME="spindle-serial-ubuntu" +DOCKERFILE="$REPO_ROOT/containers/spindle-serial-ubuntu/Dockerfile.podman" + +echo "==========================================" +echo "Building Spindle Serial Container" +echo "==========================================" +echo "" +echo "This builds the actual Spindle serial test container." +echo "Image: $IMAGE_NAME" +echo "Dockerfile: $DOCKERFILE" +echo "" + +# Build the image +podman_build "$IMAGE_NAME" "$DOCKERFILE" "$REPO_ROOT" + +echo "" +echo "==========================================" +echo "Build complete!" +echo "==========================================" +echo "" +echo "Image: $IMAGE_NAME" +echo "" +echo "Next steps:" +echo " 1. Run regular tests: ./scripts/podman/test-spindle-serial.sh" +echo " 2. Run crash tests: ./scripts/podman/test-spindle-serial-crash.sh" +echo " 3. Or manually: podman run --rm -it $IMAGE_NAME" +echo "" diff --git a/scripts/podman/build-spindle-slurm-base.sh b/scripts/podman/build-spindle-slurm-base.sh new file mode 100755 index 00000000..678bc3d6 --- /dev/null +++ b/scripts/podman/build-spindle-slurm-base.sh @@ -0,0 +1,44 @@ +#!/bin/bash +# +# Build Spindle Slurm base image for podman +# +# This builds the base image with Slurm and MPICH. +# Run this from outside the sandbox where podman is available. + +set -e + +# Get the directory containing this script +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +# Source common functions +source "$SCRIPT_DIR/common.sh" + +# Configuration +IMAGE_NAME="spindle-slurm-base" +DOCKERFILE="$REPO_ROOT/containers/spindle-slurm-ubuntu/base/Dockerfile.podman" +CONTEXT="$REPO_ROOT/containers/spindle-slurm-ubuntu/base" + +echo "==========================================" +echo "Building Spindle Slurm Base Image" +echo "==========================================" +echo "" +echo "This builds the base image with Slurm and MPICH." +echo "This will take 5-10 minutes (building Slurm from source)." +echo "Image: $IMAGE_NAME" +echo "Dockerfile: $DOCKERFILE" +echo "" + +# Build the image +podman_build "$IMAGE_NAME" "$DOCKERFILE" "$CONTEXT" + +echo "" +echo "==========================================" +echo "Build complete!" +echo "==========================================" +echo "" +echo "Image: $IMAGE_NAME" +echo "" +echo "Next step: Build the testing image" +echo " ./scripts/podman/build-spindle-slurm-srun.sh" +echo "" diff --git a/scripts/podman/build-spindle-slurm-rshlaunch.sh b/scripts/podman/build-spindle-slurm-rshlaunch.sh new file mode 100755 index 00000000..e319fb1d --- /dev/null +++ b/scripts/podman/build-spindle-slurm-rshlaunch.sh @@ -0,0 +1,66 @@ +#!/bin/bash +# +# Build Spindle Slurm rshlaunch test container for podman +# +# This builds the Slurm testing image with Spindle (rshlaunch launcher). +# Requires: spindle-slurm-base image (build with build-spindle-slurm-base.sh first) +# Run this from outside the sandbox where podman is available. + +set -e + +# Get the directory containing this script +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +# Source common functions +source "$SCRIPT_DIR/common.sh" + +# Configuration +BASE_IMAGE="spindle-slurm-base" +IMAGE_NAME="spindle-slurm-rshlaunch" +DOCKERFILE="$REPO_ROOT/containers/spindle-slurm-ubuntu/testing/Dockerfile.podman" + +echo "==========================================" +echo "Building Spindle Slurm Rshlaunch Container" +echo "==========================================" +echo "" + +# Check if base image exists (with or without localhost/ prefix) +if ! podman images --format "{{.Repository}}" | grep -qE "^(localhost/)?${BASE_IMAGE}$"; then + echo "Error: Base image $BASE_IMAGE not found" + echo "" + echo "Available images:" + podman images | grep spindle || echo " (no spindle images found)" + echo "" + echo "Build it first with:" + echo " ./scripts/podman/build-spindle-slurm-base.sh" + echo "" + exit 1 +fi + +echo "Base image: $BASE_IMAGE" +echo "Test image: $IMAGE_NAME" +echo "Dockerfile: $DOCKERFILE" +echo "" + +# Generate MariaDB configuration +echo "Generating MariaDB configuration..." +cd "$REPO_ROOT/containers/spindle-slurm-ubuntu/testing" +./generate_config.sh +echo "✓ Configuration generated" +cd "$REPO_ROOT" +echo "" + +# Build the image +podman_build "$IMAGE_NAME" "$DOCKERFILE" "$REPO_ROOT" "--build-arg replicas=4" + +echo "" +echo "==========================================" +echo "Build complete!" +echo "==========================================" +echo "" +echo "Image: $IMAGE_NAME" +echo "" +echo "Next steps:" +echo " 1. Run regular tests: ./scripts/podman/test-spindle-slurm-rshlaunch.sh" +echo "" diff --git a/scripts/podman/build-spindle-slurm-srun.sh b/scripts/podman/build-spindle-slurm-srun.sh new file mode 100755 index 00000000..a2a7e64e --- /dev/null +++ b/scripts/podman/build-spindle-slurm-srun.sh @@ -0,0 +1,66 @@ +#!/bin/bash +# +# Build Spindle Slurm srun test container for podman +# +# This builds the Slurm testing image with Spindle. +# Requires: spindle-slurm-base image (build with build-spindle-slurm-base.sh first) +# Run this from outside the sandbox where podman is available. + +set -e + +# Get the directory containing this script +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +# Source common functions +source "$SCRIPT_DIR/common.sh" + +# Configuration +BASE_IMAGE="spindle-slurm-base" +IMAGE_NAME="spindle-slurm-srun" +DOCKERFILE="$REPO_ROOT/containers/spindle-slurm-ubuntu/testing-srun/Dockerfile.podman" + +echo "==========================================" +echo "Building Spindle Slurm Srun Container" +echo "==========================================" +echo "" + +# Check if base image exists (with or without localhost/ prefix) +if ! podman images --format "{{.Repository}}" | grep -qE "^(localhost/)?${BASE_IMAGE}$"; then + echo "Error: Base image $BASE_IMAGE not found" + echo "" + echo "Available images:" + podman images | grep spindle || echo " (no spindle images found)" + echo "" + echo "Build it first with:" + echo " ./scripts/podman/build-spindle-slurm-base.sh" + echo "" + exit 1 +fi + +echo "Base image: $BASE_IMAGE" +echo "Test image: $IMAGE_NAME" +echo "Dockerfile: $DOCKERFILE" +echo "" + +# Generate MariaDB configuration +echo "Generating MariaDB configuration..." +cd "$REPO_ROOT/containers/spindle-slurm-ubuntu/testing-srun" +./generate_config.sh +echo "✓ Configuration generated" +cd "$REPO_ROOT" +echo "" + +# Build the image +podman_build "$IMAGE_NAME" "$DOCKERFILE" "$REPO_ROOT" "--build-arg replicas=4" + +echo "" +echo "==========================================" +echo "Build complete!" +echo "==========================================" +echo "" +echo "Image: $IMAGE_NAME" +echo "" +echo "Next steps:" +echo " 1. Run regular tests: ./scripts/podman/test-spindle-slurm-srun.sh" +echo "" diff --git a/scripts/podman/cleanup.sh b/scripts/podman/cleanup.sh new file mode 100755 index 00000000..c917d7c3 --- /dev/null +++ b/scripts/podman/cleanup.sh @@ -0,0 +1,85 @@ +#!/bin/bash +# +# Clean up Spindle test containers and networks +# +# Usage: cleanup.sh [run-id] +# +# With no arguments: removes ALL slurm-srun-* containers and networks +# With run-id: removes only containers/networks for that specific run + +set -e + +RUN_ID="${1}" + +if [ -n "$RUN_ID" ]; then + echo "==========================================" + echo "Cleaning up run $RUN_ID..." + echo "==========================================" + echo "" + + NAME_PREFIX="slurm-srun-${RUN_ID}" + NETWORK_NAME="slurm-srun-${RUN_ID}-net" + + # Stop and remove containers for this run + for container in ${NAME_PREFIX}-mariadb ${NAME_PREFIX}-db ${NAME_PREFIX}-head ${NAME_PREFIX}-node-{1..4}; do + if podman ps -a --format "{{.Names}}" | grep -q "^${container}$"; then + echo "Removing $container..." + podman stop "$container" 2>/dev/null || true + podman rm -f "$container" 2>/dev/null || true + fi + done + + # Remove network + if podman network exists "$NETWORK_NAME" 2>/dev/null; then + echo "Removing network $NETWORK_NAME..." + podman network rm -f "$NETWORK_NAME" 2>/dev/null || true + fi + + echo "✓ Cleanup complete for run $RUN_ID" +else + echo "==========================================" + echo "Cleaning up ALL Spindle test containers" + echo "==========================================" + echo "" + + # Find all slurm-srun containers + CONTAINERS=$(podman ps -a --format "{{.Names}}" | grep "^slurm-srun-" || true) + + if [ -n "$CONTAINERS" ]; then + COUNT=$(echo "$CONTAINERS" | wc -l) + echo "Found $COUNT containers to remove" + echo "" + + # Skip stop, just force-remove (stops and removes in one step) + # Use timeout and lower parallelism to avoid hangs + echo "$CONTAINERS" | while read container; do + echo "Removing $container..." + timeout 10 podman rm -f "$container" 2>/dev/null || echo " (timed out, skipping)" & + done + wait + else + echo "No slurm-srun containers found" + fi + + echo "" + + # Find all slurm-srun networks + NETWORKS=$(podman network ls --format "{{.Name}}" | grep "^slurm-srun-" || true) + + if [ -n "$NETWORKS" ]; then + COUNT=$(echo "$NETWORKS" | wc -l) + echo "Found $COUNT networks to remove" + echo "" + + # Remove networks sequentially with timeout to avoid hangs + echo "$NETWORKS" | while read network; do + echo "Removing $network..." + timeout 5 podman network rm -f "$network" 2>/dev/null || echo " (timed out or in use, skipping)" + done + else + echo "No slurm-srun networks found" + fi + + echo "" + echo "✓ Cleanup complete" +fi diff --git a/scripts/podman/common.sh b/scripts/podman/common.sh new file mode 100755 index 00000000..aac43ca5 --- /dev/null +++ b/scripts/podman/common.sh @@ -0,0 +1,77 @@ +#!/bin/bash +# +# Common functions for running Spindle containers with podman on LC systems +# +# LC systems require special handling for: +# 1. SSL certificates (volume mounts) +# 2. apt setgroups errors (handled via Dockerfile ARG) + +set -e + +# LC-specific SSL certificate mounts +# Build: Mount LLNL cert into ca-certificates directory +LC_CERT_BUILD_MOUNT="-v /etc/pki/ca-trust/source/anchors/PAN-cspca.llnl.gov.crt.pem:/usr/local/share/ca-certificates/cspca.crt:ro" + +# Run: Mount system CA bundle +LC_CERT_RUN_MOUNT="-v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/ssl/certs/ca-certificates.crt:ro" + +# Helper function to build images with LC-specific settings +podman_build() { + local image_name=$1 + local dockerfile=$2 + local context=${3:-.} + local extra_args="$4" # Optional extra build args + + echo "========================================" + echo "Building: $image_name" + echo "Dockerfile: $dockerfile" + echo "Context: $context" + echo "========================================" + + podman build \ + --build-arg PODMAN_BUILD=true \ + $LC_CERT_BUILD_MOUNT \ + $extra_args \ + -t "$image_name" \ + -f "$dockerfile" \ + "$context" + + local rc=$? + if [ $rc -eq 0 ]; then + echo "✓ Build successful: $image_name" + else + echo "✗ Build failed with exit code $rc" + return $rc + fi +} + +# Helper function to run containers with LC-specific settings +podman_run() { + local image_name=$1 + shift + + echo "========================================" + echo "Running: $image_name" + echo "========================================" + + podman run \ + --rm \ + $LC_CERT_RUN_MOUNT \ + "$image_name" \ + "$@" + + local rc=$? + if [ $rc -eq 0 ]; then + echo "✓ Container exited successfully" + else + echo "✗ Container exited with code $rc" + return $rc + fi +} + +# Helper function to clean up images +podman_cleanup() { + local image_name=$1 + echo "Cleaning up image: $image_name" + podman rmi "$image_name" 2>/dev/null || true +} diff --git a/scripts/podman/load-images.sh b/scripts/podman/load-images.sh new file mode 100755 index 00000000..b23748f8 --- /dev/null +++ b/scripts/podman/load-images.sh @@ -0,0 +1,58 @@ +#!/bin/bash +# +# Load Spindle podman images from tarball +# +# Run this on compute nodes to load images saved with save-images.sh + +set -e + +TARBALL="${1}" + +if [ -z "$TARBALL" ]; then + echo "Usage: $0 " + echo "" + echo "Example:" + echo " $0 /g/g24/rountree/v/rzadams/sandbox/spindle-podman-images.tar" + exit 1 +fi + +if [ ! -f "$TARBALL" ]; then + echo "Error: Tarball not found: $TARBALL" + exit 1 +fi + +echo "==========================================" +echo "Loading Spindle Podman Images" +echo "==========================================" +echo "" +echo "Source: $TARBALL" +echo "This may take a minute..." +echo "" + +# Extract and load each image separately +TEMP_DIR=$(mktemp -d) +trap "rm -rf $TEMP_DIR" EXIT + +echo "Extracting tarball..." +tar -xf "$TARBALL" -C "$TEMP_DIR" + +echo "Loading base image..." +podman load -i "$TEMP_DIR/base.tar" + +echo "Loading srun image..." +podman load -i "$TEMP_DIR/srun.tar" + +echo "Loading serial image..." +podman load -i "$TEMP_DIR/serial.tar" + +echo "Loading MariaDB image..." +podman load -i "$TEMP_DIR/mariadb.tar" + +echo "" +echo "==========================================" +echo "✓ Images loaded successfully" +echo "==========================================" +echo "" +echo "Available Spindle images:" +podman images | grep -E '(spindle|mariadb)' || echo " (none - something went wrong)" +echo "" diff --git a/scripts/podman/rebuild-and-test-workflow.sh b/scripts/podman/rebuild-and-test-workflow.sh new file mode 100755 index 00000000..850a00c2 --- /dev/null +++ b/scripts/podman/rebuild-and-test-workflow.sh @@ -0,0 +1,122 @@ +#!/bin/bash +# +# Bulletproof workflow for testing podman changes +# +# This script runs ON THE LOGIN NODE and guides you through the process +# +# Run from anywhere in the repo - it will find the right paths + +set -e + +# Get the directory containing this script +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +BRANCH=$(git -C "$REPO_ROOT" rev-parse --abbrev-ref HEAD 2>/dev/null || echo "unknown") + +echo "==========================================" +echo "Podman Testing Workflow" +echo "==========================================" +echo "" +echo "Branch: $BRANCH" +echo "Repo root: $REPO_ROOT" +echo "" +echo "This script runs on the LOGIN NODE" +echo "It will rebuild, save, and give you commands for the compute node" +echo "" + +cd "$REPO_ROOT" + +TARBALL="$REPO_ROOT/spindle-podman-images.tar" + +# Check if base image exists +if ! podman images | grep -q "spindle-slurm-base"; then + echo "Base image not found in podman." + echo "" + + # Check if tarball exists + if [ -f "$TARBALL" ]; then + echo "Found tarball: $TARBALL" + echo "Loading images from tarball..." + echo "Command: ./scripts/podman/load-images.sh $TARBALL" + echo "" + ./scripts/podman/load-images.sh "$TARBALL" 2>&1 | ts + echo "" + + # Check again if image is now available + if podman images | grep -q "spindle-slurm-base"; then + echo "✓ Base image loaded from tarball" + echo "" + else + echo "ERROR: Base image not in tarball. Need to rebuild." + echo "" + echo "Building spindle-slurm-base from source..." + echo "Command: ./scripts/podman/build-spindle-slurm-base.sh" + echo "(This takes ~6 minutes - compiles Slurm + MPICH from source)" + echo "" + ./scripts/podman/build-spindle-slurm-base.sh 2>&1 | ts + echo "" + echo "✓ Base image built" + echo "" + fi + else + echo "Tarball not found at: $TARBALL" + echo "Building spindle-slurm-base from source..." + echo "Command: ./scripts/podman/build-spindle-slurm-base.sh" + echo "(This takes ~6 minutes - compiles Slurm + MPICH from source)" + echo "" + ./scripts/podman/build-spindle-slurm-base.sh 2>&1 | ts + echo "" + echo "✓ Base image built" + echo "" + fi +else + echo "✓ Base image already loaded" + echo "" +fi + +echo "Step 1: Rebuild slurm-srun image" +echo "Command: ./scripts/podman/build-spindle-slurm-srun.sh" +./scripts/podman/build-spindle-slurm-srun.sh 2>&1 | ts + +echo "" +echo "✓ Build complete" +echo "" + +echo "Step 2: Save images to tarball" +echo "Command: ./scripts/podman/save-images.sh" +./scripts/podman/save-images.sh 2>&1 | ts + +TARBALL_PATH="$REPO_ROOT/spindle-podman-images.tar" +echo "" +echo "✓ Images saved to: $TARBALL_PATH" +echo "" + +echo "==========================================" +echo "NOW SWITCH TO COMPUTE NODE" +echo "==========================================" +echo "" +echo "Run these commands on the compute node:" +echo "" +echo " # Get allocation (if needed)" +echo " salloc -N1 -t 60" +echo "" +echo " # Setup podman" +echo " enable-podman" +echo "" +echo " # Navigate to repo" +echo " cd $REPO_ROOT" +echo "" +echo " # Load images" +echo " ./scripts/podman/load-images.sh ./spindle-podman-images.tar" +echo "" +echo " # Verify canary - should show 'ENTRYPOINT DEBUG (v2)'" +echo " podman run --rm -e SLURM_ROLE=db -e MARIADB_PASSWORD=test123 localhost/spindle-slurm-srun:latest 2>&1 | head -30" +echo "" +echo " # Run test" +echo " ./scripts/podman/test-spindle-slurm-srun.sh 1 2>&1 | tee test-output.log" +echo "" +echo "==========================================" +echo "" +echo "Copy the commands above to your compute node terminal" +echo "" diff --git a/scripts/podman/run-hello-01-basic.sh b/scripts/podman/run-hello-01-basic.sh new file mode 100755 index 00000000..b0876951 --- /dev/null +++ b/scripts/podman/run-hello-01-basic.sh @@ -0,0 +1,56 @@ +#!/bin/bash +# +# Test 01-basic: Validate podman environment +# +# This test verifies: +# - Container can build with apt-get (tests setgroups fix) +# - Container can run +# - Network connectivity works +# - SSL certificates are properly configured +# +# Run this from outside the sandbox where podman is available. + +set -e + +# Get the directory containing this script +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +# Source common functions +source "$SCRIPT_DIR/common.sh" + +# Configuration +IMAGE_NAME="spindle-hello-01-basic" +DOCKERFILE="$REPO_ROOT/containers/spindle-hello-podman/01-basic/Dockerfile" + +echo "==========================================" +echo "Spindle Hello World - 01-basic" +echo "==========================================" +echo "" +echo "This test validates the podman environment on LC systems." +echo "" + +# Build the image +podman_build "$IMAGE_NAME" "$DOCKERFILE" "$REPO_ROOT" + +echo "" + +# Run the container +podman_run "$IMAGE_NAME" + +echo "" +echo "==========================================" +echo "Test complete!" +echo "" +echo "What was tested:" +echo " ✓ apt-get works (setgroups fix applied)" +echo " ✓ Container runs successfully" +echo " ✓ Network connectivity" +echo " ✓ SSL certificates configured" +echo "" +echo "Next step: run-hello-02-user.sh" +echo "==========================================" + +# Optional: Clean up the image +# Uncomment if you want to remove the image after testing +# podman_cleanup "$IMAGE_NAME" diff --git a/scripts/podman/run-hello-02-user.sh b/scripts/podman/run-hello-02-user.sh new file mode 100755 index 00000000..c53ae455 --- /dev/null +++ b/scripts/podman/run-hello-02-user.sh @@ -0,0 +1,65 @@ +#!/bin/bash +# +# Test 02-user-switch: Validate user switching and permissions +# +# This test verifies: +# - Creating a non-root user in the container +# - Switching to that user (USER directive) +# - File permission handling +# - Sudo access when needed +# +# This pattern is used in all Spindle containers where builds and tests +# run as a non-root user for security and to match typical HPC environments. +# +# Run this from outside the sandbox where podman is available. + +set -e + +# Get the directory containing this script +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +# Source common functions +source "$SCRIPT_DIR/common.sh" + +# Configuration +IMAGE_NAME="spindle-hello-02-user" +DOCKERFILE="$REPO_ROOT/containers/spindle-hello-podman/02-user-switch/Dockerfile" + +echo "==========================================" +echo "Spindle Hello World - 02-user-switch" +echo "==========================================" +echo "" +echo "This test validates non-root user patterns used in Spindle containers." +echo "" + +# Build the image +podman_build "$IMAGE_NAME" "$DOCKERFILE" "$REPO_ROOT" + +echo "" + +# Run the container +podman_run "$IMAGE_NAME" + +echo "" +echo "==========================================" +echo "Test complete!" +echo "" +echo "What was tested:" +echo " ✓ Non-root user creation" +echo " ✓ User switching with USER directive" +echo " ✓ Home directory setup" +echo " ✓ File permission handling" +echo " ✓ Sudo access" +echo "" +echo "This pattern is used in all Spindle containers to:" +echo " - Match HPC security practices" +echo " - Test file permissions realistically" +echo " - Avoid running builds as root" +echo "" +echo "Next step: run-hello-03-filesystem.sh" +echo "==========================================" + +# Optional: Clean up the image +# Uncomment if you want to remove the image after testing +# podman_cleanup "$IMAGE_NAME" diff --git a/scripts/podman/run-hello-03-filesystem.sh b/scripts/podman/run-hello-03-filesystem.sh new file mode 100755 index 00000000..6e7c0875 --- /dev/null +++ b/scripts/podman/run-hello-03-filesystem.sh @@ -0,0 +1,132 @@ +#!/bin/bash +# +# Test 03-filesystem: Validate volume mounts and filesystem operations +# +# This test verifies: +# - Mounting host directories into containers +# - Reading files from host mounts +# - Writing files to mounted volumes +# - File ownership and permissions with :Z flag +# - Artifacts persisting after container exits +# +# This pattern is critical for Spindle containers: +# - Source code is mounted from host (read-only or read-write) +# - Build artifacts are written to mounted volumes +# - Logs persist on host for debugging +# +# Run this from outside the sandbox where podman is available. + +set -e + +# Get the directory containing this script +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +# Source common functions +source "$SCRIPT_DIR/common.sh" + +# Configuration +IMAGE_NAME="spindle-hello-03-filesystem" +DOCKERFILE="$REPO_ROOT/containers/spindle-hello-podman/03-filesystem/Dockerfile" + +# Create temporary directories for testing +TEST_DIR=$(mktemp -d) +HOST_DATA_DIR="$TEST_DIR/host-data" +OUTPUT_DIR="$TEST_DIR/output" + +mkdir -p "$HOST_DATA_DIR" +mkdir -p "$OUTPUT_DIR" + +# Create test input file +echo "Hello from the host filesystem!" > "$HOST_DATA_DIR/test-input.txt" +echo "This file was created outside the container." >> "$HOST_DATA_DIR/test-input.txt" +echo "Container should be able to read this." >> "$HOST_DATA_DIR/test-input.txt" + +# No special permissions needed - we'll use --userns=keep-id to map host UID into container + +echo "==========================================" +echo "Spindle Hello World - 03-filesystem" +echo "==========================================" +echo "" +echo "This test validates volume mount patterns used in Spindle containers." +echo "" +echo "Test setup:" +echo " Host data directory: $HOST_DATA_DIR" +echo " Output directory: $OUTPUT_DIR" +echo " Using --userns=keep-id to map host UID into container" +echo "" + +# Build the image +podman_build "$IMAGE_NAME" "$DOCKERFILE" "$REPO_ROOT" + +echo "" +echo "Running container with volume mounts..." +echo "" + +# Run the container with volume mounts +# --userns=keep-id - Map host UID into container (avoids permission issues) +# -v host:container:Z - Z flag sets SELinux context for container access +podman run \ + --rm \ + --userns=keep-id \ + $LC_CERT_RUN_MOUNT \ + -v "$HOST_DATA_DIR:/home/testuser/host-data:Z" \ + -v "$OUTPUT_DIR:/home/testuser/output:Z" \ + "$IMAGE_NAME" + +EXIT_CODE=$? + +echo "" +echo "==========================================" +echo "Container exited. Checking results..." +echo "==========================================" +echo "" + +if [ $EXIT_CODE -eq 0 ]; then + echo "✓ Container executed successfully" +else + echo "✗ Container exited with code $EXIT_CODE" +fi + +echo "" +echo "Files created by container in output directory:" +ls -lh "$OUTPUT_DIR" + +echo "" +echo "Content of container-output.txt:" +if [ -f "$OUTPUT_DIR/container-output.txt" ]; then + cat "$OUTPUT_DIR/container-output.txt" + echo "" + echo "✓ Container successfully wrote to mounted volume" +else + echo "✗ Expected output file not found" +fi + +echo "" +echo "==========================================" +echo "Test complete!" +echo "" +echo "What was tested:" +echo " ✓ Volume mounts (-v host:container:Z)" +echo " ✓ User namespace mapping (--userns=keep-id)" +echo " ✓ Reading files from host" +echo " ✓ Writing files to mounted volumes" +echo " ✓ File persistence after container exit" +echo " ✓ SELinux context handling (:Z flag)" +echo "" +echo "This pattern enables Spindle to:" +echo " - Access source code from host" +echo " - Write build artifacts to persistent storage" +echo " - Generate logs accessible after tests" +echo " - Share data between container and host" +echo "" +echo "Cleaning up test directories..." +rm -rf "$TEST_DIR" +echo "✓ Cleanup complete" +echo "" +echo "Next step: run-hello-04-networking.sh" +echo "==========================================" + +# Optional: Clean up the image +# Uncomment if you want to remove the image after testing +# podman_cleanup "$IMAGE_NAME" diff --git a/scripts/podman/run-hello-04-networking.sh b/scripts/podman/run-hello-04-networking.sh new file mode 100755 index 00000000..2a1b44ba --- /dev/null +++ b/scripts/podman/run-hello-04-networking.sh @@ -0,0 +1,155 @@ +#!/bin/bash +# +# Test 04-networking: Validate multi-container networking +# +# This test verifies: +# - Creating custom podman networks +# - Multiple containers on the same network +# - Container-to-container communication by hostname +# - Service discovery (DNS resolution) +# - Network isolation +# +# This pattern is essential for multi-container setups like Slurm clusters +# where head nodes need to communicate with compute nodes. +# +# Run this from outside the sandbox where podman is available. + +set -e + +# Get the directory containing this script +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +# Source common functions +source "$SCRIPT_DIR/common.sh" + +# Configuration +IMAGE_NAME="spindle-hello-04-networking" +DOCKERFILE="$REPO_ROOT/containers/spindle-hello-podman/04-networking/Dockerfile" +NETWORK_NAME="spindle-test-net" +SERVER_NAME="test-server" +CLIENT1_NAME="test-client1" +CLIENT2_NAME="test-client2" + +echo "==========================================" +echo "Spindle Hello World - 04-networking" +echo "==========================================" +echo "" +echo "This test validates multi-container networking for Slurm/Flux clusters." +echo "" + +# Cleanup function +cleanup() { + echo "" + echo "Cleaning up containers and network..." + podman rm -f "$SERVER_NAME" "$CLIENT1_NAME" "$CLIENT2_NAME" 2>/dev/null || true + podman network rm "$NETWORK_NAME" 2>/dev/null || true + echo "✓ Cleanup complete" +} + +# Set trap to cleanup on exit +trap cleanup EXIT + +# Build the image +podman_build "$IMAGE_NAME" "$DOCKERFILE" "$REPO_ROOT" + +echo "" +echo "==========================================" +echo "Setting up network infrastructure..." +echo "==========================================" +echo "" + +# Create a custom network +echo "Creating network: $NETWORK_NAME" +podman network create "$NETWORK_NAME" +echo "✓ Network created" + +echo "" +echo "Starting server container..." +podman run \ + --name "$SERVER_NAME" \ + --network "$NETWORK_NAME" \ + --hostname "$SERVER_NAME" \ + -d \ + "$IMAGE_NAME" +echo "✓ Server started: $SERVER_NAME" + +# Give server time to start +sleep 2 + +echo "" +echo "==========================================" +echo "Testing container networking..." +echo "==========================================" +echo "" + +echo "1. Testing DNS resolution (can clients resolve server hostname?)..." +podman run \ + --name "$CLIENT1_NAME" \ + --network "$NETWORK_NAME" \ + --hostname "$CLIENT1_NAME" \ + --rm \ + "$IMAGE_NAME" \ + /bin/bash -c " + echo 'Client: $CLIENT1_NAME' + echo 'Resolving $SERVER_NAME...' + if ping -c 1 -W 2 $SERVER_NAME > /dev/null 2>&1; then + echo '✓ DNS resolution works: $SERVER_NAME is reachable' + echo 'Server IP:' \$(getent hosts $SERVER_NAME | awk '{print \$1}') + else + echo '✗ Cannot resolve $SERVER_NAME' + exit 1 + fi + " + +echo "" +echo "2. Testing HTTP communication between containers..." +podman run \ + --name "$CLIENT2_NAME" \ + --network "$NETWORK_NAME" \ + --hostname "$CLIENT2_NAME" \ + --rm \ + "$IMAGE_NAME" \ + /bin/bash -c " + echo 'Client: $CLIENT2_NAME' + echo 'Fetching from http://$SERVER_NAME:8080...' + response=\$(curl -s --max-time 5 http://$SERVER_NAME:8080) + if [ -n \"\$response\" ]; then + echo 'Response from server:' + echo \"\$response\" + echo '✓ HTTP communication works' + else + echo '✗ No response from server' + exit 1 + fi + " + +echo "" +echo "3. Checking server logs..." +echo "Server received requests from:" +podman logs "$SERVER_NAME" 2>&1 | tail -5 + +echo "" +echo "==========================================" +echo "Test complete!" +echo "" +echo "What was tested:" +echo " ✓ Custom network creation" +echo " ✓ Multiple containers on same network" +echo " ✓ DNS resolution (hostname lookups)" +echo " ✓ Container-to-container HTTP communication" +echo " ✓ Service discovery patterns" +echo "" +echo "This pattern enables:" +echo " - Slurm head node + compute nodes" +echo " - Flux broker + worker nodes" +echo " - Server-client architectures" +echo " - Service discovery by hostname" +echo "" +echo "Next steps:" +echo " - 05-flux-multi: Multi-container Flux cluster" +echo " - 06-slurm-multi: Multi-container Slurm cluster" +echo " - Then: Real Spindle containers" +echo "==========================================" + +# Cleanup happens automatically via trap diff --git a/scripts/podman/run-hello-05-flux.sh b/scripts/podman/run-hello-05-flux.sh new file mode 100755 index 00000000..173f4bf4 --- /dev/null +++ b/scripts/podman/run-hello-05-flux.sh @@ -0,0 +1,141 @@ +#!/bin/bash +# +# Test 05-flux-multi: Validate Flux multi-node cluster +# +# This test verifies: +# - Using official fluxrm/flux-sched base image +# - Multi-node Flux cluster setup +# - Head node and worker node coordination +# - Running distributed jobs with `flux run` +# - Flux resource management +# +# This demonstrates the pattern used for Spindle's Flux tests, +# simplified to just show Flux itself working. +# +# Run this from outside the sandbox where podman is available. + +set -e + +# Get the directory containing this script +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +# Source common functions +source "$SCRIPT_DIR/common.sh" + +# Configuration +IMAGE_NAME="spindle-hello-05-flux" +DOCKERFILE="$REPO_ROOT/containers/spindle-hello-podman/05-flux-multi/Dockerfile" +NETWORK_NAME="flux-test-net" +NUM_WORKERS=3 + +echo "==========================================" +echo "Spindle Hello World - 05-flux-multi" +echo "==========================================" +echo "" +echo "This test validates a multi-node Flux cluster setup." +echo "Nodes: 1 head + ${NUM_WORKERS} workers = $(($NUM_WORKERS + 1)) total" +echo "" + +# Cleanup function +cleanup() { + echo "" + echo "Cleaning up containers and network..." + for i in $(seq 1 $NUM_WORKERS); do + podman rm -f "flux-node-$i" 2>/dev/null || true + done + podman network rm "$NETWORK_NAME" 2>/dev/null || true + echo "✓ Cleanup complete" +} + +# Set trap to cleanup on exit +trap cleanup EXIT + +echo "Building Flux container image..." +echo "(This may take a few minutes - downloading fluxrm/flux-sched base image)" +echo "" + +# Build the image +podman_build "$IMAGE_NAME" "$DOCKERFILE" "$REPO_ROOT" + +echo "" +echo "==========================================" +echo "Setting up Flux cluster..." +echo "==========================================" +echo "" + +# Create network +echo "Creating network: $NETWORK_NAME" +podman network create "$NETWORK_NAME" +echo "✓ Network created" + +echo "" +echo "Starting Flux nodes..." + +# Start head node first +echo " Starting flux-node-1 (head node)..." +podman run \ + --name "flux-node-1" \ + --hostname "flux-node-1" \ + --network "$NETWORK_NAME" \ + -d \ + "$IMAGE_NAME" + +sleep 2 + +# Start worker nodes +for i in $(seq 2 $NUM_WORKERS); do + echo " Starting flux-node-$i (worker)..." + podman run \ + --name "flux-node-$i" \ + --hostname "flux-node-$i" \ + --network "$NETWORK_NAME" \ + -d \ + "$IMAGE_NAME" + sleep 1 +done + +echo "" +echo "✓ All nodes started" +echo "" +echo "Waiting for Flux cluster to initialize..." +echo "(Munge authentication + broker connections)" +sleep 15 + +echo "" +echo "==========================================" +echo "Flux Cluster Status" +echo "==========================================" +echo "" + +# Show head node logs +echo "Head node logs:" +echo "----------------------------------------" +podman logs flux-node-1 2>&1 | tail -30 + +echo "" +echo "==========================================" +echo "Test complete!" +echo "" +echo "What was tested:" +echo " ✓ Official fluxrm/flux-sched base image" +echo " ✓ Multi-node Flux cluster (1 head + ${NUM_WORKERS} workers)" +echo " ✓ Munge authentication" +echo " ✓ Flux broker connections" +echo " ✓ Distributed job execution (flux run)" +echo " ✓ Resource management" +echo "" +echo "This demonstrates the Flux pattern used by Spindle:" +echo " - Same image for all nodes" +echo " - Head node vs worker behavior based on hostname" +echo " - Workers connect to head node" +echo " - Jobs can run across all nodes" +echo "" +echo "To interact with the cluster:" +echo " podman exec -it flux-node-1 flux resource list" +echo " podman exec -it flux-node-1 flux run -N ${NUM_WORKERS} hostname" +echo "" +echo "Next: 06-slurm-multi (Slurm cluster example)" +echo "==========================================" + +# Cleanup happens automatically via trap diff --git a/scripts/podman/run-hello-06-slurm.sh b/scripts/podman/run-hello-06-slurm.sh new file mode 100755 index 00000000..ee9e4405 --- /dev/null +++ b/scripts/podman/run-hello-06-slurm.sh @@ -0,0 +1,168 @@ +#!/bin/bash +# +# Test 06-slurm-multi: Validate Slurm multi-node cluster +# +# This test verifies: +# - Installing Slurm from Ubuntu packages +# - Multi-node Slurm cluster setup +# - Controller (slurmctld) and compute (slurmd) daemons +# - Munge authentication +# - Node registration and job submission +# +# This demonstrates a simplified Slurm pattern. The full Spindle +# Slurm setup includes MariaDB and slurmdbd for accounting. +# +# Run this from outside the sandbox where podman is available. + +set -e + +# Get the directory containing this script +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +# Source common functions +source "$SCRIPT_DIR/common.sh" + +# Configuration +IMAGE_NAME="spindle-hello-06-slurm" +DOCKERFILE="$REPO_ROOT/containers/spindle-hello-podman/06-slurm-multi/Dockerfile" +NETWORK_NAME="slurm-test-net" + +echo "==========================================" +echo "Spindle Hello World - 06-slurm-multi" +echo "==========================================" +echo "" +echo "This test validates a simplified multi-node Slurm cluster." +echo "Cluster: 1 controller + 2 compute nodes" +echo "" + +# Cleanup function +cleanup() { + echo "" + echo "Cleaning up containers and network..." + podman rm -f slurm-head slurm-node-1 slurm-node-2 2>/dev/null || true + podman network rm "$NETWORK_NAME" 2>/dev/null || true + echo "✓ Cleanup complete" +} + +# Clean up any existing resources from previous runs +cleanup + +# Set trap to cleanup on exit +trap cleanup EXIT + +echo "Building Slurm container image..." +echo "" + +# Build the image +podman_build "$IMAGE_NAME" "$DOCKERFILE" "$REPO_ROOT" + +echo "" +echo "==========================================" +echo "Setting up Slurm cluster..." +echo "==========================================" +echo "" + +# Create network (or reuse if exists) +if podman network exists "$NETWORK_NAME" 2>/dev/null; then + echo "Network $NETWORK_NAME already exists, reusing" +else + echo "Creating network: $NETWORK_NAME" + podman network create "$NETWORK_NAME" + echo "✓ Network created" +fi + +echo "" +echo "Starting Slurm controller (head node)..." +podman run \ + --name slurm-head \ + --hostname slurm-head \ + --network "$NETWORK_NAME" \ + -e SLURM_ROLE=ctl \ + -d \ + "$IMAGE_NAME" + +echo "✓ Controller started" +sleep 5 + +echo "" +echo "Starting compute nodes..." + +podman run \ + --name slurm-node-1 \ + --hostname slurm-node-1 \ + --network "$NETWORK_NAME" \ + -e SLURM_ROLE=worker \ + -d \ + "$IMAGE_NAME" +echo " ✓ slurm-node-1 started" + +podman run \ + --name slurm-node-2 \ + --hostname slurm-node-2 \ + --network "$NETWORK_NAME" \ + -e SLURM_ROLE=worker \ + -d \ + "$IMAGE_NAME" +echo " ✓ slurm-node-2 started" + +echo "" +echo "Waiting for Slurm cluster to initialize..." +echo "(Munge + node registration)" +sleep 10 + +echo "" +echo "==========================================" +echo "Slurm Cluster Status" +echo "==========================================" +echo "" + +echo "Checking node status with 'sinfo':" +podman exec slurm-head sinfo || echo " (Still initializing...)" + +echo "" +echo "Attempting to run a test job..." +echo "Running: srun -N 2 hostname" +echo "" + +# Try to run a simple job +podman exec slurm-head srun -N 2 hostname || { + echo "" + echo "Job may have failed. Checking controller logs:" + echo "----------------------------------------" + podman logs slurm-head 2>&1 | tail -20 + echo "" + echo "Compute node logs:" + echo "----------------------------------------" + podman logs slurm-node-1 2>&1 | tail -10 +} + +echo "" +echo "==========================================" +echo "Test complete!" +echo "" +echo "What was tested:" +echo " ✓ Slurm installation from Ubuntu packages" +echo " ✓ Multi-node cluster (1 controller + 2 compute)" +echo " ✓ Munge authentication" +echo " ✓ slurmctld (controller daemon)" +echo " ✓ slurmd (compute daemon)" +echo " ✓ Node registration" +echo " ✓ Job submission with srun" +echo "" +echo "This demonstrates a simplified Slurm pattern." +echo "The full Spindle Slurm setup adds:" +echo " - MariaDB for accounting database" +echo " - slurmdbd for accounting" +echo " - More complex configuration" +echo " - MPICH for MPI jobs" +echo "" +echo "To interact with the cluster:" +echo " podman exec slurm-head sinfo" +echo " podman exec slurm-head srun -N 2 hostname" +echo " podman exec slurm-head scontrol show nodes" +echo "" +echo "Ready to port real Spindle containers!" +echo "==========================================" + +# Cleanup happens automatically via trap diff --git a/scripts/podman/run-spindle-serial-interactive.sh b/scripts/podman/run-spindle-serial-interactive.sh new file mode 100755 index 00000000..81ef2e07 --- /dev/null +++ b/scripts/podman/run-spindle-serial-interactive.sh @@ -0,0 +1,78 @@ +#!/bin/bash +# +# Run Spindle serial-interactive container for manual debugging +# +# This starts the container and keeps it running for interactive access. +# Run this from outside the sandbox where podman is available. + +set -e + +# Get the directory containing this script +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +# Source common functions +source "$SCRIPT_DIR/common.sh" + +# Configuration +IMAGE_NAME="spindle-serial-interactive-ubuntu" +CONTAINER_NAME="spindlenode-interactive" + +echo "==========================================" +echo "Spindle Serial Interactive Container" +echo "==========================================" +echo "" + +# Check if container already exists +if podman ps -a --format "{{.Names}}" 2>/dev/null | grep -q "^${CONTAINER_NAME}\$"; then + echo "Container '$CONTAINER_NAME' already exists." + echo "" + + # Check if it's running + if podman ps --format "{{.Names}}" 2>/dev/null | grep -q "^${CONTAINER_NAME}\$"; then + echo "Container is running. Connect with:" + echo " podman exec -it $CONTAINER_NAME bash" + else + echo "Container exists but is stopped. Removing..." + podman rm -f "$CONTAINER_NAME" 2>/dev/null + echo "Starting fresh container..." + fi + echo "" +fi + +# Only start if not already running +if ! podman ps --format "{{.Names}}" 2>/dev/null | grep -q "^${CONTAINER_NAME}\$"; then + echo "Starting interactive container..." + echo "" + + podman run \ + --name "$CONTAINER_NAME" \ + --hostname "$CONTAINER_NAME" \ + --cap-add SYS_NICE \ + -d \ + -t \ + "$IMAGE_NAME" + + echo "✓ Container started" + sleep 3 + + echo "" + echo "==========================================" + echo "Container ready for debugging!" + echo "==========================================" + echo "" +fi + +echo "Connect to the container:" +echo " podman exec -it $CONTAINER_NAME bash" +echo "" +echo "Inside the container, you can:" +echo " 1. Check munge: munge -n | unmunge" +echo " 2. Enable core dumps: ulimit -c unlimited" +echo " 3. Run tests: cd Spindle-build/testsuite && SPINDLE_DEBUG=3 ./runTests" +echo " 4. Check cores: ls -lh /tmp/core.* Spindle-build/testsuite/core.*" +echo " 5. Debug with gdb: gdb ./test_driver /tmp/core.test_driver.PID" +echo "" +echo "When done, stop the container:" +echo " podman rm -f $CONTAINER_NAME" +echo "" diff --git a/scripts/podman/save-images.sh b/scripts/podman/save-images.sh new file mode 100755 index 00000000..9e1a956d --- /dev/null +++ b/scripts/podman/save-images.sh @@ -0,0 +1,75 @@ +#!/bin/bash +# +# Save Spindle podman images to tarball for compute node deployment +# +# Run this on the login node where images were built. +# The tarball can then be loaded on compute nodes via load-images.sh + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +OUTPUT_FILE="${1:-$REPO_ROOT/spindle-podman-images.tar}" + +echo "==========================================" +echo "Saving Spindle Podman Images" +echo "==========================================" +echo "" +echo "This saves all Spindle images to a tarball for deployment to compute nodes." +echo "Output: $OUTPUT_FILE" +echo "" + +# Check what images exist +echo "Available Spindle images:" +podman images | grep spindle || { + echo "Error: No Spindle images found. Build them first." + exit 1 +} + +echo "" +echo "Saving images to tarball..." +echo "This may take several minutes..." +echo "" + +# Save images separately to avoid parent-child ID collisions +# When multiple images share a base, podman save can collapse them incorrectly +TEMP_DIR=$(mktemp -d) +trap "rm -rf $TEMP_DIR" EXIT + +echo "Saving spindle-slurm-base..." +podman save localhost/spindle-slurm-base:latest -o "$TEMP_DIR/base.tar" + +echo "Saving spindle-slurm-srun..." +podman save localhost/spindle-slurm-srun:latest -o "$TEMP_DIR/srun.tar" + +echo "Saving spindle-serial-ubuntu..." +podman save localhost/spindle-serial-ubuntu:latest -o "$TEMP_DIR/serial.tar" + +echo "Saving MariaDB..." +podman save docker.io/library/mariadb:12 -o "$TEMP_DIR/mariadb.tar" + +echo "Combining into single tarball..." +tar -cf "$OUTPUT_FILE" -C "$TEMP_DIR" base.tar srun.tar serial.tar mariadb.tar + +# Copy mariadb.env for portable deployment +MARIADB_ENV_SOURCE="$REPO_ROOT/containers/spindle-slurm-ubuntu/testing-srun/mariadb.env" +MARIADB_ENV_DEST="$REPO_ROOT/mariadb.env" +if [ -f "$MARIADB_ENV_SOURCE" ]; then + echo "Copying mariadb.env for portable deployment..." + cp "$MARIADB_ENV_SOURCE" "$MARIADB_ENV_DEST" + echo " ✓ mariadb.env copied to repo root" +fi + +SIZE=$(du -h "$OUTPUT_FILE" | cut -f1) +echo "" +echo "==========================================" +echo "✓ Images saved successfully" +echo "==========================================" +echo "" +echo "File: $OUTPUT_FILE" +echo "Size: $SIZE" +echo "" +echo "To load on compute nodes:" +echo " ./scripts/podman/load-images.sh $OUTPUT_FILE" +echo "" diff --git a/scripts/podman/test-spindle-flux.sh b/scripts/podman/test-spindle-flux.sh new file mode 100755 index 00000000..b447e66c --- /dev/null +++ b/scripts/podman/test-spindle-flux.sh @@ -0,0 +1,167 @@ +#!/bin/bash +# +# Run Spindle Flux tests in podman +# +# This runs the Spindle testsuite in a 4-node Flux cluster. +# Based on the CI workflow and docker-compose configuration. +# +# Run this from outside the sandbox where podman is available. + +set -e + +# Get the directory containing this script +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +# Source common functions +source "$SCRIPT_DIR/common.sh" + +# Configuration +IMAGE_NAME="spindle-flux-ubuntu" +NETWORK_NAME="flux-test-net" +WORKERS=4 +MAIN_HOST="node-1" + +echo "==========================================" +echo "Spindle Flux Tests" +echo "==========================================" +echo "" +echo "This runs the Spindle testsuite in a 4-node Flux cluster." +echo "" + +# Cleanup function +cleanup() { + echo "" + echo "Cleaning up containers and network..." + for i in $(seq 1 $WORKERS); do + podman rm -f "node-$i" 2>/dev/null || true + done + podman network rm "$NETWORK_NAME" 2>/dev/null || true + echo "✓ Cleanup complete" +} + +# Set trap to cleanup on exit +trap cleanup EXIT + +# Initial cleanup +cleanup + +echo "==========================================" +echo "Setting up Flux cluster..." +echo "==========================================" +echo "" + +# Create network (or reuse if exists) +if podman network exists "$NETWORK_NAME" 2>/dev/null; then + echo "Network $NETWORK_NAME already exists, reusing" +else + echo "Creating network: $NETWORK_NAME" + podman network create "$NETWORK_NAME" + echo "✓ Network created" +fi + +echo "" +echo "Starting Flux nodes..." +echo "" + +# Start all 4 nodes +for i in $(seq 1 $WORKERS); do + NODE_NAME="node-$i" + echo "Starting $NODE_NAME..." + podman run \ + --name "$NODE_NAME" \ + --hostname "$NODE_NAME" \ + --network "$NETWORK_NAME" \ + -e mainHost="$MAIN_HOST" \ + -e workers="$WORKERS" \ + --cap-add SYS_NICE \ + -d \ + "$IMAGE_NAME" + echo " ✓ $NODE_NAME started" +done + +echo "" +echo "Waiting for Flux cluster to initialize..." +echo "(This takes ~20-30 seconds for all nodes to register)" +sleep 30 + +echo "" +echo "==========================================" +echo "Checking container status..." +echo "==========================================" +echo "" + +# Check if containers are still running +for i in $(seq 1 $WORKERS); do + NODE_NAME="node-$i" + if podman ps --filter "name=$NODE_NAME" --format "{{.Names}}" | grep -q "$NODE_NAME"; then + echo " ✓ $NODE_NAME is running" + else + echo " ✗ $NODE_NAME has exited!" + echo "" + echo "Last 30 lines of $NODE_NAME logs:" + echo "----------------------------------------" + podman logs "$NODE_NAME" 2>&1 | tail -30 + echo "----------------------------------------" + echo "" + echo "Container exited unexpectedly. Check logs above." + exit 1 + fi +done + +echo "" +echo "==========================================" +echo "Verifying munge authentication..." +echo "==========================================" +echo "" + +podman exec "$MAIN_HOST" bash -c 'munge -n | unmunge' + +echo "" +echo "✓ Munge working" + +echo "" +echo "==========================================" +echo "Verifying Flux cluster health..." +echo "==========================================" +echo "" + +echo "Checking Flux status..." +podman exec "$MAIN_HOST" bash -c 'flux resource list' || echo " (Flux may still be initializing)" + +echo "" +echo "Running flux healthcheck..." +podman exec "$MAIN_HOST" bash -c './flux_healthcheck.sh' || echo " (Some nodes may not be registered yet)" + +echo "" +echo "==========================================" +echo "Running Spindle testsuite..." +echo "==========================================" +echo "" +echo "This will take several minutes." +echo "" + +# Run the testsuite +# Based on CI: docker exec node-1 bash -c 'cd Spindle-build/testsuite && flux alloc --nodes=${workers} ./runTests --nodes=${workers} --tasks-per-node=3' +if podman exec "$MAIN_HOST" bash -c "cd Spindle-build/testsuite && flux alloc --nodes=${WORKERS} ./runTests --nodes=${WORKERS} --tasks-per-node=3"; then + echo "" + echo "==========================================" + echo "✓ All tests passed!" + echo "==========================================" + echo "" + exit 0 +else + echo "" + echo "==========================================" + echo "✗ Some tests failed" + echo "==========================================" + echo "" + echo "To inspect the cluster:" + echo " podman exec -it $MAIN_HOST bash" + echo " flux resource list" + echo " cd Spindle-build/testsuite" + echo "" + exit 1 +fi + +# Cleanup happens automatically via trap diff --git a/scripts/podman/test-spindle-serial-crash.sh b/scripts/podman/test-spindle-serial-crash.sh new file mode 100755 index 00000000..462adab8 --- /dev/null +++ b/scripts/podman/test-spindle-serial-crash.sh @@ -0,0 +1,83 @@ +#!/bin/bash +# +# Run Spindle serial crash tests in podman +# +# This runs only the crash tests (slower). +# Run this from outside the sandbox where podman is available. + +set -e + +# Get the directory containing this script +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +# Source common functions +source "$SCRIPT_DIR/common.sh" + +# Configuration +IMAGE_NAME="spindle-serial-ubuntu" +CONTAINER_NAME="spindlenode" + +echo "==========================================" +echo "Spindle Serial Crash Tests" +echo "==========================================" +echo "" + +# Cleanup function +cleanup() { + echo "" + echo "Cleaning up container..." + podman rm -f "$CONTAINER_NAME" 2>/dev/null || true + echo "✓ Cleanup complete" +} + +# Set trap to cleanup on exit +trap cleanup EXIT + +# Initial cleanup +cleanup + +echo "Starting Spindle serial container..." +echo "" + +# Start the container +podman run \ + --name "$CONTAINER_NAME" \ + --hostname "$CONTAINER_NAME" \ + --cap-add SYS_NICE \ + -d \ + -t \ + "$IMAGE_NAME" + +echo "✓ Container started" +sleep 3 + +echo "" +echo "==========================================" +echo "Running crash tests..." +echo "==========================================" +echo "" +echo "This will take several minutes." +echo "" + +# Run crash tests +if podman exec "$CONTAINER_NAME" bash -c 'cd Spindle-build/testsuite && ./run_crash_tests.sh --launcher=serial --scratch=/tmp/spindle_crash_test'; then + echo "" + echo "==========================================" + echo "✓ All crash tests passed!" + echo "==========================================" + echo "" + exit 0 +else + echo "" + echo "==========================================" + echo "✗ Some crash tests failed" + echo "==========================================" + echo "" + echo "To inspect the container:" + echo " podman exec -it $CONTAINER_NAME bash" + echo "" + exit 1 +fi + +# Cleanup happens automatically via trap diff --git a/scripts/podman/test-spindle-serial-debug.sh b/scripts/podman/test-spindle-serial-debug.sh new file mode 100755 index 00000000..6bebb7dc --- /dev/null +++ b/scripts/podman/test-spindle-serial-debug.sh @@ -0,0 +1,138 @@ +#!/bin/bash +# +# Run Spindle serial tests with debug logging enabled +# +# This demonstrates: +# 1. Running tests with SPINDLE_DEBUG=3 (verbose logging) +# 2. Extracting logs from the container to the host +# +# Run this from outside the sandbox where podman is available. + +set -e + +# Get the directory containing this script +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +# Source common functions +source "$SCRIPT_DIR/common.sh" + +# Configuration +IMAGE_NAME="spindle-serial-ubuntu" +CONTAINER_NAME="spindlenode-debug" +LOG_DIR="$REPO_ROOT/spindle-debug-logs" + +echo "==========================================" +echo "Spindle Serial Debug Test" +echo "==========================================" +echo "" +echo "This runs a single test with SPINDLE_DEBUG=3 enabled" +echo "and extracts the debug logs to the host." +echo "" + +# Cleanup function +cleanup() { + echo "" + echo "Cleaning up container..." + podman rm -f "$CONTAINER_NAME" 2>/dev/null || true + echo "✓ Cleanup complete" +} + +# Set trap to cleanup on exit +trap cleanup EXIT + +# Initial cleanup +cleanup + +# Create log directory on host +mkdir -p "$LOG_DIR" +echo "Debug logs will be saved to: $LOG_DIR" +echo "" + +echo "Starting Spindle serial container..." +echo "" + +# Start the container +podman run \ + --name "$CONTAINER_NAME" \ + --hostname "$CONTAINER_NAME" \ + --cap-add SYS_NICE \ + -d \ + -t \ + "$IMAGE_NAME" + +echo "✓ Container started" +sleep 3 + +echo "" +echo "==========================================" +echo "Running single test with SPINDLE_DEBUG=3" +echo "==========================================" +echo "" +echo "Test: ./run_driver --dependency --preload" +echo "" + +# Run a single test with SPINDLE_DEBUG=3 +# This should generate verbose logs +podman exec "$CONTAINER_NAME" bash -c 'cd Spindle-build/testsuite && SPINDLE_DEBUG=3 ./run_driver --dependency --preload' || { + echo "" + echo "Note: Test may have generated logs even if it failed" +} + +echo "" +echo "==========================================" +echo "Extracting logs from container" +echo "==========================================" +echo "" + +# Spindle log files are named: spindle_output.. +echo "Looking for spindle_output.* files in testsuite directory..." +if podman exec "$CONTAINER_NAME" bash -c 'ls -la /home/spindleuser/Spindle-build/testsuite/spindle_output.* 2>/dev/null'; then + echo "" + echo "Found Spindle log files! Extracting..." + + # Get list of log files + LOG_FILES=$(podman exec "$CONTAINER_NAME" bash -c 'cd /home/spindleuser/Spindle-build/testsuite && ls spindle_output.* 2>/dev/null' || echo "") + + if [ -n "$LOG_FILES" ]; then + for logfile in $LOG_FILES; do + echo " Copying $logfile..." + podman cp "$CONTAINER_NAME:/home/spindleuser/Spindle-build/testsuite/$logfile" "$LOG_DIR/" + done + echo "" + echo "✓ Logs extracted to: $LOG_DIR" + echo "" + echo "View logs with:" + for logfile in $LOG_FILES; do + echo " cat $LOG_DIR/$logfile" + done + fi +else + echo "No spindle_output.* files found in testsuite directory" +fi + +echo "" +echo "Checking for other spindle files..." +podman exec "$CONTAINER_NAME" bash -c 'find /home/spindleuser/Spindle-build/testsuite -name "spindle*" -type f 2>/dev/null | head -20' || echo "No other spindle files found" + +echo "" +echo "==========================================" +echo "Summary" +echo "==========================================" +echo "" +echo "Container: $CONTAINER_NAME (still running)" +echo "Log directory: $LOG_DIR" +echo "" +echo "To explore interactively:" +echo " podman exec -it $CONTAINER_NAME bash" +echo " cd /home/spindleuser/Spindle-build/testsuite" +echo " ls spindle_output.*" +echo "" +echo "When done, cleanup with:" +echo " podman rm -f $CONTAINER_NAME" +echo "" + +# Don't cleanup automatically - let user explore +trap - EXIT +echo "Note: Container left running for inspection. Clean up manually when done." +echo "" diff --git a/scripts/podman/test-spindle-serial.sh b/scripts/podman/test-spindle-serial.sh new file mode 100755 index 00000000..c1d52ae2 --- /dev/null +++ b/scripts/podman/test-spindle-serial.sh @@ -0,0 +1,94 @@ +#!/bin/bash +# +# Run Spindle serial regular tests in podman +# +# This runs the main testsuite (not crash tests). +# Run this from outside the sandbox where podman is available. + +set -e + +# Get the directory containing this script +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +# Source common functions +source "$SCRIPT_DIR/common.sh" + +# Configuration +IMAGE_NAME="spindle-serial-ubuntu" +CONTAINER_NAME="spindlenode" + +echo "==========================================" +echo "Spindle Serial Regular Tests" +echo "==========================================" +echo "" + +# Cleanup function +cleanup() { + echo "" + echo "Cleaning up container..." + podman rm -f "$CONTAINER_NAME" 2>/dev/null || true + echo "✓ Cleanup complete" +} + +# Set trap to cleanup on exit +trap cleanup EXIT + +# Initial cleanup +cleanup + +echo "Starting Spindle serial container..." +echo "" + +# Start the container +podman run \ + --name "$CONTAINER_NAME" \ + --hostname "$CONTAINER_NAME" \ + --cap-add SYS_NICE \ + -d \ + -t \ + "$IMAGE_NAME" + +echo "✓ Container started" +sleep 3 + +echo "" +echo "==========================================" +echo "Verifying munge authentication..." +echo "==========================================" +echo "" + +podman exec "$CONTAINER_NAME" bash -c 'munge -n | unmunge' + +echo "" +echo "✓ Munge working" + +echo "" +echo "==========================================" +echo "Running Spindle testsuite..." +echo "==========================================" +echo "" +echo "This will take several minutes." +echo "" + +# Run the testsuite +if podman exec "$CONTAINER_NAME" bash -c 'cd Spindle-build/testsuite && ./runTests'; then + echo "" + echo "==========================================" + echo "✓ All tests passed!" + echo "==========================================" + echo "" + exit 0 +else + echo "" + echo "==========================================" + echo "✗ Some tests failed" + echo "==========================================" + echo "" + echo "To inspect the container:" + echo " podman exec -it $CONTAINER_NAME bash" + echo "" + exit 1 +fi + +# Cleanup happens automatically via trap diff --git a/scripts/podman/test-spindle-slurm-rshlaunch.sh b/scripts/podman/test-spindle-slurm-rshlaunch.sh new file mode 100755 index 00000000..def418cc --- /dev/null +++ b/scripts/podman/test-spindle-slurm-rshlaunch.sh @@ -0,0 +1,247 @@ +#!/bin/bash +# +# Run Spindle Slurm rshlaunch tests in podman +# +# This runs the Spindle testsuite in a Slurm cluster with rshlaunch launcher. +# Based on the CI workflow and docker-compose configuration. +# +# Cluster: 1 MariaDB + 1 slurmdbd + 1 slurmctld + 4 slurmd workers +# +# CPU Pinning: Cores 24-30 (avoiding system cores 0-23) +# +# Run this from outside the sandbox where podman is available. + +set -e + +# Get the directory containing this script +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +# Source common functions +source "$SCRIPT_DIR/common.sh" + +# Configuration +IMAGE_NAME="spindle-slurm-rshlaunch" +NETWORK_NAME="slurm-rshlaunch-test-net" +WORKERS=4 + +# CPU pinning disabled - not supported in rootless podman on this system +# See PODMAN.md for details +# CPU_MARIADB=24 +# CPU_DB=25 +# CPU_HEAD=26 +# CPU_NODE_BASE=27 # nodes 1-4 get 27-30 + +echo "==========================================" +echo "Spindle Slurm Rshlaunch Tests" +echo "==========================================" +echo "" +echo "This runs the Spindle testsuite in a Slurm cluster." +echo "Cluster: MariaDB + slurmdbd + slurmctld + 4 workers" +echo "Note: CPU pinning disabled (not supported in rootless podman)" +echo "" + +# Cleanup function +cleanup() { + echo "" + echo "Cleaning up containers and network..." + # Stop and remove all containers (force removal even if running) + for container in slurm-rshlaunch-mariadb slurm-rshlaunch-db slurm-rshlaunch-head slurm-rshlaunch-node-{1..4}; do + podman stop "$container" 2>/dev/null || true + podman rm -f "$container" 2>/dev/null || true + done + podman network rm -f "$NETWORK_NAME" 2>/dev/null || true + echo "✓ Cleanup complete" +} + +# Set trap to cleanup on exit +# DISABLED for debugging - cleanup manually with: podman rm -f slurm-rshlaunch-{mariadb,db,head,node-{1..4}}; podman network rm -f slurm-rshlaunch-test-net +# trap cleanup EXIT + +# Initial cleanup +cleanup + +echo "==========================================" +echo "Setting up Slurm cluster..." +echo "==========================================" +echo "" + +# Create network +if podman network exists "$NETWORK_NAME" 2>/dev/null; then + echo "Network $NETWORK_NAME already exists, reusing" +else + echo "Creating network: $NETWORK_NAME" + podman network create "$NETWORK_NAME" + echo "✓ Network created" +fi + +echo "" +echo "Starting MariaDB..." +# Read password from generated mariadb.env +MARIADB_PASSWORD=$(grep MARIADB_PASSWORD "$REPO_ROOT/containers/spindle-slurm-ubuntu/testing/mariadb.env" | cut -d'"' -f2) +if [ -z "$MARIADB_PASSWORD" ]; then + echo "Error: Could not read password from mariadb.env" + exit 1 +fi +podman run \ + --name slurm-rshlaunch-mariadb \ + --hostname slurm-mariadb \ + --network "$NETWORK_NAME" \ + -e MYSQL_RANDOM_ROOT_PASSWORD=yes \ + -e MYSQL_DATABASE=slurm_acct_db \ + -e MYSQL_USER=slurm \ + -e MYSQL_PASSWORD="$MARIADB_PASSWORD" \ + -d \ + mariadb:12 + +echo " ✓ MariaDB started" +echo "Waiting for MariaDB to initialize..." +sleep 15 + +echo "" +echo "Starting slurmdbd (accounting daemon)..." +podman run \ + --name slurm-rshlaunch-db \ + --hostname slurm-db \ + --network "$NETWORK_NAME" \ + -e SLURM_ROLE=db \ + -e SLURM_HEAD_NODE=slurm-head \ + -e workers="$WORKERS" \ + -d \ + "$IMAGE_NAME" + +echo " ✓ slurmdbd started" +sleep 10 + +echo "" +echo "Starting slurmctld (controller)..." +podman run \ + --name slurm-rshlaunch-head \ + --hostname slurm-head \ + --network "$NETWORK_NAME" \ + -e SLURM_ROLE=ctl \ + -e SLURM_HEAD_NODE=slurm-head \ + -e workers="$WORKERS" \ + -t \ + -d \ + "$IMAGE_NAME" + +echo " ✓ slurmctld started" +sleep 10 + +echo "" +echo "Starting worker nodes..." +for i in $(seq 1 $WORKERS); do + echo "Starting slurm-node-$i..." + podman run \ + --name "slurm-rshlaunch-node-$i" \ + --hostname "slurm-node-$i" \ + --network "$NETWORK_NAME" \ + -e SLURM_ROLE=worker \ + -e SLURM_HEAD_NODE=slurm-head \ + -e workers="$WORKERS" \ + -d \ + "$IMAGE_NAME" + echo " ✓ slurm-node-$i started" +done + +echo "" +echo "Waiting for Slurm cluster to initialize..." +echo "(This takes ~30 seconds for all daemons and nodes)" +sleep 30 + +echo "" +echo "==========================================" +echo "Checking container status..." +echo "==========================================" +echo "" + +# Check if containers are still running +ALL_RUNNING=true +for container in slurm-rshlaunch-mariadb slurm-rshlaunch-db slurm-rshlaunch-head slurm-rshlaunch-node-{1..4}; do + if podman ps --filter "name=$container" --format "{{.Names}}" | grep -q "$container"; then + echo " ✓ $container is running" + else + echo " ✗ $container has exited!" + ALL_RUNNING=false + echo "" + echo "Last 30 lines of $container logs:" + echo "----------------------------------------" + podman logs "$container" 2>&1 | tail -30 + echo "----------------------------------------" + fi +done + +if [ "$ALL_RUNNING" = false ]; then + echo "" + echo "Some containers exited. Check logs above." + exit 1 +fi + +echo "" +echo "==========================================" +echo "Verifying munge authentication..." +echo "==========================================" +echo "" + +podman exec slurm-rshlaunch-head bash -c 'munge -n | unmunge' + +echo "" +echo "✓ Munge working" + +echo "" +echo "==========================================" +echo "Verifying Slurm cluster..." +echo "==========================================" +echo "" + +echo "Checking node status with sinfo:" +podman exec slurm-rshlaunch-head sinfo || echo " (Nodes may still be registering)" + +echo "" +echo "Checking cluster status with scontrol:" +podman exec slurm-rshlaunch-head scontrol show nodes || echo " (Still initializing)" + +echo "" +echo "==========================================" +echo "Running Spindle testsuite..." +echo "==========================================" +echo "" +echo "This will take several minutes." +echo "" + +# Run the testsuite +# Based on CI: docker exec slurm-rshlaunch-head bash -c 'cd Spindle-build/testsuite && salloc -n${workers} -N${workers} ./runTests ${workers}' +if podman exec slurm-rshlaunch-head bash -c "cd Spindle-build/testsuite && salloc -n${WORKERS} -N${WORKERS} ./runTests ${WORKERS}"; then + echo "" + echo "==========================================" + echo "✓ All tests passed!" + echo "==========================================" + echo "" + exit 0 +else + echo "" + echo "==========================================" + echo "✗ Some tests failed" + echo "==========================================" + echo "" + echo "To inspect the cluster:" + echo " podman exec -it slurm-rshlaunch-head bash" + echo " sinfo" + echo " scontrol show nodes" + echo " cd Spindle-build/testsuite" + echo "" + echo "Manual cleanup when done:" + echo " podman rm -f slurm-rshlaunch-mariadb slurm-rshlaunch-db slurm-rshlaunch-head slurm-rshlaunch-node-{1..4}" + echo " podman network rm -f slurm-rshlaunch-test-net" + echo "" + exit 1 +fi + +echo "" +echo "Manual cleanup when done:" +echo " podman rm -f slurm-rshlaunch-mariadb slurm-rshlaunch-db slurm-rshlaunch-head slurm-rshlaunch-node-{1..4}" +echo " podman network rm -f slurm-rshlaunch-test-net" +echo "" + +# Cleanup disabled for debugging - do manually diff --git a/scripts/podman/test-spindle-slurm-srun-parallel.sh b/scripts/podman/test-spindle-slurm-srun-parallel.sh new file mode 100755 index 00000000..6e05eef2 --- /dev/null +++ b/scripts/podman/test-spindle-slurm-srun-parallel.sh @@ -0,0 +1,261 @@ +#!/bin/bash +# +# Run Spindle Slurm srun test with unique container names for parallel execution +# +# Usage: test-spindle-slurm-srun-parallel.sh +# +# The run-id is appended to all container and network names to avoid conflicts +# when running multiple tests in parallel. + +set -e + +RUN_ID="${1}" + +if [ -z "$RUN_ID" ]; then + echo "Usage: $0 " + echo "" + echo "Example: $0 42" + echo " Creates containers: slurm-srun-42-mariadb, slurm-srun-42-head, etc." + exit 1 +fi + +# Get the directory containing this script +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +# Source common functions +source "$SCRIPT_DIR/common.sh" + +# Configuration - all names include RUN_ID for uniqueness +IMAGE_NAME="spindle-slurm-srun" +NETWORK_NAME="slurm-srun-${RUN_ID}-net" +WORKERS=4 +NAME_PREFIX="slurm-srun-${RUN_ID}" + +echo "==========================================" +echo "Spindle Slurm Srun Tests (Run ID: $RUN_ID)" +echo "==========================================" +echo "" +echo "Container prefix: $NAME_PREFIX" +echo "Network: $NETWORK_NAME" +echo "" + +# Cleanup function +cleanup() { + echo "" + echo "Cleaning up run $RUN_ID..." + # Stop all containers in parallel for faster cleanup + for container in ${NAME_PREFIX}-mariadb ${NAME_PREFIX}-db ${NAME_PREFIX}-head ${NAME_PREFIX}-node-{1..4}; do + (podman stop "$container" 2>/dev/null || true) & + done + wait + # Remove all containers in parallel + for container in ${NAME_PREFIX}-mariadb ${NAME_PREFIX}-db ${NAME_PREFIX}-head ${NAME_PREFIX}-node-{1..4}; do + (podman rm -f "$container" 2>/dev/null || true) & + done + wait + podman network rm -f "$NETWORK_NAME" 2>/dev/null || true + echo "✓ Cleanup complete for run $RUN_ID" +} + +# Set trap to cleanup on exit +trap cleanup EXIT + +# Initial cleanup +cleanup + +echo "==========================================" +echo "Setting up Slurm cluster..." +echo "==========================================" +echo "" + +# Create network +if podman network exists "$NETWORK_NAME" 2>/dev/null; then + echo "Network $NETWORK_NAME already exists, reusing" +else + echo "Creating network: $NETWORK_NAME" + podman network create "$NETWORK_NAME" + echo "✓ Network created" +fi + +echo "" +echo "Starting MariaDB..." +# Read password from mariadb.env +# First check repo root (portable deployment), then fall back to source location +MARIADB_ENV="" +if [ -f "$REPO_ROOT/mariadb.env" ]; then + MARIADB_ENV="$REPO_ROOT/mariadb.env" +elif [ -f "$REPO_ROOT/containers/spindle-slurm-ubuntu/testing-srun/mariadb.env" ]; then + MARIADB_ENV="$REPO_ROOT/containers/spindle-slurm-ubuntu/testing-srun/mariadb.env" +else + echo "Error: Could not find mariadb.env" + echo " Looked in:" + echo " $REPO_ROOT/mariadb.env" + echo " $REPO_ROOT/containers/spindle-slurm-ubuntu/testing-srun/mariadb.env" + exit 1 +fi + +MARIADB_PASSWORD=$(grep MARIADB_PASSWORD "$MARIADB_ENV" | cut -d'"' -f2) +if [ -z "$MARIADB_PASSWORD" ]; then + echo "Error: Could not read password from $MARIADB_ENV" + exit 1 +fi +echo " Using password from: $MARIADB_ENV" +podman run \ + --name "${NAME_PREFIX}-mariadb" \ + --hostname slurm-mariadb \ + --network "$NETWORK_NAME" \ + -e MYSQL_RANDOM_ROOT_PASSWORD=yes \ + -e MYSQL_DATABASE=slurm_acct_db \ + -e MYSQL_USER=slurm \ + -e MYSQL_PASSWORD="$MARIADB_PASSWORD" \ + -d \ + mariadb:12 + +echo " ✓ MariaDB started" +echo "Waiting for MariaDB to initialize..." +sleep 15 + +echo "" +echo "Starting slurmdbd (accounting daemon)..." +podman run \ + --name "${NAME_PREFIX}-db" \ + --hostname slurm-db \ + --network "$NETWORK_NAME" \ + -e SLURM_ROLE=db \ + -e SLURM_HEAD_NODE=slurm-head \ + -e workers="$WORKERS" \ + -d \ + "$IMAGE_NAME" + +echo " ✓ slurmdbd started" +sleep 10 + +echo "" +echo "Starting slurmctld (controller)..." +podman run \ + --name "${NAME_PREFIX}-head" \ + --hostname slurm-head \ + --network "$NETWORK_NAME" \ + -e SLURM_ROLE=ctl \ + -e SLURM_HEAD_NODE=slurm-head \ + -e workers="$WORKERS" \ + -t \ + -d \ + "$IMAGE_NAME" + +echo " ✓ slurmctld started" +sleep 10 + +echo "" +echo "Starting worker nodes..." +for i in $(seq 1 $WORKERS); do + echo "Starting slurm-node-$i..." + podman run \ + --name "${NAME_PREFIX}-node-$i" \ + --hostname "slurm-node-$i" \ + --network "$NETWORK_NAME" \ + -e SLURM_ROLE=worker \ + -e SLURM_HEAD_NODE=slurm-head \ + -e workers="$WORKERS" \ + -d \ + "$IMAGE_NAME" + echo " ✓ slurm-node-$i started" +done + +echo "" +echo "Waiting for Slurm cluster to initialize..." +echo "(This takes ~60 seconds for all daemons and nodes)" +sleep 60 + +echo "" +echo "==========================================" +echo "Checking container status..." +echo "==========================================" +echo "" + +# Check if containers are still running +ALL_RUNNING=true +for container in ${NAME_PREFIX}-mariadb ${NAME_PREFIX}-db ${NAME_PREFIX}-head ${NAME_PREFIX}-node-{1..4}; do + if podman ps --filter "name=$container" --format "{{.Names}}" | grep -q "$container"; then + echo " ✓ $container is running" + else + echo " ✗ $container has exited!" + ALL_RUNNING=false + echo "" + echo "Last 30 lines of $container logs:" + echo "----------------------------------------" + podman logs "$container" 2>&1 | tail -30 + echo "----------------------------------------" + fi +done + +if [ "$ALL_RUNNING" = false ]; then + echo "" + echo "Some containers exited. Check logs above." + exit 1 +fi + +echo "" +echo "==========================================" +echo "Verifying munge authentication..." +echo "==========================================" +echo "" + +podman exec "${NAME_PREFIX}-head" bash -c 'munge -n | unmunge' + +echo "" +echo "✓ Munge working" + +echo "" +echo "==========================================" +echo "Verifying Slurm cluster..." +echo "==========================================" +echo "" + +echo "Checking node status with sinfo:" +if podman exec "${NAME_PREFIX}-head" sinfo; then + echo " ✓ Slurm cluster ready" +else + echo "" + echo "✗ Slurm cluster verification FAILED" + echo "" + echo "Containers are still running for debugging." + echo "Press ENTER to cleanup and exit, or Ctrl-C to keep them running." + echo "" + echo "Useful debug commands:" + echo " podman logs ${NAME_PREFIX}-head" + echo " podman logs ${NAME_PREFIX}-db" + echo " podman exec ${NAME_PREFIX}-head sinfo" + echo " podman exec ${NAME_PREFIX}-head scontrol show nodes" + echo "" + read -r + exit 1 +fi + +echo "" +echo "==========================================" +echo "Running Spindle testsuite..." +echo "==========================================" +echo "" +echo "This will take several minutes." +echo "" + +# Run the testsuite +if podman exec "${NAME_PREFIX}-head" bash -c "cd Spindle-build/testsuite && salloc -n${WORKERS} -N${WORKERS} ./runTests ${WORKERS}"; then + echo "" + echo "==========================================" + echo "✓ All tests passed! (Run $RUN_ID)" + echo "==========================================" + echo "" + exit 0 +else + echo "" + echo "==========================================" + echo "✗ Some tests failed (Run $RUN_ID)" + echo "==========================================" + echo "" + exit 1 +fi + +# Cleanup happens automatically via trap diff --git a/scripts/podman/test-spindle-slurm-srun.sh b/scripts/podman/test-spindle-slurm-srun.sh new file mode 100755 index 00000000..23680ad0 --- /dev/null +++ b/scripts/podman/test-spindle-slurm-srun.sh @@ -0,0 +1,370 @@ +#!/bin/bash +# +# Run N parallel Spindle Slurm srun tests in podman +# +# Usage: test-spindle-slurm-srun.sh +# +# Creates N independent Slurm clusters and runs tests in parallel. +# Each instance outputs to both stdout and out. with timestamps. +# +# Run this from outside the sandbox where podman is available. + +set -e + +NUM_INSTANCES="${1}" + +if [ -z "$NUM_INSTANCES" ]; then + echo "Usage: $0 " + echo "" + echo "Example: $0 10" + echo " Creates 10 independent Slurm clusters and runs tests in parallel" + exit 1 +fi + +if ! [[ "$NUM_INSTANCES" =~ ^[0-9]+$ ]] || [ "$NUM_INSTANCES" -lt 1 ]; then + echo "Error: num-instances must be a positive integer" + exit 1 +fi + +# Get the directory containing this script +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +# Source common functions +source "$SCRIPT_DIR/common.sh" + +# Configuration +IMAGE_NAME="spindle-slurm-srun" +WORKERS=4 + +echo "==========================================" +echo "Spindle Slurm Srun Parallel Tests" +echo "==========================================" +echo "" +echo "Instances: $NUM_INSTANCES" +echo "Each cluster: MariaDB + slurmdbd + slurmctld + 4 workers" +echo "Output: stdout + out. files (timestamped)" +echo "" + +# Function to run a single test instance +run_instance() { + local INSTANCE_ID=$1 + local NAME_PREFIX="slurm-srun-${INSTANCE_ID}" + + # Instance-specific hostnames for shared network + local MARIADB_HOST="${NAME_PREFIX}-mariadb" + local DB_HOST="${NAME_PREFIX}-db" + local HEAD_NODE="${NAME_PREFIX}-head" + local NODE_PREFIX="${NAME_PREFIX}-node" + + # All output from this function goes through ts and tee + { + echo "[Instance $INSTANCE_ID] Starting test" + echo "" + + echo "[Instance $INSTANCE_ID] Setting up Slurm cluster..." + echo "" + + # Generate random password for this cluster + MARIADB_PASSWORD=$(openssl rand -base64 16) + echo "[Instance $INSTANCE_ID] Generated MariaDB password" + echo "" + + # Start MariaDB + echo "[Instance $INSTANCE_ID] Starting MariaDB..." + podman run \ + --name "${NAME_PREFIX}-mariadb" \ + --hostname "$MARIADB_HOST" \ + --network "$SHARED_NETWORK" \ + -e MYSQL_RANDOM_ROOT_PASSWORD=yes \ + -e MYSQL_DATABASE=slurm_acct_db \ + -e MYSQL_USER=slurm \ + -e MYSQL_PASSWORD="$MARIADB_PASSWORD" \ + -d \ + mariadb:12 >/dev/null + echo "[Instance $INSTANCE_ID] MariaDB started" + echo "[Instance $INSTANCE_ID] Waiting for MariaDB to initialize (15s)..." + sleep 15 + echo "" + + # Start worker nodes FIRST so they're available for DNS resolution + # when slurmctld starts and tries to resolve node addresses + echo "[Instance $INSTANCE_ID] Starting worker nodes..." + for i in $(seq 1 $WORKERS); do + echo "[Instance $INSTANCE_ID] Starting ${NODE_PREFIX}-$i..." + podman run \ + --name "${NAME_PREFIX}-node-$i" \ + --hostname "${NODE_PREFIX}-$i" \ + --network "$SHARED_NETWORK" \ + -e SLURM_ROLE=worker \ + -e SLURM_HEAD_NODE="$HEAD_NODE" \ + -e SLURM_DB_HOST="$DB_HOST" \ + -e SLURM_NODE_PREFIX="$NODE_PREFIX" \ + -e workers="$WORKERS" \ + -d \ + "$IMAGE_NAME" >/dev/null + echo "[Instance $INSTANCE_ID] ${NODE_PREFIX}-$i started" + done + echo "[Instance $INSTANCE_ID] Waiting for workers to initialize (5s)..." + sleep 5 + echo "" + + # Start slurmdbd + echo "[Instance $INSTANCE_ID] Starting slurmdbd..." + podman run \ + --name "${NAME_PREFIX}-db" \ + --hostname "$DB_HOST" \ + --network "$SHARED_NETWORK" \ + -e SLURM_ROLE=db \ + -e SLURM_HEAD_NODE="$HEAD_NODE" \ + -e SLURM_DB_HOST="$DB_HOST" \ + -e SLURM_MARIADB_HOST="$MARIADB_HOST" \ + -e SLURM_NODE_PREFIX="$NODE_PREFIX" \ + -e workers="$WORKERS" \ + -e MARIADB_PASSWORD="$MARIADB_PASSWORD" \ + -d \ + "$IMAGE_NAME" >/dev/null + echo "[Instance $INSTANCE_ID] slurmdbd started" + sleep 10 + echo "" + + # Start slurmctld LAST so all other containers are reachable via DNS + echo "[Instance $INSTANCE_ID] Starting slurmctld..." + podman run \ + --name "${NAME_PREFIX}-head" \ + --hostname "$HEAD_NODE" \ + --network "$SHARED_NETWORK" \ + -e SLURM_ROLE=ctl \ + -e SLURM_HEAD_NODE="$HEAD_NODE" \ + -e SLURM_DB_HOST="$DB_HOST" \ + -e SLURM_NODE_PREFIX="$NODE_PREFIX" \ + -e workers="$WORKERS" \ + -t \ + -d \ + "$IMAGE_NAME" >/dev/null + echo "[Instance $INSTANCE_ID] slurmctld started" + sleep 10 + echo "" + + echo "[Instance $INSTANCE_ID] Waiting for Slurm cluster to initialize (60s)..." + sleep 60 + echo "" + + # Verify cluster + echo "[Instance $INSTANCE_ID] Verifying cluster..." + if podman exec "${NAME_PREFIX}-head" sinfo >/dev/null 2>&1; then + echo "[Instance $INSTANCE_ID] Cluster ready" + else + echo "[Instance $INSTANCE_ID] WARNING: sinfo failed" + echo "" + echo "[Instance $INSTANCE_ID] ========== DIAGNOSTICS ==========" + + # Check container status + echo "[Instance $INSTANCE_ID] Container status:" + for container in ${NAME_PREFIX}-mariadb ${NAME_PREFIX}-db ${NAME_PREFIX}-head; do + if podman ps --filter "name=$container" --format "{{.Names}}" | grep -q "$container"; then + echo "[Instance $INSTANCE_ID] ✓ $container is running" + else + echo "[Instance $INSTANCE_ID] ✗ $container has exited!" + fi + done + echo "" + + # Show MariaDB logs + echo "[Instance $INSTANCE_ID] MariaDB logs (last 20 lines):" + podman logs "${NAME_PREFIX}-mariadb" 2>&1 | tail -20 | sed "s/^/[Instance $INSTANCE_ID] /" + echo "" + + # Show slurmdbd logs + echo "[Instance $INSTANCE_ID] slurmdbd logs (last 30 lines):" + podman logs "${NAME_PREFIX}-db" 2>&1 | tail -30 | sed "s/^/[Instance $INSTANCE_ID] /" + echo "" + + # Show slurmctld logs + echo "[Instance $INSTANCE_ID] slurmctld logs (last 30 lines):" + podman logs "${NAME_PREFIX}-head" 2>&1 | tail -30 | sed "s/^/[Instance $INSTANCE_ID] /" + echo "" + + # Test MariaDB connectivity + echo "[Instance $INSTANCE_ID] Testing MariaDB connectivity:" + if podman exec "${NAME_PREFIX}-mariadb" mysqladmin ping 2>&1 | grep -q "mysqld is alive"; then + echo "[Instance $INSTANCE_ID] ✓ MariaDB is responding" + else + echo "[Instance $INSTANCE_ID] ✗ MariaDB not responding" + fi + echo "" + + # Check if slurmdbd can resolve MariaDB hostname + echo "[Instance $INSTANCE_ID] DNS check from slurmdbd:" + podman exec "${NAME_PREFIX}-db" getent hosts slurm-mariadb 2>&1 | sed "s/^/[Instance $INSTANCE_ID] /" || echo "[Instance $INSTANCE_ID] ✗ Cannot resolve slurm-mariadb" + echo "" + + echo "[Instance $INSTANCE_ID] ========== END DIAGNOSTICS ==========" + echo "" + echo "[Instance $INSTANCE_ID] Continuing with tests anyway..." + fi + echo "" + + # Run tests + echo "[Instance $INSTANCE_ID] Running Spindle testsuite..." + echo "[Instance $INSTANCE_ID] Press any key to run tests..." + read -n 1 -s -r < /dev/tty + echo "" + if podman exec "${NAME_PREFIX}-head" bash -c "cd Spindle-build/testsuite && export SPINDLE_DEBUG=3 && salloc -n${WORKERS} -N${WORKERS} ./runTests ${WORKERS}"; then + echo "" + echo "[Instance $INSTANCE_ID] ==========================================" + echo "[Instance $INSTANCE_ID] ALL TESTS PASSED" + echo "[Instance $INSTANCE_ID] ==========================================" + RESULT=0 + else + echo "" + echo "[Instance $INSTANCE_ID] ==========================================" + echo "[Instance $INSTANCE_ID] SOME TESTS FAILED" + echo "[Instance $INSTANCE_ID] ==========================================" + RESULT=1 + fi + echo "" + echo "[Instance $INSTANCE_ID] Test complete" + echo "[Instance $INSTANCE_ID] Containers are running. You can now:" + echo "[Instance $INSTANCE_ID] podman exec -it ${NAME_PREFIX}-head bash" + echo "[Instance $INSTANCE_ID] podman exec -it ${NAME_PREFIX}-node-1 bash" + echo "[Instance $INSTANCE_ID] Press any key to cleanup and exit..." + read -n 1 -s -r < /dev/tty + echo "" + echo "[Instance $INSTANCE_ID] Cleanup will happen in serial phase" + echo "" + + exit $RESULT + } 2>&1 | ts | tee "out.${INSTANCE_ID}" +} + +# Shared network for all instances (avoids 30s timeout per instance) +SHARED_NETWORK="slurm-srun-shared" + +# Serial phase: Verify prerequisites and cleanup +echo "==========================================" +echo "Serial Phase: Prerequisites & Cleanup" +echo "==========================================" +echo "" + +echo "Checking for required images..." +if ! podman images | grep -q "spindle-slurm-srun"; then + echo "ERROR: spindle-slurm-srun image not found" + echo "Build it first with: ./build-spindle-slurm-srun.sh" + exit 1 +fi + +if ! podman images | grep -q "mariadb.*12"; then + echo "ERROR: mariadb:12 image not found" + echo "Pull it first with: podman pull mariadb:12" + echo "Or load from tarball with: ./load-images.sh " + exit 1 +fi +echo "✓ All required images present" +echo "" + +echo "Cleaning up any existing test containers and network..." +for i in $(seq 1 $NUM_INSTANCES); do + NAME_PREFIX="slurm-srun-${i}" + + # Remove containers for this instance + for container in ${NAME_PREFIX}-mariadb ${NAME_PREFIX}-db ${NAME_PREFIX}-head ${NAME_PREFIX}-node-{1..4}; do + podman rm -f "$container" 2>/dev/null || true + done +done + +# Remove shared network +podman network rm -f "$SHARED_NETWORK" 2>/dev/null || true +echo "✓ Cleanup complete" +echo "" + +echo "Creating shared network: $SHARED_NETWORK" +podman network create "$SHARED_NETWORK" >/dev/null +echo "✓ Shared network created (this may take ~30s due to systemd session bus timeout)" +echo "" + +# Parallel phase: Launch all instances +echo "==========================================" +echo "Parallel Phase: Launching $NUM_INSTANCES instances" +echo "==========================================" +echo "" + +PIDS=() +for i in $(seq 1 $NUM_INSTANCES); do + echo "Launching instance $i..." + run_instance $i & + PIDS+=($!) +done + +echo "" +echo "All instances launched. Waiting for completion..." +echo "(Output to stdout and out. files)" +echo "" + +# Wait for all instances and collect results +FAILED=0 +for i in $(seq 1 $NUM_INSTANCES); do + if ! wait ${PIDS[$((i-1))]}; then + FAILED=$((FAILED + 1)) + fi +done + +echo "" +echo "==========================================" +echo "Serial Phase: Cleanup" +echo "==========================================" +echo "" + +echo "Cleaning up $NUM_INSTANCES test clusters..." +for i in $(seq 1 $NUM_INSTANCES); do + NAME_PREFIX="slurm-srun-${i}" + + echo "Cleaning up instance $i..." + + # Stop containers + for container in ${NAME_PREFIX}-mariadb ${NAME_PREFIX}-db ${NAME_PREFIX}-head ${NAME_PREFIX}-node-{1..4}; do + podman stop "$container" 2>/dev/null || true + done + + # Remove containers + for container in ${NAME_PREFIX}-mariadb ${NAME_PREFIX}-db ${NAME_PREFIX}-head ${NAME_PREFIX}-node-{1..4}; do + podman rm -f "$container" 2>/dev/null || true + done +done +echo "✓ Cleanup complete" +echo "" + +# Remove shared network +echo "Removing shared network..." +podman network rm -f "$SHARED_NETWORK" 2>/dev/null || true +echo "✓ Network removed" +echo "" + +# Serial phase: Summary +echo "==========================================" +echo "Serial Phase: Summary" +echo "==========================================" +echo "" +echo "Total instances: $NUM_INSTANCES" +echo "Passed: $((NUM_INSTANCES - FAILED))" +echo "Failed: $FAILED" +echo "" + +if [ $FAILED -eq 0 ]; then + echo "✓ All instances passed!" + exit 0 +else + echo "✗ Some instances failed" + echo "" + echo "Check individual logs:" + for i in $(seq 1 $NUM_INSTANCES); do + if grep -q "SOME TESTS FAILED" "out.$i" 2>/dev/null; then + echo " out.$i - FAILED" + elif grep -q "ALL TESTS PASSED" "out.$i" 2>/dev/null; then + echo " out.$i - PASSED" + else + echo " out.$i - UNKNOWN" + fi + done + exit 1 +fi diff --git a/spindle-podman b/spindle-podman new file mode 100755 index 00000000..05b2cbd3 --- /dev/null +++ b/spindle-podman @@ -0,0 +1,641 @@ +#!/bin/bash +# +# spindle-podman - Consolidated CLI for Spindle podman operations +# +# Usage: ./spindle-podman [options] +# +# This replaces the scattered collection of build/test/cleanup scripts +# with a single entry point that tracks state and provides visibility. + +set -euo pipefail + +# Determine repo root +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +STATE_FILE="$REPO_ROOT/.podman-state.json" +METRICS_LOG="$REPO_ROOT/.podman-metrics.log" +LOG_DIR="$REPO_ROOT/.podman-logs" + +# Colors for output +if [[ -t 1 ]]; then + GREEN='\033[0;32m' + YELLOW='\033[1;33m' + RED='\033[0;31m' + BLUE='\033[0;34m' + BOLD='\033[1m' + RESET='\033[0m' +else + GREEN='' YELLOW='' RED='' BLUE='' BOLD='' RESET='' +fi + +# Source common podman utilities +if [[ -f "$REPO_ROOT/scripts/podman/common.sh" ]]; then + source "$REPO_ROOT/scripts/podman/common.sh" +fi + +#============================================================================== +# State Management +#============================================================================== + +init_state() { + if [[ ! -f "$STATE_FILE" ]]; then + cat > "$STATE_FILE" <<'EOF' +{ + "version": "1.0", + "images": {}, + "tests": {}, + "background_jobs": {} +} +EOF + fi +} + +get_state() { + local key="$1" + jq -r "$key // empty" "$STATE_FILE" 2>/dev/null || echo "" +} + +update_state() { + local key="$1" + local value="$2" + local tmp + tmp=$(mktemp) + # Remove leading dot and split by dot to create path array + local cleaned_key="${key#.}" + jq --arg k "$cleaned_key" --argjson v "$value" 'setpath($k | split("."); $v)' "$STATE_FILE" > "$tmp" + mv "$tmp" "$STATE_FILE" +} + +#============================================================================== +# Logging and Metrics +#============================================================================== + +log_metric() { + local operation="$1" + local target="$2" + local duration="$3" + local status="$4" + + mkdir -p "$(dirname "$METRICS_LOG")" + printf '%s,%s,%s,%s,%s\n' \ + "$(date -Iseconds)" \ + "$operation" \ + "$target" \ + "$duration" \ + "$status" >> "$METRICS_LOG" +} + +log_dir_for_run() { + local run_id + run_id="$(date +%Y%m%d-%H%M%S)" + echo "$LOG_DIR/$run_id" +} + +#============================================================================== +# Status Command +#============================================================================== + +cmd_status() { + local verbose=false + + if [[ "${1:-}" == "--verbose" ]] || [[ "${1:-}" == "-v" ]]; then + verbose=true + fi + + init_state + + echo -e "${BOLD}=== Spindle Podman Status ===${RESET}\n" + + # Check podman availability + if command -v podman &>/dev/null; then + echo -e "${GREEN}✓${RESET} Podman available: $(podman --version | head -1)" + else + echo -e "${RED}✗${RESET} Podman not found" + return 1 + fi + + # Check images + echo -e "\n${BOLD}Images:${RESET}" + local has_images=false + + for target in serial flux slurm-srun slurm-base; do + local image_name + case "$target" in + serial) image_name="localhost/spindle-serial-ubuntu" ;; + flux) image_name="localhost/spindle-flux-ubuntu" ;; + slurm-srun) image_name="localhost/spindle-slurm-srun" ;; + slurm-base) image_name="localhost/spindle-slurm-base" ;; + esac + + if podman images --format "{{.Repository}}" 2>/dev/null | grep -q "^${image_name}\$"; then + has_images=true + local image_id size created + image_id=$(podman images --format "{{.ID}}" "$image_name:latest" 2>/dev/null | head -1) + size=$(podman images --format "{{.Size}}" "$image_name:latest" 2>/dev/null | head -1) + created=$(podman images --format "{{.CreatedAt}}" "$image_name:latest" 2>/dev/null | head -1) + + echo -e " ${GREEN}✓${RESET} ${BOLD}$target${RESET}: $size (created: $created)" + + if $verbose; then + echo " Image ID: $image_id" + fi + else + echo -e " ${YELLOW}○${RESET} $target: not built" + fi + done + + if ! $has_images; then + echo -e " ${YELLOW}No images built yet${RESET}" + fi + + # Check running containers + echo -e "\n${BOLD}Running Containers:${RESET}" + local container_count + container_count=$(podman ps --format "{{.Names}}" 2>/dev/null | grep -c "spindle" || true) + + if [[ $container_count -gt 0 ]]; then + podman ps --filter "name=spindle" --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}" 2>/dev/null + else + echo " None" + fi + + # Check test status + echo -e "\n${BOLD}Test Status:${RESET}" + local last_test_target last_test_status last_test_time + last_test_target=$(get_state '.tests | keys | .[-1]') + + if [[ -n "$last_test_target" ]]; then + last_test_status=$(get_state ".tests[\"$last_test_target\"].status") + last_test_time=$(get_state ".tests[\"$last_test_target\"].last_run") + local passed failed + passed=$(get_state ".tests[\"$last_test_target\"].passed") + failed=$(get_state ".tests[\"$last_test_target\"].failed") + + if [[ "$last_test_status" == "passed" ]]; then + echo -e " ${GREEN}✓${RESET} ${BOLD}$last_test_target${RESET}: $passed passed, $failed failed" + else + echo -e " ${RED}✗${RESET} ${BOLD}$last_test_target${RESET}: $last_test_status" + fi + echo " Last run: $last_test_time" + else + echo " No tests run yet" + fi + + # Check logs + if [[ -d "$LOG_DIR" ]]; then + local log_count + log_count=$(find "$LOG_DIR" -mindepth 1 -maxdepth 1 -type d 2>/dev/null | wc -l) + if [[ $log_count -gt 0 ]]; then + echo -e "\n${BOLD}Logs:${RESET} $log_count sessions in $LOG_DIR" + if $verbose; then + echo " Recent sessions:" + find "$LOG_DIR" -mindepth 1 -maxdepth 1 -type d | sort -r | head -3 | while read -r dir; do + echo " $(basename "$dir")" + done + fi + fi + fi + + echo "" +} + +#============================================================================== +# Build Command +#============================================================================== + +cmd_build() { + local target="${1:-}" + local background=false + + if [[ -z "$target" ]]; then + echo "Error: build target required" + echo "Usage: $0 build " + return 1 + fi + + shift + while [[ $# -gt 0 ]]; do + case "$1" in + --background|-b) background=true; shift ;; + *) echo "Unknown option: $1"; return 1 ;; + esac + done + + init_state + + local build_script="$REPO_ROOT/scripts/podman/build-spindle-${target}.sh" + + if [[ ! -f "$build_script" ]]; then + echo -e "${RED}Error:${RESET} No build script found for target '$target'" + echo "Available targets: serial, flux, slurm-srun, slurm-base" + return 1 + fi + + echo -e "${BOLD}Building $target...${RESET}\n" + + local start_time + start_time=$(date +%s) + local log_dir + log_dir=$(log_dir_for_run) + mkdir -p "$log_dir" + local log_file="$log_dir/build-${target}.log" + + if $background; then + echo "Starting background build..." + echo "Log: $log_file" + + bash "$build_script" > "$log_file" 2>&1 & + local pid=$! + + echo "Background job PID: $pid" + echo -e "${BLUE}→${RESET} Use '$0 status' to check progress" + + # Update state with background job info + update_state "background_jobs.build-${target}" \ + "{\"pid\": $pid, \"started\": \"$(date -Iseconds)\", \"log\": \"$log_file\"}" + + return 0 + else + # Run in foreground with live output + if bash "$build_script" 2>&1 | tee "$log_file"; then + local end_time duration + end_time=$(date +%s) + duration=$((end_time - start_time)) + + echo -e "\n${GREEN}✓${RESET} Build completed in ${duration}s" + + # Log metric + log_metric "build" "$target" "${duration}s" "success" + + # Update state + update_state "images.$target" \ + "{\"built\": \"$(date -Iseconds)\", \"build_time_sec\": $duration}" + + # Check if build exceeded baseline (more lenient for compute nodes) + local baseline=250 # Observed: 227s on compute node, allow 10% margin + if [[ $duration -gt $baseline ]]; then + echo -e "${YELLOW}⚠${RESET} Build time (${duration}s) exceeded baseline (${baseline}s)" + fi + + return 0 + else + local end_time duration + end_time=$(date +%s) + duration=$((end_time - start_time)) + + echo -e "\n${RED}✗${RESET} Build failed after ${duration}s" + echo "Log: $log_file" + + log_metric "build" "$target" "${duration}s" "failed" + return 1 + fi + fi +} + +#============================================================================== +# Test Command +#============================================================================== + +cmd_test() { + local target="${1:-}" + local repeat=1 + local background=false + + if [[ -z "$target" ]]; then + echo "Error: test target required" + echo "Usage: $0 test [--repeat N] [--background]" + return 1 + fi + + shift + while [[ $# -gt 0 ]]; do + case "$1" in + --repeat|-r) repeat="$2"; shift 2 ;; + --background|-b) background=true; shift ;; + *) echo "Unknown option: $1"; return 1 ;; + esac + done + + init_state + + local test_script="$REPO_ROOT/scripts/podman/test-spindle-${target}.sh" + + if [[ ! -f "$test_script" ]]; then + echo -e "${RED}Error:${RESET} No test script found for target '$target'" + echo "Available targets: serial, flux, slurm-srun" + return 1 + fi + + # Check if image exists + local image_name + case "$target" in + serial) image_name="localhost/spindle-serial-ubuntu" ;; + flux) image_name="localhost/spindle-flux-ubuntu" ;; + slurm-srun) image_name="localhost/spindle-slurm-srun" ;; + esac + + if ! podman images --format "{{.Repository}}" 2>/dev/null | grep -q "^${image_name}\$"; then + echo -e "${RED}Error:${RESET} Image '$image_name' not found" + echo "Build it first with: $0 build $target" + return 1 + fi + + echo -e "${BOLD}Testing $target...${RESET}\n" + + local start_time + start_time=$(date +%s) + local log_dir + log_dir=$(log_dir_for_run) + mkdir -p "$log_dir" + local log_file="$log_dir/test-${target}.log" + + if $background; then + echo "Starting background test..." + echo "Log: $log_file" + + bash "$test_script" > "$log_file" 2>&1 & + local pid=$! + + echo "Background job PID: $pid" + echo -e "${BLUE}→${RESET} Use '$0 status' to check results" + + update_state "background_jobs.test-${target}" \ + "{\"pid\": $pid, \"started\": \"$(date -Iseconds)\", \"log\": \"$log_file\"}" + + return 0 + else + local success=true + + for ((i=1; i<=repeat; i++)); do + if [[ $repeat -gt 1 ]]; then + echo -e "${BOLD}Run $i of $repeat${RESET}" + fi + + if bash "$test_script" 2>&1 | tee -a "$log_file"; then + if [[ $repeat -gt 1 ]]; then + echo -e "${GREEN}✓${RESET} Run $i passed\n" + fi + else + echo -e "${RED}✗${RESET} Run $i failed\n" + success=false + break + fi + done + + local end_time duration + end_time=$(date +%s) + duration=$((end_time - start_time)) + + if $success; then + echo -e "\n${GREEN}✓${RESET} All tests passed in ${duration}s" + + log_metric "test" "$target" "${duration}s" "passed" + + # Parse test output for counts (looking for "ALL TESTS PASSED") + local passed failed + if grep -q "ALL TESTS PASSED" "$log_file"; then + passed=$(grep -c "^Running:" "$log_file" || echo "0") + failed=0 + else + passed=0 + failed=1 + fi + + update_state "tests.$target" \ + "{\"last_run\": \"$(date -Iseconds)\", \"status\": \"passed\", \"duration_sec\": $duration, \"passed\": $passed, \"failed\": $failed, \"log\": \"$log_file\"}" + + return 0 + else + echo -e "\n${RED}✗${RESET} Tests failed after ${duration}s" + echo "Log: $log_file" + + log_metric "test" "$target" "${duration}s" "failed" + + update_state "tests.$target" \ + "{\"last_run\": \"$(date -Iseconds)\", \"status\": \"failed\", \"duration_sec\": $duration, \"log\": \"$log_file\"}" + + return 1 + fi + fi +} + +#============================================================================== +# Cleanup Command +#============================================================================== + +cmd_cleanup() { + local force=false + + if [[ "${1:-}" == "--force" ]] || [[ "${1:-}" == "-f" ]]; then + force=true + fi + + echo -e "${BOLD}Cleaning up Spindle podman resources...${RESET}\n" + + # Stop running containers + local containers + containers=$(podman ps -a --filter "name=spindle" --format "{{.Names}}" 2>/dev/null || true) + + if [[ -n "$containers" ]]; then + echo "Stopping containers..." + echo "$containers" | while read -r container; do + if $force; then + echo " Force removing: $container" + timeout 2 podman rm -f "$container" 2>/dev/null || true + else + echo " Stopping: $container" + if ! timeout 10 podman stop "$container" 2>/dev/null; then + echo " Timeout, force removing..." + podman rm -f "$container" 2>/dev/null || true + fi + fi + done + echo -e "${GREEN}✓${RESET} Containers cleaned\n" + else + echo "No running containers to clean" + fi + + # Clean up dangling resources + echo "Cleaning dangling resources..." + podman container prune -f 2>/dev/null || true + + # Clean up /tmp directories + echo "Cleaning /tmp/spindle* directories..." + local tmp_dirs + tmp_dirs=$(find /tmp -maxdepth 1 -name "spindle*" -type d 2>/dev/null || true) + if [[ -n "$tmp_dirs" ]]; then + echo "$tmp_dirs" | while read -r dir; do + if $force; then + echo " Force removing: $dir" + rm -rf "$dir" 2>/dev/null || true + else + echo " Removing: $dir" + rm -rf "$dir" 2>/dev/null || true + fi + done + fi + + echo -e "\n${GREEN}✓${RESET} Cleanup complete" +} + +cmd_defenestrate() { + echo -e "${RED}${BOLD}Nuclear cleanup - killing everything!${RESET}\n" + + # Kill all spindle containers immediately + podman ps -a --filter "name=spindle" --format "{{.Names}}" 2>/dev/null | while read -r container; do + echo "Killing: $container" + podman rm -f "$container" 2>/dev/null || true + done + + # Prune everything + podman container prune -f 2>/dev/null || true + podman network prune -f 2>/dev/null || true + + # Nuke /tmp directories + echo "Nuking /tmp/spindle* directories..." + find /tmp -maxdepth 1 -name "spindle*" -type d -exec rm -rf {} \; 2>/dev/null || true + + echo -e "\n${GREEN}✓${RESET} All resources destroyed" +} + +#============================================================================== +# Logs Command +#============================================================================== + +cmd_logs() { + local count="${1:-5}" + + if [[ ! -d "$LOG_DIR" ]]; then + echo "No logs directory found" + return 0 + fi + + echo -e "${BOLD}Recent log sessions (last $count):${RESET}\n" + + find "$LOG_DIR" -mindepth 1 -maxdepth 1 -type d | sort -r | head -n "$count" | while read -r dir; do + local session + session=$(basename "$dir") + echo -e "${BOLD}$session${RESET}" + + # Check for errors + local has_errors=false + find "$dir" -name "*.log" -type f | while read -r log; do + if grep -qi "error\|failed" "$log" 2>/dev/null; then + has_errors=true + fi + + local log_name + log_name=$(basename "$log") + local size + size=$(du -h "$log" | cut -f1) + + if $has_errors; then + echo -e " ${RED}✗${RESET} $log_name ($size) - contains errors" + else + echo -e " ${GREEN}✓${RESET} $log_name ($size)" + fi + done + echo "" + done + + echo "Log directory: $LOG_DIR" +} + +#============================================================================== +# Help +#============================================================================== + +show_help() { + cat <<'EOF' +spindle-podman - Consolidated Spindle podman operations + +Usage: + spindle-podman [options] + +Commands: + status [--verbose] Show current state of images, containers, tests + build [--background] Build a container image + test [options] Run tests in a container + cleanup [--force] Clean up containers (gentle by default) + defenestrate Nuclear cleanup (kill -9 everything) + logs [N] Show recent log sessions (default: 5) + help Show this help + +Build/Test Targets: + serial Serial resource manager + flux Flux resource manager + slurm-srun Slurm with srun + slurm-base Slurm base image (for building others) + +Test Options: + --repeat N, -r N Run tests N times + --background, -b Run in background + +Examples: + ./spindle-podman status + ./spindle-podman build serial + ./spindle-podman test serial + ./spindle-podman test serial --repeat 10 + ./spindle-podman cleanup + ./spindle-podman logs 3 + +State and logs: + State: .podman-state.json + Metrics: .podman-metrics.log + Logs: .podman-logs/ + +EOF +} + +#============================================================================== +# Main +#============================================================================== + +main() { + local command="${1:-help}" + + case "$command" in + status|st) + shift + cmd_status "$@" + ;; + build|b) + shift + cmd_build "$@" + ;; + test|t) + shift + cmd_test "$@" + ;; + cleanup|clean) + shift + cmd_cleanup "$@" + ;; + defenestrate|nuke) + shift + cmd_defenestrate "$@" + ;; + logs|log) + shift + cmd_logs "$@" + ;; + help|--help|-h) + show_help + ;; + *) + echo "Unknown command: $command" + echo "" + show_help + return 1 + ;; + esac +} + +# Check for required tools +for tool in podman jq; do + if ! command -v "$tool" &>/dev/null; then + echo "Error: Required tool '$tool' not found" + exit 1 + fi +done + +main "$@" diff --git a/src/cobo/cobo_comm.c b/src/cobo/cobo_comm.c index 8f59e0cb..7f2084b2 100644 --- a/src/cobo/cobo_comm.c +++ b/src/cobo/cobo_comm.c @@ -241,3 +241,61 @@ int write_msg(int fd, ldcs_message_t *msg) return 0; } + +/* ll_send_nosignal is a variation on ll_write which uses + * send(..., MSG_NOSIGNAL) to avoid SIGPIPE when writing to + * a broken socket. + */ +int ll_send_nosignal(int fd, void *buf, size_t count) +{ + int error; + ssize_t result; + size_t pos = 0; + unsigned char *cbuf = (unsigned char *) buf; + + debug_printf3("Have %lu bytes at %p to write to network\n", count, buf); + + while (pos < count) { + result = send(fd, cbuf + pos, count - pos, MSG_NOSIGNAL); + debug_printf3("Wrote %d bytes at %p to network: 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x...\n", (int) result, cbuf + pos, + result > 0 ? ((int) cbuf[pos+0]) : 0, + result > 1 ? ((int) cbuf[pos+1]) : 0, + result > 2 ? ((int) cbuf[pos+2]) : 0, + result > 3 ? ((int) cbuf[pos+3]) : 0, + result > 4 ? ((int) cbuf[pos+4]) : 0, + result > 5 ? ((int) cbuf[pos+5]) : 0, + result > 6 ? ((int) cbuf[pos+6]) : 0, + result > 7 ? ((int) cbuf[pos+7]) : 0); + if (result == -1 && (errno == EINTR || errno == EAGAIN)) + continue; + if (result <= 0) { + error = errno; + err_printf("Error writing to cobo FD %d: %s\n", fd, strerror(error)); + return -1; + } + pos += result; + } + debug_printf3("Sent %lu bytes to fd %d\n", count, fd); + return 0; +} + +/* write_msg_nosignal is a variation on write_msg + * which uses ll_send_nosignal instead of ll_send + * to avoid SIGPIPE when writing to a broken socket. + */ +int write_msg_nosignal(int fd, ldcs_message_t *msg) +{ + int result = ll_send_nosignal(fd, msg, sizeof(*msg)); + if (result == -1) { + return -1; + } + + if (msg->header.len && msg->data) { + result = ll_send_nosignal(fd, msg->data, msg->header.len); + if (result == -1) { + return -1; + } + } + + return 0; +} diff --git a/src/cobo/cobo_comm.h b/src/cobo/cobo_comm.h index 38a72511..f6612ad9 100644 --- a/src/cobo/cobo_comm.h +++ b/src/cobo/cobo_comm.h @@ -24,7 +24,9 @@ Place, Suite 330, Boston, MA 02111-1307 USA int ldcs_cobo_read_fd(int fd, void* buf, int size); int ldcs_cobo_write_fd(int fd, void* buf, int size); int ll_write(int fd, void *buf, size_t count); +int ll_send_nosignal(int fd, void *buf, size_t count); int ll_read(int fd, void *buf, size_t count); int write_msg(int fd, ldcs_message_t *msg); +int write_msg_nosignal(int fd, ldcs_message_t *msg); #endif /* _COBO_COMM_H */ diff --git a/src/cobo/handshake.c b/src/cobo/handshake.c index 74645cab..3614022b 100644 --- a/src/cobo/handshake.c +++ b/src/cobo/handshake.c @@ -684,6 +684,10 @@ static int munge_encrypt_packet(void *packet, size_t packet_size, } return return_result; #else + (void)packet; + (void)packet_size; + (void)packet_buffer; + (void)packet_buffer_size; error_printf("Handshake not compiled with munge support\n"); return HSHAKE_INTERNAL_ERROR; #endif @@ -1073,6 +1077,10 @@ static int munge_decrypt_packet(void *recvd_packet, size_t recvd_packet_size, return return_result; #else + (void)recvd_packet; + (void)recvd_packet_size; + (void)recvd_buffer; + (void)recvd_buffer_size; error_printf("Handshake not compiled with munge support\n"); return HSHAKE_INTERNAL_ERROR; #endif diff --git a/src/fe/comlib/cobo_fe_comm.c b/src/fe/comlib/cobo_fe_comm.c index 297fc1d3..0df792e8 100644 --- a/src/fe/comlib/cobo_fe_comm.c +++ b/src/fe/comlib/cobo_fe_comm.c @@ -178,7 +178,9 @@ int ldcs_audit_server_fe_md_close ( void *data ) { out_msg.data = NULL; cobo_server_get_root_socket(&root_fd); - write_msg(root_fd, &out_msg); + /* We use write_msg_nosignal to avoid SIGPIPE if the socket is broken. + * We're exiting here, so ignore if the server already exited. */ + write_msg_nosignal(root_fd, &out_msg); return cobo_server_close(); } diff --git a/src/fe/startup/launch_slurm.cc b/src/fe/startup/launch_slurm.cc index 07408f55..c7aac443 100644 --- a/src/fe/startup/launch_slurm.cc +++ b/src/fe/startup/launch_slurm.cc @@ -210,10 +210,27 @@ bool SlurmLauncher::spawnJob(app_id_t id, int app_argc, char **app_argv) return false; } else if (pid == 0) { - execvp(*app_argv, app_argv); + // Inject --export to propagate SPINDLE_DEBUG and SPINDLE_TEST to remote nodes + // Build new argv with --export=SPINDLE_DEBUG,SPINDLE_TEST,TMPDIR,TEMPDIR inserted after srun + int new_argc = app_argc + 1; + char **new_argv = (char **) malloc((new_argc + 1) * sizeof(char *)); + new_argv[0] = app_argv[0]; // srun + new_argv[1] = const_cast("--export=SPINDLE_DEBUG,SPINDLE_TEST,TMPDIR,TEMPDIR"); + for (int i = 1; i < app_argc; i++) { + new_argv[i + 1] = app_argv[i]; + } + new_argv[new_argc] = NULL; + + debug_printf("Execing srun with --export: "); + for (int i = 0; new_argv[i]; i++) { + bare_printf("%s ", new_argv[i]); + } + bare_printf("\n"); + + execvp(*new_argv, new_argv); int error = errno; - fprintf(stderr, "Spindle failed to run %s: %s\n", app_argv[0], strerror(error)); - err_printf("Failed to run application %s: %s\n", app_argv[0], strerror(error)); + fprintf(stderr, "Spindle failed to run %s: %s\n", new_argv[0], strerror(error)); + err_printf("Failed to run application %s: %s\n", new_argv[0], strerror(error)); exit(-1); } app_pids[pid] = id; diff --git a/src/slurm_plugin/plugin_utils.c b/src/slurm_plugin/plugin_utils.c index f77531e4..58ebda9e 100644 --- a/src/slurm_plugin/plugin_utils.c +++ b/src/slurm_plugin/plugin_utils.c @@ -240,7 +240,7 @@ char **getHostAddrSinfo(unsigned int num_hosts, char **hostlist) free(sinfo_cmdline); if (!ret && hostaddrlist) { for (i = 0; i < num_hosts; i++) free(hostaddrlist[i]); - free(hostlist); + free(hostaddrlist); } return ret; } @@ -254,11 +254,11 @@ int isFEHost(char **hostlist, unsigned int num_hosts) int feresult = -1; for (i = 0; i < num_hosts; i++) { - if (!last_host || strcmp(hostlist[i], last_host) == 1) { + if (!last_host || strcmp(hostlist[i], last_host) > 0) { last_host = hostlist[i]; } } - sdprintf(2, "last_host = %s\n", last_host ? last_host : NULL); + sdprintf(2, "last_host = %s\n", last_host ? last_host : "(null)"); if (!last_host) { error = errno; sdprintf(1, "ERROR: Could not get current system's hostname: %s\n", strerror(error)); @@ -360,6 +360,12 @@ static int createFEExitSocket(char *socket_path) int result, sock = -1, retval = -1; debug_printf("Creating unix socket for session at %s\n", socket_path); + + if (strlen(socket_path) > sizeof(local.sun_path)-1) { + err_printf("Session exit socket path too long: %s\n", socket_path); + goto done; + } + sock = socket(AF_UNIX, SOCK_STREAM, 0); if (sock == -1) { int error = errno; @@ -367,8 +373,14 @@ static int createFEExitSocket(char *socket_path) goto done; } + memset(&local, 0, sizeof(local)); local.sun_family = AF_UNIX; strncpy(local.sun_path, socket_path, sizeof(local.sun_path)-1); + + /* If there's an exit socket left over from a previous run that + * failed before removing it, remove it here */ + unlink(socket_path); + result = bind(sock, (struct sockaddr *) &local, sizeof(local)); if (result == -1) { int error = errno; @@ -419,32 +431,39 @@ int waitForSpankSessionEnd(spindle_args_t *params) goto done; sockfd = createFEExitSocket(socket_path); - if (sockfd == -1) + if (sockfd == -1) goto done; - - fd = accept(sockfd, NULL, NULL); - if (fd == -1) { - error = errno; - err_printf("Could not accept session exit socket connection: %s\n", strerror(error)); - goto done; - } - do { - result = read(fd, &msg, 1); - } while (result == -1 && errno == EINTR); - if (result == -1) { - error = errno; - err_printf("Failed to read from session exit socket: %s\n", strerror(error)); - goto done; - } - if (msg != 'q') { - error = errno; - err_printf("Recieved incorrect msg character: %c\n", msg); - goto done; + for (;;) { + fd = accept(sockfd, NULL, NULL); + if (fd == -1) { + error = errno; + err_printf("Could not accept session exit socket connection: %s\n", strerror(error)); + goto done; + } + + msg = 0; + do { + result = read(fd, &msg, 1); + } while (result == -1 && errno == EINTR); + + if (result == 1 && msg == 'q') { + close(fd); + fd = -1; + sdprintf(2, "Received session exit message\n"); + retval = 0; + break; + } + + if (result == -1) { + error = errno; + err_printf("Failed read from session exit socket: %s\n", strerror(error)); + } else { + sdprintf(2, "Received message other than exit on exit socket"); + } + close(fd); + fd = -1; } - - sdprintf(2, "Received session exit message\n"); - retval = 0; done: if (fd != -1) @@ -478,6 +497,12 @@ int signalSpankSessionEnd(spindle_args_t *params) goto done; } + if (strlen(socket_path) > sizeof(saddr.sun_path)-1) { + err_printf("Session exit socket path too long: %s\n", socket_path); + goto done; + } + + memset(&saddr, 0, sizeof(saddr)); saddr.sun_family = AF_UNIX; strncpy(saddr.sun_path, socket_path, sizeof(saddr.sun_path)-1); @@ -485,7 +510,7 @@ int signalSpankSessionEnd(spindle_args_t *params) if (result == -1) { int error = errno; err_printf("Failed to connect to session exit socket: %s\n", strerror(error)); - return -1; + goto done; } msg = 'q'; @@ -515,6 +540,15 @@ int signalSpankSessionEnd(spindle_args_t *params) char *unique_file = NULL; +void cleanup_unique_file() +{ + if (unique_file) { + unlink(unique_file); + free(unique_file); + unique_file = NULL; + } +} + #define UNIQUE_FILE_NAME "spindle_unique" int isBEProc(spindle_args_t *params, unsigned int exit_phase) @@ -526,6 +560,10 @@ int isBEProc(spindle_args_t *params, unsigned int exit_phase) int fd = -1, error; realized_dir = locSpecificDir(params); + if (!realized_dir) { + sdprintf(1, "ERROR: Could not resolve location directory in isBEProc\n"); + goto done; + } gethostname(hostname, sizeof(hostname)); hostname[sizeof(hostname)-1] = '\0'; @@ -540,7 +578,7 @@ int isBEProc(spindle_args_t *params, unsigned int exit_phase) strlen(hostname) + 1 + strlen(session_id_str) + 1; - unique_file = (char *) malloc(sizeof(char*) * unique_file_len); + unique_file = (char *) malloc(sizeof(char) * unique_file_len); snprintf(unique_file, unique_file_len, "%s/%s.%s.%s.%s", realized_dir, UNIQUE_FILE_NAME, phase_name, hostname, session_id_str); spindle_mkdir(realized_dir); @@ -687,8 +725,10 @@ void push_env(spank_t spank, saved_env_t **env) e->new_spindledebug = readSpankEnv(spank, "SPINDLE_DEBUG"); e->old_spindledebug = getenv("SPINDLE_DEBUG"); - if (e->new_pwd) - chdir(e->new_pwd); + if (e->new_pwd) { + if (chdir(e->new_pwd) == -1) + sdprintf(1, "WARNING: Could not chdir to %s: %s\n", e->new_pwd, strerror(errno)); + } if (e->new_home) setenv("HOME", e->new_home, 1); @@ -727,8 +767,10 @@ void pop_env(saved_env_t *env) else unsetenv("SPINDLE_DEBUG"); - if (env->old_pwd) - chdir(env->old_pwd); + if (env->old_pwd) { + if (chdir(env->old_pwd) == -1) + sdprintf(1, "WARNING: Could not chdir to %s: %s\n", env->old_pwd, strerror(errno)); + } if (env->new_home) free(env->new_home); @@ -815,7 +857,7 @@ int dropPrivilegeAndRun(dpr_function_t func, uid_t uid, void *input, char **outp exit(-1); } if (output_len) { - result = safe_write(pipe_fds[1], output_str, output_len+1); + result = safe_write(pipe_fds[1], child_output_str, output_len+1); if (result != output_len+1) { error = errno; fprintf(stderr, "Spindle error. Could not write result string to pipe: %s\n", strerror(error)); @@ -989,7 +1031,11 @@ pid_t grandchild_fork() int result, fork_result = -1; pipe_fds[0] = pipe_fds[1] = -1; - pipe(pipe_fds); + result = pipe(pipe_fds); + if (result == -1) { + sdprintf(1, "ERROR: pipe() failed in grandchild_fork. Aborting spindle\n"); + return -1; + } child_pid = fork(); if (child_pid == -1) { @@ -1008,7 +1054,7 @@ pid_t grandchild_fork() sdprintf(1, "ERROR collecting pid after fork. Aborting spindle\n"); goto done; } - if (!WIFEXITED(status) && WEXITSTATUS(status) != 0) { + if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { sdprintf(1, "ERROR with invalid child exit during grandchild fork. Aborting spindle\n"); goto done; } @@ -1280,7 +1326,7 @@ char *readSpankEnv(spank_t spank, const char *envname) free(buffer); buffer = (char *) malloc(buffer_size); } - if (err == ESPANK_ENV_NOEXIST) { + if (err == ESPANK_ENV_NOEXIST || err == ESPANK_NOT_REMOTE || err == ESPANK_BAD_ARG) { free(buffer); buffer = getenv(envname); return buffer ? strdup(buffer) : NULL; diff --git a/src/slurm_plugin/plugin_utils.h b/src/slurm_plugin/plugin_utils.h index 0ac2c8c8..a29b9805 100644 --- a/src/slurm_plugin/plugin_utils.h +++ b/src/slurm_plugin/plugin_utils.h @@ -45,6 +45,7 @@ char **getHostsParse(unsigned int num_hosts, const char *shortlist); int isFEHost(char **hostlist, unsigned int num_hosts); extern char *unique_file; +void cleanup_unique_file(); int isBEProc(spindle_args_t *params, unsigned int exit_phase); int doesFEExitSocketExist(spindle_args_t *params); diff --git a/src/slurm_plugin/slurm_plugin.c b/src/slurm_plugin/slurm_plugin.c index 0989299c..1778acbf 100644 --- a/src/slurm_plugin/slurm_plugin.c +++ b/src/slurm_plugin/slurm_plugin.c @@ -27,6 +27,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA #include #include +#include #include #include "spindle_launch.h" @@ -102,6 +103,7 @@ static __thread spank_t current_spank; static const char *user_options = NULL; static int enable_spindle = 0; static int start_session = 0; +static int prolog_alloc_mode = 0; extern char *parse_location(char *loc, number_t number); extern char *realize(char *path); @@ -156,7 +158,14 @@ static int should_use_session(spank_t spank) { if (session_env) return 1; err = spank_option_getopt(spank, &session_option, NULL); - return (err == ESPANK_SUCCESS); + return (err == ESPANK_SUCCESS); + } + + /* In the allocator, we are running in the same process that + * handled the command line arguments and can check the + * flag directly. */ + if (context == S_CTX_ALLOCATOR) { + return start_session; } return 0; @@ -168,6 +177,23 @@ int slurm_spank_init(spank_t spank, int ac, char *argv[]) { if (context == S_CTX_ALLOCATOR) { spank_option_register(spank, &session_option); } + +#if defined(PROLOG_FLAG_ALLOC) + /* Check whether Slurm is configured to run the prolog at + * allocation time (PROLOG_FLAG_ALLOC), or the default mode + * where the prolog runs on the first step. */ + if (!prolog_alloc_mode) { + slurm_conf_t *conf = NULL; + if (slurm_load_ctl_conf(0, &conf) == SLURM_SUCCESS) { + if (conf->prolog_flags & PROLOG_FLAG_ALLOC) + prolog_alloc_mode = 1; + slurm_free_ctl_conf(conf); + } else { + sdprintf(1, "Could not read Slurm config, falling back to non-prolog launch.\n"); + } + } +#endif + return 0; } @@ -186,6 +212,14 @@ int slurm_spank_init_post_opt(spank_t spank, int ac, char *argv[]) { if (start_session) { setenv(SPANK_SPINDLE_USE_SESSION, "1", 1); } + + if (start_session) { + int result = forward_environment_to_job_control(spank); + if (result == -1) { + slurm_error("ERROR: Spindle plugin error. Unable to forward environment variables to job control.\n"); + return result; + } + } } return 0; } @@ -314,10 +348,13 @@ int slurm_spank_local_user_init(spank_t spank, int ac, char *argv[]) goto done; } - use_session = should_use_session(spank); + use_session = should_use_session(spank); if (!use_session) goto done; + if (prolog_alloc_mode) + goto done; + result = process_spindle_args(spank, ac, argv, ¶ms, NULL, NULL, use_session); if (result == -1) { slurm_error("ERROR: Spindle plugin error. Could not process spindle args in local user init.\n"); @@ -375,8 +412,8 @@ int slurm_spank_job_prolog(spank_t spank, int ac, char *argv[]) { start_params_t start_params; spank_err_t err; int result, use_session; - char *result_str, *work_dir, *envVal; - + char *result_str = NULL, *work_dir, *envVal; + handle_forwarded_environment(); use_session = should_use_session(spank); @@ -384,15 +421,18 @@ int slurm_spank_job_prolog(spank_t spank, int ac, char *argv[]) { return 0; - envVal = getenv("SPANK_SPINDLE_RSHLAUNCH"); - if (envVal && strcmp(envVal, "1") == 0) - return 0; + if (!prolog_alloc_mode) { + envVal = getenv("SPANK_SPINDLE_RSHLAUNCH"); + if (envVal && strcmp(envVal, "1") == 0) + return 0; + } // The prolog starts in the user's home directory. // Change to $SLURM_JOB_WORK_DIR so logs go to right place. work_dir = getenv("SLURM_JOB_WORK_DIR"); if (work_dir) { - chdir(work_dir); + if (chdir(work_dir) == -1) + sdprintf(1, "WARNING: Could not chdir to %s: %s\n", work_dir, strerror(errno)); } err = spank_get_item(spank, S_JOB_UID, &userid); @@ -412,19 +452,24 @@ int slurm_spank_job_prolog(spank_t spank, int ac, char *argv[]) { return -1; } + if (result_str) + free(result_str); + return 0; } -/* job_epilog called on every compute node when allocation ends, +/* job_epilog called on every compute node when allocation ends, * regardless of whether any step ever ran on that node and * even if the prolog never ran. */ int slurm_spank_job_epilog(spank_t spank, int ac, char *argv[]) { int result, use_session; - char *result_str; + char *result_str = NULL; spank_err_t err; uid_t userid; exit_params_t exit_params; + handle_forwarded_environment(); + use_session = should_use_session(spank); if (!use_session) @@ -443,12 +488,15 @@ int slurm_spank_job_epilog(spank_t spank, int ac, char *argv[]) { exit_params.site_argv = argv; result = dropPrivilegeAndRun(handleExit, userid, &exit_params, &result_str); - + if (result == -1) { slurm_error("Failed to run handleExit. Spindle may not shutdown properly\n"); return -1; } - + + if (result_str) + free(result_str); + return 0; } @@ -505,16 +553,22 @@ int slurm_spank_task_init(spank_t spank, int site_argc, char *site_argv[]) } if (params.opts & OPT_OFF) { + pop_env(env); return 0; } - /* When using a session without RSHLAUNCH, handle start in job prolog, not here. */ - if ((!use_session) || (params.opts & OPT_RSHLAUNCH)) { + /* When using a session without RSHLAUNCH, handle start in job prolog, not here. + With PrologFlags=Alloc, session+RSHLAUNCH is also handled in the prolog. */ + if ((!use_session) || ((params.opts & OPT_RSHLAUNCH) && !prolog_alloc_mode)) { start_params.spank = spank; start_params.site_argc = site_argc; start_params.site_argv = site_argv; result = handleStart(&start_params, &result_str); + if (result == -1) { + sdprintf(1, "Error launching spindle. Aborting spindle\n"); + goto done; + } } result = prepApp(spank, ¶ms); @@ -560,8 +614,9 @@ static int handleStart(void *params, char **output_str) return 0; } - // Only initialize a session once - if (use_session && (args.opts & OPT_RSHLAUNCH)) { + /* Only initialize a session once. In prolog context (S_CTX_JOB_SCRIPT), + * there is no step yet, so skip the step ID check. */ + if (use_session && (args.opts & OPT_RSHLAUNCH) && spank_context() != S_CTX_JOB_SCRIPT) { err = get_stepid(spank, &stepid); if (err != ESPANK_SUCCESS) { slurm_error("ERROR: Spindle plugin error. Could not get step id."); @@ -584,7 +639,7 @@ static int handleStart(void *params, char **output_str) int slurm_spank_task_exit(spank_t spank, int site_argc, char *site_argv[]) { spank_context_t context; - char *result_str; + char *result_str = NULL; int result, use_session; uid_t userid; spank_err_t err; @@ -625,11 +680,15 @@ int slurm_spank_task_exit(spank_t spank, int site_argc, char *site_argv[]) push_env(spank, &saved_env); result = dropPrivilegeAndRun(handleExit, userid, &exit_params, &result_str); pop_env(saved_env); - + if (result == -1) { slurm_error("ERROR: Failed to run handleExit. Spindle may not shutdown properly\n"); return -1; } + + if (result_str) + free(result_str); + return 0; } @@ -638,18 +697,21 @@ static spank_err_t get_stepid(spank_t spank, uint32_t *stepid) { char *slurm_step_id_s; spank_err_t err; - uint64_t combined; - - slurm_step_id_s = getenv("SLURM_STEP_ID"); - if (slurm_step_id_s) { - *stepid = (uint32_t) atol(slurm_step_id_s); - } else { - err = spank_get_item(spank, S_JOB_STEPID, stepid); - if (err != ESPANK_SUCCESS) { - return err; + + /* Only get step ID from env var in job script context */ + if (spank_context() == S_CTX_JOB_SCRIPT) { + slurm_step_id_s = getenv("SLURM_STEP_ID"); + if (slurm_step_id_s) { + *stepid = (uint32_t) atol(slurm_step_id_s); + return ESPANK_SUCCESS; } } + err = spank_get_item(spank, S_JOB_STEPID, stepid); + if (err != ESPANK_SUCCESS) { + return err; + } + return ESPANK_SUCCESS; } @@ -707,6 +769,8 @@ static int fillInArgs(spank_t spank, spindle_args_t *args, int argc, char **argv char *symbolic_commpath, *orig_commpath; char *err_string; + current_spank = spank; + args->unique_id = unique_id; args->number = (number_t) args->unique_id; result = fillInSpindleArgsCmdlineFE(args, SPINDLE_FILLARGS_NOUNIQUEID | SPINDLE_FILLARGS_NONUMBER, @@ -738,8 +802,13 @@ static int fillInArgs(spank_t spank, spindle_args_t *args, int argc, char **argv return -1; } args->commpath = realize(orig_commpath); - - current_spank = spank; + if (args->commpath != orig_commpath) + free(orig_commpath); + if (!args->commpath) { + slurm_error("Spindle Options Error: Could not resolve commpath location\n"); + sdprintf(1, "ERROR: Could not realize commpath from '%s'\n", symbolic_commpath); + return -1; + } return 0; } @@ -844,6 +913,7 @@ static int get_num_hosts_step(spank_t spank) num_hosts_str = readSpankEnv(spank, "SLURM_STEP_NUM_NODES"); if (num_hosts_str) { result = atoi(num_hosts_str); + free(num_hosts_str); if (result > 0) return (int) result; } @@ -972,7 +1042,7 @@ static int get_spindle_args(spank_t spank, spindle_args_t *params) goto done; } - result = fillInArgs(spank, params, spindle_argc, spindle_argv, unique_id); + result = fillInArgs(spank, params, spindle_argc, spindle_argv, unique_id, 0); if (result == -1) goto done; @@ -1075,7 +1145,7 @@ static int launch_spindle(spank_t spank, spindle_args_t *params) free(hostlist_job); } if (hostaddrlist) { - for (i = 0; i < num_hosts; i++) free(hostaddrlist[i]); + for (i = 0; i < num_hosts_fe; i++) free(hostaddrlist[i]); free(hostaddrlist); } @@ -1175,9 +1245,7 @@ static int launchBE(spank_t spank, spindle_args_t *params) else sdprintf(1, "spindleRunBE completed. Session finishing.\n"); - if (unique_file) unlink(unique_file); - free(unique_file); - unique_file = NULL; + cleanup_unique_file(); exit(result); @@ -1273,7 +1341,11 @@ static int handleExit(void *params, char **output_str) // The task_exit callback is run for _each proc_, so we use // isBEProc to pick only one proc per node to call spindleExitBE. is_be_leader = isBEProc(&args, 1); - if (is_be_leader) { + if (is_be_leader == -1) { + sdprintf(1, "ERROR: Could not determine BE leader in handleExit\n"); + return -1; + } + if (is_be_leader) { if (use_session || (args.opts & OPT_RSHLAUNCH)) { result = signalSpankSessionEnd(&args); if (result == -1) { diff --git a/testsuite/run_driver_template b/testsuite/run_driver_template index fddf2d3e..a486766b 100644 --- a/testsuite/run_driver_template +++ b/testsuite/run_driver_template @@ -10,6 +10,40 @@ export SPINDLE_TEST=1 export SPINDLE=SPINDLE_EXEC export PATH=$PATH:. +SESSION_ACTIVE=false +if [ "x$SESSION_ID" != "x" ] || [ "x$SPANK_SPINDLE_USE_SESSION" != "x" ] ; then + SESSION_ACTIVE=true +fi + +# If we're not running in a session, wait, on every node in the job, for +# the log daemon to exit before proceeding to the actual test; after a +# delay, forcibly kill it. This ensures that each test connects to a +# fresh daemon. +if [ $SESSION_ACTIVE == false ] && [ "x$1" != "x--end-session" ] ; then + # This is the command run on each node. + LOGD_WAIT_CMD=' +SECONDS=0 +TMP="${TMPDIR:-${TEMPDIR:-/tmp}}" +LOCK="$TMP/spindle_log_lock" +TIMEOUT="${SPINDLE_LOGD_SHUTDOWN_TIMEOUT:-15}" +while PID=$(cat "$LOCK" 2>/dev/null) && kill -0 "$PID" 2>/dev/null ; do + if [ $SECONDS -ge $TIMEOUT ]; then + echo "WARNING: $(hostname): spindle_logd (pid $PID) still running after $TIMEOUT sec; killing it" >&2 + kill -9 "$PID" 2>/dev/null + break + fi + sleep 0.1 +done +rm -f "$TMP/spindle_log" "$TMP/spindle_test" "$TMP/spindle_log_lock" "$TMP/spindle_log_reset" +' + bash -c "$LOGD_WAIT_CMD" + if [ "x$TEST_RM" == "xslurm" -o "x$TEST_RM" == "xslurm-plugin" ] && [ "x$SLURM_NNODES" != "x" ] ; then + srun --overlap -N $SLURM_NNODES -n $SLURM_NNODES bash -c "$LOGD_WAIT_CMD" + elif [ "x$TEST_RM" == "xflux" ] ; then + flux exec -r all bash -c "$LOGD_WAIT_CMD" + fi +fi + if [ $1 == --start-session ] ; then # With SPANK plugin, sessions are started by argument to salloc/sbatch instead if [ "x$TEST_RM" == "xslurm-plugin" ] ; then