Podman port

.containerignore

@@ -0,0 +1,26 @@
+# Podman/Docker ignore file
+# Excludes files from being copied into container images during build
+
+# Exclude the saved images tarball (6+ GB)
+spindle-podman-images.tar
+
+# Exclude build artifacts
+workspace-Spindle/build/
+workspace-Spindle/install/
+*.o
+*.a
+*.so
+
+# Exclude git metadata
+.git/
+.gitignore
+
+# Exclude test output
+out.*
+*.log
+
+# Exclude investigation directory
+podman-concurrency-investigation/
+
+# Exclude debugging artifacts
+debugging/

.dockerignore

@@ -0,0 +1 @@
+.containerignore

.github/workflows/ci.yml

@@ -25,10 +25,10 @@ jobs:
     timeout-minutes: 20
     steps:
       - name: Check out Spindle
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
 
       - name: Setup Docker Compose
-        uses: docker/setup-compose-action@16feee727cbdc83b6a014e6cc26fec4a79bcf30c
+        uses: docker/setup-compose-action@4eb059ff7f16592f9c84d5ca339c53cb7c5064e2
         with:
           version: latest
 
@@ -78,10 +78,10 @@ jobs:
     timeout-minutes: 30
     steps:
       - name: Check out Spindle
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
 
       - name: Setup Docker Compose
-        uses: docker/setup-compose-action@16feee727cbdc83b6a014e6cc26fec4a79bcf30c
+        uses: docker/setup-compose-action@4eb059ff7f16592f9c84d5ca339c53cb7c5064e2
         with:
           version: latest
 
@@ -131,10 +131,10 @@ jobs:
     timeout-minutes: 20
     steps:
       - name: Check out Spindle
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
 
       - name: Setup Docker Compose
-        uses: docker/setup-compose-action@16feee727cbdc83b6a014e6cc26fec4a79bcf30c
+        uses: docker/setup-compose-action@4eb059ff7f16592f9c84d5ca339c53cb7c5064e2
         with:
           version: latest
 
@@ -184,16 +184,16 @@ jobs:
     timeout-minutes: 20
     steps:
       - name: Check out Spindle
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
 
       - name: Setup Docker Compose
-        uses: docker/setup-compose-action@16feee727cbdc83b6a014e6cc26fec4a79bcf30c
+        uses: docker/setup-compose-action@4eb059ff7f16592f9c84d5ca339c53cb7c5064e2
         with:
           version: latest
 
       - name: Login to GitHub Container Registry
         if: ${{ !env.ACT }}
-        uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee
+        uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
@@ -251,16 +251,16 @@ jobs:
     timeout-minutes: 20
     steps:
       - name: Check out Spindle
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
 
       - name: Setup Docker Compose
-        uses: docker/setup-compose-action@16feee727cbdc83b6a014e6cc26fec4a79bcf30c
+        uses: docker/setup-compose-action@4eb059ff7f16592f9c84d5ca339c53cb7c5064e2
         with:
           version: latest
 
       - name: Login to GitHub Container Registry
         if: ${{ !env.ACT }}
-        uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee
+        uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
@@ -309,16 +309,16 @@ jobs:
     timeout-minutes: 20
     steps:
       - name: Check out Spindle
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
 
       - name: Setup Docker Compose
-        uses: docker/setup-compose-action@16feee727cbdc83b6a014e6cc26fec4a79bcf30c
+        uses: docker/setup-compose-action@4eb059ff7f16592f9c84d5ca339c53cb7c5064e2
         with:
           version: latest
 
       - name: Login to GitHub Container Registry
         if: ${{ !env.ACT }}
-        uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee
+        uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f
         with:
           registry: ghcr.io
           username: ${{ github.actor }}

.github/workflows/container.yml

@@ -11,20 +11,20 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5
+        uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c
 
       - name: Login to GitHub Container Registry
-        uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee
+        uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Build & Push Slurm Base Image
-        uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf
+        uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a
         with:
           context: ./containers/spindle-slurm-ubuntu/base
           platforms: linux/amd64

.gitignore

@@ -35,3 +35,16 @@ run_driver
 run_driver_rm
 preload_file_list
 build
+
+# Generated by generate_config.sh
+containers/spindle-slurm-ubuntu/testing/conf/slurmdbd.conf
+containers/spindle-slurm-ubuntu/testing/mariadb.env
+containers/spindle-slurm-ubuntu/testing-srun/conf/slurmdbd.conf
+containers/spindle-slurm-ubuntu/testing-srun/mariadb.env
+
+# spindle-podman
+.podman-state.json
+.podman-metrics.log
+.podman-logs
+
+

DEBUGGING.txt

@@ -0,0 +1,21 @@
+# Build
+podman rm -f spindlenode-interactive
+./scripts/podman/build-spindle-serial-interactive.sh
+# Start
+./scripts/podman/run-spindle-serial-interactive.sh
+# Connect
+podman exec -it spindlenode-interactive bash
+# Enable core dumps
+ulimit -c unlimited
+# Set up spindle config file
+mkdir -p /home/spindleuser/Spindle-inst/etc/spindle
+cat > /home/spindleuser/Spindle-inst/etc/spindle/spindle.conf <<'EOF'
+noclean = true
+EOF
+# cd
+cd Spindle-build/testsuite
+# Run the problematic test
+SPINDLE_DEBUG=3 SPINDLE_FLAGS="--noclean" ./run_driver --thrdopen --push
+# Core file analysis
+RELOCATED=$(find /tmp/commpath/cachepath -name "*test_driver" -type f | grep -v "\.so" | head -1)
+gdb ${RELOCATED} /tmp/core.test_driver.$(ls -t *.core | head -1 )

containers/spindle-flux-ubuntu/Dockerfile.podman

@@ -0,0 +1,97 @@
+# This is based on the Flux Container Tutorial
+# See https://flux-framework.readthedocs.io/en/latest/tutorials/containers
+ARG flux_sched_version=noble-v0.48.0-amd64
+FROM fluxrm/flux-sched:${flux_sched_version} AS builder
+ARG replicas=4
+ENV workers=${replicas}
+USER root
+ENV TMPDIR=/tmp
+RUN echo 'TMPDIR="/tmp"' >> /etc/environment
+ENV SPINDLE_TEST_CONTAINER=1
+
+# LC-specific fix for podman setgroups issue
+ARG PODMAN_BUILD=false
+RUN if [ "$PODMAN_BUILD" = "true" ]; then \
+      echo 'APT::Sandbox::User root;' > /etc/apt/apt.conf.d/00-apt-sandbox; \
+    fi
+
+RUN DEBIAN_FRONTEND="noninteractive" apt-get update \
+    && apt-get -qq install -y --no-install-recommends \
+        autotools-dev \
+        autoconf \
+        automake \
+        cmake \
+        git \
+        python3 \
+        openssh-server \
+        openssh-client \
+        libdb-dev \
+        apt-utils \
+        dnsutils \
+        iputils-ping \
+        python3-pip \
+        libgcrypt20 \
+        libgcrypt20-dev \
+        gdb \
+        libc6-dbg \
+        software-properties-common
+
+ARG USER=fluxuser
+ARG CONFIG_ROOT=containers/spindle-flux-ubuntu
+
+# Allow fluxuser to run as other users so it can start munged
+RUN sh -c "printf \"${USER} ALL=(ALL) NOPASSWD: ALL\\n\" >> /etc/sudoers"
+
+# Configure flux
+ENV STATE_DIR=/var/lib/flux
+RUN mkdir -p ${STATE_DIR} /etc/flux/system /etc/flux/system/cron.d /etc/flux/config /run/flux /etc/flux/imp/conf.d
+COPY ${CONFIG_ROOT}/flux/imp.toml /etc/flux/imp/conf.d/
+COPY ${CONFIG_ROOT}/flux/broker.toml /etc/flux/config/
+RUN mkdir -p /etc/flux/system/cron.d && \
+    mkdir -p /mnt/curve && \
+    flux keygen /mnt/curve/curve.cert && \
+    flux R encode --hosts="node-[1-${workers}]" > /etc/flux/system/R && \
+    chmod -R a+rX /etc/flux && \
+    chown -R ${USER}:${USER} /run/flux ${STATE_DIR} /mnt/curve/curve.cert
+
+# Build Spindle
+WORKDIR /home/${USER}
+# Copy the whole git repo into the container
+COPY . /home/${USER}/Spindle
+
+# Fix permissions on Spindle source (configure needs +x)
+RUN chmod -R u+rwX /home/${USER}/Spindle && \
+    chown -R ${USER}:${USER} /home/${USER}/Spindle
+
+# Copy and prepare build script as root
+COPY ${CONFIG_ROOT}/scripts/build_spindle.sh /home/${USER}/build_spindle.sh
+RUN chmod +rx /home/${USER}/build_spindle.sh && \
+    chown ${USER}:${USER} /home/${USER}/build_spindle.sh
+
+# Switch to user to run the build
+USER ${USER}
+RUN bash ./build_spindle.sh
+
+# Copy scripts as root
+USER root
+RUN chown -R ${USER}:${USER} /home/fluxuser && \
+    chown -R ${USER}:${USER} /run/flux
+
+COPY ${CONFIG_ROOT}/scripts/flux_healthcheck.sh /home/${USER}/flux_healthcheck.sh
+COPY ${CONFIG_ROOT}/scripts/entrypoint.sh.podman /home/${USER}/entrypoint.sh
+RUN chmod +rx /home/${USER}/flux_healthcheck.sh /home/${USER}/entrypoint.sh && \
+    chown ${USER}:${USER} /home/${USER}/flux_healthcheck.sh /home/${USER}/entrypoint.sh
+
+USER ${USER}
+WORKDIR /home/${USER}
+ENV PATH /home/${USER}/Spindle-inst/bin:${PATH}
+# Make libfabric work with fork.
+ENV RDMAV_FORK_SAFE 1
+# Silence warning from hwloc about unsupported PCI device
+# on GitHub-hosted runners.
+ENV HWLOC_HIDE_ERRORS 2
+# Suppress UCX warnings about /proc/sys tuning parameters
+# These are expected in rootless podman containers
+ENV UCX_LOG_LEVEL=error
+
+ENTRYPOINT /bin/bash ./entrypoint.sh

containers/spindle-flux-ubuntu/scripts/entrypoint.sh.podman

@@ -0,0 +1,51 @@
+#!/bin/bash
+#
+# Podman-compatible entrypoint for Flux containers
+# Starts munged and the flux broker.
+#
+# For documentation on running Flux in containers, see
+# https://flux-framework.readthedocs.io/en/latest/tutorials/containers
+
+set -x  # Debug output
+
+brokerOptions="-Scron.directory=/etc/flux/system/cron.d \
+  -Stbon.fanout=256 \
+  -Srundir=/run/flux \
+  -Sstatedir=${STATE_DIRECTORY:-/var/lib/flux} \
+  -Slocal-uri=local:///run/flux/local \
+  -Slog-stderr-level=6 \
+  -Slog-stderr-mode=local"
+
+# Get the hostname that will resolve for the Docker bridge network.
+address=$(echo $( nslookup "$( hostname -i )" | head -n 1 ))
+parts=(${address//=/ })
+hostName=${parts[2]}
+thisHost=(${hostName//./ })
+thisHost=${thisHost[0]}
+echo "This host: $thisHost"
+echo "Main host: $mainHost"
+export FLUX_FAKE_HOSTNAME=$thisHost
+
+if [ -d /shared ]; then
+    sudo chown -R "$(id -un):$(id -gn)" /shared
+    sudo chmod 755 /shared
+fi
+
+# Start munged
+echo "Starting munged..."
+sudo -u munge /usr/sbin/munged
+
+# Give munge time to start
+sleep 2
+
+if [ "${thisHost}" != "${mainHost}" ]; then
+    # Worker node -- wait for head node before connecting
+    echo "Worker node: waiting for head node..."
+    sleep 15
+    echo "Starting flux broker (worker)..."
+    exec flux start -o --config /etc/flux/config ${brokerOptions} sleep inf
+else
+    # Head node
+    echo "Head node: starting flux broker..."
+    exec flux start -o --config /etc/flux/config ${brokerOptions} sleep inf
+fi

containers/spindle-hello-podman/01-basic/Dockerfile

@@ -0,0 +1,28 @@
+FROM ubuntu:noble
+
+# LC-specific fix for podman setgroups issue
+# This is a no-op in Docker, but required for podman on LC systems
+ARG PODMAN_BUILD=false
+RUN if [ "$PODMAN_BUILD" = "true" ]; then \
+      echo 'APT::Sandbox::User root;' > /etc/apt/apt.conf.d/00-apt-sandbox; \
+    fi
+
+# Install curl to test both apt-get and SSL connectivity
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    curl \
+    ca-certificates \
+ && rm -rf /var/lib/apt/lists/*
+
+# Test script that validates:
+# 1. Container runs
+# 2. Network access works
+# 3. SSL certificates work
+CMD echo "=== Hello from Podman! ===" && \
+    echo "Container is running successfully." && \
+    echo "" && \
+    echo "Testing network and SSL connectivity..." && \
+    curl -I https://www.google.com 2>&1 | head -5 && \
+    echo "" && \
+    echo "If you see HTTP headers above, SSL is working!" && \
+    echo "" && \
+    echo "=== 01-basic test complete ==="