Size the worker pool from the CPU allocation

docker/Dockerfile

@@ -37,6 +37,14 @@ LABEL org.opencontainers.image.licenses="Apache-2.0"
 # Every one has a default, so the container starts with no environment set.
 ENV DIMS_DOWNLOAD_TIMEOUT=60000
 ENV DIMS_IMAGEMAGICK_TIMEOUT=20000
+# Worker pool. The default auto sizes each value from the container's CPU
+# allocation at start. Set a number to pin it. See the resources page.
+ENV DIMS_SERVER_LIMIT=auto
+ENV DIMS_THREADS_PER_CHILD=auto
+ENV DIMS_MAX_WORKERS=auto
+# Recycle a child after this many connections, so its memory returns to the
+# system. 0 keeps a child forever.
+ENV DIMS_MAX_CONNECTIONS_PER_CHILD=10000
 ENV DIMS_CLIENT=development
 ENV DIMS_NO_IMAGE_URL="-"
 ENV DIMS_DEFAULT_IMAGE_URL="-"
@@ -73,6 +81,7 @@ COPY --from=build /usr/local/imagemagick /usr/local/imagemagick
 COPY docker/policy.xml /usr/local/imagemagick/etc/ImageMagick-7/policy.xml
 COPY profiles/ /usr/local/share/mod_dims/profiles/
 COPY docker/dims.conf /usr/local/apache2/conf/extra/dims.conf
+COPY --chmod=0755 docker/entrypoint.sh /usr/local/bin/dims-entrypoint
 
 ENV LD_LIBRARY_PATH=/usr/local/imagemagick/lib
 
@@ -100,3 +109,7 @@ USER 33
 # without reaching an origin.
 HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \
     CMD /usr/local/imagemagick/bin/curl -fsS -o /dev/null http://127.0.0.1:8000/dims-status/ || exit 1
+
+# Size the worker pool from the CPU allocation, then run httpd.
+ENTRYPOINT ["/usr/local/bin/dims-entrypoint"]
+CMD ["httpd-foreground"]

docker/dims.conf

@@ -8,6 +8,24 @@ PidFile /tmp/httpd.pid
 ErrorLog /proc/self/fd/2
 CustomLog /proc/self/fd/1 common
 
+# Worker pool. The entrypoint sizes ServerLimit, ThreadsPerChild, and
+# MaxRequestWorkers from the container's CPU allocation, so a larger container
+# scales without a change here. A soak test found that throughput peaks near
+# one worker per vCPU, with two threads per child split across processes. Set
+# DIMS_SERVER_LIMIT, DIMS_THREADS_PER_CHILD, or DIMS_MAX_WORKERS to a number to
+# pin one. MaxConnectionsPerChild recycles a child after many connections, so
+# its memory returns to the system.
+<IfModule mpm_event_module>
+    ServerLimit            ${DIMS_SERVER_LIMIT}
+    StartServers           ${DIMS_SERVER_LIMIT}
+    ThreadLimit            ${DIMS_THREADS_PER_CHILD}
+    ThreadsPerChild        ${DIMS_THREADS_PER_CHILD}
+    MinSpareThreads        1
+    MaxSpareThreads        ${DIMS_MAX_WORKERS}
+    MaxRequestWorkers      ${DIMS_MAX_WORKERS}
+    MaxConnectionsPerChild ${DIMS_MAX_CONNECTIONS_PER_CHILD}
+</IfModule>
+
 DimsDownloadTimeout ${DIMS_DOWNLOAD_TIMEOUT}
 DimsImagemagickTimeout ${DIMS_IMAGEMAGICK_TIMEOUT}
 

docker/entrypoint.sh

@@ -0,0 +1,53 @@
+#!/bin/sh
+# Size the httpd worker pool from the container's CPU allocation, so a larger
+# container scales without a configuration change. A soak test found that
+# throughput peaks near one worker per vCPU, with two threads per child split
+# across processes. This sets ServerLimit to half the vCPU count, at least two,
+# and MaxRequestWorkers to ServerLimit times ThreadsPerChild.
+#
+# An explicit DIMS_SERVER_LIMIT, DIMS_THREADS_PER_CHILD, or DIMS_MAX_WORKERS
+# keeps its value. The default of each is "auto".
+set -e
+
+# The effective vCPU count. nproc reports the host cores, not the container
+# limit, so read the cgroup CPU quota first.
+detect_vcpus() {
+    if [ -r /sys/fs/cgroup/cpu.max ]; then                       # cgroup v2
+        read -r quota period < /sys/fs/cgroup/cpu.max
+        if [ "$quota" != max ] && [ "${period:-0}" -gt 0 ] 2>/dev/null; then
+            echo $(( (quota + period - 1) / period ))
+            return
+        fi
+    fi
+    if [ -r /sys/fs/cgroup/cpu/cpu.cfs_quota_us ]; then          # cgroup v1
+        quota=$(cat /sys/fs/cgroup/cpu/cpu.cfs_quota_us)
+        period=$(cat /sys/fs/cgroup/cpu/cpu.cfs_period_us)
+        if [ "${quota:-0}" -gt 0 ] && [ "${period:-0}" -gt 0 ] 2>/dev/null; then
+            echo $(( (quota + period - 1) / period ))
+            return
+        fi
+    fi
+    nproc 2>/dev/null || echo 1
+}
+
+tpc=${DIMS_THREADS_PER_CHILD:-auto}
+[ "$tpc" = auto ] && tpc=2
+
+sl=${DIMS_SERVER_LIMIT:-auto}
+if [ "$sl" = auto ]; then
+    vcpus=$(detect_vcpus)
+    [ "${vcpus:-0}" -ge 1 ] 2>/dev/null || vcpus=1
+    sl=$(( vcpus / 2 ))
+    [ "$sl" -lt 2 ] && sl=2
+fi
+
+mw=${DIMS_MAX_WORKERS:-auto}
+[ "$mw" = auto ] && mw=$(( sl * tpc ))
+
+export DIMS_SERVER_LIMIT=$sl
+export DIMS_THREADS_PER_CHILD=$tpc
+export DIMS_MAX_WORKERS=$mw
+
+echo "mod_dims workers: ServerLimit=$sl ThreadsPerChild=$tpc MaxRequestWorkers=$mw" >&2
+
+exec "$@"

docs/docs/configuration/resources.md

@@ -59,8 +59,49 @@ DimsImagemagickAreaSize 128
 Megabytes any single image may use. Default 128. An image larger than this is
 cached on disk rather than in memory.
 
+## Workers
+
+The container sizes the httpd worker pool from its CPU allocation at start, so
+a larger container scales without a change. Three environment variables control
+it. Each one defaults to `auto`.
+
+| Variable | Directive | Default |
+| --- | --- | --- |
+| DIMS_SERVER_LIMIT | ServerLimit | auto |
+| DIMS_THREADS_PER_CHILD | ThreadsPerChild | auto |
+| DIMS_MAX_WORKERS | MaxRequestWorkers | auto |
+
+Image work is CPU-bound. A soak test on several container sizes found that
+throughput peaks near one worker per vCPU and drops with more. Two threads per
+child, split across processes, give the lowest tail latency, because two
+processes share less ImageMagick lock state than one process with more threads.
+One thread per child is slow: the single thread cannot both accept and process.
+
+So `auto` sets `ThreadsPerChild` to 2 and `ServerLimit` to half the vCPU count,
+at least 2. `MaxRequestWorkers` is their product. Set a variable to a number to
+pin it. `DIMS_MAX_WORKERS` must equal `DIMS_SERVER_LIMIT` times
+`DIMS_THREADS_PER_CHILD`.
+
+| Container | ServerLimit | ThreadsPerChild | MaxRequestWorkers | Throughput |
+| --- | --- | --- | --- | --- |
+| 2 vCPU, 1 GB | 2 | 2 | 4 | 18 req/s |
+| 4 vCPU, 2 GB | 2 | 2 | 4 | 35 req/s |
+| 6 vCPU, 3 GB | 3 | 2 | 6 | 40 req/s |
+| 8 vCPU, 4 GB | 4 | 2 | 8 | — |
+
+The throughput is for a mixed resize, crop, format, and watermark load on full
+size photos. A lighter load or a smaller source reaches a higher number. The 6
+vCPU row is indicative, and the 8 vCPU row follows the rule, not a measurement,
+because the test host has eight cores and the load generator needs some.
+
+`DIMS_MAX_CONNECTIONS_PER_CHILD` recycles a child after that many connections,
+which returns its memory to the system. The default is 10000. Set it to 0 to
+keep a child for the life of the server.
+
 ## Choosing values
 
-With `ServerLimit 16` and the defaults, sixteen processes may each hold 512 MB
-of pixel cache and write 2 GB to disk. Divide the memory a host has by the
-worker count, and set the limits from that.
+Each worker may hold as much pixel cache and disk as the limits above allow.
+With the default four workers, four processes may each hold the memory limit at
+once. Divide the memory the container has by the worker count from the Workers
+section, and set the limits from that. At the worker counts above the memory is
+not the limit; each optimum held under 250 MB, and CPU sets the rate.

test/soak/.gitignore

@@ -0,0 +1,6 @@
+# Downloaded fixtures and run output. The images come from pexels at run time.
+origin/
+results.tsv
+results-*.tsv
+urls.txt
+*.log

test/soak/Dockerfile.wrk

@@ -0,0 +1,11 @@
+# Throwaway load generator. wrk is not a module dependency and never ships in
+# the server image. It is built here only to drive the soak test.
+FROM debian:bookworm-slim
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+        build-essential libssl-dev zlib1g-dev git unzip ca-certificates && \
+    git clone --depth 1 https://github.com/wg/wrk.git /src && \
+    make -C /src -j"$(nproc)" && \
+    cp /src/wrk /usr/local/bin/wrk && \
+    rm -rf /src /var/lib/apt/lists/*
+ENTRYPOINT ["wrk"]

test/soak/README.md

@@ -0,0 +1,49 @@
+# Soak test
+
+This directory finds the Apache worker configuration that gives the most
+throughput on a fixed CPU and memory budget with no failures. It is an
+operational tool. It is not part of `make test`, and nothing in the module or
+the C test harness depends on it.
+
+## What it does
+
+`run.sh` runs the production server image under a hard CPU and memory limit,
+drives it with a mix of resize, crop, thumbnail, format, strip, and watermark
+requests, and sweeps the Apache MPM sizing. For each configuration it records
+throughput, tail latency, HTTP errors, socket errors, peak memory, and the
+kernel OOM-kill count.
+
+The images are three sizes of one pexels photo, downloaded once and served from
+a local origin. The load generator is `wrk`, built in a throwaway image. `wrk`
+is never a module or test dependency and never ships in the server image.
+
+## Requirements
+
+- Docker.
+- Internet on the first run, to download the pexels images and build `wrk`.
+  Later runs reuse both.
+
+## Run
+
+```
+bash test/soak/run.sh
+```
+
+Environment variables tune the run:
+
+- `CPUS`, `MEM` — the container limits. Default `2` and `1g`.
+- `DUR`, `WARM` — the measure and warm-up time per point. Default `30s`, `10s`.
+- `CONN`, `WT` — the wrk connection and thread count. Default `32`, `4`.
+- `SOAK_GRID` — the sweep, a semicolon-separated list of `ServerLimit
+  ThreadsPerChild` pairs, e.g. `SOAK_GRID="2 2;4 2"`.
+
+Results print as a table and land in `results.tsv`.
+
+## Read the table
+
+- `req_s` — throughput. Higher is better.
+- `p99_ms` — the 99th percentile latency under the offered load.
+- `non2xx`, `sockerr` — request failures. A shipped configuration holds both at
+  zero.
+- `oom` — kernel OOM-kills. Any value above zero fails the configuration.
+- `peak_MiB` — the peak container memory.

test/soak/run.sh

@@ -0,0 +1,149 @@
+#!/usr/bin/env bash
+# Soak test. Find the Apache MPM sizing that gives the most throughput on a
+# 2 vCPU / 1 GB container with no failures.
+#
+# It serves pexels images locally, drives production-style command shapes with wrk, and
+# for each MPM configuration records throughput, tail latency, errors, the peak
+# memory, and the kernel OOM-kill count. A configuration fails if it OOM-kills a
+# worker or returns any non-2xx or socket error.
+set -euo pipefail
+
+HERE="$(cd "$(dirname "$0")" && pwd)"
+ROOT="$(cd "$HERE/../.." && pwd)"
+NET=soak-net
+ORIGIN=soak-origin
+DIMS=soak-dims
+IMG_DIR="$HERE/origin"
+DIMS_IMAGE=mod-dims:soak
+WRK_IMAGE=soak-wrk
+CPUS="${CPUS:-2}"
+MEM="${MEM:-1g}"
+DUR="${DUR:-30s}"
+WARM="${WARM:-10s}"
+CONN="${CONN:-32}"
+WT="${WT:-4}"
+
+# ServerLimit ThreadsPerChild pairs to sweep. Override with SOAK_GRID, a
+# semicolon-separated list, e.g. SOAK_GRID="2 2;4 1".
+if [ -n "${SOAK_GRID:-}" ]; then
+  IFS=";" read -ra GRID <<< "$SOAK_GRID"
+else
+  GRID=( "1 2" "1 3" "1 4" "1 6" "1 8" "2 2" "2 4" "1 16" "1 32" )
+fi
+
+log(){ printf '\n=== %s ===\n' "$*" >&2; }
+cread(){ docker exec -u 0 "$DIMS" cat "$1" 2>/dev/null || echo 0; }
+
+# 1. Images: three sizes of a pexels photo, downloaded once.
+mkdir -p "$IMG_DIR"
+declare -A PEX=(
+  [big.jpg]="https://images.pexels.com/photos/1539116/pexels-photo-1539116.jpeg?auto=compress&cs=tinysrgb&w=3840"
+  [medium.jpg]="https://images.pexels.com/photos/1539116/pexels-photo-1539116.jpeg?auto=compress&cs=tinysrgb&w=1920"
+  [small.jpg]="https://images.pexels.com/photos/1539116/pexels-photo-1539116.jpeg?auto=compress&cs=tinysrgb&w=960"
+)
+for name in "${!PEX[@]}"; do
+  [ -s "$IMG_DIR/$name" ] || { log "download $name"; curl -fsSL --max-time 90 -o "$IMG_DIR/$name" "${PEX[$name]}"; }
+done
+ls -lh "$IMG_DIR" >&2
+
+# 1b. Signed request list. Bash signs each dims4 request, because Lua has no
+# md5:  md5(expires + secret + commands + "/" + raw_url [+ encoded overlay]).
+# The list mixes resize, crop, thumbnail, format, strip, and watermark shapes
+# over three image sizes, and varies the width so requests miss the cache and
+# decode the source. The list is cached; delete it to rebuild.
+cp -f "$ROOT/test/origin/overlay.png" "$IMG_DIR/overlay.png"
+CLIENT=TEST; SECRET=t3stk3y; EXPIRES=2147483647
+URLS="$HERE/urls.txt"
+urlenc(){ local s=$1 o= c i h; for ((i=0;i<${#s};i++)); do c=${s:i:1}; case $c in [a-zA-Z0-9]) o+=$c;; *) printf -v h '%%%02X' "'$c"; o+=$h;; esac; done; printf '%s' "$o"; }
+md5f(){ printf '%s' "$1" | md5sum | cut -d' ' -f1; }
+gen_urls(){
+  : > "$URLS"
+  local ov_enc; ov_enc=$(urlenc "http://origin/overlay.png")
+  local img raw enc w cmds sig
+  for img in big.jpg medium.jpg small.jpg; do
+    raw="http://origin/$img"; enc=$(urlenc "$raw")
+    for w in $(seq 200 5 1200); do
+      for cmds in \
+        "resize/${w}x/quality/85" \
+        "resize/${w}x/quality/90/format/webp" \
+        "crop/1600x900/resize/${w}x450/quality/85" \
+        "thumbnail/${w}x300" \
+        "strip/true/resize/${w}x/format/jpg/quality/80"; do
+        sig=$(md5f "${EXPIRES}${SECRET}${cmds}/${raw}")
+        printf '/dims4/%s/%s/%s/%s/?url=%s\n' "$CLIENT" "$sig" "$EXPIRES" "$cmds" "$enc" >> "$URLS"
+      done
+      cmds="resize/${w}x/watermark/0.3,0.4,se"
+      sig=$(md5f "${EXPIRES}${SECRET}${cmds}/${raw}${ov_enc}")
+      printf '/dims4/%s/%s/%s/%s/?url=%s&overlay=%s&_keys=overlay\n' "$CLIENT" "$sig" "$EXPIRES" "$cmds" "$enc" "$ov_enc" >> "$URLS"
+    done
+  done
+  shuf "$URLS" -o "$URLS"
+}
+[ -s "$URLS" ] || { log "sign request list"; gen_urls; }
+log "$(wc -l < "$URLS") signed request paths"
+
+# 2. Build the server image and the load generator.
+log "build $DIMS_IMAGE"; docker build -q -f "$ROOT/docker/Dockerfile" -t "$DIMS_IMAGE" "$ROOT" >/dev/null
+log "build $WRK_IMAGE"; docker build -q -f "$HERE/Dockerfile.wrk" -t "$WRK_IMAGE" "$HERE" >/dev/null
+
+# 3. Network and origin.
+docker network create "$NET" >/dev/null 2>&1 || true
+docker rm -f "$ORIGIN" >/dev/null 2>&1 || true
+docker run -d --name "$ORIGIN" --network "$NET" --network-alias origin \
+  -v "$IMG_DIR":/usr/local/apache2/htdocs:ro httpd:2.4.68 >/dev/null
+
+cleanup(){ docker rm -f "$DIMS" "$ORIGIN" >/dev/null 2>&1 || true; docker network rm "$NET" >/dev/null 2>&1 || true; }
+trap cleanup EXIT
+
+RESULTS="$HERE/results.tsv"
+printf 'SL\tTPC\tworkers\treq_s\tp99_ms\tnon2xx\tsockerr\toom\tpeak_MiB\n' > "$RESULTS"
+
+wrk_run(){ # dur
+  docker run --rm --network "$NET" \
+    -v "$HERE/urls.lua":/urls.lua:ro -v "$URLS":/urls.txt:ro "$WRK_IMAGE" \
+    -t"$WT" -c"$CONN" -d"$1" --timeout 20s --latency -s /urls.lua http://dims:8000 2>&1
+}
+
+run_point(){
+  local SL=$1 TPC=$2 MRW=$(( $1 * $2 ))
+  docker rm -f "$DIMS" >/dev/null 2>&1 || true
+  docker run -d --name "$DIMS" --network "$NET" --network-alias dims \
+    --cpus="$CPUS" --memory="$MEM" --memory-swap="$MEM" \
+    -e DIMS_CLIENT=TEST -e DIMS_SECRET=t3stk3y \
+    -e DIMS_WHITELIST=origin -e DIMS_ALLOW_PRIVATE_ADDRESSES=on \
+    "$DIMS_IMAGE" \
+    /usr/local/apache2/bin/httpd -DFOREGROUND \
+      -c "ServerLimit $SL" -c "ThreadLimit $TPC" -c "StartServers $SL" \
+      -c "ThreadsPerChild $TPC" -c "MaxRequestWorkers $MRW" >/dev/null
+
+  local ok=0
+  for _ in $(seq 1 30); do
+    if docker exec "$DIMS" /usr/local/imagemagick/bin/curl -fsS -o /dev/null http://127.0.0.1:8000/dims-status/ 2>/dev/null; then ok=1; break; fi
+    sleep 1
+  done
+  [ "$ok" = 1 ] || { log "dims did not start for $SL/$TPC"; docker logs --tail 20 "$DIMS" >&2 || true; return; }
+
+  wrk_run "$WARM" >/dev/null 2>&1 || true
+
+  local out; out="$(wrk_run "$DUR" || true)"
+  local req_s p99 non2xx sockerr peak oom
+  req_s=$(printf '%s' "$out"  | awk '/Requests\/sec/{print $2}')
+  p99=$(printf '%s'   "$out"  | awk '/^[[:space:]]*99%/{print $2; exit}')
+  non2xx=$(printf '%s' "$out" | awk '/Non-2xx or 3xx/{print $NF}'); non2xx=${non2xx:-0}
+  sockerr=$(printf '%s' "$out"| awk -F'[:,]' '/Socket errors/{c=$0; gsub(/[^0-9 ]/,"",c); n=split(c,a," "); s=0; for(i=1;i<=n;i++)s+=a[i]; print s}'); sockerr=${sockerr:-0}
+  peak=$(( $(cread /sys/fs/cgroup/memory.peak) / 1048576 ))
+  oom=$(cread /sys/fs/cgroup/memory.events | awk '/oom_kill /{print $2}'); oom=${oom:-0}
+
+  printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \
+    "$SL" "$TPC" "$MRW" "${req_s:-0}" "${p99:-?}" "$non2xx" "$sockerr" "$oom" "$peak" | tee -a "$RESULTS" >&2
+}
+
+for point in "${GRID[@]}"; do
+  # shellcheck disable=SC2086
+  set -- $point
+  log "ServerLimit=$1 ThreadsPerChild=$2"
+  run_point "$1" "$2"
+done
+
+log "results"
+column -t "$RESULTS" >&2

test/soak/urls.lua

@@ -0,0 +1,11 @@
+-- wrk script. Reads the signed request paths that run.sh precomputes and
+-- rotates through them. Bash signs the dims4 requests because Lua has no md5.
+local paths = {}
+for line in io.lines("/urls.txt") do
+  if #line > 0 then paths[#paths + 1] = line end
+end
+local n = 0
+request = function()
+  n = (n % #paths) + 1
+  return wrk.format("GET", paths[n])
+end