From 791baff0ad42bf0e79fb69e1f3f33aa727502699 Mon Sep 17 00:00:00 2001 From: Jeremy Collins Date: Fri, 28 Aug 2026 17:57:17 -0400 Subject: [PATCH 1/2] Add a soak test for the worker configuration --- test/soak/.gitignore | 6 ++ test/soak/Dockerfile.wrk | 11 +++ test/soak/README.md | 49 +++++++++++++ test/soak/run.sh | 149 +++++++++++++++++++++++++++++++++++++++ test/soak/urls.lua | 11 +++ 5 files changed, 226 insertions(+) create mode 100644 test/soak/.gitignore create mode 100644 test/soak/Dockerfile.wrk create mode 100644 test/soak/README.md create mode 100755 test/soak/run.sh create mode 100644 test/soak/urls.lua diff --git a/test/soak/.gitignore b/test/soak/.gitignore new file mode 100644 index 0000000..25e2d23 --- /dev/null +++ b/test/soak/.gitignore @@ -0,0 +1,6 @@ +# Downloaded fixtures and run output. The images come from pexels at run time. +origin/ +results.tsv +results-*.tsv +urls.txt +*.log diff --git a/test/soak/Dockerfile.wrk b/test/soak/Dockerfile.wrk new file mode 100644 index 0000000..a760b22 --- /dev/null +++ b/test/soak/Dockerfile.wrk @@ -0,0 +1,11 @@ +# Throwaway load generator. wrk is not a module dependency and never ships in +# the server image. It is built here only to drive the soak test. +FROM debian:bookworm-slim +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + build-essential libssl-dev zlib1g-dev git unzip ca-certificates && \ + git clone --depth 1 https://github.com/wg/wrk.git /src && \ + make -C /src -j"$(nproc)" && \ + cp /src/wrk /usr/local/bin/wrk && \ + rm -rf /src /var/lib/apt/lists/* +ENTRYPOINT ["wrk"] diff --git a/test/soak/README.md b/test/soak/README.md new file mode 100644 index 0000000..f8ad6a0 --- /dev/null +++ b/test/soak/README.md @@ -0,0 +1,49 @@ +# Soak test + +This directory finds the Apache worker configuration that gives the most +throughput on a fixed CPU and memory budget with no failures. It is an +operational tool. It is not part of `make test`, and nothing in the module or +the C test harness depends on it. + +## What it does + +`run.sh` runs the production server image under a hard CPU and memory limit, +drives it with a mix of resize, crop, thumbnail, format, strip, and watermark +requests, and sweeps the Apache MPM sizing. For each configuration it records +throughput, tail latency, HTTP errors, socket errors, peak memory, and the +kernel OOM-kill count. + +The images are three sizes of one pexels photo, downloaded once and served from +a local origin. The load generator is `wrk`, built in a throwaway image. `wrk` +is never a module or test dependency and never ships in the server image. + +## Requirements + +- Docker. +- Internet on the first run, to download the pexels images and build `wrk`. + Later runs reuse both. + +## Run + +``` +bash test/soak/run.sh +``` + +Environment variables tune the run: + +- `CPUS`, `MEM` — the container limits. Default `2` and `1g`. +- `DUR`, `WARM` — the measure and warm-up time per point. Default `30s`, `10s`. +- `CONN`, `WT` — the wrk connection and thread count. Default `32`, `4`. +- `SOAK_GRID` — the sweep, a semicolon-separated list of `ServerLimit + ThreadsPerChild` pairs, e.g. `SOAK_GRID="2 2;4 2"`. + +Results print as a table and land in `results.tsv`. + +## Read the table + +- `req_s` — throughput. Higher is better. +- `p99_ms` — the 99th percentile latency under the offered load. +- `non2xx`, `sockerr` — request failures. A shipped configuration holds both at + zero. +- `oom` — kernel OOM-kills. Any value above zero fails the configuration. +- `peak_MiB` — the peak container memory. diff --git a/test/soak/run.sh b/test/soak/run.sh new file mode 100755 index 0000000..6569291 --- /dev/null +++ b/test/soak/run.sh @@ -0,0 +1,149 @@ +#!/usr/bin/env bash +# Soak test. Find the Apache MPM sizing that gives the most throughput on a +# 2 vCPU / 1 GB container with no failures. +# +# It serves pexels images locally, drives production-style command shapes with wrk, and +# for each MPM configuration records throughput, tail latency, errors, the peak +# memory, and the kernel OOM-kill count. A configuration fails if it OOM-kills a +# worker or returns any non-2xx or socket error. +set -euo pipefail + +HERE="$(cd "$(dirname "$0")" && pwd)" +ROOT="$(cd "$HERE/../.." && pwd)" +NET=soak-net +ORIGIN=soak-origin +DIMS=soak-dims +IMG_DIR="$HERE/origin" +DIMS_IMAGE=mod-dims:soak +WRK_IMAGE=soak-wrk +CPUS="${CPUS:-2}" +MEM="${MEM:-1g}" +DUR="${DUR:-30s}" +WARM="${WARM:-10s}" +CONN="${CONN:-32}" +WT="${WT:-4}" + +# ServerLimit ThreadsPerChild pairs to sweep. Override with SOAK_GRID, a +# semicolon-separated list, e.g. SOAK_GRID="2 2;4 1". +if [ -n "${SOAK_GRID:-}" ]; then + IFS=";" read -ra GRID <<< "$SOAK_GRID" +else + GRID=( "1 2" "1 3" "1 4" "1 6" "1 8" "2 2" "2 4" "1 16" "1 32" ) +fi + +log(){ printf '\n=== %s ===\n' "$*" >&2; } +cread(){ docker exec -u 0 "$DIMS" cat "$1" 2>/dev/null || echo 0; } + +# 1. Images: three sizes of a pexels photo, downloaded once. +mkdir -p "$IMG_DIR" +declare -A PEX=( + [big.jpg]="https://images.pexels.com/photos/1539116/pexels-photo-1539116.jpeg?auto=compress&cs=tinysrgb&w=3840" + [medium.jpg]="https://images.pexels.com/photos/1539116/pexels-photo-1539116.jpeg?auto=compress&cs=tinysrgb&w=1920" + [small.jpg]="https://images.pexels.com/photos/1539116/pexels-photo-1539116.jpeg?auto=compress&cs=tinysrgb&w=960" +) +for name in "${!PEX[@]}"; do + [ -s "$IMG_DIR/$name" ] || { log "download $name"; curl -fsSL --max-time 90 -o "$IMG_DIR/$name" "${PEX[$name]}"; } +done +ls -lh "$IMG_DIR" >&2 + +# 1b. Signed request list. Bash signs each dims4 request, because Lua has no +# md5: md5(expires + secret + commands + "/" + raw_url [+ encoded overlay]). +# The list mixes resize, crop, thumbnail, format, strip, and watermark shapes +# over three image sizes, and varies the width so requests miss the cache and +# decode the source. The list is cached; delete it to rebuild. +cp -f "$ROOT/test/origin/overlay.png" "$IMG_DIR/overlay.png" +CLIENT=TEST; SECRET=t3stk3y; EXPIRES=2147483647 +URLS="$HERE/urls.txt" +urlenc(){ local s=$1 o= c i h; for ((i=0;i<${#s};i++)); do c=${s:i:1}; case $c in [a-zA-Z0-9]) o+=$c;; *) printf -v h '%%%02X' "'$c"; o+=$h;; esac; done; printf '%s' "$o"; } +md5f(){ printf '%s' "$1" | md5sum | cut -d' ' -f1; } +gen_urls(){ + : > "$URLS" + local ov_enc; ov_enc=$(urlenc "http://origin/overlay.png") + local img raw enc w cmds sig + for img in big.jpg medium.jpg small.jpg; do + raw="http://origin/$img"; enc=$(urlenc "$raw") + for w in $(seq 200 5 1200); do + for cmds in \ + "resize/${w}x/quality/85" \ + "resize/${w}x/quality/90/format/webp" \ + "crop/1600x900/resize/${w}x450/quality/85" \ + "thumbnail/${w}x300" \ + "strip/true/resize/${w}x/format/jpg/quality/80"; do + sig=$(md5f "${EXPIRES}${SECRET}${cmds}/${raw}") + printf '/dims4/%s/%s/%s/%s/?url=%s\n' "$CLIENT" "$sig" "$EXPIRES" "$cmds" "$enc" >> "$URLS" + done + cmds="resize/${w}x/watermark/0.3,0.4,se" + sig=$(md5f "${EXPIRES}${SECRET}${cmds}/${raw}${ov_enc}") + printf '/dims4/%s/%s/%s/%s/?url=%s&overlay=%s&_keys=overlay\n' "$CLIENT" "$sig" "$EXPIRES" "$cmds" "$enc" "$ov_enc" >> "$URLS" + done + done + shuf "$URLS" -o "$URLS" +} +[ -s "$URLS" ] || { log "sign request list"; gen_urls; } +log "$(wc -l < "$URLS") signed request paths" + +# 2. Build the server image and the load generator. +log "build $DIMS_IMAGE"; docker build -q -f "$ROOT/docker/Dockerfile" -t "$DIMS_IMAGE" "$ROOT" >/dev/null +log "build $WRK_IMAGE"; docker build -q -f "$HERE/Dockerfile.wrk" -t "$WRK_IMAGE" "$HERE" >/dev/null + +# 3. Network and origin. +docker network create "$NET" >/dev/null 2>&1 || true +docker rm -f "$ORIGIN" >/dev/null 2>&1 || true +docker run -d --name "$ORIGIN" --network "$NET" --network-alias origin \ + -v "$IMG_DIR":/usr/local/apache2/htdocs:ro httpd:2.4.68 >/dev/null + +cleanup(){ docker rm -f "$DIMS" "$ORIGIN" >/dev/null 2>&1 || true; docker network rm "$NET" >/dev/null 2>&1 || true; } +trap cleanup EXIT + +RESULTS="$HERE/results.tsv" +printf 'SL\tTPC\tworkers\treq_s\tp99_ms\tnon2xx\tsockerr\toom\tpeak_MiB\n' > "$RESULTS" + +wrk_run(){ # dur + docker run --rm --network "$NET" \ + -v "$HERE/urls.lua":/urls.lua:ro -v "$URLS":/urls.txt:ro "$WRK_IMAGE" \ + -t"$WT" -c"$CONN" -d"$1" --timeout 20s --latency -s /urls.lua http://dims:8000 2>&1 +} + +run_point(){ + local SL=$1 TPC=$2 MRW=$(( $1 * $2 )) + docker rm -f "$DIMS" >/dev/null 2>&1 || true + docker run -d --name "$DIMS" --network "$NET" --network-alias dims \ + --cpus="$CPUS" --memory="$MEM" --memory-swap="$MEM" \ + -e DIMS_CLIENT=TEST -e DIMS_SECRET=t3stk3y \ + -e DIMS_WHITELIST=origin -e DIMS_ALLOW_PRIVATE_ADDRESSES=on \ + "$DIMS_IMAGE" \ + /usr/local/apache2/bin/httpd -DFOREGROUND \ + -c "ServerLimit $SL" -c "ThreadLimit $TPC" -c "StartServers $SL" \ + -c "ThreadsPerChild $TPC" -c "MaxRequestWorkers $MRW" >/dev/null + + local ok=0 + for _ in $(seq 1 30); do + if docker exec "$DIMS" /usr/local/imagemagick/bin/curl -fsS -o /dev/null http://127.0.0.1:8000/dims-status/ 2>/dev/null; then ok=1; break; fi + sleep 1 + done + [ "$ok" = 1 ] || { log "dims did not start for $SL/$TPC"; docker logs --tail 20 "$DIMS" >&2 || true; return; } + + wrk_run "$WARM" >/dev/null 2>&1 || true + + local out; out="$(wrk_run "$DUR" || true)" + local req_s p99 non2xx sockerr peak oom + req_s=$(printf '%s' "$out" | awk '/Requests\/sec/{print $2}') + p99=$(printf '%s' "$out" | awk '/^[[:space:]]*99%/{print $2; exit}') + non2xx=$(printf '%s' "$out" | awk '/Non-2xx or 3xx/{print $NF}'); non2xx=${non2xx:-0} + sockerr=$(printf '%s' "$out"| awk -F'[:,]' '/Socket errors/{c=$0; gsub(/[^0-9 ]/,"",c); n=split(c,a," "); s=0; for(i=1;i<=n;i++)s+=a[i]; print s}'); sockerr=${sockerr:-0} + peak=$(( $(cread /sys/fs/cgroup/memory.peak) / 1048576 )) + oom=$(cread /sys/fs/cgroup/memory.events | awk '/oom_kill /{print $2}'); oom=${oom:-0} + + printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \ + "$SL" "$TPC" "$MRW" "${req_s:-0}" "${p99:-?}" "$non2xx" "$sockerr" "$oom" "$peak" | tee -a "$RESULTS" >&2 +} + +for point in "${GRID[@]}"; do + # shellcheck disable=SC2086 + set -- $point + log "ServerLimit=$1 ThreadsPerChild=$2" + run_point "$1" "$2" +done + +log "results" +column -t "$RESULTS" >&2 diff --git a/test/soak/urls.lua b/test/soak/urls.lua new file mode 100644 index 0000000..54d404b --- /dev/null +++ b/test/soak/urls.lua @@ -0,0 +1,11 @@ +-- wrk script. Reads the signed request paths that run.sh precomputes and +-- rotates through them. Bash signs the dims4 requests because Lua has no md5. +local paths = {} +for line in io.lines("/urls.txt") do + if #line > 0 then paths[#paths + 1] = line end +end +local n = 0 +request = function() + n = (n % #paths) + 1 + return wrk.format("GET", paths[n]) +end From ac19e5563a9102f30512015f7ec66348f76e69b8 Mon Sep 17 00:00:00 2001 From: Jeremy Collins Date: Fri, 28 Aug 2026 17:57:17 -0400 Subject: [PATCH 2/2] Size the worker pool from the CPU allocation - Read the cgroup CPU quota at start and set ServerLimit, ThreadsPerChild, and MaxRequestWorkers, so a larger container scales without a change. - Recycle a child after DIMS_MAX_CONNECTIONS_PER_CHILD connections, so its memory returns to the system. --- docker/Dockerfile | 13 +++++++ docker/dims.conf | 18 ++++++++++ docker/entrypoint.sh | 53 ++++++++++++++++++++++++++++ docs/docs/configuration/resources.md | 47 ++++++++++++++++++++++-- 4 files changed, 128 insertions(+), 3 deletions(-) create mode 100755 docker/entrypoint.sh diff --git a/docker/Dockerfile b/docker/Dockerfile index 68a0cfb..7278738 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -37,6 +37,14 @@ LABEL org.opencontainers.image.licenses="Apache-2.0" # Every one has a default, so the container starts with no environment set. ENV DIMS_DOWNLOAD_TIMEOUT=60000 ENV DIMS_IMAGEMAGICK_TIMEOUT=20000 +# Worker pool. The default auto sizes each value from the container's CPU +# allocation at start. Set a number to pin it. See the resources page. +ENV DIMS_SERVER_LIMIT=auto +ENV DIMS_THREADS_PER_CHILD=auto +ENV DIMS_MAX_WORKERS=auto +# Recycle a child after this many connections, so its memory returns to the +# system. 0 keeps a child forever. +ENV DIMS_MAX_CONNECTIONS_PER_CHILD=10000 ENV DIMS_CLIENT=development ENV DIMS_NO_IMAGE_URL="-" ENV DIMS_DEFAULT_IMAGE_URL="-" @@ -73,6 +81,7 @@ COPY --from=build /usr/local/imagemagick /usr/local/imagemagick COPY docker/policy.xml /usr/local/imagemagick/etc/ImageMagick-7/policy.xml COPY profiles/ /usr/local/share/mod_dims/profiles/ COPY docker/dims.conf /usr/local/apache2/conf/extra/dims.conf +COPY --chmod=0755 docker/entrypoint.sh /usr/local/bin/dims-entrypoint ENV LD_LIBRARY_PATH=/usr/local/imagemagick/lib @@ -100,3 +109,7 @@ USER 33 # without reaching an origin. HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \ CMD /usr/local/imagemagick/bin/curl -fsS -o /dev/null http://127.0.0.1:8000/dims-status/ || exit 1 + +# Size the worker pool from the CPU allocation, then run httpd. +ENTRYPOINT ["/usr/local/bin/dims-entrypoint"] +CMD ["httpd-foreground"] diff --git a/docker/dims.conf b/docker/dims.conf index d12a1f1..5962583 100644 --- a/docker/dims.conf +++ b/docker/dims.conf @@ -8,6 +8,24 @@ PidFile /tmp/httpd.pid ErrorLog /proc/self/fd/2 CustomLog /proc/self/fd/1 common +# Worker pool. The entrypoint sizes ServerLimit, ThreadsPerChild, and +# MaxRequestWorkers from the container's CPU allocation, so a larger container +# scales without a change here. A soak test found that throughput peaks near +# one worker per vCPU, with two threads per child split across processes. Set +# DIMS_SERVER_LIMIT, DIMS_THREADS_PER_CHILD, or DIMS_MAX_WORKERS to a number to +# pin one. MaxConnectionsPerChild recycles a child after many connections, so +# its memory returns to the system. + + ServerLimit ${DIMS_SERVER_LIMIT} + StartServers ${DIMS_SERVER_LIMIT} + ThreadLimit ${DIMS_THREADS_PER_CHILD} + ThreadsPerChild ${DIMS_THREADS_PER_CHILD} + MinSpareThreads 1 + MaxSpareThreads ${DIMS_MAX_WORKERS} + MaxRequestWorkers ${DIMS_MAX_WORKERS} + MaxConnectionsPerChild ${DIMS_MAX_CONNECTIONS_PER_CHILD} + + DimsDownloadTimeout ${DIMS_DOWNLOAD_TIMEOUT} DimsImagemagickTimeout ${DIMS_IMAGEMAGICK_TIMEOUT} diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh new file mode 100755 index 0000000..419a8ff --- /dev/null +++ b/docker/entrypoint.sh @@ -0,0 +1,53 @@ +#!/bin/sh +# Size the httpd worker pool from the container's CPU allocation, so a larger +# container scales without a configuration change. A soak test found that +# throughput peaks near one worker per vCPU, with two threads per child split +# across processes. This sets ServerLimit to half the vCPU count, at least two, +# and MaxRequestWorkers to ServerLimit times ThreadsPerChild. +# +# An explicit DIMS_SERVER_LIMIT, DIMS_THREADS_PER_CHILD, or DIMS_MAX_WORKERS +# keeps its value. The default of each is "auto". +set -e + +# The effective vCPU count. nproc reports the host cores, not the container +# limit, so read the cgroup CPU quota first. +detect_vcpus() { + if [ -r /sys/fs/cgroup/cpu.max ]; then # cgroup v2 + read -r quota period < /sys/fs/cgroup/cpu.max + if [ "$quota" != max ] && [ "${period:-0}" -gt 0 ] 2>/dev/null; then + echo $(( (quota + period - 1) / period )) + return + fi + fi + if [ -r /sys/fs/cgroup/cpu/cpu.cfs_quota_us ]; then # cgroup v1 + quota=$(cat /sys/fs/cgroup/cpu/cpu.cfs_quota_us) + period=$(cat /sys/fs/cgroup/cpu/cpu.cfs_period_us) + if [ "${quota:-0}" -gt 0 ] && [ "${period:-0}" -gt 0 ] 2>/dev/null; then + echo $(( (quota + period - 1) / period )) + return + fi + fi + nproc 2>/dev/null || echo 1 +} + +tpc=${DIMS_THREADS_PER_CHILD:-auto} +[ "$tpc" = auto ] && tpc=2 + +sl=${DIMS_SERVER_LIMIT:-auto} +if [ "$sl" = auto ]; then + vcpus=$(detect_vcpus) + [ "${vcpus:-0}" -ge 1 ] 2>/dev/null || vcpus=1 + sl=$(( vcpus / 2 )) + [ "$sl" -lt 2 ] && sl=2 +fi + +mw=${DIMS_MAX_WORKERS:-auto} +[ "$mw" = auto ] && mw=$(( sl * tpc )) + +export DIMS_SERVER_LIMIT=$sl +export DIMS_THREADS_PER_CHILD=$tpc +export DIMS_MAX_WORKERS=$mw + +echo "mod_dims workers: ServerLimit=$sl ThreadsPerChild=$tpc MaxRequestWorkers=$mw" >&2 + +exec "$@" diff --git a/docs/docs/configuration/resources.md b/docs/docs/configuration/resources.md index 106da77..874cfa3 100644 --- a/docs/docs/configuration/resources.md +++ b/docs/docs/configuration/resources.md @@ -59,8 +59,49 @@ DimsImagemagickAreaSize 128 Megabytes any single image may use. Default 128. An image larger than this is cached on disk rather than in memory. +## Workers + +The container sizes the httpd worker pool from its CPU allocation at start, so +a larger container scales without a change. Three environment variables control +it. Each one defaults to `auto`. + +| Variable | Directive | Default | +| --- | --- | --- | +| DIMS_SERVER_LIMIT | ServerLimit | auto | +| DIMS_THREADS_PER_CHILD | ThreadsPerChild | auto | +| DIMS_MAX_WORKERS | MaxRequestWorkers | auto | + +Image work is CPU-bound. A soak test on several container sizes found that +throughput peaks near one worker per vCPU and drops with more. Two threads per +child, split across processes, give the lowest tail latency, because two +processes share less ImageMagick lock state than one process with more threads. +One thread per child is slow: the single thread cannot both accept and process. + +So `auto` sets `ThreadsPerChild` to 2 and `ServerLimit` to half the vCPU count, +at least 2. `MaxRequestWorkers` is their product. Set a variable to a number to +pin it. `DIMS_MAX_WORKERS` must equal `DIMS_SERVER_LIMIT` times +`DIMS_THREADS_PER_CHILD`. + +| Container | ServerLimit | ThreadsPerChild | MaxRequestWorkers | Throughput | +| --- | --- | --- | --- | --- | +| 2 vCPU, 1 GB | 2 | 2 | 4 | 18 req/s | +| 4 vCPU, 2 GB | 2 | 2 | 4 | 35 req/s | +| 6 vCPU, 3 GB | 3 | 2 | 6 | 40 req/s | +| 8 vCPU, 4 GB | 4 | 2 | 8 | — | + +The throughput is for a mixed resize, crop, format, and watermark load on full +size photos. A lighter load or a smaller source reaches a higher number. The 6 +vCPU row is indicative, and the 8 vCPU row follows the rule, not a measurement, +because the test host has eight cores and the load generator needs some. + +`DIMS_MAX_CONNECTIONS_PER_CHILD` recycles a child after that many connections, +which returns its memory to the system. The default is 10000. Set it to 0 to +keep a child for the life of the server. + ## Choosing values -With `ServerLimit 16` and the defaults, sixteen processes may each hold 512 MB -of pixel cache and write 2 GB to disk. Divide the memory a host has by the -worker count, and set the limits from that. +Each worker may hold as much pixel cache and disk as the limits above allow. +With the default four workers, four processes may each hold the memory limit at +once. Divide the memory the container has by the worker count from the Workers +section, and set the limits from that. At the worker counts above the memory is +not the limit; each optimum held under 250 MB, and CPU sets the rate.