Add a metrics endpoint

.github/workflows/pr.yml

@@ -191,6 +191,22 @@ jobs:
             "http://127.0.0.1:8000/dims3/development/resize/100x100/?url=http://$origin/grid.png"
           docker exec dims-ci sh -c 'test -s /tmp/out.png'
 
+      # The metrics endpoint serves its own port, and only when the
+      # environment turns it on.
+      - name: Scrape the metrics endpoint
+        run: |
+          docker rm -f dims-ci
+          docker run -d --name dims-ci -p 8001:8001 \
+            -e DIMS_METRICS_ENABLED=on mod-dims:ci
+          for i in $(seq 30); do
+            if curl -fsS -o /dev/null http://127.0.0.1:8001/metrics; then
+              break
+            fi
+            sleep 1
+          done
+          curl -fsS http://127.0.0.1:8001/metrics | grep -q dims_requests_total
+          curl -fsS http://127.0.0.1:8001/metrics | grep -q dims_httpd_workers
+
       # The runtime user must not be able to rewrite the module it runs.
       - name: Check the file permissions
         run: |

CMakeLists.txt

@@ -82,6 +82,8 @@ add_library(mod_dims MODULE
     src/signature.c
     src/svgguard.c
     src/url.c
+    src/metrics.c
+    src/metrics_report.c
     src/status.c
     src/mod_dims.c
     src/module.c

docker/Dockerfile

@@ -65,6 +65,7 @@ ENV DIMS_ALLOW_PRIVATE_ADDRESSES=off
 ENV DIMS_STATUS_VERBOSE=off
 ENV DIMS_ENABLE_DIMS3=off
 ENV DIMS_ENABLE_DIMS4=on
+ENV DIMS_METRICS_ENABLED=off
 ENV DIMS_ENCRYPTION_ALGORITHM=AES/GCM/NoPadding
 ENV LC_ALL="C"
 
@@ -105,7 +106,7 @@ RUN rm -rf /usr/local/apache2/build \
     chown -R root:root /usr/local/apache2 /usr/local/imagemagick && \
     chmod -R go-w /usr/local/apache2 /usr/local/imagemagick
 
-EXPOSE 8000
+EXPOSE 8000 8001
 USER 33
 
 # A real request, not a configuration check. The status handler answers

docker/README.md

@@ -36,6 +36,16 @@ $ docker run -e DIMS_SECRET=mysecret mod-dims:latest
 $ docker run -e DIMS_ENABLE_DIMS3=on -e DIMS_WHITELIST="images.pexels.com" mod-dims:latest
 ```
 
+## Set DIMS_METRICS_ENABLED to scrape the server
+
+```shell
+$ docker run -p 8000:8000 -p 8001:8001 -e DIMS_METRICS_ENABLED=on mod-dims:latest
+$ curl http://localhost:8001/metrics
+```
+
+Port 8001 serves `/metrics` and `/dims-status/`. Publish 8000 to callers and
+keep 8001 for the monitoring network.
+
 # Configuration
 
 | Environment Variables | Description | Default |
@@ -45,6 +55,7 @@ $ docker run -e DIMS_ENABLE_DIMS3=on -e DIMS_WHITELIST="images.pexels.com" mod-d
 | `DIMS_WHITELIST` | Hosts /dims3/ may fetch from, separated by spaces. Empty matches no host. | "" |
 | `DIMS_ENABLE_DIMS3` | Whether the image serves /dims3/. It accepts an unsigned request, so the image ships it off. | off |
 | `DIMS_ENABLE_DIMS4` | Whether the image serves /dims4/. | on |
+| `DIMS_METRICS_ENABLED` | Whether the image serves Prometheus metrics at `/metrics` on port 8001. | off |
 | `DIMS_DEFAULT_IMAGE_URL` | URL to the image sent when a request fails. A single dash means none. | "-" |
 | `DIMS_SECRET` | Shared secret for /dims4/ signatures. A single dash means none, and /dims4/ then refuses every request. | "-" |
 | `DIMS_DOWNLOAD_TIMEOUT` | Max time allowed for downloading source images, in milliseconds. | 60000 |

docker/dims.conf

@@ -87,3 +87,23 @@ DimsAddWhitelist ${DIMS_WHITELIST}
 <Location /dims-status/>
     SetHandler dims-status
 </Location>
+
+# The metrics listener. A Service exposes 8000 and a scrape targets 8001, so
+# no ingress route reaches /metrics. Behind an ingress controller httpd reads
+# the controller address rather than the caller address, so a second listener
+# moves the decision to the network.
+#
+# /dims-status/ serves both ports, because the health check reads it on 8000.
+Listen 8001
+
+DimsMetricsEnabled ${DIMS_METRICS_ENABLED}
+
+<VirtualHost *:8001>
+    <Location /metrics>
+        SetHandler dims-metrics
+    </Location>
+
+    <Location /dims-status/>
+        SetHandler dims-status
+    </Location>
+</VirtualHost>

docs/docs/configuration/index.md

@@ -71,6 +71,7 @@ Two more are worth enabling as your deployment allows.
 | `DimsImagemagickMemorySize` | 512 |
 | `DimsImagemagickTimeout` | 3000 |
 | `DimsIncludeDisposition` | false |
+| `DimsMetricsEnabled` | `Off` |
 | `DimsMaxSourceBytes` | 0 |
 | `DimsNoImageCacheExpire` | 60 |
 | `DimsOptimizeResize` | 0 |

docs/docs/configuration/metrics.md

@@ -0,0 +1,33 @@
+# Metrics
+
+## DimsMetricsEnabled
+
+```apacheconf
+DimsMetricsEnabled on
+```
+
+Whether the module serves [`/metrics`](/endpoints/metrics). Default `Off`.
+
+The directive is the switch, and a `<Location>` names the path. Both are
+needed:
+
+```apacheconf
+DimsMetricsEnabled on
+
+<Location /metrics>
+    SetHandler dims-metrics
+</Location>
+```
+
+With the directive `Off` the handler declines the request and httpd answers
+404, so a configuration that keeps the location stops publishing when the
+directive goes off.
+
+The container reads `DIMS_METRICS_ENABLED`. The image sets it to `off`. It
+serves metrics on port 8001.
+
+:::note
+The endpoint reports the counters, the uptime, and the library versions. Serve
+it on a port your callers cannot reach, or restrict the location to your
+monitoring host. `DimsStatusVerbose Off` drops the version labels.
+:::

docs/docs/endpoints/dims3.md

@@ -19,6 +19,11 @@ any image, so the host allowlist is what decides which images are reachable.
 It reads the same query parameters as [`/dims4/`](/endpoints/dims4), apart from
 the signature.
 
+The `client` segment is a legacy of `/dims3/` and `/dims4/`.
+[`/dims5/`](/endpoints/dims5) dropped it. A server that serves several clients
+gives each its own `<VirtualHost>`, which separates the clients, the allowlist,
+and the keys.
+
 ## The allowlist
 
 [`DimsAddWhitelist`](/configuration/image-sources) lists the hosts an image may

docs/docs/endpoints/dims4.md

@@ -18,6 +18,11 @@ neither can be changed without the secret.
 | `expires` | when the URL stops working, in seconds since the epoch |
 | `commands` | the [operations](/operations/) to run |
 
+The `client` segment is a legacy of `/dims3/` and `/dims4/`.
+[`/dims5/`](/endpoints/dims5) dropped it. A server that serves several clients
+gives each its own `<VirtualHost>`, which separates the clients, the allowlist,
+and the keys.
+
 ## Query parameters
 
 | Name | Meaning |

docs/docs/endpoints/index.md

@@ -1,13 +1,14 @@
 # Endpoints
 
-Four handlers, each set on a location.
+Five handlers, each set on a location.
 
 | Handler | URL | Checked by |
 |---|---|---|
 | [`dims5`](/endpoints/dims5) | `/dims5/<commands>/?url=&sig=` | a signature |
 | [`dims4`](/endpoints/dims4) | `/dims4/<client>/<signature>/<expires>/<commands>/?url=` | a signature |
 | [`dims3`](/endpoints/dims3) | `/dims3/<client>/<commands>/?url=` | the host allowlist |
 | [`dims-status`](/endpoints/status) | `/dims-status/` | nothing |
+| [`dims-metrics`](/endpoints/metrics) | `/metrics` | nothing |
 | [`dims-local`](/endpoints/local) | a file on disk | nothing |
 
 `/dims5/` is the one to use. Its signature covers the commands, the image URL,

docs/docs/endpoints/metrics.md

@@ -0,0 +1,189 @@
+# /metrics
+
+```
+/metrics
+```
+
+Reports the module, ImageMagick, and httpd in the Prometheus text format. A
+Prometheus server scrapes it, and so does an OpenTelemetry Collector through
+its `prometheus` receiver.
+
+Two lines turn it on. The directive is the switch, and the location names the
+path.
+
+```apacheconf
+DimsMetricsEnabled on
+
+<Location /metrics>
+    SetHandler dims-metrics
+</Location>
+```
+
+[`DimsMetricsEnabled`](/configuration/metrics) defaults to `Off`, so a server
+that upgrades publishes no metrics until an operator turns it on.
+
+## What it reports
+
+```
+# HELP dims_requests_total Requests by endpoint and outcome.
+# TYPE dims_requests_total counter
+dims_requests_total{endpoint="dims5",outcome="success"} 18423
+dims_requests_total{endpoint="dims5",outcome="download_timeout"} 4
+
+# HELP dims_request_duration_seconds Time to serve a request.
+# TYPE dims_request_duration_seconds histogram
+dims_request_duration_seconds_bucket{endpoint="dims5",le="0.025"} 17155
+dims_request_duration_seconds_bucket{endpoint="dims5",le="+Inf"} 18427
+dims_request_duration_seconds_sum{endpoint="dims5"} 314.288000
+dims_request_duration_seconds_count{endpoint="dims5"} 18427
+```
+
+| Metric | Type | Labels |
+|---|---|---|
+| `dims_requests_total` | counter | `endpoint`, `outcome` |
+| `dims_responses_total` | counter | `endpoint`, `code` |
+| `dims_requests_in_flight` | gauge | `endpoint` |
+| `dims_request_duration_seconds` | histogram | `endpoint` |
+| `dims_source_fetch_duration_seconds` | histogram | |
+| `dims_source_bytes` | histogram | |
+| `dims_source_bytes_total` | counter | |
+| `dims_source_format_total` | counter | `format` |
+| `dims_origin_responses_total` | counter | `code` |
+| `dims_imagemagick_duration_seconds` | histogram | |
+| `dims_source_frames` | histogram | |
+| `dims_output_bytes` | histogram | |
+| `dims_output_bytes_total` | counter | |
+| `dims_output_format_total` | counter | `format` |
+| `dims_source_fetch_total` | counter | `result` |
+| `dims_source_fetch_errors_total` | counter | `code` |
+| `dims_netguard_refusals_total` | counter | `reason` |
+| `dims_allowlist_checks_total` | counter | `mode`, `result` |
+| `dims_signature_checks_total` | counter | `endpoint`, `result` |
+| `dims_eurl_decrypt_total` | counter | `result` |
+| `dims_operations_total` | counter | `operation`, `trigger` |
+| `dims_operation_failures_total` | counter | `operation` |
+| `dims_operation_duration_seconds` | histogram | `operation` |
+| `dims_imagemagick_exceptions_total` | counter | `kind`, `severity` |
+| `dims_overlay_cache_lookups_total` | counter | `result` |
+| `dims_overlay_cache_evictions_total` | counter | |
+| `dims_error_images_total` | counter | `source` |
+| `dims_error_image_failures_total` | counter | |
+| `dims_imagemagick_resource_bytes` | gauge | `resource` |
+| `dims_imagemagick_resource_max_bytes` | gauge | `resource` |
+| `dims_imagemagick_resource_limit_bytes` | gauge | `resource` |
+| `dims_process_resident_bytes` | gauge | |
+| `dims_process_resident_max_bytes` | gauge | |
+| `dims_process_virtual_bytes` | gauge | |
+| `dims_workers` | gauge | |
+| `dims_httpd_workers` | gauge | `state` |
+| `dims_httpd_processes` | gauge | |
+| `dims_httpd_processes_limit` | gauge | |
+| `dims_httpd_threads_per_process` | gauge | |
+| `dims_httpd_connections` | gauge | `state` |
+| `dims_httpd_generation` | gauge | |
+
+`endpoint` holds `dims3`, `dims4`, `dims5`, or `local`.
+
+`outcome` holds `success`, `failure`, `download_timeout`,
+`imagemagick_timeout`, `bad_client`, `bad_url`, `bad_arguments`,
+`hostname_not_in_whitelist`, `file_not_found`, or `network_refused`.
+
+`code` holds `200`, `400`, `403`, `404`, `429`, `500`, `502`, `503`, `504`, or
+`other`. The list bounds the label, because
+[`DimsOriginStatusMode`](/configuration/cache-control) `forward` reports the
+status the origin returned.
+
+`format` holds `jpeg`, `png`, `gif`, `webp`, `avif`, `heic`, `tiff`, `svg`, or
+`other`.
+
+`resource` holds `area`, `memory`, `map`, or `disk`.
+
+`result` on `dims_source_fetch_total` holds `ok`, `timeout`,
+`transport_error`, `refused`, or `http_error`. `code` on
+`dims_source_fetch_errors_total` holds a libcurl name such as
+`couldnt_resolve_host` or `ssl_connect_error`, and `other` for the rest.
+
+`reason` holds `bad_scheme`, `bad_url`, `reserved_address`,
+`private_address`, `host_not_allowed`, or `too_many_redirects`.
+
+`mode` holds `skip`, `log`, or `enforce`. Under
+[`DimsAllowlistSigned`](/configuration/image-sources) `log` the module records
+what enforcing would refuse and serves the request, so
+`dims_allowlist_checks_total{mode="log",result="refused"}` is the number to
+watch before setting `enforce`.
+
+`result` on a signature holds `ok`, `mismatch`, `expired`, `too_far_future`,
+`missing_key`, or `bad_client`.
+
+`operation` holds one of the seventeen command names. `trigger` holds
+`request` or `default`. `strip` runs when the command list holds none, and
+`format` runs under [`DimsDefaultOutputFormat`](/configuration/output), so
+both report `default`.
+
+`kind` holds an ImageMagick exception kind such as `resource_limit`,
+`corrupt_image`, or `policy`. `severity` holds `warning`, `error`, or
+`fatal`.
+
+`source` on an error image holds `drawn` for
+[`DimsErrorBackground`](/configuration/cache-control), `fetched` for
+[`DimsDefaultImageURL`](/configuration/cache-control), or `none`.
+
+## The whole server
+
+Every metric describes the server, with no `client` label. The client id is a
+legacy of `/dims3/` and `/dims4/`, and `/dims5/` dropped it.
+
+A deployment that serves several clients gives each its own `<VirtualHost>`,
+which separates the clients, the allowlist, and the keys. The counters stay
+shared, because the module creates one block for the process. Separate metrics
+need separate instances.
+
+## Per process and per server
+
+Most metrics come from a block every worker adds to, so they describe the whole
+server.
+
+`dims_imagemagick_resource_bytes`, `dims_process_resident_bytes`, and
+`dims_process_virtual_bytes` are per worker. Each child writes its own numbers
+when a request ends, and this endpoint sums the live workers. The matching
+`_max_bytes` gauge reports the widest single worker, so one worker at its
+ceiling is visible beside three idle ones. `dims_workers` counts the workers in
+the sum.
+
+`dims_process_resident_bytes` and `dims_process_virtual_bytes` read
+`/proc/self/statm`. A platform without it omits both.
+
+## Kubernetes
+
+The container serves images on 8000 and metrics on 8001, so a Service exposes
+8000 and a scrape targets 8001.
+
+```shell
+docker run -p 8000:8000 -p 8001:8001 -e DIMS_METRICS_ENABLED=on \
+  ghcr.io/beetlebugorg/mod_dims:latest
+```
+
+An OpenTelemetry Collector reads it through the `prometheus` receiver:
+
+```yaml
+receivers:
+  prometheus:
+    config:
+      scrape_configs:
+        - job_name: mod_dims
+          static_configs:
+            - targets: ['mod-dims:8001']
+```
+
+## Worker headroom
+
+`dims_httpd_workers{state="ready"}` against `dims_httpd_processes_limit` times
+`dims_httpd_threads_per_process` is the headroom the pool has left. Ready
+workers near zero while `dims_requests_in_flight` climbs means the pool is
+full.
+
+:::note
+`dims_build_info` names the mod_dims, ImageMagick, and libcurl versions.
+[`DimsStatusVerbose`](/configuration/output) `Off` drops it, the same way it
+drops the version lines from [`/dims-status/`](/endpoints/status).
+:::

docs/sidebars.js

@@ -7,7 +7,7 @@ const sidebars = {
       type: 'category',
       label: 'Endpoints',
       link: {type: 'doc', id: 'endpoints/index'},
-      items: ['endpoints/dims5', 'endpoints/dims4', 'endpoints/dims3', 'endpoints/status', 'endpoints/local'],
+      items: ['endpoints/dims5', 'endpoints/dims4', 'endpoints/dims3', 'endpoints/status', 'endpoints/metrics', 'endpoints/local'],
     },
     {
       type: 'category',
@@ -60,6 +60,7 @@ const sidebars = {
         'configuration/cache-control',
         'configuration/output',
         'configuration/resources',
+        'configuration/metrics',
       ],
     },
   ],