diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1873a3f..658ec21 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -144,6 +144,37 @@ jobs: -v "$PWD:/repo" -w /repo "$AM_IMAGE" \ check-config "$STACK/alertmanager/alertmanager.yaml" + # check-config proves the tree parses and that every route names a + # receiver that exists. It does not say WHICH receiver an alert reaches, + # and the tree is first-match-wins with no `continue` — so demoting a + # category route below the bare `severity` routes stops it matching while + # check-config still reports SUCCESS. Verified: that mutation passes + # check-config and fails the assertions below. + # + # Same six assertions as scripts/validate.sh. Both places, one commit: + # this pair has drifted before (#68). + - name: amtool config routes test + run: | + fail=0 + while read -r expected labels; do + [ -n "$expected" ] || continue + # shellcheck disable=SC2086 + docker run --rm --entrypoint amtool \ + -v "$PWD:/repo" -w /repo "$AM_IMAGE" \ + config routes test \ + --config.file="$STACK/alertmanager/alertmanager.yaml" \ + --verify.receivers="$expected" $labels \ + || { echo "::error::expected $expected for $labels"; fail=1; } + done <<'ROUTES' + urgent severity=critical category=power + security severity=critical category=security + security severity=warning category=security + urgent severity=critical category=availability + default severity=warning category=capacity + null severity=info category=correctness + ROUTES + exit $fail + # `fmt --test` exits non-zero if the file is not canonically formatted, and # fails outright on a syntax error. It does not validate that components # are configured correctly — Alloy has no `validate` subcommand, so that is diff --git a/docs/observability.md b/docs/observability.md index 0cc694e..9eff662 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -119,20 +119,47 @@ against the broken rule too. Coverage is one rule of 34 so far: `ContainerHighMemory`. The other 33 are still validated for syntax only, which is exactly the standing #63 had. -Routing is by `severity` and `category` (see `alertmanager/alertmanager.yaml`). -`critical` + `category=power` pages immediately and repeats every 30 minutes; -other criticals repeat every 4 hours; warnings every 12; `info` is recorded but -never notified. - -Inhibit rules stop cascades: a down host suppresses its own disk warnings, and a -dead `snmp-exporter` suppresses the "every device is unreachable" storm that -would otherwise follow. - Disk alerting is predictive rather than a fixed threshold — `predict_linear` over a 6-hour window, firing when the extrapolation reaches zero within a day *and* free space is already under 30%. A disk sitting at 86% and stable is not an emergency; one climbing fast at 60% is. +### Routing + +Three receivers, **three separate destinations** (see +`alertmanager/alertmanager.yaml`). They were three names for one webhook URL +until [#66](https://github.com/Gerrrt/HomeLab/issues/66), which meant `urgent` +and `default` differed only in how often they repeated — a UPS on battery and a +slow scrape landed in the same place. The routing tree decides which alert is +urgent; only a distinct destination makes that difference audible, because +per-topic sound and do-not-disturb settings live on the receiving end. + +| Matches | Receiver | First notification | Repeats | +| --- | --- | --- | --- | +| `critical` + `category=power` | `urgent` | immediately | 30m | +| `critical` + `category=security` | `security` | immediately | 1h | +| `warning` + `category=security` | `security` | 30s | 4h | +| `critical` (anything else) | `urgent` | 10s | 4h | +| `warning` (anything else) | `default` | 30s | 12h | +| `info` | `null` | never | — | + +Security has its own destination at both severities because ten rules carry +`category: security` — SSH brute force, a terminal segment reaching the internal +network, IoT lateral movement, priority-1 Suricata — and routed on `severity` +alone, the warning-severity half of that list arrived in the default channel on +a 12-hour repeat, indistinguishable from a disk filling up. `category=power` +was the precedent. + +**First match wins and nothing sets `continue`, so the order of those rows is +the design.** A category route moved below the bare `severity` rows silently +stops matching, and `amtool check-config` still reports SUCCESS — that mutation +was tried. `scripts/validate.sh` and CI therefore assert the table itself with +`amtool config routes test --verify.receivers`, one assertion per row. + +Inhibit rules stop cascades: a down host suppresses its own disk warnings, and a +dead `snmp-exporter` suppresses the "every device is unreachable" storm that +would otherwise follow. + ## Adding a monitored device See [`runbooks/add-monitored-device.md`](runbooks/add-monitored-device.md). In diff --git a/scripts/render-config.sh b/scripts/render-config.sh index 82a17aa..00720b1 100755 --- a/scripts/render-config.sh +++ b/scripts/render-config.sh @@ -43,6 +43,8 @@ REQUIRED=( GRAFANA_ADMIN_PASSWORD GRAFANA_RENDERER_TOKEN ALERTMANAGER_WEBHOOK_URL + ALERTMANAGER_URGENT_WEBHOOK_URL + ALERTMANAGER_SECURITY_WEBHOOK_URL SNMP_COMMUNITY_PFSENSE SNMP_COMMUNITY_APC SNMP_COMMUNITY_MOKERLINK @@ -101,27 +103,67 @@ if [[ -f "${SNMP_SRC}" ]]; then fi # --------------------------------------------------------------------------- -# Render the Alertmanager webhook URL +# Render the Alertmanager receiver URLs # # Alertmanager does not expand environment variables in its config. `url_file` -# is the supported mechanism, so the URL is written to a file that compose +# is the supported mechanism, so each URL is written to a file that compose # mounts read-only. +# +# One file per channel, driven by this list rather than three copies of the +# same four lines — the three receivers in alertmanager.yaml were three names +# for one URL until #66, and the shape that made that easy was a block written +# out once and never generalised. Adding a fourth channel is now one entry here +# plus one in REQUIRED above. +# +# The variable name is not derived from the filename (or vice versa): the +# default channel's variable predates the other two and is not +# ALERTMANAGER_DEFAULT_WEBHOOK_URL, and renaming a key in an encrypted file to +# suit a loop is a worse trade than writing the pair out. # --------------------------------------------------------------------------- +AM_CHANNELS=( + "ALERTMANAGER_WEBHOOK_URL:webhook_url" + "ALERTMANAGER_URGENT_WEBHOOK_URL:urgent_url" + "ALERTMANAGER_SECURITY_WEBHOOK_URL:security_url" +) AM_OUT_DIR="${STACK_DIR}/alertmanager/.rendered" if [[ -f "${STACK_DIR}/alertmanager/alertmanager.yaml" ]]; then - info "rendering alertmanager webhook_url" + info "rendering ${#AM_CHANNELS[@]} alertmanager receiver URL(s)" mkdir -p "${AM_OUT_DIR}" - printf '%s' "${ALERTMANAGER_WEBHOOK_URL}" > "${AM_OUT_DIR}/webhook_url" - chmod 600 "${AM_OUT_DIR}/webhook_url" + chmod 700 "${AM_OUT_DIR}" + for channel in "${AM_CHANNELS[@]}"; do + var="${channel%%:*}" + file="${channel##*:}" + # No trailing newline: Alertmanager takes the file's whole content as the + # URL, and a newline in a URL is a delivery error rather than a warning. + printf '%s' "${!var}" > "${AM_OUT_DIR}/${file}" + chmod 600 "${AM_OUT_DIR}/${file}" + done + unset channel var file + + # Every url_file alertmanager.yaml names must be one this loop just wrote. A + # url_file that does not exist is not a config error — Alertmanager reads it + # at notify time, so the stack starts, amtool check-config passes, and the + # first real alert is the thing that discovers the missing file. Adding a + # receiver and forgetting AM_CHANNELS now fails here at deploy time instead. + while read -r wanted; do + [[ -f "${AM_OUT_DIR}/${wanted}" ]] \ + || die "alertmanager.yaml reads ${wanted}, which nothing in AM_CHANNELS renders" + # + # Anchored on `url_file:` rather than on the path fragment. A bare + # `secrets/[a-z_]+` also matched `secrets/observability.sops.yaml` in this + # file's own header comment, and reported the header as a missing channel. + done < <(grep -oE 'url_file:[[:space:]]*/etc/alertmanager/secrets/[a-z_]+' \ + "${STACK_DIR}/alertmanager/alertmanager.yaml" \ + | sed 's|.*/||' | sort -u) fi # --------------------------------------------------------------------------- # Write .env for compose interpolation # # Only the values compose actually interpolates are written here. The SNMP -# communities go into the rendered snmp.yaml and the webhook URL into -# webhook_url; copying them into .env as well would spread the same secret -# across three files for no benefit. +# communities go into the rendered snmp.yaml and the receiver URLs into their +# own files under alertmanager/.rendered; copying them into .env as well would +# spread the same secret across two files for no benefit. # --------------------------------------------------------------------------- COMPOSE_VARS=(GRAFANA_ADMIN_USER GRAFANA_ADMIN_PASSWORD GRAFANA_RENDERER_TOKEN) ENV_FILE="${STACK_DIR}/.env" diff --git a/scripts/validate.sh b/scripts/validate.sh index afa7340..c6dbde1 100755 --- a/scripts/validate.sh +++ b/scripts/validate.sh @@ -143,6 +143,42 @@ if ((${#AMTOOL[@]})); then "${AMTOOL[@]}" check-config "${STACK}/alertmanager/alertmanager.yaml" fail "amtool check-config" fi + + # check-config proves the tree parses and that every route names a receiver + # that exists. It says nothing about WHICH receiver an alert reaches, and the + # routing tree is first-match-wins with no `continue` anywhere — so moving a + # category route below the bare `severity` routes stops it matching while + # check-config still passes. That is the #66 failure shape exactly: a config + # that loads, validates, and delivers to the wrong place. + # + # Each line below is one row of the table in alertmanager.yaml. + # --verify.receivers exits non-zero when the resolved receiver differs. + routes_ok=1 + while read -r expected labels; do + [[ -n "${expected}" ]] || continue + # shellcheck disable=SC2086 # labels is a deliberate word-split list + if ! "${AMTOOL[@]}" config routes test \ + --config.file="${STACK}/alertmanager/alertmanager.yaml" \ + --verify.receivers="${expected}" ${labels} >/dev/null 2>&1; then + printf 'expected %s for %s, got: ' "${expected}" "${labels}" >&2 + "${AMTOOL[@]}" config routes test \ + --config.file="${STACK}/alertmanager/alertmanager.yaml" ${labels} >&2 + routes_ok=0 + fi + done <<'ROUTES' +urgent severity=critical category=power +security severity=critical category=security +security severity=warning category=security +urgent severity=critical category=availability +default severity=warning category=capacity +null severity=info category=correctness +ROUTES + + if ((routes_ok)); then + pass "amtool config routes test (6 assertions)" + else + fail "amtool config routes test" + fi else skip "no amtool and no docker daemon" fi diff --git a/scripts/verify-key-backup.sh b/scripts/verify-key-backup.sh index c1f85fd..3c888b7 100755 --- a/scripts/verify-key-backup.sh +++ b/scripts/verify-key-backup.sh @@ -216,6 +216,8 @@ fi REQUIRED=( GRAFANA_ADMIN_PASSWORD ALERTMANAGER_WEBHOOK_URL + ALERTMANAGER_URGENT_WEBHOOK_URL + ALERTMANAGER_SECURITY_WEBHOOK_URL SNMP_COMMUNITY_PFSENSE SNMP_COMMUNITY_APC SNMP_COMMUNITY_MOKERLINK diff --git a/secrets/README.md b/secrets/README.md index 72037d6..df8a7d7 100644 --- a/secrets/README.md +++ b/secrets/README.md @@ -84,10 +84,11 @@ file, exports the values as environment variables, and: (the Grafana credentials); - renders `snmp-exporter/snmp.yaml`'s `${SNMP_COMMUNITY_*}` placeholders into `snmp-exporter/.rendered/snmp.yaml`, which is what the container mounts; -- writes `alertmanager/.rendered/webhook_url`, because Alertmanager does not +- writes one file per notification channel into `alertmanager/.rendered/` — + `webhook_url`, `urgent_url` and `security_url` — because Alertmanager does not expand environment variables and reads receiver URLs via `url_file`. -All three are gitignored, and each secret is written to exactly one of them. +All of them are gitignored, and each secret is written to exactly one of them. Nothing writes a secret into a tracked file. ## Rotating diff --git a/secrets/observability.example.yaml b/secrets/observability.example.yaml index d6bb7b1..b241f31 100644 --- a/secrets/observability.example.yaml +++ b/secrets/observability.example.yaml @@ -37,4 +37,18 @@ SNMP_COMMUNITY_ILO: change-me-ilo # Where Alertmanager delivers notifications. Any webhook receiver works — ntfy, # Gotify, a Home Assistant automation, or a Discord/Slack incoming webhook. +# +# THREE URLS, THREE DESTINATIONS. These were one URL under three receiver names +# until #66, which made `urgent` and `default` differ only in how often they +# repeated: a UPS on battery and a slow scrape arrived in the same place. The +# routing tree in alertmanager/alertmanager.yaml decides which alert is urgent; +# only a separate destination can make that difference audible, because that is +# where the per-topic sound and do-not-disturb settings live. +# +# With ntfy, three topics on one server is the whole setup — the topic is the +# last path segment, and a topic needs no creation step. Point them all at the +# same topic if you would rather have one channel; nothing breaks, but then the +# receiver names are lying again, so say so in docs/observability.md. ALERTMANAGER_WEBHOOK_URL: https://ntfy.example.invalid/homelab-alerts +ALERTMANAGER_URGENT_WEBHOOK_URL: https://ntfy.example.invalid/homelab-urgent +ALERTMANAGER_SECURITY_WEBHOOK_URL: https://ntfy.example.invalid/homelab-security diff --git a/stacks/observability/alertmanager/alertmanager.yaml b/stacks/observability/alertmanager/alertmanager.yaml index b710213..315da4b 100644 --- a/stacks/observability/alertmanager/alertmanager.yaml +++ b/stacks/observability/alertmanager/alertmanager.yaml @@ -1,15 +1,22 @@ --- # Alert routing. # -# The receiver URL is a secret (a webhook URL is a bearer credential — anyone -# holding it can post to it), so it is not in this file. +# Receiver URLs are secrets (a webhook URL is a bearer credential — anyone +# holding it can post to it), so they are not in this file. # # Alertmanager does NOT expand environment variables in its config. The # supported mechanism is `url_file`, which reads the URL from disk at notify # time. scripts/render-config.sh decrypts secrets/observability.sops.yaml and -# writes .rendered/webhook_url, which compose mounts read-only at -# /etc/alertmanager/secrets/. Both the rendered directory and its contents are -# gitignored. +# writes one file per channel into .rendered/, which compose mounts read-only +# at /etc/alertmanager/secrets/. Both the rendered directory and its contents +# are gitignored. +# +# There are three channels and three destinations. They used to be three names +# for one URL, which meant `urgent` and `default` differed only in how often +# they repeated — a UPS on battery and a slow scrape landed in the same place +# (#66). Point them at three separate topics on whatever receiver you use, so +# the phone can treat them differently: the routing tree below decides which +# alert is urgent, but only a distinct destination can make that audible. global: resolve_timeout: 5m @@ -22,6 +29,11 @@ route: # Don't re-nag every hour for something already known to be broken. repeat_interval: 12h + # FIRST MATCH WINS, and none of these set `continue`, so ORDER IS THE DESIGN. + # Every category-specific route has to sit above the bare `severity` routes + # at the bottom; move one below them and it silently stops matching while the + # config still loads and still validates. scripts/validate.sh asserts the + # whole table with `amtool config routes test` for exactly that reason. routes: # Power events page immediately and repeat aggressively — a UPS on battery # with no spare battery installed is a short fuse. @@ -32,6 +44,29 @@ route: group_wait: 0s repeat_interval: 30m + # Security gets its own destination at both severities. There are ten rules + # labelled category=security across security.rules.yaml and + # network.rules.yaml — SSH brute force, a terminal segment reaching the + # internal network, IoT lateral movement, priority-1 Suricata. Routed on + # severity alone, the warning-severity half of that list arrived in the + # default channel on a 12-hour repeat, indistinguishable from a disk + # filling up. `category = "power"` above is the precedent. + - receiver: security + matchers: + - severity = "critical" + - category = "security" + group_wait: 0s + repeat_interval: 1h + + # Still 4h rather than the default 12h: a brute-force attempt that is still + # running four hours later is a different situation from one that stopped. + - receiver: security + matchers: + - severity = "warning" + - category = "security" + group_wait: 30s + repeat_interval: 4h + - receiver: urgent matchers: - severity = "critical" @@ -71,5 +106,10 @@ receivers: - name: urgent webhook_configs: - - url_file: /etc/alertmanager/secrets/webhook_url + - url_file: /etc/alertmanager/secrets/urgent_url + send_resolved: true + + - name: security + webhook_configs: + - url_file: /etc/alertmanager/secrets/security_url send_resolved: true diff --git a/stacks/observability/compose.yaml b/stacks/observability/compose.yaml index b52322c..860e61c 100644 --- a/stacks/observability/compose.yaml +++ b/stacks/observability/compose.yaml @@ -73,9 +73,9 @@ services: - --web.external-url=http://${MONITORING_HOST:-10.0.99.20}:${ALERTMANAGER_PORT:-9093} volumes: - ./alertmanager/alertmanager.yaml:/etc/alertmanager/alertmanager.yaml:ro - # Contains webhook_url, written by scripts/render-config.sh. Alertmanager - # does not expand environment variables, so the receiver URL is read from - # a file via `url_file`. + # One file per notification channel, written by scripts/render-config.sh. + # Alertmanager does not expand environment variables, so each receiver + # URL is read from a file via `url_file`. - ./alertmanager/.rendered:/etc/alertmanager/secrets:ro - alertmanager-data:/alertmanager ports: