From ae695aed716e810221fd8cf8a19fcf8b79e61dea Mon Sep 17 00:00:00 2001
From: Mike Langmayr <1809691+mikelangmayr@users.noreply.github.com>
Date: Thu, 17 Sep 2026 18:08:21 -0700
Subject: [PATCH] Deploy the keyword archiver: shim, config, systemd instance
 and a secrets file

---
 README.md                               |   2 +-
 config/hispec/hispec_keygrabber.yaml    | 134 ++++++++++++++++++++++++
 daemons/generic/keygrabber              |  16 +++
 pyproject.toml                          |   3 +-
 systemd/README.md                       |  43 +++++++-
 systemd/hispec-daemon@.service          |   4 +
 systemd/install.sh                      |  15 +++
 systemd/instances/hispec_keygrabber.env |  16 +++
 8 files changed, 229 insertions(+), 4 deletions(-)
 create mode 100644 config/hispec/hispec_keygrabber.yaml
 create mode 100755 daemons/generic/keygrabber
 create mode 100644 systemd/instances/hispec_keygrabber.env

diff --git a/README.md b/README.md
index b57869d..b207f72 100644
--- a/README.md
+++ b/README.md
@@ -17,7 +17,7 @@ or the [daemon inventory][inventory] to find a specific mechanism.
 
 | Path | Contents |
 | --- | --- |
-| `daemons/` | Deployable device daemons. `generic/` are config-driven and shared across subsystems; `hsfei/`, `hscal/` are subsystem-specific. |
+| `daemons/` | Deployable device daemons. `generic/` are config-driven and shared across subsystems; `hsfei/`, `hscal/` are subsystem-specific. `generic/keygrabber` is the exception that drives no hardware: it records the other daemons' keywords into InfluxDB for Grafana. |
 | `config/` | One YAML file per deployed daemon instance, organised by subsystem. |
 | `src/hispec/` | Installable package: the `HispecDaemon` base class and the `driver/` submodules. |
 | `systemd/` | Template unit, per-instance env files and installer. See [systemd/README.md](systemd/README.md). |
diff --git a/config/hispec/hispec_keygrabber.yaml b/config/hispec/hispec_keygrabber.yaml
new file mode 100644
index 0000000..a2af913
--- /dev/null
+++ b/config/hispec/hispec_keygrabber.yaml
@@ -0,0 +1,134 @@
+# Keyword Archiver — polls the HISPEC daemons and writes to InfluxDB for Grafana
+#
+# Usage:
+#   daemons/generic/keygrabber -c config/hispec/hispec_keygrabber.yaml
+#
+# Needs libby's optional extra on the host: pip install 'libby[influxdb]'
+# The InfluxDB token comes from the environment, never from this file; the
+# systemd instance file is where it gets set.
+
+peer_id: keygrabber
+group_id: hispec
+
+sink:
+  type: influxdb
+  url: http://localhost:8086
+  org: hispec
+  bucket: telemetry
+  token_env: HISPEC_INFLUX_TOKEN
+
+# Concurrent reads. Collections are read in parallel up to this many at once,
+# but a single daemon still answers one request at a time, so raising this only
+# helps spread reads across peers.
+workers: 4
+
+defaults:
+  interval_s: 30.0
+  timeout_s: 2.0
+  refresh_s: 300.0
+
+# Each collection reads one peer. "%" takes everything readable, minus the
+# defaults libby excludes (uptime, lasterror). Narrow a collection's keywords
+# list once a dashboard shows which ones matter; that costs nothing to change,
+# since a reload re-reads this file.
+collections:
+
+  # Motion: positions and limits move during an observation, so read faster
+  adc:
+    peer: hsfei.adc
+    keywords: ["%"]
+    interval_s: 5.0
+  atcl:
+    peer: hsfei.atcl
+    keywords: ["%"]
+    interval_s: 5.0
+  atcp:
+    peer: hsfei.atcp
+    keywords: ["%"]
+    interval_s: 5.0
+  feipo:
+    peer: hsfei.feipo
+    keywords: ["%"]
+    interval_s: 5.0
+  lsm:
+    peer: hsfei.lsm
+    keywords: ["%"]
+    interval_s: 5.0
+  ms:
+    peer: hsfei.ms
+    keywords: ["%"]
+    interval_s: 5.0
+  piaagimb:
+    peer: hsfei.piaagimb
+    keywords: ["%"]
+    interval_s: 5.0
+  piaagimr:
+    peer: hsfei.piaagimr
+    keywords: ["%"]
+    interval_s: 5.0
+
+  piaadeploy:
+    peer: hsfei.piaadeploy
+    keywords: ["%"]
+    interval_s: 5.0
+  yjfam:
+    peer: hsfei.yjfam
+    keywords: ["%"]
+    interval_s: 5.0
+  hkfam:
+    peer: hsfei.hkfam
+    keywords: ["%"]
+    interval_s: 5.0
+
+  # Filter wheels and attenuators: discrete positions, changed rarely
+  atcfw:
+    peer: hsfei.atcfw
+    keywords: ["%"]
+  hkcalfwheel1:
+    peer: hscal.hkcalfwheel1
+    keywords: ["%"]
+  hkcalfwheel2:
+    peer: hscal.hkcalfwheel2
+    keywords: ["%"]
+  hkgcellfwheel:
+    peer: hscal.hkgcellfwheel
+    keywords: ["%"]
+  yjcalfwheel1:
+    peer: hscal.yjcalfwheel1
+    keywords: ["%"]
+  yjcalfwheel2:
+    peer: hscal.yjcalfwheel2
+    keywords: ["%"]
+  hketatten:
+    peer: hscal.hketatten
+    keywords: ["%"]
+
+  # Thermal: slow-moving, but the long-term trend is the point
+  atctherm:
+    peer: hsfei.atctherm
+    keywords: ["%"]
+  hkettherm:
+    peer: hscal.hkettherm
+    keywords: ["%"]
+  yjettherm:
+    peer: hscal.yjettherm
+    keywords: ["%"]
+  gcellheater1:
+    peer: hscal.gcellheater1
+    keywords: ["%"]
+  gcellheater2:
+    peer: hscal.gcellheater2
+    keywords: ["%"]
+
+  # Cryo and vacuum: slowest of all, and the trend that matters over weeks
+  atccryo:
+    peer: hsfei.atccryo
+    keywords: ["%"]
+    interval_s: 60.0
+  atcpress:
+    peer: hsfei.atcpress
+    keywords: ["%"]
+    interval_s: 60.0
+
+logging:
+  level: INFO
diff --git a/daemons/generic/keygrabber b/daemons/generic/keygrabber
new file mode 100755
index 0000000..07b8888
--- /dev/null
+++ b/daemons/generic/keygrabber
@@ -0,0 +1,16 @@
+#!/usr/bin/env python3
+"""
+Keyword Archiver Daemon
+
+Polls keywords from the other HISPEC daemons and writes them to InfluxDB for
+Grafana. The daemon itself lives in libby, since nothing about it is
+instrument-specific; this shim exists so the systemd unit template, which runs
+``daemons/<subdir>/<script>``, can launch it like any other HISPEC daemon.
+
+Needs libby's optional extra: ``pip install 'libby[influxdb]'``.
+"""
+
+from libby.keygrabber.main import main
+
+if __name__ == '__main__':
+    raise SystemExit(main())
diff --git a/pyproject.toml b/pyproject.toml
index 3b71267..3d32e93 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -24,7 +24,8 @@ dependencies = [
     "libximc",
     "scipy",
     "pyyaml",
-    "libby@git+https://github.com/CaltechOpticalObservatories/libby.git",
+    # influxdb extra: needed by daemons/generic/keygrabber
+    "libby[influxdb]@git+https://github.com/CaltechOpticalObservatories/libby.git",
     "hardware_device_base@git+https://github.com/COO-Utilities/hardware_device_base"
 ]
 
diff --git a/systemd/README.md b/systemd/README.md
index f7d7b79..5b280eb 100644
--- a/systemd/README.md
+++ b/systemd/README.md
@@ -24,8 +24,9 @@ sudo ./systemd/install.sh
 
 `install.sh` creates an unprivileged `hispec` user, a `hispec-ops` group,
 `/etc/hispec/{,instances/}` and `/var/log/hispec/` (group-writable by
-`hispec-ops`), a venv at `/opt/hispec/venv` with the repo `pip install -e`d
-into it, installs `hispec-daemon@.service`, and installs a polkit rule
+`hispec-ops`), a root-only `/etc/hispec/secrets.env`, a venv at
+`/opt/hispec/venv` with the repo `pip install -e`d into it, installs
+`hispec-daemon@.service`, and installs a polkit rule
 (`systemd/polkit/49-hispec-daemons.rules`) letting `hispec-ops` members
 start/stop/restart `hispec-daemon@*` units without sudo. Override
 `HISPEC_REPO_DIR` / `HISPEC_VENV_DIR` for different paths.
@@ -59,6 +60,44 @@ ship with the driver, and the path in the config is relative to the installed
 `hispec` package, so there is nothing extra to deploy and nothing that
 depends on the unit's working directory.
 
+### Secrets
+
+`/etc/hispec/instances/*.env` is group-readable by every `hispec-ops` member,
+which is right for configs and wrong for a credential. Every unit also reads
+`/etc/hispec/secrets.env` if it exists, which `install.sh` creates root-only
+(0600), so a secret goes there instead:
+
+```bash
+sudo tee -a /etc/hispec/secrets.env <<'EOF'
+HISPEC_INFLUX_TOKEN=<InfluxDB write token>
+EOF
+sudo systemctl restart hispec-daemon@hispec_keygrabber
+```
+
+Only `generic/keygrabber` needs one today. A config file names the variable it
+expects rather than holding the value, so the value never reaches git.
+
+### The keyword archiver
+
+`hispec_keygrabber` is the one instance that reads the other daemons rather
+than any hardware, writing their keywords to InfluxDB for Grafana. Two extra
+steps beyond the recipe above:
+
+```bash
+/opt/hispec/venv/bin/pip install 'libby[influxdb]'   # optional extra
+sudo tee -a /etc/hispec/secrets.env                  # the token, as above
+```
+
+It needs no hardware, owns no device, and can be restarted freely. Pausing it
+does not need a restart at all:
+
+```bash
+libby modify hispec.keygrabber.enabled=false     # stop collecting, stay up
+libby show   hispec.keygrabber.%                 # counters and health
+libby show   hispec.keygrabber.%.%               # per-collection cadence
+libby modify hispec.keygrabber.reload=1          # re-read the config file
+```
+
 For a new daemon/config not yet in the table, add its config under
 `config/<subsystem>/`, write a matching `systemd/instances/<name>.env`
 (`HISPEC_DAEMON=...`, `HISPEC_CONFIG=...`), deploy both the same way, then
diff --git a/systemd/hispec-daemon@.service b/systemd/hispec-daemon@.service
index 6f42020..335d844 100644
--- a/systemd/hispec-daemon@.service
+++ b/systemd/hispec-daemon@.service
@@ -29,6 +29,10 @@ User=hispec
 Group=hispec
 
 EnvironmentFile=/etc/hispec/instances/%i.env
+# Optional, root-only, shared by every instance: for secrets that must not sit
+# in the instance files above, since /etc/hispec/instances is group-readable by
+# hispec-ops. systemd reads this as root before dropping to User= below.
+EnvironmentFile=-/etc/hispec/secrets.env
 # Drivers write their own .log next to the working directory, so this has to
 # be writable by User= above; the repo checkout is not
 WorkingDirectory=/var/log/hispec
diff --git a/systemd/install.sh b/systemd/install.sh
index a70ea8f..17177a1 100755
--- a/systemd/install.sh
+++ b/systemd/install.sh
@@ -42,6 +42,21 @@ install -d -o hispec -g hispec-ops -m 2775 /etc/hispec
 install -d -o hispec -g hispec-ops -m 2775 /etc/hispec/instances
 install -d -o hispec -g hispec-ops -m 2750 /var/log/hispec
 
+# Shared secrets, read by every unit if present. Root-only on purpose: the
+# instance files above are group-readable by hispec-ops, which is right for
+# configs and wrong for a database token.
+if [[ ! -e /etc/hispec/secrets.env ]]; then
+    cat > /etc/hispec/secrets.env <<'EOF'
+# Environment for every hispec-daemon@ instance. Root-only; keep secrets here
+# rather than in /etc/hispec/instances/*.env, which operators can read.
+#
+# HISPEC_INFLUX_TOKEN=<InfluxDB write token, for generic/keygrabber>
+EOF
+    echo "created /etc/hispec/secrets.env"
+fi
+chown root:root /etc/hispec/secrets.env
+chmod 0600 /etc/hispec/secrets.env
+
 # Python environment (editable install so `git pull` picks up code changes
 # without reinstalling).
 if [[ ! -x "$VENV_DIR/bin/python3" ]]; then
diff --git a/systemd/instances/hispec_keygrabber.env b/systemd/instances/hispec_keygrabber.env
new file mode 100644
index 0000000..ecd955c
--- /dev/null
+++ b/systemd/instances/hispec_keygrabber.env
@@ -0,0 +1,16 @@
+# Keyword Archiver — polls the HISPEC daemons into InfluxDB for Grafana
+#
+# systemctl enable --now hispec-daemon@hispec_keygrabber
+#
+# Deploy the config alongside this file:
+#   config/hispec/hispec_keygrabber.yaml -> /etc/hispec/hispec_keygrabber.yaml
+#
+# The InfluxDB token does NOT belong here: this directory is group-readable by
+# every hispec-ops member. Put it in /etc/hispec/secrets.env instead, which is
+# root-only and which the unit reads if present:
+#   HISPEC_INFLUX_TOKEN=<token>
+#
+# Needs libby's optional extra in the venv:
+#   /opt/hispec/venv/bin/pip install 'libby[influxdb]'
+HISPEC_DAEMON=generic/keygrabber
+HISPEC_CONFIG=/etc/hispec/hispec_keygrabber.yaml