tl-signer: private keys out of the API process, per-node keys, key history (1.12.0)

CHANGELOG.md

@@ -6,6 +6,35 @@ Versions follow [Semantic Versioning](https://semver.org/).
 
 ---
 
+## [1.12.0] — 2026-09-24
+
+### Added
+- `tl-signer` (`signer/tl_signer.py`): a separate service that holds the private keys and signs over a Unix
+  socket. Closed operations only (chain hash, reputation statement, JWS built by the signer, Rekor artifact),
+  bounded inputs, no operation returns key material. Keys are born in the signer, one per node.
+- Signer mode: with `TL_SIGNER_SOCKET` set, the Trust Layer reads and creates no key file, takes its public keys
+  from the socket at startup, and refuses to start if its key is missing from the published history. Unset, the
+  legacy `.pem` keys are used as before.
+- Key history (`trust_layer/published_keys.json`): `/v1/pubkey` adds `kid`, `rekor_kid`, `keys` and `rekor_keys`;
+  `/.well-known/did.json` lists every key, the node key first, and only keys not retired may assert.
+- Proofs carry `arkforge_kid`, attestations and reputation scores `signature_kid`. The kid is added after the
+  chain hash, like `arkforge_pubkey`.
+- `scripts/verify_proof.py` picks the key a proof names (or, for older proofs, the key it carries) from the
+  history, and refuses a key retired before the proof's date. Rekor entries are attributed to any published
+  Rekor key of the history.
+
+- Under its own user, the service reads the CEO vault from systemd credentials (`LoadCredential=vault.json.enc`,
+  `vault_key`); the master key is only in the environment while the vault loads.
+- `/v1/health` adds `signing` (`mode`, `kid`, `self_test`). In signer mode the Trust Layer signs and verifies a
+  fixed hash at startup and refuses to start if it fails.
+- Deploy script: the standby and primary canaries require `signing.self_test == ok` and log the signing kid.
+- A paid request never loses its money to an unavailable signer: `/v1/proxy` checks the signer before counting or
+  charging (503 `signing_unavailable`, nothing charged), and refunds the debit if the signer is lost between the
+  charge and the signature (`refund` credit transaction).
+
+### Changed
+- CTEF verdicts: the JWS header (`alg`, `kid`) is set by the signer, no longer a hardcoded `#key-1`.
+
 ## [1.11.4] — 2026-09-21
 
 ### Fixed

scripts/deploy_trust_layer_prod.sh

@@ -107,6 +107,15 @@ except Exception: print('')
 }
 
 # --- Health of a node: "status version role" ---
+# Signing on a node, from /v1/health (1.12.0+): "self_test kid mode".
+signing_of() {
+    python3 -c "
+import sys, json
+try:
+    s = json.load(sys.stdin).get('signing') or {}
+    print(s.get('self_test', ''), s.get('kid', ''), s.get('mode', ''))
+except Exception: print('')"
+}
 local_health() { curl -s --max-time 5 "$LOCAL_URL/v1/health" 2>/dev/null || true; }
 standby_health() { $SSH "$STANDBY_HOST" "curl -s --max-time 5 $LOCAL_URL/v1/health" 2>/dev/null || true; }
 standby_http_code() { $SSH "$STANDBY_HOST" "curl -s -o /dev/null -w '%{http_code}' --max-time 5 $LOCAL_URL$1" 2>/dev/null || echo "000"; }
@@ -325,6 +334,14 @@ if [ "$STANDBY_OK" = true ]; then
     done
 fi
 
+# The standby signs with its own key after a failover: its startup
+# self-test must have signed and verified with the key it publishes.
+if [ "$STANDBY_OK" = true ]; then
+    SIG=$(standby_health | signing_of)
+    log "Phase 2a canary: standby signing = $SIG"
+    case "$SIG" in ok\ *) ;; *) STANDBY_OK=false ;; esac
+fi
+
 if [ "$STANDBY_OK" = false ]; then
     rollback_standby
     rollback_local_tree
@@ -350,7 +367,9 @@ for i in $(seq 1 6); do
     H=$(local_health)
     log "Phase 2b attempt $i/6: status=$(echo "$H" | json_field status) version=$(echo "$H" | json_field version) role=$(echo "$H" | json_field role)"
     if [ "$(echo "$H" | json_field status)" = "ok" ] && [ "$(echo "$H" | json_field version)" = "$NEW_VERSION" ]; then
-        PRIMARY_OK=true
+        SIG=$(echo "$H" | signing_of)
+        log "Phase 2b: primary signing = $SIG"
+        case "$SIG" in ok\ *) PRIMARY_OK=true ;; esac
         break
     fi
 done

scripts/verify_proof.py

@@ -16,7 +16,8 @@
                          internal consistency. On its own it is NOT evidence:
                          whoever fabricates a proof produces coherent hashes.
   2. Ed25519 signature — ArkForge's own signature over the chain hash, checked
-                         against the key published at /.well-known/did.json.
+                         against ArkForge's published key history (/v1/pubkey):
+                         the key the proof names, not retired at the proof's date.
                          Proves ArkForge issued it. Still not independent.
   3. Batch anchor      — the chain hash is a leaf of the batch Merkle tree whose
                          root was anchored. Self-consistency again, but it is what
@@ -57,6 +58,7 @@
 import sys
 import tempfile
 import urllib.request
+from datetime import datetime
 from pathlib import Path
 
 # Overridable so the procedure can be run verbatim against another instance —
@@ -410,25 +412,64 @@ def _b64url_decode(s):
     return base64.urlsafe_b64decode(s + "=" * (-len(s) % 4))
 
 
+def _utc(ts):
+    """Seconds precision is enough to order a proof and a key retirement (UTC)."""
+    return datetime.strptime(ts[:19], "%Y-%m-%dT%H:%M:%S")
+
+
+def published_ed25519_keys():
+    """ArkForge's key history (/v1/pubkey `keys`). A server that predates key
+    rotation publishes a single key, the first verification method of did.json."""
+    pub = json.loads(fetch(f"{TRUST_LAYER_BASE}/v1/pubkey"))
+    if pub.get("keys"):
+        return pub["keys"]
+    did = json.loads(fetch(f"{TRUST_LAYER_BASE}/.well-known/did.json"))
+    x = did["verificationMethod"][0]["publicKeyJwk"]["x"]
+    return [{"kid": "key-1", "public": "ed25519:" + x, "retired_at": None}]
+
+
+def _published_key_for(proof, keys):
+    """(public key, None) for the proof, or (None, reason). The proof names its key
+    by `arkforge_kid`; proofs issued before rotation carry only `arkforge_pubkey`.
+    A key is valid for proofs dated before its retirement, never after."""
+    kid = proof.get("arkforge_kid")
+    embedded = proof.get("arkforge_pubkey") or ""
+    if kid:
+        entry = next((k for k in keys if k.get("kid") == kid), None)
+        if entry is None:
+            return None, f"key {kid} named by the proof is not published"
+    else:
+        entry = next((k for k in keys if k.get("public") == embedded), None)
+        if entry is None:
+            return None, "key in proof does not match any key published (/v1/pubkey, did.json)"
+    if embedded and embedded != entry.get("public"):
+        return None, f"key in proof is not the key published as {entry.get('kid')}"
+    retired = entry.get("retired_at")
+    if retired:
+        ts = proof.get("timestamp")
+        if not ts or _utc(ts) >= _utc(retired):
+            return None, f"key {entry.get('kid')} was retired at {retired}, proof dated {ts}"
+    return entry["public"], None
+
+
 def check_ed25519(proof, chain_hex, rep, offline):
     sig_str = proof.get("arkforge_signature")
     if not sig_str:
         rep.add("Ed25519 (ArkForge)", SKIP, "proof carries no signature")
         return
-    published = None
-    if not offline:
+    if offline:
+        pub = proof.get("arkforge_pubkey") or ""
+    else:
         try:
-            did = json.loads(fetch(f"{TRUST_LAYER_BASE}/.well-known/did.json"))
-            published = did["verificationMethod"][0]["publicKeyJwk"]["x"]
+            keys = published_ed25519_keys()
         except Exception as e:
-            rep.add("Ed25519 (ArkForge)", FAIL, f"cannot fetch published key: {e}")
+            rep.add("Ed25519 (ArkForge)", FAIL, f"cannot fetch published keys: {e}")
             return
-    embedded = (proof.get("arkforge_pubkey") or "").replace("ed25519:", "")
-    if published and embedded and published != embedded:
-        rep.add("Ed25519 (ArkForge)", FAIL,
-                "key in proof does not match the key published at did.json")
-        return
-    pub_b64 = published or embedded
+        pub, reason = _published_key_for(proof, keys)
+        if reason:
+            rep.add("Ed25519 (ArkForge)", FAIL, reason)
+            return
+    pub_b64 = pub.replace("ed25519:", "")
     if not pub_b64:
         rep.add("Ed25519 (ArkForge)", SKIP, "no public key available")
         return
@@ -712,13 +753,19 @@ def _check_rekor(proof, chain_hex, rep, offline):
         rep.add("Sigstore Rekor", FAIL, "entry signature does not verify")
         return
 
-    # 4c. Attribution: is the submitting key the one ArkForge publishes?
+    # 4c. Attribution: is the submitting key one ArkForge publishes? Any Rekor key of
+    # the history counts, provided it was not retired before the proof's date.
     attributed = None
     try:
         published = json.loads(fetch(f"{TRUST_LAYER_BASE}/v1/pubkey"))
-        pub_rekor = published.get("rekor_pubkey")
-        if pub_rekor:
-            attributed = _normalise_pem(pub_rekor) == _normalise_pem(submitter_pem.decode())
+        ts = proof.get("timestamp")
+        candidates = [published.get("rekor_pubkey")] + [
+            k.get("public_pem") for k in published.get("rekor_keys") or []
+            if not k.get("retired_at") or (ts and _utc(ts) < _utc(k["retired_at"]))
+        ]
+        candidates = [_normalise_pem(c) for c in candidates if c]
+        if candidates:
+            attributed = _normalise_pem(submitter_pem.decode()) in candidates
     except Exception:
         attributed = None
 

signer/tl_signer.py

@@ -0,0 +1,193 @@
+"""tl-signer: holds the Trust Layer private keys and signs through a Unix socket.
+
+Runs as its own user, with no network, from a root-owned file (arkforge-infra, role
+tl_signer). The Trust Layer reaches it through /run/tl-signer/sign.sock; whoever can
+open the socket can sign, nobody can read a key. Keys are born here on first start,
+one per node, and never leave the state directory: a lost key is replaced by a new
+one, published next to the old ones (proof-spec, key history).
+
+Protocol: one JSON object per line in, one JSON object per line out. Closed set of
+operations, each with a bounded, validated input. No operation returns private key
+material.
+
+Depends on the standard library and `cryptography` only (python3-cryptography from
+the distribution), so the host does not need a venv.
+"""
+
+import base64
+import json
+import logging
+import os
+import re
+import socket
+import socketserver
+import sys
+import threading
+from pathlib import Path
+
+from cryptography.hazmat.primitives import hashes, serialization
+from cryptography.hazmat.primitives.asymmetric import ec
+from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
+
+MAX_REQUEST = 16 * 1024
+MAX_JWS_PAYLOAD = 8 * 1024
+CHAIN_HASH = re.compile(r"[0-9a-f]{64}")
+# reputation.py signs "{agent_id}:{score}:{computed_at}".
+REPUTATION = re.compile(
+    r"sha256:[0-9a-f]{64}:\d{1,3}:\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d{1,6})?(\+00:00|Z)"
+)
+
+log = logging.getLogger("tl-signer")
+
+
+def _b64url(data: bytes) -> str:
+    return base64.urlsafe_b64encode(data).rstrip(b"=").decode("ascii")
+
+
+def _load_or_create(path: Path, generate):
+    """Load a PKCS8 key, or create it once. The file never leaves this process."""
+    if path.exists():
+        return serialization.load_pem_private_key(path.read_bytes(), password=None)
+    key = generate()
+    pem = key.private_bytes(
+        serialization.Encoding.PEM,
+        serialization.PrivateFormat.PKCS8,
+        serialization.NoEncryption(),
+    )
+    fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
+    with os.fdopen(fd, "wb") as f:
+        f.write(pem)
+    log.info("generated %s", path.name)
+    return key
+
+
+class Signer:
+    def __init__(self, state_dir: Path, kid: str, rekor_kid: str, did: str):
+        state_dir.mkdir(mode=0o700, parents=True, exist_ok=True)
+        self.kid, self.rekor_kid, self.did = kid, rekor_kid, did
+        self._ed = _load_or_create(state_dir / "ed25519.pem", Ed25519PrivateKey.generate)
+        self._rekor = _load_or_create(
+            state_dir / "rekor.pem", lambda: ec.generate_private_key(ec.SECP256R1())
+        )
+        self._lock = threading.Lock()
+        self.counts = {}
+        raw = self._ed.public_key().public_bytes(
+            serialization.Encoding.Raw, serialization.PublicFormat.Raw
+        )
+        self.ed_public = f"ed25519:{_b64url(raw)}"
+        self.rekor_public_pem = self._rekor.public_key().public_bytes(
+            serialization.Encoding.PEM, serialization.PublicFormat.SubjectPublicKeyInfo
+        ).decode("ascii")
+
+    def _ed_sign(self, message: str) -> str:
+        return f"ed25519:{_b64url(self._ed.sign(message.encode('utf-8')))}"
+
+    def handle(self, req: dict) -> dict:
+        op = req.get("op")
+        if op == "pubkeys":
+            return {
+                "ed25519": {"kid": self.kid, "public": self.ed_public},
+                "rekor": {"kid": self.rekor_kid, "public_pem": self.rekor_public_pem},
+                "did": self.did,
+            }
+        if op == "sign_chain_hash":
+            chain_hash = req.get("chain_hash")
+            if not isinstance(chain_hash, str) or not CHAIN_HASH.fullmatch(chain_hash):
+                raise ValueError("chain_hash must be 64 lowercase hex characters")
+            return {"kid": self.kid, "signature": self._ed_sign(chain_hash)}
+        if op == "sign_reputation":
+            payload = req.get("payload")
+            if not isinstance(payload, str) or not REPUTATION.fullmatch(payload):
+                raise ValueError("payload is not a reputation statement")
+            return {"kid": self.kid, "signature": self._ed_sign(payload)}
+        if op == "sign_jws":
+            # The header is ours, never the caller's.
+            payload = req.get("payload")
+            if not isinstance(payload, dict):
+                raise ValueError("payload must be a JSON object")
+            p = json.dumps(payload, separators=(",", ":")).encode("utf-8")
+            if len(p) > MAX_JWS_PAYLOAD:
+                raise ValueError("payload too large")
+            h = json.dumps({"alg": "EdDSA", "kid": f"{self.did}#{self.kid}"},
+                           separators=(",", ":")).encode("utf-8")
+            signing_input = f"{_b64url(h)}.{_b64url(p)}"
+            sig = self._ed.sign(signing_input.encode("ascii"))
+            return {"kid": self.kid, "jws": f"{signing_input}.{_b64url(sig)}"}
+        if op == "sign_rekor":
+            # hashedrekord artifact = the chain hash itself (rekor.py).
+            chain_hash = req.get("chain_hash")
+            if not isinstance(chain_hash, str) or not CHAIN_HASH.fullmatch(chain_hash):
+                raise ValueError("chain_hash must be 64 lowercase hex characters")
+            der = self._rekor.sign(chain_hash.encode("utf-8"), ec.ECDSA(hashes.SHA256()))
+            return {"kid": self.rekor_kid, "signature": base64.b64encode(der).decode("ascii")}
+        raise ValueError("unknown operation")
+
+    def count(self, op: str) -> int:
+        with self._lock:
+            self.counts[op] = self.counts.get(op, 0) + 1
+            return self.counts[op]
+
+
+class _Handler(socketserver.StreamRequestHandler):
+    def handle(self):
+        line = self.rfile.readline(MAX_REQUEST + 1)
+        signer: Signer = self.server.signer
+        try:
+            if len(line) > MAX_REQUEST or not line.endswith(b"\n"):
+                raise ValueError("request too long or not terminated")
+            req = json.loads(line)
+            if not isinstance(req, dict):
+                raise ValueError("request must be a JSON object")
+            resp = signer.handle(req)
+            n = signer.count(req["op"])
+            log.info("op=%s n=%d peer_uid=%s", req["op"], n, _peer_uid(self.request))
+        except (ValueError, json.JSONDecodeError) as e:
+            resp = {"error": str(e)}
+            log.warning("refused: %s peer_uid=%s", e, _peer_uid(self.request))
+        self.wfile.write((json.dumps(resp, separators=(",", ":")) + "\n").encode())
+
+
+def _peer_uid(sock) -> str:
+    try:
+        creds = sock.getsockopt(socket.SOL_SOCKET, socket.SO_PEERCRED, 12)
+        return str(int.from_bytes(creds[4:8], sys.byteorder))
+    except OSError:
+        return "?"
+
+
+class _Server(socketserver.ThreadingMixIn, socketserver.UnixStreamServer):
+    daemon_threads = True
+
+
+def build_server(state_dir: Path, sock_path: Path, kid: str, rekor_kid: str, did: str,
+                 listen_fd: int | None = None) -> _Server:
+    """Server on sock_path, or on an inherited socket (systemd socket activation)."""
+    if listen_fd is None:
+        server = _Server(str(sock_path), _Handler)
+    else:
+        server = _Server(str(sock_path), _Handler, bind_and_activate=False)
+        server.socket = socket.socket(fileno=listen_fd)
+    server.signer = Signer(Path(state_dir), kid, rekor_kid, did)
+    return server
+
+
+def main() -> int:
+    logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s")
+    env = os.environ
+    listen_fd = 3 if env.get("LISTEN_FDS") == "1" and env.get("LISTEN_PID") == str(os.getpid()) else None
+    server = build_server(
+        state_dir=Path(env["STATE_DIRECTORY"]),
+        sock_path=Path(env.get("TL_SIGNER_SOCKET", "/run/tl-signer/sign.sock")),
+        kid=env["TL_SIGNER_KID"],
+        rekor_kid=env["TL_SIGNER_REKOR_KID"],
+        did=env["TL_SIGNER_DID"],
+        listen_fd=listen_fd,
+    )
+    log.info("serving kid=%s rekor_kid=%s public=%s", server.signer.kid,
+             server.signer.rekor_kid, server.signer.ed_public)
+    server.serve_forever()
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())