From 84e2577862d51e22894eecc9a6e03b23ac5b54cb Mon Sep 17 00:00:00 2001
From: echoomegaprime <314902331+echoomegaprime@users.noreply.github.com>
Date: Fri, 25 Sep 2026 01:52:50 -0500
Subject: [PATCH] fix(deploy): make readiness waits configurable (default 90s)

The staging boot, rollback-restore and production health waits each polled
/healthz for only 20s (40 x 0.5s). A cold uvicorn boot on a loaded FORGE
exceeds that: it aborted the d8ad9fd security release at staging, and the
same budget can report a healthy rollback as failed or roll back a good
production release.

- CERTFORGE_READY_TIMEOUT_S (default 90) drives all three waits; a healthy
  service still returns on the first successful probe.
- On a staging readiness failure, print the /healthz probe result and keep
  service.log under $STATE_ROOT/deploy-logs/ before the EXIT trap removes
  the scratch directory.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0152mN1wMwj9vE2YmV2xZF4F
---
 deploy/deploy_forge.sh | 22 +++++++++++++++++-----
 1 file changed, 17 insertions(+), 5 deletions(-)

diff --git a/deploy/deploy_forge.sh b/deploy/deploy_forge.sh
index c280297..a266eba 100644
--- a/deploy/deploy_forge.sh
+++ b/deploy/deploy_forge.sh
@@ -14,6 +14,11 @@ EXPECTED_COMMIT_SHA="${CERTFORGE_EXPECTED_COMMIT_SHA:-}"
 RELEASE_ROOT="${CERTFORGE_RELEASE_ROOT:-/home/forge/echo-certification-forge-releases}"
 CURRENT_LINK="${CERTFORGE_CURRENT_LINK:-/home/forge/echo-certification-forge-current}"
 STATE_ROOT="${CERTFORGE_STATE_ROOT:-/home/forge/echo-certification-forge/var}"
+# Readiness waits poll /healthz every 0.5s and return on the first healthy probe.
+# 20s proved too short for a cold uvicorn boot on a loaded FORGE (staging abort,
+# false rollback-restore failure, needless production rollback).
+READY_TIMEOUT_S="${CERTFORGE_READY_TIMEOUT_S:-90}"
+READY_POLLS=$(( READY_TIMEOUT_S * 2 ))
 ADAPTER_DIR="${ECHO_CERTFORGE_PROD_ADAPTER_DIR:-$STATE_ROOT/p5}"
 ADAPTER_MODE="${CERTFORGE_ADAPTER_MODE:-required}"
 TRUSTED_MANIFEST_SHA256="${ECHO_CERTFORGE_TRUSTED_MANIFEST_SHA256:-7dc98e0e95e6dd2c000ec069a8c46c4d1d49a4fe869ad4eae25e059d103644f4}"
@@ -221,10 +226,12 @@ ECHO_CERTFORGE_API_KEY_PEPPER="$STAGING_PEPPER" \
 STAGING_PID=$!
 
 ready=0
-for _ in $(seq 1 40); do
+for _ in $(seq 1 "$READY_POLLS"); do
   kill -0 "$STAGING_PID" 2>/dev/null || {
     echo "!! staging process exited before readiness"
-    tail -20 "$STAGING_ROOT/service.log"
+    mkdir -p "$STATE_ROOT/deploy-logs" &&
+      cp -f "$STAGING_ROOT/service.log" "$STATE_ROOT/deploy-logs/staging-$RELEASE_ID.log" 2>/dev/null || true
+    tail -40 "$STAGING_ROOT/service.log"
     exit 1
   }
   curl -sf "http://127.0.0.1:$STAGING_PORT/healthz" >/dev/null 2>&1 && {
@@ -240,7 +247,12 @@ for _ in $(seq 1 40); do
 done
 if [ "$ready" != 1 ]; then
   echo "!! staging never became healthy"
-  tail -20 "$STAGING_ROOT/service.log"
+  echo "   waited ${READY_TIMEOUT_S}s; set CERTFORGE_READY_TIMEOUT_S to adjust"
+  curl -sS -m 5 -o /dev/null -w "   /healthz probe: HTTP %{http_code}\n" \
+    "http://127.0.0.1:$STAGING_PORT/healthz" 2>&1 || true
+  mkdir -p "$STATE_ROOT/deploy-logs" &&
+    cp -f "$STAGING_ROOT/service.log" "$STATE_ROOT/deploy-logs/staging-$RELEASE_ID.log" 2>/dev/null || true
+  tail -40 "$STAGING_ROOT/service.log"
   exit 1
 fi
 
@@ -383,7 +395,7 @@ rollback_production() {
     if [ "$PREV_ACTIVE" = "active" ]; then
       sudo systemctl start "$SERVICE.service" || rollback_status=1
       restored=0
-      for _ in $(seq 1 40); do
+      for _ in $(seq 1 "$READY_POLLS"); do
         service_owns_port "$PROD_PORT" &&
           curl -sf "http://127.0.0.1:$PROD_PORT/healthz" >/dev/null 2>&1 && {
           restored=1
@@ -585,7 +597,7 @@ sudo systemctl enable "$DISPATCH_SERVICE.service"
 
 echo "== [8/9] production health + live-smoke =="
 ready=0
-for _ in $(seq 1 40); do
+for _ in $(seq 1 "$READY_POLLS"); do
   service_owns_port "$PROD_PORT" &&
     curl -sf "http://127.0.0.1:$PROD_PORT/healthz" >/dev/null 2>&1 && {
     ready=1