diff --git a/custom-domain/dstack-ingress/scripts/entrypoint.sh b/custom-domain/dstack-ingress/scripts/entrypoint.sh index 2f1eed9..eaefd70 100644 --- a/custom-domain/dstack-ingress/scripts/entrypoint.sh +++ b/custom-domain/dstack-ingress/scripts/entrypoint.sh @@ -356,9 +356,11 @@ process_domain() { set_alias_record "$domain" set_txt_record "$domain" - renew-certificate.sh "$domain" || echo "First certificate renewal failed for $domain, will retry after set CAA record" + # renew-certificate.sh exits 0 on renewal, 2 when nothing needed renewal + # (not a failure — this script runs under `set -e`), 1 on real failure. + renew-certificate.sh "$domain" || [ $? -eq 2 ] || echo "First certificate renewal failed for $domain, will retry after set CAA record" set_caa_record "$domain" - renew-certificate.sh "$domain" + renew-certificate.sh "$domain" || [ $? -eq 2 ] } bootstrap() { @@ -400,8 +402,12 @@ else generate-evidences.sh fi -# Build combined PEM files for haproxy +# Build combined PEM files for haproxy. Stamp the build so the renewal +# daemon's certs_pending_apply check starts from "everything applied" — +# haproxy loads these fresh PEMs itself at startup, no reload pending. build-combined-pems.sh +mkdir -p /var/run/haproxy +touch /var/run/haproxy/renewal-applied.stamp # Generate haproxy config if [ -n "$ROUTING_MAP" ]; then diff --git a/custom-domain/dstack-ingress/scripts/renew-certificate.sh b/custom-domain/dstack-ingress/scripts/renew-certificate.sh index 8c822cb..56a24a5 100755 --- a/custom-domain/dstack-ingress/scripts/renew-certificate.sh +++ b/custom-domain/dstack-ingress/scripts/renew-certificate.sh @@ -8,11 +8,17 @@ SCRIPT_DIR="$(dirname "$(readlink -f "$0")")" python3 "$SCRIPT_DIR/certman.py" auto --domain "$DOMAIN" --email "$CERTBOT_EMAIL" CERT_STATUS=$? -if [ $CERT_STATUS -eq 1 ]; then - echo "Certificate management failed" >&2 - exit 1 -elif [ $CERT_STATUS -eq 2 ]; then +if [ $CERT_STATUS -eq 2 ]; then echo "No certificates need renewal, skipping evidence generation" + # Propagate "nothing renewed" so the renewal daemon does not rebuild + # PEMs and reload haproxy on every check cycle. + exit 2 +elif [ $CERT_STATUS -ne 0 ]; then + # 1 is certman's failure exit; anything else (127 missing interpreter, + # 130 signal, ...) is just as much a failure — never let it fall through + # to "renewed", which would trigger a spurious PEM rebuild + reload. + echo "Certificate management failed (status $CERT_STATUS)" >&2 + exit 1 fi exit 0 \ No newline at end of file diff --git a/custom-domain/dstack-ingress/scripts/renewal-daemon.sh b/custom-domain/dstack-ingress/scripts/renewal-daemon.sh index d20614d..afedd25 100755 --- a/custom-domain/dstack-ingress/scripts/renewal-daemon.sh +++ b/custom-domain/dstack-ingress/scripts/renewal-daemon.sh @@ -1,5 +1,20 @@ #!/bin/bash +# Written after renewed certificates have been fully applied (evidences + +# combined PEMs rebuilt + haproxy reloaded). Certificate material newer than +# this stamp means a renewal has not reached haproxy yet, so the apply step +# retries on the next cycle instead of waiting for the next renewal (~60 +# days) — without it, a single failed reload would leave haproxy serving the +# old certificate until it expires. The entrypoint touches the stamp after +# the initial PEM build, so a fresh container start does not trigger a +# spurious reload. +STAMP=/var/run/haproxy/renewal-applied.stamp + +certs_pending_apply() { + [ -f "$STAMP" ] || return 0 + [ -n "$(find /etc/letsencrypt/live -name fullchain.pem -newer "$STAMP" -print -quit 2>/dev/null)" ] +} + while true; do echo "[$(date)] Checking for certificate renewal" @@ -10,28 +25,43 @@ while true; do while IFS= read -r domain; do [[ -n "$domain" ]] || continue echo "[$(date)] Checking renewal for domain: $domain" - if renew-certificate.sh "$domain"; then + renew-certificate.sh "$domain" + status=$? + # 0 = a certificate was actually renewed; 2 = nothing to renew. + # Only a real renewal warrants a PEM rebuild + haproxy reload — + # reloading on every check cycle replaced the worker (and its + # accumulated state) twice a day for no reason. + if [ $status -eq 0 ]; then renewal_occurred=true - else - echo "Certificate renewal check failed for $domain with status $?" + elif [ $status -ne 2 ]; then + echo "Certificate renewal check failed for $domain with status $status" fi done <<<"$all_domains" - if [ "$renewal_occurred" = true ]; then - echo "[$(date)] Generating evidence files after renewals..." - generate-evidences.sh || echo "Evidence generation failed" + if [ "$renewal_occurred" = true ] || certs_pending_apply; then + apply_ok=true + echo "[$(date)] Applying renewed certificates..." + generate-evidences.sh || { echo "Evidence generation failed"; apply_ok=false; } # Rebuild combined PEM files for haproxy - build-combined-pems.sh || echo "Combined PEM build failed" + build-combined-pems.sh || { echo "Combined PEM build failed"; apply_ok=false; } # Graceful reload: send SIGUSR2 to haproxy master process if [ ! -f /var/run/haproxy/haproxy.pid ]; then echo "HAProxy reload failed: PID file /var/run/haproxy/haproxy.pid not found" >&2 + apply_ok=false elif ! kill -USR2 "$(cat /var/run/haproxy/haproxy.pid)"; then echo "HAProxy reload failed: SIGUSR2 to PID $(cat /var/run/haproxy/haproxy.pid) failed" >&2 + apply_ok=false else echo "Certificate renewed and HAProxy reloaded successfully" fi + + if [ "$apply_ok" = true ]; then + touch "$STAMP" + else + echo "[$(date)] Certificate apply incomplete; will retry on the next check cycle" + fi fi else echo "[$(date)] No domains configured for renewal"