From 26f40725273fed80cad51e72b9aba06de59c3b0a Mon Sep 17 00:00:00 2001 From: pacoyang Date: Tue, 28 Jul 2026 01:45:25 -0700 Subject: [PATCH] fix(dstack-ingress): retry applying a renewed certificate until it sticks Renewing and applying were treated as one step: if [ "$changed" -eq 1 ]; then collect_evidence || echo "Evidence generation failed" >&2 build_combined_pems || echo "Combined PEM build failed" >&2 haproxy_reload || true fi Every failure there is swallowed, and none of them set `failed`, so the pass still reports success. Twelve hours later the ACME client sees a fresh certificate on disk, reports "nothing to renew", `changed` is 0, and the apply is never attempted again. HAProxy goes on serving the pre-renewal certificate until it expires -- which is soon, because being close to expiry is why it renewed in the first place. The symptom arrives weeks after the cause, as an expired certificate, with nothing in the logs at that moment to connect the two. Separate the two: `APPLY_PENDING` is set when something renewed and only cleared once evidence, PEMs and the reload have all succeeded, so a failed apply is retried on the next pass instead of waiting for the next renewal. Quiet passes stay quiet. With nothing renewed and nothing pending the block does not run at all, so this does not bring back the reload-on- every-check behaviour that made HAProxy replace its worker twice a day. Kept in both mode scripts rather than lifted into a library: the apply steps are mode-specific (certbot and lego lay certificates out differently), so a shared helper would have to call back into functions its callers define, which is harder to follow than the dozen lines it would save. --- custom-domain/dstack-ingress/scripts/dns01.sh | 25 ++++++++++++++++--- .../dstack-ingress/scripts/tlsalpn.sh | 25 ++++++++++++++++--- 2 files changed, 44 insertions(+), 6 deletions(-) diff --git a/custom-domain/dstack-ingress/scripts/dns01.sh b/custom-domain/dstack-ingress/scripts/dns01.sh index ada2353..ab7e538 100644 --- a/custom-domain/dstack-ingress/scripts/dns01.sh +++ b/custom-domain/dstack-ingress/scripts/dns01.sh @@ -288,6 +288,10 @@ collect_evidence() { evidence_finalize } +# Set when certificates have been renewed but not yet applied to haproxy. +# Survives across passes, so a failed apply is retried on the next one. +APPLY_PENDING=false + # One pass over every domain: make the DNS records right, then issue or renew. # Idempotent, so the first pass and every later one are the same code. run_pass() { @@ -306,9 +310,24 @@ run_pass() { done <<<"$(get-all-domains.sh)" if [ "$changed" -eq 1 ]; then - collect_evidence || echo "Evidence generation failed" >&2 - build_combined_pems || echo "Combined PEM build failed" >&2 - haproxy_reload || true + APPLY_PENDING=true + fi + + # Applying is separate from renewing, and retried until it sticks. A + # renewal that reaches disk but not haproxy is invisible: the next pass + # sees a fresh certificate, reports "nothing to renew", and never reloads + # again -- so haproxy would serve the pre-renewal certificate until it + # expires, which is soon, because being near expiry is why it renewed. + if [ "$APPLY_PENDING" = true ]; then + local applied=true + collect_evidence || { echo "Evidence generation failed" >&2; applied=false; } + build_combined_pems || { echo "Combined PEM build failed" >&2; applied=false; } + haproxy_reload || applied=false + if [ "$applied" = true ]; then + APPLY_PENDING=false + else + echo "[$(date)] Certificate apply incomplete; retrying on the next pass" >&2 + fi fi [ "$failed" -eq 0 ] } diff --git a/custom-domain/dstack-ingress/scripts/tlsalpn.sh b/custom-domain/dstack-ingress/scripts/tlsalpn.sh index 485bce3..f5f2fbb 100644 --- a/custom-domain/dstack-ingress/scripts/tlsalpn.sh +++ b/custom-domain/dstack-ingress/scripts/tlsalpn.sh @@ -262,6 +262,10 @@ process_domain() { legoman.py auto --domain "$domain" ${ACME_EMAIL:+--email "$ACME_EMAIL"} } +# Set when certificates have been renewed but not yet applied to haproxy. +# Survives across passes, so a failed apply is retried on the next one. +APPLY_PENDING=false + # One pass over every domain: make sure the records exist, then issue or renew. # Idempotent, so the first pass and every later one are the same code. There is # deliberately only one of these -- an earlier split between a one-shot @@ -285,9 +289,24 @@ run_pass() { done <<<"$(get-all-domains.sh)" if [ "$changed" -eq 1 ]; then - collect_evidence || echo "Evidence generation failed" >&2 - build_combined_pems || echo "Combined PEM build failed" >&2 - haproxy_reload || true + APPLY_PENDING=true + fi + + # Applying is separate from renewing, and retried until it sticks. A + # renewal that reaches disk but not haproxy is invisible: the next pass + # sees a fresh certificate, reports "nothing to renew", and never reloads + # again -- so haproxy would serve the pre-renewal certificate until it + # expires, which is soon, because being near expiry is why it renewed. + if [ "$APPLY_PENDING" = true ]; then + local applied=true + collect_evidence || { echo "Evidence generation failed" >&2; applied=false; } + build_combined_pems || { echo "Combined PEM build failed" >&2; applied=false; } + haproxy_reload || applied=false + if [ "$applied" = true ]; then + APPLY_PENDING=false + else + echo "[$(date)] Certificate apply incomplete; retrying on the next pass" >&2 + fi fi FIRST_PASS=false [ "$failed" -eq 0 ]