feat(dstack-ingress): ACME DNS-01 challenge delegation (CNAME alias) for least-privilege DNS tokens

custom-domain/dstack-ingress/README.md

@@ -179,9 +179,50 @@ environment:
 | `EVIDENCE_SERVER` | `true` | Serve evidence files at `/evidences/` on the TLS port |
 | `EVIDENCE_PORT` | `80` | Internal port for evidence HTTP server |
 | `ALPN` | | TLS ALPN protocols (e.g. `h2,http/1.1`). Only set if backends support h2c |
+| `ACME_CHALLENGE_ALIAS` | | Delegate the ACME DNS-01 challenge to this zone (see below) so the DNS token needs no access to the served domain's own zone |
+| `ACME_CHALLENGE_PROPAGATION_SECONDS` | `30` | Wait after writing the delegated challenge TXT before validation (only with `ACME_CHALLENGE_ALIAS`). Keep well under ~250s — certbot is killed after a 300s per-run timeout |
+| `ALLOW_MISSING_CAA` | `false` | In delegation mode, continue even if the required `accounturi` CAA cannot be confirmed. Default fails closed (see below) |
 
 For DNS provider credentials, see [DNS_PROVIDERS.md](DNS_PROVIDERS.md).
 
+### ACME challenge delegation (least-privilege DNS token)
+
+By default the DNS token must be able to edit the **served domain's own zone**
+(to write `_acme-challenge`, and — with `SET_CAA` — the CAA record). If the
+served name lives under a shared production zone (e.g. `svc.example.com` under
+`example.com`), that token can edit every record in the zone, which may be more
+privilege than you want.
+
+Set `ACME_CHALLENGE_ALIAS=<delegation-zone>` to answer the DNS-01 challenge in a
+separate zone that your token controls, so the token never touches the served
+domain's zone. In this mode dstack-ingress **only** manages the challenge TXT in
+the delegation zone; you set the following records **once, statically**, in the
+served domain's production zone (the container prints the exact values on start):
+
+```
+svc.example.com                       CNAME  <GATEWAY_DOMAIN>
+_dstack-app-address.svc.example.com   TXT    "<app-id>:<port>"
+_acme-challenge.svc.example.com       CNAME  _acme-challenge.svc.example.com.<delegation-zone>
+svc.example.com                       CAA    0 issue "letsencrypt.org;validationmethods=dns-01;accounturi=<account-uri>"
+```
+
+> **Security note.** The `accounturi` CAA restricts issuance to this enclave's
+> ACME account and is the control that prevents forged certificates. In
+> delegation mode dstack-ingress cannot set it for you (no token for the served
+> zone), so it prints the record and verifies its presence via DoH. **If the
+> CAA is confirmed absent the container fails to start** (set `ALLOW_MISSING_CAA=true`
+> to override; a transient DoH failure only warns, it does not block). Without
+> this CAA, anyone who can satisfy the delegated challenge could obtain a
+> certificate for the domain.
+>
+> Use a **dedicated** delegation zone and scope the token to only that zone — a
+> zone shared with other tenants lets anyone with write access to it complete
+> the challenge. `SET_CAA` does not affect delegation mode (this CAA path always
+> runs). If a certificate was previously issued with the standard DNS-plugin
+> authenticator, delete `/etc/letsencrypt/renewal/<domain>.conf` before enabling
+> delegation, otherwise `certbot renew` reuses the old plugin (which needs the
+> production-zone token this mode avoids).
+
 ## Evidence & Attestation
 
 Evidence files are served at `https://your-domain.com/evidences/` by default (via payload inspection in HAProxy's TCP mode). They can also be accessed by the backend application through the shared `/evidences` volume.

custom-domain/dstack-ingress/scripts/acme-dns-alias-hook.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+# certbot --manual auth/cleanup hook for ACME DNS-01 challenge delegation.
+#
+# Instead of writing the `_acme-challenge` TXT into the served domain's own
+# zone (which requires a DNS token for that zone), this writes it into a
+# delegated zone that our token controls (ACME_CHALLENGE_ALIAS). The served
+# domain's zone only needs one static, operator-managed CNAME:
+#
+#   _acme-challenge.<domain>  CNAME  _acme-challenge.<domain>.<ACME_CHALLENGE_ALIAS>
+#
+# Let's Encrypt follows that CNAME during validation and reads the TXT from the
+# delegated zone, so the enclave's token never needs access to the served
+# domain's (production) zone.
+#
+# certbot invokes this with $CERTBOT_DOMAIN and $CERTBOT_VALIDATION set, and
+# runs it as either:  acme-dns-alias-hook.sh auth  |  acme-dns-alias-hook.sh cleanup
+set -euo pipefail
+
+action="${1:-}"
+
+if [ -z "${ACME_CHALLENGE_ALIAS:-}" ]; then
+    echo "acme-dns-alias-hook: ACME_CHALLENGE_ALIAS is not set" >&2
+    exit 1
+fi
+if [ -z "${CERTBOT_DOMAIN:-}" ]; then
+    echo "acme-dns-alias-hook: CERTBOT_DOMAIN not set (must be invoked by certbot)" >&2
+    exit 1
+fi
+
+# certbot always passes the base domain (no leading "*.") in CERTBOT_DOMAIN.
+record="_acme-challenge.${CERTBOT_DOMAIN}.${ACME_CHALLENGE_ALIAS}"
+
+case "$action" in
+    auth)
+        : "${CERTBOT_VALIDATION:?CERTBOT_VALIDATION not set}"
+        echo "acme-dns-alias-hook: writing challenge TXT to delegated record $record"
+        dnsman.py set_txt --domain "$record" --content "$CERTBOT_VALIDATION"
+        # certbot asks Let's Encrypt to validate immediately after this hook
+        # returns, so wait for the record to propagate on the delegated zone's
+        # authoritative servers before returning.
+        sleep "${ACME_CHALLENGE_PROPAGATION_SECONDS:-30}"
+        ;;
+    cleanup)
+        echo "acme-dns-alias-hook: removing challenge TXT $record"
+        # Non-fatal: a failed cleanup leaves a stale TXT in the delegated zone,
+        # which the next auth overwrites; it must not fail the whole run.
+        dnsman.py unset_txt --domain "$record" || true
+        ;;
+    *)
+        echo "acme-dns-alias-hook: usage: $0 <auth|cleanup>" >&2
+        exit 1
+        ;;
+esac

custom-domain/dstack-ingress/scripts/certman.py

@@ -265,13 +265,40 @@ def setup_credentials(self) -> bool:
 
     def _build_certbot_command(self, action: str, domain: str, email: str) -> List[str]:
         """Build certbot command using provider configuration."""
+        certbot_cmd = self._get_certbot_command()
+
+        # Challenge-delegation mode: when ACME_CHALLENGE_ALIAS is set, answer the
+        # DNS-01 challenge in a delegated zone via a manual hook instead of the
+        # provider's certbot plugin, so our DNS token never needs access to the
+        # served domain's own zone. See scripts/acme-dns-alias-hook.sh.
+        if os.environ.get("ACME_CHALLENGE_ALIAS", "").strip():
+            hook = "/scripts/acme-dns-alias-hook.sh"
+            base_cmd = certbot_cmd + [action, "--non-interactive", "-v"]
+            if action == "certonly":
+                base_cmd.extend([
+                    "--manual",
+                    "--preferred-challenges=dns",
+                    f"--manual-auth-hook={hook} auth",
+                    f"--manual-cleanup-hook={hook} cleanup",
+                    "--agree-tos", "--no-eff-email",
+                    "--email", email, "-d", domain,
+                ])
+            # For `renew`, certbot reuses the authenticator + hooks saved in the
+            # renewal config from the initial `certonly`, so we don't re-specify
+            # them here (and must not fall back to the DNS plugin).
+            if os.environ.get("CERTBOT_STAGING", "false") == "true":
+                base_cmd.append("--staging")
+            masked = [a if not (i > 0 and base_cmd[i - 1] == "--email") else "<email>"
+                      for i, a in enumerate(base_cmd)]
+            print(f"Executing (challenge-delegation): {' '.join(masked)}")
+            return base_cmd
+
         plugin = self.provider.CERTBOT_PLUGIN
         if not plugin:
             raise ValueError(
                 f"No certbot plugin configured for {self.provider_type}")
 
         # Use Python module execution to ensure same environment
-        certbot_cmd = self._get_certbot_command()
         base_cmd = certbot_cmd + [action, "-a",
                                   plugin, "--non-interactive", "-v"]
 

custom-domain/dstack-ingress/scripts/dns_providers/base.py

@@ -247,6 +247,28 @@ def set_txt_record(self, name: str, content: str, ttl: int = 60) -> bool:
         )
         return self.create_dns_record(new_record)
 
+    def unset_txt_record(self, name: str) -> bool:
+        """Delete all TXT records for a name.
+
+        Used to clean up ACME DNS-01 challenge records (e.g. from a certbot
+        --manual cleanup hook). Missing records are treated as success.
+
+        Args:
+            name: The record name
+
+        Returns:
+            True if all matching records were removed (or none existed).
+        """
+        ok = True
+        for record in self.get_dns_records(name, RecordType.TXT):
+            if not record.id:
+                print(f"Warning: TXT record for {name} has no id; cannot delete")
+                ok = False
+                continue
+            if not self.delete_dns_record(record.id, name):
+                ok = False
+        return ok
+
     def set_caa_record(
         self,
         name: str,

custom-domain/dstack-ingress/scripts/dnsman.py

@@ -14,7 +14,7 @@ def main():
     )
     parser.add_argument(
         "action",
-        choices=["set_cname", "set_alias", "set_txt", "set_caa"],
+        choices=["set_cname", "set_alias", "set_txt", "unset_txt", "set_caa"],
         help="Action to perform",
     )
     parser.add_argument("--domain", required=True, help="Domain name")
@@ -67,6 +67,13 @@ def main():
                 sys.exit(1)
             print(f"Successfully set TXT record for {args.domain}")
 
+        elif args.action == "unset_txt":
+            success = provider.unset_txt_record(args.domain)
+            if not success:
+                print(f"Failed to unset TXT record for {args.domain}", file=sys.stderr)
+                sys.exit(1)
+            print(f"Successfully unset TXT record for {args.domain}")
+
         elif args.action == "set_caa":
             if not args.caa_tag or not args.caa_value:
                 print(

custom-domain/dstack-ingress/scripts/entrypoint.sh

@@ -268,6 +268,16 @@ backend ${be_name}
 
 set_alias_record() {
     local domain="$1"
+    if [ -n "${ACME_CHALLENGE_ALIAS:-}" ]; then
+        # certbot validates the challenge at the base name even for a wildcard,
+        # so the _acme-challenge CNAME must use the base domain (strip "*.").
+        local base="${domain#\*.}"
+        echo "[challenge-delegation] Not touching ${domain}'s own zone (token is scoped to the delegated zone)."
+        echo "[challenge-delegation] Set these in your production zone yourself (static, one-time):"
+        echo "    ${domain}  CNAME  ${GATEWAY_DOMAIN}"
+        echo "    _acme-challenge.${base}  CNAME  _acme-challenge.${base}.${ACME_CHALLENGE_ALIAS}"
+        return
+    fi
     echo "Setting alias record for $domain"
     dnsman.py set_alias \
         --domain "$domain" \
@@ -301,6 +311,12 @@ set_txt_record() {
         txt_domain="${TXT_PREFIX}.${domain}"
     fi
 
+    if [ -n "${ACME_CHALLENGE_ALIAS:-}" ]; then
+        echo "[challenge-delegation] Set this in your production zone yourself (static, one-time):"
+        echo "    ${txt_domain}  TXT  \"$APP_ID:$PORT\""
+        return
+    fi
+
     dnsman.py set_txt \
         --domain "$txt_domain" \
         --content "$APP_ID:$PORT"
@@ -311,37 +327,101 @@ set_txt_record() {
     fi
 }
 
+# caa_domain_and_tag DOMAIN -> prints "caa_domain caa_tag" (strips a wildcard).
+caa_domain_and_tag() {
+    local domain="$1"
+    if [[ "$domain" == \*.* ]]; then
+        echo "${domain#\*.} issuewild"
+    else
+        echo "$domain issue"
+    fi
+}
+
+# In delegation mode we have NO token for the served domain's zone, so we cannot
+# set the accounturi CAA ourselves. That CAA is the forge-prevention (only this
+# enclave's ACME account may issue), so this path is deliberately NOT gated by
+# SET_CAA and fails closed if the record is confirmed absent — otherwise anyone
+# who can satisfy the delegated DNS-01 challenge could obtain a cert for the
+# domain. Set ALLOW_MISSING_CAA=true to override (accept the risk).
+verify_delegation_caa() {
+    local domain="$1"
+    local account_file account_uri caa_domain caa_tag caa_value resp status found
+
+    if ! account_file=$(get_letsencrypt_account_file); then
+        echo "ERROR: cannot read the Let's Encrypt account file to determine the required accounturi CAA" >&2
+        caa_fail_or_allow "$domain"
+        return
+    fi
+    read -r caa_domain caa_tag < <(caa_domain_and_tag "$domain")
+    account_uri=$(jq -j '.uri' "$account_file")
+    caa_value="letsencrypt.org;validationmethods=dns-01;accounturi=$account_uri"
+
+    echo "[challenge-delegation] Set this CAA in your production zone (static, one-time):"
+    echo "    ${caa_domain}  CAA  0 ${caa_tag} \"${caa_value}\""
+
+    # Verify via DNS-over-HTTPS (dig is not installed in the image; curl+jq are).
+    resp=$(curl -s --max-time 10 "https://dns.google/resolve?name=${caa_domain}&type=257" 2>/dev/null || true)
+    if [ -z "$resp" ]; then
+        echo "WARNING: could not reach dns.google to verify the CAA (network issue) — NOT confirmed; continuing"
+        return
+    fi
+    status=$(echo "$resp" | jq -r '.Status // empty' 2>/dev/null || true)
+    if [ "$status" != "0" ]; then
+        echo "WARNING: CAA DoH query for ${caa_domain} returned status ${status:-unknown} — NOT confirmed; continuing"
+        return
+    fi
+    # Literal (grep -F) match on the CAA data fields — the account URI contains
+    # '/' and '.', which must not be treated as regex.
+    found=$(echo "$resp" | jq -r '.Answer // [] | .[] | .data' 2>/dev/null | grep -F "accounturi=$account_uri" || true)
+    if [ -n "$found" ]; then
+        echo "[challenge-delegation] Verified: accounturi CAA is present for $caa_domain"
+        return
+    fi
+    echo "ERROR: the accounturi CAA is NOT present for $caa_domain." >&2
+    echo "ERROR: without it, anyone who can satisfy the delegated DNS-01 challenge could obtain a" >&2
+    echo "ERROR: publicly-trusted certificate for this domain (forged TLS termination)." >&2
+    caa_fail_or_allow "$domain"
+}
+
+caa_fail_or_allow() {
+    local domain="$1"
+    if [ "${ALLOW_MISSING_CAA:-false}" = "true" ]; then
+        echo "WARNING: ALLOW_MISSING_CAA=true — continuing without a verified accounturi CAA for $domain"
+        return 0
+    fi
+    echo "ERROR: refusing to continue without the accounturi CAA for $domain." >&2
+    echo "ERROR: set the CAA record shown above, or ALLOW_MISSING_CAA=true to override." >&2
+    exit 1
+}
+
 set_caa_record() {
     local domain="$1"
+
+    # Delegation mode is handled separately and is NOT gated by SET_CAA.
+    if [ -n "${ACME_CHALLENGE_ALIAS:-}" ]; then
+        verify_delegation_caa "$domain"
+        return
+    fi
+
     if [ "$SET_CAA" != "true" ]; then
         echo "Skipping CAA record setup"
         return
     fi
 
-    local ACCOUNT_URI
-    local account_file
-
+    local account_file account_uri caa_domain caa_tag
     if ! account_file=$(get_letsencrypt_account_file); then
         echo "Warning: Cannot set CAA record - account file not found"
         echo "This is not critical - certificates can still be issued without CAA records"
         return
     fi
+    read -r caa_domain caa_tag < <(caa_domain_and_tag "$domain")
+    account_uri=$(jq -j '.uri' "$account_file")
 
-    local caa_domain caa_tag
-    if [[ "$domain" == \*.* ]]; then
-        caa_domain="${domain#\*.}"
-        caa_tag="issuewild"
-    else
-        caa_domain="$domain"
-        caa_tag="issue"
-    fi
-
-    ACCOUNT_URI=$(jq -j '.uri' "$account_file")
-    echo "Adding CAA record ($caa_tag) for $caa_domain, accounturi=$ACCOUNT_URI"
+    echo "Adding CAA record ($caa_tag) for $caa_domain, accounturi=$account_uri"
     dnsman.py set_caa \
         --domain "$caa_domain" \
         --caa-tag "$caa_tag" \
-        --caa-value "letsencrypt.org;validationmethods=dns-01;accounturi=$ACCOUNT_URI"
+        --caa-value "letsencrypt.org;validationmethods=dns-01;accounturi=$account_uri"
 
     if [ $? -ne 0 ]; then
         echo "Warning: Failed to set CAA record for $domain"