diff --git a/custom-domain/dstack-ingress/README.md b/custom-domain/dstack-ingress/README.md index 03befed6..ef16d2ea 100644 --- a/custom-domain/dstack-ingress/README.md +++ b/custom-domain/dstack-ingress/README.md @@ -179,9 +179,50 @@ environment: | `EVIDENCE_SERVER` | `true` | Serve evidence files at `/evidences/` on the TLS port | | `EVIDENCE_PORT` | `80` | Internal port for evidence HTTP server | | `ALPN` | | TLS ALPN protocols (e.g. `h2,http/1.1`). Only set if backends support h2c | +| `ACME_CHALLENGE_ALIAS` | | Delegate the ACME DNS-01 challenge to this zone (see below) so the DNS token needs no access to the served domain's own zone | +| `ACME_CHALLENGE_PROPAGATION_SECONDS` | `30` | Wait after writing the delegated challenge TXT before validation (only with `ACME_CHALLENGE_ALIAS`). Keep well under ~250s — certbot is killed after a 300s per-run timeout | +| `ALLOW_MISSING_CAA` | `false` | In delegation mode, continue even if the required `accounturi` CAA cannot be confirmed. Default fails closed (see below) | For DNS provider credentials, see [DNS_PROVIDERS.md](DNS_PROVIDERS.md). +### ACME challenge delegation (least-privilege DNS token) + +By default the DNS token must be able to edit the **served domain's own zone** +(to write `_acme-challenge`, and — with `SET_CAA` — the CAA record). If the +served name lives under a shared production zone (e.g. `svc.example.com` under +`example.com`), that token can edit every record in the zone, which may be more +privilege than you want. + +Set `ACME_CHALLENGE_ALIAS=` to answer the DNS-01 challenge in a +separate zone that your token controls, so the token never touches the served +domain's zone. In this mode dstack-ingress **only** manages the challenge TXT in +the delegation zone; you set the following records **once, statically**, in the +served domain's production zone (the container prints the exact values on start): + +``` +svc.example.com CNAME +_dstack-app-address.svc.example.com TXT ":" +_acme-challenge.svc.example.com CNAME _acme-challenge.svc.example.com. +svc.example.com CAA 0 issue "letsencrypt.org;validationmethods=dns-01;accounturi=" +``` + +> **Security note.** The `accounturi` CAA restricts issuance to this enclave's +> ACME account and is the control that prevents forged certificates. In +> delegation mode dstack-ingress cannot set it for you (no token for the served +> zone), so it prints the record and verifies its presence via DoH. **If the +> CAA is confirmed absent the container fails to start** (set `ALLOW_MISSING_CAA=true` +> to override; a transient DoH failure only warns, it does not block). Without +> this CAA, anyone who can satisfy the delegated challenge could obtain a +> certificate for the domain. +> +> Use a **dedicated** delegation zone and scope the token to only that zone — a +> zone shared with other tenants lets anyone with write access to it complete +> the challenge. `SET_CAA` does not affect delegation mode (this CAA path always +> runs). If a certificate was previously issued with the standard DNS-plugin +> authenticator, delete `/etc/letsencrypt/renewal/.conf` before enabling +> delegation, otherwise `certbot renew` reuses the old plugin (which needs the +> production-zone token this mode avoids). + ## Evidence & Attestation Evidence files are served at `https://your-domain.com/evidences/` by default (via payload inspection in HAProxy's TCP mode). They can also be accessed by the backend application through the shared `/evidences` volume. diff --git a/custom-domain/dstack-ingress/scripts/acme-dns-alias-hook.sh b/custom-domain/dstack-ingress/scripts/acme-dns-alias-hook.sh new file mode 100755 index 00000000..751ec2b3 --- /dev/null +++ b/custom-domain/dstack-ingress/scripts/acme-dns-alias-hook.sh @@ -0,0 +1,53 @@ +#!/bin/bash +# certbot --manual auth/cleanup hook for ACME DNS-01 challenge delegation. +# +# Instead of writing the `_acme-challenge` TXT into the served domain's own +# zone (which requires a DNS token for that zone), this writes it into a +# delegated zone that our token controls (ACME_CHALLENGE_ALIAS). The served +# domain's zone only needs one static, operator-managed CNAME: +# +# _acme-challenge. CNAME _acme-challenge.. +# +# Let's Encrypt follows that CNAME during validation and reads the TXT from the +# delegated zone, so the enclave's token never needs access to the served +# domain's (production) zone. +# +# certbot invokes this with $CERTBOT_DOMAIN and $CERTBOT_VALIDATION set, and +# runs it as either: acme-dns-alias-hook.sh auth | acme-dns-alias-hook.sh cleanup +set -euo pipefail + +action="${1:-}" + +if [ -z "${ACME_CHALLENGE_ALIAS:-}" ]; then + echo "acme-dns-alias-hook: ACME_CHALLENGE_ALIAS is not set" >&2 + exit 1 +fi +if [ -z "${CERTBOT_DOMAIN:-}" ]; then + echo "acme-dns-alias-hook: CERTBOT_DOMAIN not set (must be invoked by certbot)" >&2 + exit 1 +fi + +# certbot always passes the base domain (no leading "*.") in CERTBOT_DOMAIN. +record="_acme-challenge.${CERTBOT_DOMAIN}.${ACME_CHALLENGE_ALIAS}" + +case "$action" in + auth) + : "${CERTBOT_VALIDATION:?CERTBOT_VALIDATION not set}" + echo "acme-dns-alias-hook: writing challenge TXT to delegated record $record" + dnsman.py set_txt --domain "$record" --content "$CERTBOT_VALIDATION" + # certbot asks Let's Encrypt to validate immediately after this hook + # returns, so wait for the record to propagate on the delegated zone's + # authoritative servers before returning. + sleep "${ACME_CHALLENGE_PROPAGATION_SECONDS:-30}" + ;; + cleanup) + echo "acme-dns-alias-hook: removing challenge TXT $record" + # Non-fatal: a failed cleanup leaves a stale TXT in the delegated zone, + # which the next auth overwrites; it must not fail the whole run. + dnsman.py unset_txt --domain "$record" || true + ;; + *) + echo "acme-dns-alias-hook: usage: $0 " >&2 + exit 1 + ;; +esac diff --git a/custom-domain/dstack-ingress/scripts/certman.py b/custom-domain/dstack-ingress/scripts/certman.py index 52126d23..d12527f8 100644 --- a/custom-domain/dstack-ingress/scripts/certman.py +++ b/custom-domain/dstack-ingress/scripts/certman.py @@ -265,13 +265,40 @@ def setup_credentials(self) -> bool: def _build_certbot_command(self, action: str, domain: str, email: str) -> List[str]: """Build certbot command using provider configuration.""" + certbot_cmd = self._get_certbot_command() + + # Challenge-delegation mode: when ACME_CHALLENGE_ALIAS is set, answer the + # DNS-01 challenge in a delegated zone via a manual hook instead of the + # provider's certbot plugin, so our DNS token never needs access to the + # served domain's own zone. See scripts/acme-dns-alias-hook.sh. + if os.environ.get("ACME_CHALLENGE_ALIAS", "").strip(): + hook = "/scripts/acme-dns-alias-hook.sh" + base_cmd = certbot_cmd + [action, "--non-interactive", "-v"] + if action == "certonly": + base_cmd.extend([ + "--manual", + "--preferred-challenges=dns", + f"--manual-auth-hook={hook} auth", + f"--manual-cleanup-hook={hook} cleanup", + "--agree-tos", "--no-eff-email", + "--email", email, "-d", domain, + ]) + # For `renew`, certbot reuses the authenticator + hooks saved in the + # renewal config from the initial `certonly`, so we don't re-specify + # them here (and must not fall back to the DNS plugin). + if os.environ.get("CERTBOT_STAGING", "false") == "true": + base_cmd.append("--staging") + masked = [a if not (i > 0 and base_cmd[i - 1] == "--email") else "" + for i, a in enumerate(base_cmd)] + print(f"Executing (challenge-delegation): {' '.join(masked)}") + return base_cmd + plugin = self.provider.CERTBOT_PLUGIN if not plugin: raise ValueError( f"No certbot plugin configured for {self.provider_type}") # Use Python module execution to ensure same environment - certbot_cmd = self._get_certbot_command() base_cmd = certbot_cmd + [action, "-a", plugin, "--non-interactive", "-v"] diff --git a/custom-domain/dstack-ingress/scripts/dns_providers/base.py b/custom-domain/dstack-ingress/scripts/dns_providers/base.py index 608cf16b..e8e89570 100644 --- a/custom-domain/dstack-ingress/scripts/dns_providers/base.py +++ b/custom-domain/dstack-ingress/scripts/dns_providers/base.py @@ -247,6 +247,28 @@ def set_txt_record(self, name: str, content: str, ttl: int = 60) -> bool: ) return self.create_dns_record(new_record) + def unset_txt_record(self, name: str) -> bool: + """Delete all TXT records for a name. + + Used to clean up ACME DNS-01 challenge records (e.g. from a certbot + --manual cleanup hook). Missing records are treated as success. + + Args: + name: The record name + + Returns: + True if all matching records were removed (or none existed). + """ + ok = True + for record in self.get_dns_records(name, RecordType.TXT): + if not record.id: + print(f"Warning: TXT record for {name} has no id; cannot delete") + ok = False + continue + if not self.delete_dns_record(record.id, name): + ok = False + return ok + def set_caa_record( self, name: str, diff --git a/custom-domain/dstack-ingress/scripts/dnsman.py b/custom-domain/dstack-ingress/scripts/dnsman.py index 98d8392c..20dfc360 100755 --- a/custom-domain/dstack-ingress/scripts/dnsman.py +++ b/custom-domain/dstack-ingress/scripts/dnsman.py @@ -14,7 +14,7 @@ def main(): ) parser.add_argument( "action", - choices=["set_cname", "set_alias", "set_txt", "set_caa"], + choices=["set_cname", "set_alias", "set_txt", "unset_txt", "set_caa"], help="Action to perform", ) parser.add_argument("--domain", required=True, help="Domain name") @@ -67,6 +67,13 @@ def main(): sys.exit(1) print(f"Successfully set TXT record for {args.domain}") + elif args.action == "unset_txt": + success = provider.unset_txt_record(args.domain) + if not success: + print(f"Failed to unset TXT record for {args.domain}", file=sys.stderr) + sys.exit(1) + print(f"Successfully unset TXT record for {args.domain}") + elif args.action == "set_caa": if not args.caa_tag or not args.caa_value: print( diff --git a/custom-domain/dstack-ingress/scripts/entrypoint.sh b/custom-domain/dstack-ingress/scripts/entrypoint.sh index 2f1eed97..c1c88e34 100644 --- a/custom-domain/dstack-ingress/scripts/entrypoint.sh +++ b/custom-domain/dstack-ingress/scripts/entrypoint.sh @@ -268,6 +268,16 @@ backend ${be_name} set_alias_record() { local domain="$1" + if [ -n "${ACME_CHALLENGE_ALIAS:-}" ]; then + # certbot validates the challenge at the base name even for a wildcard, + # so the _acme-challenge CNAME must use the base domain (strip "*."). + local base="${domain#\*.}" + echo "[challenge-delegation] Not touching ${domain}'s own zone (token is scoped to the delegated zone)." + echo "[challenge-delegation] Set these in your production zone yourself (static, one-time):" + echo " ${domain} CNAME ${GATEWAY_DOMAIN}" + echo " _acme-challenge.${base} CNAME _acme-challenge.${base}.${ACME_CHALLENGE_ALIAS}" + return + fi echo "Setting alias record for $domain" dnsman.py set_alias \ --domain "$domain" \ @@ -301,6 +311,12 @@ set_txt_record() { txt_domain="${TXT_PREFIX}.${domain}" fi + if [ -n "${ACME_CHALLENGE_ALIAS:-}" ]; then + echo "[challenge-delegation] Set this in your production zone yourself (static, one-time):" + echo " ${txt_domain} TXT \"$APP_ID:$PORT\"" + return + fi + dnsman.py set_txt \ --domain "$txt_domain" \ --content "$APP_ID:$PORT" @@ -311,37 +327,101 @@ set_txt_record() { fi } +# caa_domain_and_tag DOMAIN -> prints "caa_domain caa_tag" (strips a wildcard). +caa_domain_and_tag() { + local domain="$1" + if [[ "$domain" == \*.* ]]; then + echo "${domain#\*.} issuewild" + else + echo "$domain issue" + fi +} + +# In delegation mode we have NO token for the served domain's zone, so we cannot +# set the accounturi CAA ourselves. That CAA is the forge-prevention (only this +# enclave's ACME account may issue), so this path is deliberately NOT gated by +# SET_CAA and fails closed if the record is confirmed absent — otherwise anyone +# who can satisfy the delegated DNS-01 challenge could obtain a cert for the +# domain. Set ALLOW_MISSING_CAA=true to override (accept the risk). +verify_delegation_caa() { + local domain="$1" + local account_file account_uri caa_domain caa_tag caa_value resp status found + + if ! account_file=$(get_letsencrypt_account_file); then + echo "ERROR: cannot read the Let's Encrypt account file to determine the required accounturi CAA" >&2 + caa_fail_or_allow "$domain" + return + fi + read -r caa_domain caa_tag < <(caa_domain_and_tag "$domain") + account_uri=$(jq -j '.uri' "$account_file") + caa_value="letsencrypt.org;validationmethods=dns-01;accounturi=$account_uri" + + echo "[challenge-delegation] Set this CAA in your production zone (static, one-time):" + echo " ${caa_domain} CAA 0 ${caa_tag} \"${caa_value}\"" + + # Verify via DNS-over-HTTPS (dig is not installed in the image; curl+jq are). + resp=$(curl -s --max-time 10 "https://dns.google/resolve?name=${caa_domain}&type=257" 2>/dev/null || true) + if [ -z "$resp" ]; then + echo "WARNING: could not reach dns.google to verify the CAA (network issue) — NOT confirmed; continuing" + return + fi + status=$(echo "$resp" | jq -r '.Status // empty' 2>/dev/null || true) + if [ "$status" != "0" ]; then + echo "WARNING: CAA DoH query for ${caa_domain} returned status ${status:-unknown} — NOT confirmed; continuing" + return + fi + # Literal (grep -F) match on the CAA data fields — the account URI contains + # '/' and '.', which must not be treated as regex. + found=$(echo "$resp" | jq -r '.Answer // [] | .[] | .data' 2>/dev/null | grep -F "accounturi=$account_uri" || true) + if [ -n "$found" ]; then + echo "[challenge-delegation] Verified: accounturi CAA is present for $caa_domain" + return + fi + echo "ERROR: the accounturi CAA is NOT present for $caa_domain." >&2 + echo "ERROR: without it, anyone who can satisfy the delegated DNS-01 challenge could obtain a" >&2 + echo "ERROR: publicly-trusted certificate for this domain (forged TLS termination)." >&2 + caa_fail_or_allow "$domain" +} + +caa_fail_or_allow() { + local domain="$1" + if [ "${ALLOW_MISSING_CAA:-false}" = "true" ]; then + echo "WARNING: ALLOW_MISSING_CAA=true — continuing without a verified accounturi CAA for $domain" + return 0 + fi + echo "ERROR: refusing to continue without the accounturi CAA for $domain." >&2 + echo "ERROR: set the CAA record shown above, or ALLOW_MISSING_CAA=true to override." >&2 + exit 1 +} + set_caa_record() { local domain="$1" + + # Delegation mode is handled separately and is NOT gated by SET_CAA. + if [ -n "${ACME_CHALLENGE_ALIAS:-}" ]; then + verify_delegation_caa "$domain" + return + fi + if [ "$SET_CAA" != "true" ]; then echo "Skipping CAA record setup" return fi - local ACCOUNT_URI - local account_file - + local account_file account_uri caa_domain caa_tag if ! account_file=$(get_letsencrypt_account_file); then echo "Warning: Cannot set CAA record - account file not found" echo "This is not critical - certificates can still be issued without CAA records" return fi + read -r caa_domain caa_tag < <(caa_domain_and_tag "$domain") + account_uri=$(jq -j '.uri' "$account_file") - local caa_domain caa_tag - if [[ "$domain" == \*.* ]]; then - caa_domain="${domain#\*.}" - caa_tag="issuewild" - else - caa_domain="$domain" - caa_tag="issue" - fi - - ACCOUNT_URI=$(jq -j '.uri' "$account_file") - echo "Adding CAA record ($caa_tag) for $caa_domain, accounturi=$ACCOUNT_URI" + echo "Adding CAA record ($caa_tag) for $caa_domain, accounturi=$account_uri" dnsman.py set_caa \ --domain "$caa_domain" \ --caa-tag "$caa_tag" \ - --caa-value "letsencrypt.org;validationmethods=dns-01;accounturi=$ACCOUNT_URI" + --caa-value "letsencrypt.org;validationmethods=dns-01;accounturi=$account_uri" if [ $? -ne 0 ]; then echo "Warning: Failed to set CAA record for $domain"