diff --git a/custom-domain/dstack-ingress/Dockerfile b/custom-domain/dstack-ingress/Dockerfile index a018ac95..a6452448 100644 --- a/custom-domain/dstack-ingress/Dockerfile +++ b/custom-domain/dstack-ingress/Dockerfile @@ -34,6 +34,22 @@ RUN --mount=type=bind,source=pinned-packages.txt,target=/tmp/pinned-packages.txt mini-httpd && \ rm -rf /var/lib/apt/lists/* /var/log/* /var/cache/ldconfig/aux-cache +# lego, for the tls-alpn-01 challenge. certbot cannot do tls-alpn-01: its +# standalone plugin is HTTP-01 only and the acme library removed the challenge +# in 4.2.0. Pinned by checksum so the build stays reproducible; bump both the +# version and the digest together. +ARG LEGO_VERSION=5.3.1 +ARG LEGO_SHA256=b3c71b122ee1947eacfe0b809b955647f6377239fe4bfc49f73b1a091ae1252a +RUN set -eu; \ + url="https://github.com/go-acme/lego/releases/download/v${LEGO_VERSION}/lego_v${LEGO_VERSION}_linux_amd64.tar.gz"; \ + curl -fsSL -o /tmp/lego.tar.gz "$url"; \ + echo "${LEGO_SHA256} /tmp/lego.tar.gz" | sha256sum -c -; \ + tar -xzf /tmp/lego.tar.gz -C /usr/local/bin lego; \ + rm -f /tmp/lego.tar.gz; \ + chmod 755 /usr/local/bin/lego; \ + touch -d @0 /usr/local/bin/lego; \ + lego --version + RUN mkdir -p \ /etc/letsencrypt \ /var/www/certbot \ diff --git a/custom-domain/dstack-ingress/README.md b/custom-domain/dstack-ingress/README.md index ef16d2ea..a0db066e 100644 --- a/custom-domain/dstack-ingress/README.md +++ b/custom-domain/dstack-ingress/README.md @@ -159,7 +159,7 @@ environment: | `DOMAIN` | Your domain (single-domain mode). Supports wildcards (`*.example.com`) | | `TARGET_ENDPOINT` | Backend address, e.g. `app:80` or `http://app:80` | | `GATEWAY_DOMAIN` | dstack gateway domain (e.g. `_.dstack-prod5.phala.network`) | -| `CERTBOT_EMAIL` | Email for Let's Encrypt registration | +| `ACME_EMAIL` | *(optional)* ACME contact address, in either mode. `CERTBOT_EMAIL` is the historical name and still works. See below — it is optional, and published | | `DNS_PROVIDER` | DNS provider (`cloudflare`, `linode`, `namecheap`) | ### Optional @@ -169,9 +169,21 @@ environment: | `PORT` | `443` | HAProxy listen port | | `DOMAINS` | | Multiple domains, one per line | | `ROUTING_MAP` | | Multi-domain routing: `domain=host:port` per line | -| `SET_CAA` | `false` | Enable CAA DNS record | +| `SET_CAA` | `false` | Enable CAA DNS record (dns-01 only; tls-alpn-01 cannot write DNS) | | `TXT_PREFIX` | `_dstack-app-address` | DNS TXT record prefix | -| `CERTBOT_STAGING` | `false` | Use Let's Encrypt staging server | +| `ACME_STAGING` | `false` | Use Let's Encrypt staging, in either mode. `CERTBOT_STAGING` is the historical name and still works | +| `CHALLENGE_TYPE` | `dns-01` | `dns-01` (certbot + DNS credentials) or `tls-alpn-01` (lego, no DNS credentials) | +| `DNS_SETUP_MODE` | `wait` | tls-alpn-01 only: `wait`, `print` or `webhook` — see below | +| `DNS_SETUP_TIMEOUT` | `1800` | tls-alpn-01 only: seconds to wait for the records to appear | +| `DNS_SETUP_INTERVAL` | `15` | tls-alpn-01 only: seconds between DNS checks | +| `DNS_WEBHOOK_URL` | | tls-alpn-01 + `DNS_SETUP_MODE=webhook`: endpoint to notify | +| `DNS_WEBHOOK_TOKEN` | | Shared secret; the payload is HMAC-SHA256 signed with it | +| `DOH_RESOLVERS` | Google + Cloudflare | Comma-separated DoH endpoints used to verify records | +| `TLSALPN_PORT` | `9443` | Loopback port lego's ACME responder binds to | +| `TLS_TERMINATE_PORT` | `9444` | Loopback port the TLS frontend moves to in tls-alpn-01 mode | +| `RENEW_DAYS_BEFORE` | client default | Days of remaining lifetime that trigger renewal. Applies to both modes: passed to lego as `--renew-days`, and written to certbot's `renew_before_expiry` | +| `RENEW_INTERVAL` | `43200` | Seconds between successful certificate passes | +| `DNS_SETTLE_SECONDS` | `30` | Wait after DNS verifies, so the gateway's own TXT cache expires | | `MAXCONN` | `4096` | HAProxy max connections | | `TIMEOUT_CONNECT` | `10s` | Backend connect timeout | | `TIMEOUT_CLIENT` | `86400s` | Client-side timeout (24h for long-lived connections) | @@ -266,3 +278,140 @@ Permission is hereby granted, free of charge, to any person obtaining a copy of The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +## Certificates without DNS credentials (tls-alpn-01) + +The default `dns-01` flow needs a DNS API token inside the CVM: the container +creates the CNAME, TXT and CAA records itself. `CHALLENGE_TYPE=tls-alpn-01` +removes that requirement — nothing in the container can touch your DNS zone — +at the cost of you creating three records by hand (or via a webhook). + +```yaml +services: + dstack-ingress: + image: dstacktee/dstack-ingress: + environment: + - CHALLENGE_TYPE=tls-alpn-01 + - DOMAIN=app.example.com + - TARGET_ENDPOINT=http://app:80 + # Printed as the CNAME target, and used to verify that the hostname + # really resolves to the gateway before issuance starts. + - GATEWAY_DOMAIN=_.dstack-prod5.phala.network + # - ACME_EMAIL=you@example.com # optional, and published (see below) + # - DNS_SETUP_MODE=wait # default; blocks until the records exist + ports: + - "443:443" + volumes: + - /var/run/dstack.sock:/var/run/dstack.sock + - cert-data:/etc/letsencrypt + - evidences:/evidences +``` + +On first start the container prints the exact records to create, then polls +public DNS until they are visible: + +``` +========================================================================== + DNS records required for app.example.com +========================================================================== + CNAME app.example.com + -> _.dstack-prod5.phala.network + TXT _dstack-app-address.app.example.com + -> b1ea785543bbbb19ce9de33744321360992bf63b:443 + CAA app.example.com + -> 0 issue "letsencrypt.org;validationmethods=tls-alpn-01;accounturi=https://..." +========================================================================== +``` + +`DNS_SETUP_MODE` picks what happens after printing: `wait` (default) blocks +until the records resolve or `DNS_SETUP_TIMEOUT` elapses; `print` continues +immediately; `webhook` POSTs the records to `DNS_WEBHOOK_URL` first and then +waits, so a service of yours can create them automatically. + +### The TXT record names an *instance*, not an app + +Under `dns-01` the TXT record carries the **app ID** and the gateway +load-balances across every instance of the app. tls-alpn-01 cannot work that +way. The challenge is answered by whichever instance holds the ACME order, and +Let's Encrypt validates from several vantage points at once (5 distinct source +IPs within ~2 seconds, measured), while the gateway races connections across the +app's instances. The challenge would land on the wrong replica almost every +time. So this mode publishes the **instance ID** instead, pinning the hostname +to one instance. + +Two consequences: + +- **tls-alpn-01 mode is effectively single-instance.** All traffic for the + hostname goes to the pinned instance; you lose the gateway's failover. +- **The TXT record changes when the CVM instance is replaced.** Redeploying + means updating DNS. `DNS_SETUP_MODE=webhook` exists so this can be automated; + doing it by hand means downtime on every redeploy. + +## The ACME contact address is optional, and public + +`ACME_EMAIL` (or `CERTBOT_EMAIL`) may be left unset in **either** mode. RFC 8555 +makes the ACME `contact` field optional, and Let's Encrypt stopped sending expiry +notification mail in 2025, so setting one buys little. + +It also does not stay private. The ACME account document is published as +attestation evidence at `/evidences/acme-account.json`, so an address set here is +readable by anyone who fetches the evidence endpoint. Leave it unset unless you +specifically want a contact on the account. + +Under the hood the two clients differ: lego simply omits the flag, while certbot +needs `--register-unsafely-without-email` — its "unsafely" naming predates Let's +Encrypt dropping expiry mail, and the container passes it for you. + +### Limitations + +- **No wildcards.** RFC 8737 forbids tls-alpn-01 for wildcard identifiers, and + the CA will not offer the challenge. The container refuses to start rather + than serve a name it can never obtain a certificate for, so a wildcard + anywhere in `DOMAIN`/`DOMAINS` is a startup error even if the other names are + fine. Use `dns-01` for `*.example.com`. +- **The gateway must be reachable on port 443.** The CA connects to port 443 of + whatever the CNAME resolves to; the port is fixed by the protocol. +- **CAA must permit `tls-alpn-01`.** A record left over from a `dns-01` + deployment says `validationmethods=dns-01` and will make the CA refuse. The + container checks this before asking for a certificate, so you get a clear + message instead of a failed validation. +- **Losing the `cert-data` volume changes the account URI.** With `dns-01` the + container just rewrites the CAA record; here it cannot, so a pinned + `accounturi` would start rejecting issuance until you update it by hand. + +### Webhook payload + +`DNS_SETUP_MODE=webhook` POSTs this envelope to `DNS_WEBHOOK_URL`: + +```json +{ + "payload": "{\"app_id\":\"…\",\"challenge\":\"tls-alpn-01\",\"domain\":\"app.example.com\",\"instance_id\":\"…\",\"records\":[…],\"timestamp\":1234567890,\"version\":1}", + "hmac_sha256": "…", + "attestation": { "quote": "…", "report_data": "…" } +} +``` + +`payload` is a *string* so you sign and hash exactly the bytes you received. +Verify `hmac_sha256` with `DNS_WEBHOOK_TOKEN`, and — since this request asks you +to point a hostname at the instance it names — verify `attestation` too: the +quote's `report_data` is `sha256(payload)`, so it proves which enclave produced +those records. Check the app ID and measurements in it before changing DNS. + +### Why lego and not certbot + +certbot cannot do tls-alpn-01. Its standalone plugin is HTTP-01 only, the +maintainers declined to implement the challenge +([certbot#6724](https://github.com/certbot/certbot/issues/6724)), and the `acme` +library removed it outright in 4.2.0 — the last release carrying +`acme.standalone.TLSALPN01Server` is 4.1.1, where it is already marked +deprecated. This path therefore runs [lego](https://github.com/go-acme/lego), +pinned by checksum in the Dockerfile. The `dns-01` path still uses certbot and +is untouched. + +Because haproxy owns the public port, the proxy peeks at each ClientHello and +forwards only connections advertising the `acme-tls/1` ALPN protocol to lego's +responder on loopback; everything else goes to the normal TLS frontend. Issuance +and renewal therefore never interrupt serving traffic. In this mode haproxy +starts on a self-signed placeholder certificate — it has to be listening before +the first certificate can be issued — and reloads onto the real one as soon as +it arrives. diff --git a/custom-domain/dstack-ingress/TESTING.md b/custom-domain/dstack-ingress/TESTING.md new file mode 100644 index 00000000..4ba1d820 --- /dev/null +++ b/custom-domain/dstack-ingress/TESTING.md @@ -0,0 +1,281 @@ +# Testing dstack-ingress + +This component's entire job is to obtain and serve TLS certificates. Almost +everything that can go wrong involves a party we do not control — a CA, a DNS +resolver, a gateway, a TEE — so a test that mocks those tests very little. What +follows is how to exercise it against the real things, cheaply enough to do +routinely. + +Use **Let's Encrypt staging** throughout (`CERTBOT_STAGING=true` / +`ACME_STAGING=true`). Its rate limits are generous and its certificates are +untrusted, which is exactly what you want: a test certificate that browsers +reject cannot be mistaken for a production one. + +## Three tiers, in increasing cost + +Pick the cheapest tier that can actually observe what you changed. + +| Tier | Setup | Can test | Cannot test | +|---|---|---|---| +| **1. Container + simulator** | Docker, dstack simulator | dns-01 end to end, config generation, negative paths, `DNS_SETUP_MODE`, evidence, webhook | Anything needing inbound traffic | +| **2. Tier 1 + real DNS credentials** | + a zone you control | Everything the DNS provider code does | Same | +| **3. Real CVM behind a gateway** | + TDX host, `dstack-vmm`, `dstack-gateway`, public port 443 | tls-alpn-01, ALPN routing, real TDX quotes, gateway interaction | — | + +dns-01 needs no inbound connection: the CA reads a TXT record and never talks to +you. That is why the whole dns-01 path fits in tier 1/2. tls-alpn-01 is the +opposite — the CA connects to port 443 and completes a TLS handshake — so it can +only be tested for real in tier 3. + +## Tier 1/2: containers + +The container asks the guest agent for its app and instance ID and for TDX +quotes, so it needs a socket at `/var/run/dstack.sock`. Outside a CVM, use the +simulator: + +```bash +cd dstack/sdk/simulator && ./build.sh && ./dstack-simulator & +``` + +Then mount its socket: + +```yaml +volumes: + - /path/to/sdk/simulator/dstack.sock:/var/run/dstack.sock +``` + +Keep provider credentials out of the compose file — put them in a `.env` that +compose reads, and `chmod 600` it: + +```bash +umask 077 && printf 'CLOUDFLARE_API_TOKEN=%s\n' "$TOKEN" > .env +``` + +Give each scenario its own project name and its own volumes +(`docker compose -p `), so a leftover certificate from a previous run +cannot make the next one pass for the wrong reason. This matters more than it +sounds — see "Start from a clean zone" below. + +## Tier 3: a real CVM + +You need a TDX host running `dstack-vmm`, `dstack-kms` and `dstack-gateway`, and +the gateway must be reachable from the internet on **port 443** — the CA picks +that port and RFC 8737 gives you no say. If the gateway listens elsewhere, +redirect: + +```bash +sudo iptables -t nat -A PREROUTING -d -p tcp --dport 443 \ + -m comment --comment 'ingress-lab' -j REDIRECT --to-ports +``` + +Tag every rule with a `--comment` so teardown can find them all later. + +Point the test hostname at the gateway (an `A` record works; the container's +CNAME check compares resolved addresses, not record types), then publish the +routing TXT: + +``` +_dstack-app-address. TXT ":443" +``` + +**Instance ID, not app ID.** In tls-alpn-01 mode the challenge is answered by +whichever instance holds the ACME order, while the gateway load-balances across +every instance of the app and the CA validates from several vantage points at +once — five distinct source IPs within ~2 seconds, in practice. Publishing the +app ID sends the challenge to the wrong replica almost every time. + +Reading container logs inside a CVM: with `--public-logs`, the guest agent serves +them over the gateway's WireGuard network. + +```bash +curl -s 'http://10.44.0.:8090/logs/dstack-dstack-ingress-1?text&bare&tail=500' +``` + +The container name is the compose-mangled one (`dstack--1`), not the +service name. Find the instance's WireGuard address by probing the peers listed +in `wg show allowed-ips`; stale peers from removed VMs stay in the +list, so probe for an open port rather than trusting it. + +## Exercising each area + +### Issuance + +Nothing special — start the container and wait. What to assert: + +- the certificate is served on port 443 and its issuer is a staging CA; +- SAN matches the requested name; +- for `ROUTING_MAP`, each hostname reaches *its own* backend. Routing is by SNI, + so send the SNI, not just a `Host` header. + +### The ALPN split (tls-alpn-01) + +The peek frontend forwards `acme-tls/1` to lego and everything else to the TLS +frontend. You can drive both sides by hand: + +```bash +openssl s_client -connect :443 -servername -alpn acme-tls/1 +openssl s_client -connect :443 -servername -alpn h2,http/1.1 +``` + +and then confirm in the container's haproxy log that they took different paths: + +``` +tls_peek be_acme/lego ← the ACME probe +tls_peek be_tls_terminate/local ← normal traffic +``` + +This is the mechanism the whole mode rests on, and it is worth checking directly +rather than inferring it from "issuance worked". + +### Renewal — both branches + +Renewal is the part most likely to break in production and the least likely to be +covered, because a fresh certificate is not due for ~60 days. Both clients take a +renewal window, and `RENEW_DAYS_BEFORE` sets it in either mode: + +- **not due**: leave `RENEW_DAYS_BEFORE` unset and restart the container. Expect + lego `Skip renewal` / certbot `No certificates need renewal`, **no haproxy + reload, and no new evidence**. A pass that reloads on every check is a bug — + it means the renewal decision is not being read correctly. +- **due**: set `RENEW_DAYS_BEFORE=365` and restart. Expect a new certificate, + a reload, and regenerated evidence. + +Then check the certificate actually changed on the wire — the serial number, not +just the log line: + +```bash +openssl s_client -connect :443 -servername /dev/null \ + | openssl x509 -noout -serial -dates +``` + +Shorten `RENEW_INTERVAL` (default 43200s) if you want the loop to come round +again without restarting — but note that `RENEW_DAYS_BEFORE=365` makes the +certificate *permanently* due, so every pass renews. Combined with a short +interval that is a loop issuing certificates as fast as the CA will allow. Set +one or the other, unset it once you have seen the branch you came for, and never +carry it into production. + +Do not assert on the ACME client's log wording. lego 4.x wrote `no renewal` and +5.x writes `Skip renewal`; certbot's phrasing has moved too. The container uses +lego's `--deploy-hook`, which fires only on an actual renewal, and certbot's exit +behaviour — test the observable effect (reload / no reload), not the prose. + +### Evidence + +`/evidences/` is served on the TLS port. The chain to verify: + +``` +quote.report_data == sha256(sha256sum.txt) +sha256sum.txt covers acme-account.json and every cert-.pem +cert-.pem == the certificate actually served +``` + +so: + +```bash +curl -sk https:///evidences/sha256sum.txt -o s.txt && sha256sum -c s.txt +python3 -c "import json,hashlib;print(json.load(open('quote.json'))['report_data'][:64] \ + == hashlib.sha256(open('s.txt','rb').read()).hexdigest())" +``` + +Compare the evidence copy against a live `openssl s_client` fingerprint — the +point of the chain is that the quote commits to what is *being served*, and only +that comparison tests it. + +To verify the quote itself rather than trusting it, run it through `dcap-qvl` +against Intel collateral and expect `status = UpToDate`. Re-run the whole chain +after a renewal: the evidence must track the new certificates, not go stale at +first issuance. + +### DNS setup modes (tls-alpn-01) + +- `wait` — the default. Start the container *before* creating the records and + confirm it blocks and says which record is missing. Then create them and + confirm it proceeds. This is also the only mode that runs the CAA + compatibility check. +- `print` — skips all verification, including CAA. Confirm it continues + immediately. +- `webhook` — set `DNS_WEBHOOK_URL` and `DNS_WEBHOOK_TOKEN` and point them at a + throwaway HTTP server. The body carries the records, an HMAC-SHA256 over the + payload, and a TDX quote whose `report_data` is `sha256(payload)`. Verify both: + the HMAC proves the shared secret, the quote proves the enclave. + +### Negative paths + +These fail fast and are cheap, so run them on every change: + +| Scenario | Expected | +|---|---| +| tls-alpn-01 + a wildcard domain | Refused before any ACME call, citing RFC 8737 | +| A CAA record with `validationmethods=dns-01`, mode tls-alpn-01 | Blocked with the restriction quoted back | +| TXT holding the wrong value | Reported as `want , saw `, not "missing" | +| An instance ID the gateway does not know | The CA reports a connection error — the gateway will not route to it | + +The last one is worth keeping: it is the cross-tenant impersonation defence, and +it is observable rather than merely argued. + +## Traps + +**Start from a clean zone.** Records left over from an earlier run change which +branch executes. A stale CAA record naming a previous ACME account makes dns-01's +first issuance attempt fail and the second succeed — which happens to be the path +where the code was correct, hiding a bug in the ordinary path where the first +attempt succeeds. Delete test records between runs and verify against the +authoritative nameserver, not a cached resolver. + +**One resolver is not enough to trust either answer.** Public resolvers cache +negative answers for the zone's SOA minimum, so a resolver queried before a +record existed can keep saying "no such record" for up to an hour. They also +disagree on wire format: Cloudflare returns CAA in RFC 3597 generic hex +(`\# 47 00 05 69 73 73 75 65 ...`), Google in presentation form. The container +queries two resolvers with deliberately opposite quorums — propagation needs all +of them to agree, CAA takes the union so any resolver seeing a restriction stops +issuance — and if you are checking DNS by hand you should do the same. + +**Public DNS being correct is not the same as the gateway acting on it.** The +gateway caches the app-address TXT for its TTL, so immediately after the value +changes it can still route to the previous target. `DNS_SETTLE_SECONDS` +(default 30, first pass only) covers this. A challenge that fails with +`Error getting validation data` right after a DNS change is usually this, not a +broken challenge. + +**lego buffers its output until it exits.** A renewal can show nothing at all in +the log for several minutes. Check whether the process is still running before +concluding it hung. + +**`ALPN` is not validated against the backend.** Advertising `h2` in front of an +HTTP/1.1-only backend makes haproxy negotiate HTTP/2 on its behalf and the +connection then breaks. If a test client fails but `--http1.1` works, suspect +this before suspecting the proxy. + +## Unit tests + +```bash +python3 scripts/tests/test_dnsguide.py # DNS/CAA parsing and record building +bash scripts/tests/test_sanitizers.sh # env var validation +``` + +The parsing tests are built from rdata real resolvers actually returned, in both +encodings. When you touch DNS handling, add the real string you saw rather than +one you composed — the encodings are the part that surprises people. + +## haproxy config equivalence + +The proxy config is assembled by `haproxy-lib.sh` and composed differently by +each mode. When refactoring that assembly, generate the config both before and +after your change for all four combinations — `dns-01` and `tls-alpn-01`, each +with a single domain and with `ROUTING_MAP` — and diff them. Byte-identical +output is a much stronger statement than "the tests still pass", and it is easy +to obtain because the emitters only write to stdout. + +## Cleanup + +Leftovers from a certificate test are unusually annoying: a stray CAA record can +block issuance for a domain weeks later, and a stray DNAT rule silently +intercepts port 443. Tear down in this order and verify each: + +1. containers and volumes (`docker compose -p down -v`); +2. the CVM (`vmm-cli remove`); +3. iptables rules — delete by comment tag, in a loop, until none match; +4. DNS records — list them from the provider API afterwards, not by `dig`, + which can serve a cached answer for a record you already deleted; +5. the credentials file, the simulator, and any images built for the test. diff --git a/custom-domain/dstack-ingress/scripts/build-combined-pems.sh b/custom-domain/dstack-ingress/scripts/build-combined-pems.sh deleted file mode 100644 index 6e10fcb4..00000000 --- a/custom-domain/dstack-ingress/scripts/build-combined-pems.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash -# build-combined-pems.sh - Concatenate Let's Encrypt cert files into -# HAProxy combined PEM format (fullchain + privkey in one file). - -set -e - -source /scripts/functions.sh - -CERT_DIR="/etc/haproxy/certs" -mkdir -p "$CERT_DIR" - -all_domains=$(get-all-domains.sh) - -while IFS= read -r domain; do - [[ -n "$domain" ]] || continue - le_dir="/etc/letsencrypt/live/$(cert_dir_name "$domain")" - combined="${CERT_DIR}/${domain}.pem" - if [ -f "${le_dir}/fullchain.pem" ] && [ -f "${le_dir}/privkey.pem" ]; then - cat "${le_dir}/fullchain.pem" "${le_dir}/privkey.pem" > "$combined" - chmod 600 "$combined" - echo "Combined PEM created: ${combined}" - else - echo "Warning: Cert files missing for ${domain}, skipping" - fi -done <<< "$all_domains" diff --git a/custom-domain/dstack-ingress/scripts/certman.py b/custom-domain/dstack-ingress/scripts/certman.py index d12527f8..f9b1ede8 100644 --- a/custom-domain/dstack-ingress/scripts/certman.py +++ b/custom-domain/dstack-ingress/scripts/certman.py @@ -3,6 +3,7 @@ from dns_providers import DNSProviderFactory import argparse import os +import re import subprocess import sys import pkg_resources @@ -12,6 +13,17 @@ sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +def staging_enabled() -> bool: + """Whether to use Let's Encrypt staging. + + entrypoint.sh normalises the two names, but certman.py is also runnable on + its own, so accept both here as well. ACME_STAGING wins: the setting is + about the ACME server, not about certbot. + """ + value = os.environ.get("ACME_STAGING") or os.environ.get("CERTBOT_STAGING", "false") + return value == "true" + + class CertManager: """Certificate management using DNS provider infrastructure.""" @@ -281,12 +293,17 @@ def _build_certbot_command(self, action: str, domain: str, email: str) -> List[s f"--manual-auth-hook={hook} auth", f"--manual-cleanup-hook={hook} cleanup", "--agree-tos", "--no-eff-email", - "--email", email, "-d", domain, ]) + # Same optional-contact handling as the plugin path below. + if email: + base_cmd.extend(["--email", email]) + else: + base_cmd.append("--register-unsafely-without-email") + base_cmd.extend(["-d", domain]) # For `renew`, certbot reuses the authenticator + hooks saved in the # renewal config from the initial `certonly`, so we don't re-specify # them here (and must not fall back to the DNS plugin). - if os.environ.get("CERTBOT_STAGING", "false") == "true": + if staging_enabled(): base_cmd.append("--staging") masked = [a if not (i > 0 and base_cmd[i - 1] == "--email") else "" for i, a in enumerate(base_cmd)] @@ -313,9 +330,19 @@ def _build_certbot_command(self, action: str, domain: str, email: str) -> List[s f"Credentials file does not exist: {credentials_file}") if action == "certonly": - base_cmd.extend(["--agree-tos", "--no-eff-email", - "--email", email, "-d", domain]) - if os.environ.get("CERTBOT_STAGING", "false") == "true": + base_cmd.extend(["--agree-tos", "--no-eff-email"]) + # The ACME contact address is optional (RFC 8555 section 7.3), and + # it is published: the account document is served as attestation + # evidence, so an address set here is readable by anyone who + # fetches it. certbot will not simply omit the flag, so ask for a + # contactless account explicitly. Its "unsafely" naming predates + # Let's Encrypt dropping expiry notification mail in 2025. + if email: + base_cmd.extend(["--email", email]) + else: + base_cmd.append("--register-unsafely-without-email") + base_cmd.extend(["-d", domain]) + if staging_enabled(): base_cmd.extend(["--staging"]) if getattr(self.provider, 'CERTBOT_PROPAGATION_SECONDS'): @@ -404,6 +431,7 @@ def renew_certificate(self, domain: str) -> Tuple[bool, bool]: print(f"Failed to install plugin for renewal", file=sys.stderr) return False, False + self.apply_renewal_window(domain) cmd = self._build_certbot_command("renew", domain, "") try: @@ -449,6 +477,73 @@ def certificate_exists(self, domain: str) -> bool: cert_path = f"/etc/letsencrypt/live/{domain}/fullchain.pem" return os.path.isfile(cert_path) + @staticmethod + def _lineage_name(domain: str) -> str: + """certbot stores a wildcard lineage under the bare name.""" + return domain[2:] if domain.startswith("*.") else domain + + def apply_renewal_window(self, domain: str) -> None: + """Make RENEW_DAYS_BEFORE mean the same thing here as it does for lego. + + lego takes the renewal window as a flag (`--renew-days`). certbot has no + equivalent: it reads `renew_before_expiry` out of the lineage's renewal + config, so the setting has to be written there before `certbot renew` + runs. Without this the variable is silently tls-alpn-01 only, which also + left the dns-01 renewal branch with no way to be exercised on demand. + + Leaving it unset keeps certbot's own default (30 days). + """ + days = os.environ.get("RENEW_DAYS_BEFORE", "").strip() + if not days: + return + if not days.isdigit() or int(days) < 1: + print( + f"Warning: ignoring invalid RENEW_DAYS_BEFORE={days!r} " + f"(expected a positive number of days)", + file=sys.stderr, + ) + return + + path = f"/etc/letsencrypt/renewal/{self._lineage_name(domain)}.conf" + if not os.path.isfile(path): + # No lineage yet: the first issuance has not happened, and certbot + # writes this file itself. Nothing to do. + return + + setting = f"renew_before_expiry = {days} days" + try: + with open(path, encoding="utf-8") as fh: + lines = fh.read().splitlines() + except OSError as exc: + print(f"Warning: cannot read {path}: {exc}", file=sys.stderr) + return + + out, replaced = [], False + for line in lines: + # The key ships commented out, so match both forms. + if re.match(r"\s*#?\s*renew_before_expiry\s*=", line): + if not replaced: + out.append(setting) + replaced = True + continue + out.append(line) + + if not replaced: + # Must land before the first section header; the key is top-level. + insert_at = next( + (i for i, line in enumerate(out) if line.strip().startswith("[")), + len(out), + ) + out.insert(insert_at, setting) + + try: + with open(path, "w", encoding="utf-8") as fh: + fh.write("\n".join(out) + "\n") + except OSError as exc: + print(f"Warning: cannot write {path}: {exc}", file=sys.stderr) + return + print(f"Renewal window for {domain} set to {days} days before expiry") + def acme_account_exists(self) -> bool: """Check if an ACME account exists for the current server (staging or production). @@ -461,7 +556,7 @@ def acme_account_exists(self) -> bool: """ import glob api_endpoint = "acme-v02.api.letsencrypt.org" - if os.environ.get("CERTBOT_STAGING", "false") == "true": + if staging_enabled(): api_endpoint = "acme-staging-v02.api.letsencrypt.org" pattern = f"/etc/letsencrypt/accounts/{api_endpoint}/directory/*/regr.json" return len(glob.glob(pattern)) > 0 @@ -528,15 +623,11 @@ def main(): ) sys.exit(1) - # Email is required for obtain and auto actions - if args.action in ["obtain", "auto"] and not args.email: - if not os.environ.get("CERTBOT_EMAIL"): - print( - "Error: --email is required or set CERTBOT_EMAIL environment variable", - file=sys.stderr, - ) - sys.exit(1) - args.email = os.environ["CERTBOT_EMAIL"] + # The contact address is optional; see _build_certbot_command. + if not args.email: + args.email = os.environ.get("ACME_EMAIL") or os.environ.get( + "CERTBOT_EMAIL", "" + ) success, needs_evidence = manager.run_action( args.domain, args.email, args.action diff --git a/custom-domain/dstack-ingress/scripts/dns01.sh b/custom-domain/dstack-ingress/scripts/dns01.sh new file mode 100644 index 00000000..ada2353e --- /dev/null +++ b/custom-domain/dstack-ingress/scripts/dns01.sh @@ -0,0 +1,355 @@ +#!/bin/bash +# dns01.sh - certificates via certbot and a DNS provider API. +# +# entrypoint.sh validates the shared settings and then execs this file. There is +# no shared orchestration above it and no interface it has to implement -- this +# is simply the program that runs for dns-01, top to bottom. + +set -e + +source /scripts/functions.sh +source /scripts/haproxy-lib.sh +source /scripts/evidence-lib.sh + +# ACME_EMAIL / ACME_STAGING are normalised by entrypoint.sh and exported. The +# contact address is optional; certman.py asks for a contactless account when +# none is given. + +# certbot stores certificates under /etc/letsencrypt/live//. +cert_fullchain_path() { echo "/etc/letsencrypt/live/$(cert_dir_name "$1")/fullchain.pem"; } +cert_privkey_path() { echo "/etc/letsencrypt/live/$(cert_dir_name "$1")/privkey.pem"; } + +setup_py_env() { + if [ ! -d /opt/app-venv ]; then + echo "Creating application virtual environment" + python3 -m venv --system-site-packages /opt/app-venv + fi + + # Activate venv for subsequent steps + # shellcheck disable=SC1091 + source /opt/app-venv/bin/activate + + if [ ! -f /.venv_bootstrapped ]; then + echo "Bootstrapping certbot dependencies" + pip install --upgrade pip + pip install certbot requests boto3 botocore + touch /.venv_bootstrapped + fi + + ln -sf /opt/app-venv/bin/certbot /usr/local/bin/certbot + echo 'source /opt/app-venv/bin/activate' > /etc/profile.d/app-venv.sh +} + +setup_certbot_env() { + # Ensure the virtual environment is active for certbot configuration + # shellcheck disable=SC1091 + source /opt/app-venv/bin/activate + + if [ "${DNS_PROVIDER}" = "route53" ]; then + mkdir -p /root/.aws + + cat </root/.aws/config +[profile certbot] +role_arn=${AWS_ROLE_ARN} +source_profile=certbot-source +region=${AWS_REGION:-us-east-1} +EOF + + cat </root/.aws/credentials +[certbot-source] +aws_access_key_id=${AWS_ACCESS_KEY_ID} +aws_secret_access_key=${AWS_SECRET_ACCESS_KEY} +EOF + + unset AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN + export AWS_PROFILE=certbot + fi + + # Use the unified certbot manager to install plugins and setup credentials + echo "Installing DNS plugins and setting up credentials" + certman.py setup + if [ $? -ne 0 ]; then + echo "Error: Failed to setup certbot environment" + exit 1 + fi +} + +set_alias_record() { + local domain="$1" + if [ -n "${ACME_CHALLENGE_ALIAS:-}" ]; then + # certbot validates the challenge at the base name even for a wildcard, + # so the _acme-challenge CNAME must use the base domain (strip "*."). + local base="${domain#\*.}" + echo "[challenge-delegation] Not touching ${domain}'s own zone (token is scoped to the delegated zone)." + echo "[challenge-delegation] Set these in your production zone yourself (static, one-time):" + echo " ${domain} CNAME ${GATEWAY_DOMAIN}" + echo " _acme-challenge.${base} CNAME _acme-challenge.${base}.${ACME_CHALLENGE_ALIAS}" + return + fi + echo "Setting alias record for $domain" + dnsman.py set_alias \ + --domain "$domain" \ + --content "$GATEWAY_DOMAIN" + + if [ $? -ne 0 ]; then + echo "Error: Failed to set alias record for $domain" + exit 1 + fi + echo "Alias record set for $domain" +} + +txt_record_value() { echo "${APP_ID:-$DSTACK_APP_ID}:${PORT}"; } + +set_txt_record() { + local domain="$1" + local txt_domain + txt_domain=$(txt_record_name "$domain") + + if [ -n "${ACME_CHALLENGE_ALIAS:-}" ]; then + echo "[challenge-delegation] Set this in your production zone yourself (static, one-time):" + echo " ${txt_domain} TXT \"$(txt_record_value)\"" + return + fi + + dnsman.py set_txt \ + --domain "$txt_domain" \ + --content "$(txt_record_value)" + + if [ $? -ne 0 ]; then + echo "Error: Failed to set TXT record for $domain" + exit 1 + fi +} + +# In delegation mode we have NO token for the served domain's zone, so we cannot +# set the accounturi CAA ourselves. That CAA is the forge-prevention (only this +# enclave's ACME account may issue), so this path is deliberately NOT gated by +# SET_CAA and fails closed if the record is confirmed absent -- otherwise anyone +# who can satisfy the delegated DNS-01 challenge could obtain a cert for the +# domain. Set ALLOW_MISSING_CAA=true to override (accept the risk). +verify_delegation_caa() { + local domain="$1" + local account_file account_uri caa_domain caa_tag caa_value resp status found + + if ! account_file=$(get_letsencrypt_account_file); then + echo "ERROR: cannot read the Let's Encrypt account file to determine the required accounturi CAA" >&2 + caa_fail_or_allow "$domain" + return + fi + caa_domain="${domain#\*.}" + caa_tag=$(caa_tag_for "$domain") + account_uri=$(jq -j '.uri' "$account_file") + caa_value="letsencrypt.org;validationmethods=dns-01;accounturi=$account_uri" + + echo "[challenge-delegation] Set this CAA in your production zone (static, one-time):" + echo " ${caa_domain} CAA 0 ${caa_tag} \"${caa_value}\"" + + # Verify via DNS-over-HTTPS (dig is not installed in the image; curl+jq are). + resp=$(curl -s --max-time 10 "https://dns.google/resolve?name=${caa_domain}&type=257" 2>/dev/null || true) + if [ -z "$resp" ]; then + echo "WARNING: could not reach dns.google to verify the CAA (network issue) — NOT confirmed; continuing" + return + fi + status=$(echo "$resp" | jq -r '.Status // empty' 2>/dev/null || true) + if [ "$status" != "0" ]; then + echo "WARNING: CAA DoH query for ${caa_domain} returned status ${status:-unknown} — NOT confirmed; continuing" + return + fi + # Literal (grep -F) match on the CAA data fields — the account URI contains + # '/' and '.', which must not be treated as regex. + found=$(echo "$resp" | jq -r '.Answer // [] | .[] | .data' 2>/dev/null | grep -F "accounturi=$account_uri" || true) + if [ -n "$found" ]; then + echo "[challenge-delegation] Verified: accounturi CAA is present for $caa_domain" + return + fi + echo "ERROR: the accounturi CAA is NOT present for $caa_domain." >&2 + echo "ERROR: without it, anyone who can satisfy the delegated DNS-01 challenge could obtain a" >&2 + echo "ERROR: publicly-trusted certificate for this domain (forged TLS termination)." >&2 + caa_fail_or_allow "$domain" +} + +caa_fail_or_allow() { + local domain="$1" + if [ "${ALLOW_MISSING_CAA:-false}" = "true" ]; then + echo "WARNING: ALLOW_MISSING_CAA=true — continuing without a verified accounturi CAA for $domain" + return 0 + fi + echo "ERROR: refusing to continue without the accounturi CAA for $domain." >&2 + echo "ERROR: set the CAA record shown above, or ALLOW_MISSING_CAA=true to override." >&2 + exit 1 +} + +set_caa_record() { + local domain="$1" + + # Delegation mode is handled separately and is NOT gated by SET_CAA. + if [ -n "${ACME_CHALLENGE_ALIAS:-}" ]; then + verify_delegation_caa "$domain" + return + fi + + if [ "$SET_CAA" != "true" ]; then + echo "Skipping CAA record setup" + return + fi + + local ACCOUNT_URI + local account_file + + if ! account_file=$(get_letsencrypt_account_file); then + echo "Warning: Cannot set CAA record - account file not found" + echo "This is not critical - certificates can still be issued without CAA records" + return + fi + + local caa_domain caa_tag + caa_domain="${domain#\*.}" + caa_tag=$(caa_tag_for "$domain") + + ACCOUNT_URI=$(jq -j '.uri' "$account_file") + echo "Adding CAA record ($caa_tag) for $caa_domain, accounturi=$ACCOUNT_URI" + dnsman.py set_caa \ + --domain "$caa_domain" \ + --caa-tag "$caa_tag" \ + --caa-value "letsencrypt.org;validationmethods=dns-01;accounturi=$ACCOUNT_URI" + + if [ $? -ne 0 ]; then + echo "Warning: Failed to set CAA record for $domain" + echo "This is not critical - certificates can still be issued without CAA records" + echo "Consider disabling CAA records by setting SET_CAA=false if this continues to fail" + fi +} + +process_domain() { + local domain="$1" first status + + set_alias_record "$domain" + set_txt_record "$domain" + + # The CAA record names our ACME account, so the account has to exist first: + # try once, write CAA, try again. The first attempt is expected to fail when + # a CAA record from an earlier account is still in place. + if issue_certificate "$domain"; then first=0; else first=$?; fi + if [ "$first" -eq 1 ]; then + echo "First certificate attempt failed for $domain, retrying after the CAA record is set" + fi + + set_caa_record "$domain" + + if issue_certificate "$domain"; then status=0; else status=$?; fi + + # Either attempt producing a certificate counts as a change. Reporting only + # the second one hid the common case: on a clean first deployment attempt + # one obtains the certificate and attempt two reports "nothing to renew", + # so the pass looked quiet and skipped evidence generation entirely. + if [ "$first" -eq 0 ] || [ "$status" -eq 0 ]; then + return 0 + fi + return "$status" +} + +issue_certificate() { + source /opt/app-venv/bin/activate + # The contact address is optional; certman.py asks for a contactless + # account when none is given. + certman.py auto --domain "$1" ${ACME_EMAIL:+--email "$ACME_EMAIL"} +} + +build_combined_pems() { + local domain fullchain privkey combined + mkdir -p /etc/haproxy/certs + while IFS= read -r domain; do + [[ -n "$domain" ]] || continue + fullchain=$(cert_fullchain_path "$domain") + privkey=$(cert_privkey_path "$domain") + combined="/etc/haproxy/certs/${domain}.pem" + if [ -f "$fullchain" ] && [ -f "$privkey" ]; then + cat "$fullchain" "$privkey" > "$combined" + chmod 600 "$combined" + echo "Combined PEM created: ${combined}" + else + echo "Warning: Cert files missing for ${domain}, skipping" + fi + done <<<"$(get-all-domains.sh)" +} + +collect_evidence() { + local account domain + if ! account=$(get_letsencrypt_account_file); then + echo "Error: cannot generate evidence without the ACME account file" >&2 + return 1 + fi + evidence_reset + evidence_collect_account "$account" + while IFS= read -r domain; do + [[ -n "$domain" ]] || continue + evidence_collect_cert "$domain" "$(cert_fullchain_path "$domain")" + done <<<"$(get-all-domains.sh)" + evidence_finalize +} + +# One pass over every domain: make the DNS records right, then issue or renew. +# Idempotent, so the first pass and every later one are the same code. +run_pass() { + local domain status failed=0 changed=0 + while IFS= read -r domain; do + [[ -n "$domain" ]] || continue + # `if` context, not a bare command: process_domain returns 2 for + # "nothing to do", and under `set -e` a bare non-zero would kill the + # whole script on every quiet renewal pass. + if ( process_domain "$domain" ); then status=0; else status=$?; fi + case $status in + 0) changed=1 ;; + 2) ;; + *) echo "Certificate management failed for $domain" >&2; failed=1 ;; + esac + done <<<"$(get-all-domains.sh)" + + if [ "$changed" -eq 1 ]; then + collect_evidence || echo "Evidence generation failed" >&2 + build_combined_pems || echo "Combined PEM build failed" >&2 + haproxy_reload || true + fi + [ "$failed" -eq 0 ] +} + +# $1: seconds to wait before the first pass. dns-01 issues before it serves, so +# the caller has already run a pass by the time we get here; without this the +# loop would immediately repeat it -- two full passes, and for a single domain +# four certbot invocations plus a duplicate round of DNS writes, on every start. +cert_loop() { + local attempt=0 delay="${1:-0}" + while true; do + if [ "$delay" -gt 0 ]; then + echo "[$(date)] Next certificate check in ${delay}s" + sleep "$delay" + fi + if run_pass; then + attempt=0 + delay=${RENEW_INTERVAL:-43200} + else + attempt=$((attempt + 1)) + delay=$((60 * attempt)) + [ "$delay" -gt 1800 ] && delay=1800 + echo "[$(date)] Pass failed (attempt ${attempt}); retrying" >&2 + fi + done +} + +setup_py_env +setup_certbot_env +load_dstack_identity + +# dns-01 needs no inbound traffic, so keep the original order: certificate +# first, serving second. A failure here is fatal, as it always was. +run_pass +build_combined_pems + +haproxy_emit_global +haproxy_emit_tls_frontend ":${PORT}" +haproxy_emit_backends + +evidence_start_server +cert_loop "${RENEW_INTERVAL:-43200}" & + +exec "$@" diff --git a/custom-domain/dstack-ingress/scripts/dnsguide.py b/custom-domain/dstack-ingress/scripts/dnsguide.py new file mode 100644 index 00000000..7971e864 --- /dev/null +++ b/custom-domain/dstack-ingress/scripts/dnsguide.py @@ -0,0 +1,465 @@ +#!/usr/bin/env python3 +"""Guide the operator through the DNS records dstack-ingress needs. + +In tls-alpn-01 mode the container holds no DNS credentials, so it cannot create +the CNAME / TXT / CAA records itself. Instead it computes exactly which records +are required, prints them, optionally pushes them to a webhook, and then waits +until they are observable in public DNS before letting certificate issuance +proceed. + +Waiting matters more than it looks. If the TXT record is missing the gateway has +no route for the hostname, so the CA's connection never reaches this container +and the failure surfaces as an opaque timeout -- while still burning Let's +Encrypt's "failed validations" budget (5 per account per hostname per hour). + +Resolution goes through DoH rather than the container resolver: it sidesteps +local caching and needs no extra package (python3-requests is already present). +This is a convenience check for operator workflow, not a security control -- the +authoritative CAA evaluation is the CA's own, at issuance time. +""" + +import argparse +import json +import os +import sys +import time +from dataclasses import dataclass, asdict +from typing import Dict, List, Optional, Sequence, Tuple + +import requests + +# Two independent resolvers, because one is not enough to trust either answer. +# Observed while testing this: a resolver that was asked for a CAA record before +# the record existed kept serving the negative answer well past the record's +# TTL, while the other resolver already had it. A single resolver would have +# reported "no CAA, issuance unrestricted" for a name that in fact restricted +# issuance -- exactly the check we cannot afford to get wrong. +# +# So the two record classes are read with opposite quorums: +# propagation checks (CNAME, TXT) require *every* resolver to agree, which is +# what "wait until it is really out there" means; +# the CAA permission check takes the *union*, so any resolver seeing a +# restriction is enough to stop us. +DEFAULT_DOH = "https://dns.google/resolve,https://cloudflare-dns.com/dns-query" +DEFAULT_TIMEOUT = 1800 +DEFAULT_INTERVAL = 15 +QUERY_TIMEOUT = 10 + +RR_A = 1 +RR_CNAME = 5 +RR_TXT = 16 +RR_AAAA = 28 +RR_CAA = 257 + +LE_ISSUER = "letsencrypt.org" + + +@dataclass +class Record: + """A DNS record the operator has to create.""" + + type: str + name: str + value: str + note: str = "" + + +class ResolveError(RuntimeError): + """A DoH query failed in a way that is not simply "no such record".""" + + +def doh_query(name: str, rr_type: int, resolver: str) -> List[str]: + """Return the rdata strings for name/type, or [] when there are none.""" + try: + resp = requests.get( + resolver, + params={"name": name, "type": str(rr_type)}, + headers={"accept": "application/dns-json"}, + timeout=QUERY_TIMEOUT, + ) + except requests.RequestException as exc: + raise ResolveError(f"DoH query for {name} failed: {exc}") from exc + + if resp.status_code != 200: + raise ResolveError(f"DoH query for {name} returned HTTP {resp.status_code}") + + try: + payload = resp.json() + except ValueError as exc: + raise ResolveError(f"DoH response for {name} is not JSON: {exc}") from exc + + # Status 3 is NXDOMAIN: a definite "no such name", not an error for us. + status = payload.get("Status") + if status not in (0, 3): + raise ResolveError(f"DoH query for {name} returned DNS status {status}") + + answers = payload.get("Answer") or [] + return [a["data"] for a in answers if a.get("type") == rr_type and "data" in a] + + +def split_resolvers(spec: str) -> List[str]: + return [r.strip() for r in spec.split(",") if r.strip()] + + +def query_per_resolver(name: str, rr_type: int, resolvers: str) -> List[Tuple[str, List[str]]]: + """Query every resolver; raise only if none of them answered.""" + results = [] + errors = [] + for resolver in split_resolvers(resolvers): + try: + results.append((resolver, doh_query(name, rr_type, resolver))) + except ResolveError as exc: + errors.append(str(exc)) + if not results: + raise ResolveError("; ".join(errors) or f"no resolvers configured for {name}") + return results + + +def query_union(name: str, rr_type: int, resolvers: str) -> List[str]: + seen: List[str] = [] + for _resolver, values in query_per_resolver(name, rr_type, resolvers): + for value in values: + if value not in seen: + seen.append(value) + return seen + + +def _unquote_txt(value: str) -> str: + """Normalise TXT rdata, which resolvers hand back quoted and possibly split.""" + value = value.strip() + if value.startswith('"') and value.endswith('"'): + value = value[1:-1] + # A long TXT value arrives as several quoted chunks that concatenate. + return value.replace('" "', "") + + +def _fqdn(name: str) -> str: + return name.rstrip(".").lower() + + +def check_txt(record: Record, resolvers: str) -> Tuple[bool, str]: + lagging = [] + observed: List[str] = [] + for resolver, raw in query_per_resolver(record.name, RR_TXT, resolvers): + values = [_unquote_txt(v) for v in raw] + observed.extend(v for v in values if v not in observed) + if record.value not in values: + lagging.append(resolver) + if not lagging: + return True, "ok" + if not observed: + return False, f"no TXT record found (checked {len(lagging)} resolver(s))" + return False, ( + f"TXT present but wrong value: {observed!r} (want {record.value!r})" + if len(lagging) == len(split_resolvers(resolvers)) + else f"not yet propagated to {', '.join(lagging)} (saw {observed!r})" + ) + + +def check_alias(record: Record, resolvers: str) -> Tuple[bool, str]: + """Verify the hostname ends up at the gateway. + + Accepts either a CNAME pointing at the gateway or an address record whose + value matches one of the gateway's, so a flattened / ALIAS record at a zone + apex is not rejected. + """ + target = _fqdn(record.value) + + cnames = [_fqdn(v) for v in query_union(record.name, RR_CNAME, resolvers)] + if target in cnames: + return True, "ok (CNAME)" + + own_addrs = set(query_union(record.name, RR_A, resolvers)) | set( + query_union(record.name, RR_AAAA, resolvers) + ) + if not own_addrs: + return False, "hostname does not resolve yet" + + gw_addrs = set(query_union(target, RR_A, resolvers)) | set( + query_union(target, RR_AAAA, resolvers) + ) + if gw_addrs and own_addrs & gw_addrs: + return True, "ok (address matches gateway)" + if cnames: + return False, f"CNAME points at {cnames!r}, expected {target!r}" + return False, ( + f"resolves to {sorted(own_addrs)!r} which does not match the gateway " + f"{sorted(gw_addrs)!r}" + ) + + +def _parse_caa(rdata: str) -> Optional[Tuple[int, str, str]]: + """Parse CAA rdata in either form a DoH resolver may hand back. + + Google returns presentation format -- `0 issue "letsencrypt.org;..."`. + Cloudflare returns RFC 3597 generic format -- `\\# 47 00 05 69 73 73 75 65 + ...` -- i.e. the length followed by hex octets of the wire encoding. Parsing + only the first shape silently degrades a "CAA forbids this" answer into "no + CAA found", which is the wrong direction to fail in. + """ + rdata = rdata.strip() + + if rdata.startswith("\\#"): + # wire format: flags(1) tag-length(1) tag(tag-length) value(rest) + try: + raw = bytes.fromhex("".join(rdata.split()[2:])) + except ValueError: + return None + if len(raw) < 2: + return None + flags, tag_len = raw[0], raw[1] + tag = raw[2 : 2 + tag_len].decode("ascii", "replace").lower() + value = raw[2 + tag_len :].decode("utf-8", "replace") + return flags, tag, value + + parts = rdata.split(None, 2) + if len(parts) != 3: + return None + flags_s, tag, value = parts + try: + flags = int(flags_s) + except ValueError: + return None + return flags, tag.lower(), value.strip().strip('"') + + +def _caa_permits(value: str, tag: str, want_method: str, account_uri: str) -> Tuple[bool, str]: + """Does one CAA value allow us to issue via want_method for account_uri?""" + fields = [f.strip() for f in value.split(";")] + issuer = _fqdn(fields[0]) if fields else "" + if issuer != LE_ISSUER: + return False, f"{tag} allows {issuer!r}, not {LE_ISSUER}" + + params: Dict[str, str] = {} + for field in fields[1:]: + if "=" in field: + k, v = field.split("=", 1) + params[k.strip().lower()] = v.strip() + + methods = params.get("validationmethods") + if methods is not None: + allowed = {m.strip() for m in methods.split(",")} + if want_method not in allowed: + return False, ( + f"{tag} restricts validationmethods to {sorted(allowed)}, " + f"which excludes {want_method}" + ) + + uri = params.get("accounturi") + if uri is not None and account_uri and uri != account_uri: + return False, f"{tag} pins accounturi={uri}, but this container's account is {account_uri}" + + return True, "ok" + + +def check_caa( + domain: str, want_tag: str, want_method: str, account_uri: str, resolvers: str +) -> Tuple[bool, str]: + """Check CAA for domain, walking up to the closest ancestor that has one. + + An absent CAA record set means every CA may issue (RFC 8659), so "no CAA + anywhere" is a pass, not a failure. + """ + labels = _fqdn(domain).split(".") + for i in range(len(labels) - 1): + candidate = ".".join(labels[i:]) + # Union: one resolver seeing a restriction is enough to honour it. + rdatas = query_union(candidate, RR_CAA, resolvers) + # Deduplicate after parsing, not before: resolvers disagree on the wire + # format (one returns presentation form, another RFC 3597 generic hex), + # so the same record arrives as two distinct strings and would otherwise + # be reported, and counted, twice. + parsed = [] + for rdata in rdatas: + record = _parse_caa(rdata) + if record is not None and record not in parsed: + parsed.append(record) + relevant = [p for p in parsed if p[1] == want_tag] + if not parsed: + continue # keep climbing; this level has no CAA record set + + # This is the closest CAA record set; it is the one that governs. + if not relevant: + return False, ( + f"{candidate} has a CAA record set but no {want_tag} tag, " + f"which forbids issuance" + ) + reasons = [] + for _flags, tag, value in relevant: + ok, reason = _caa_permits(value, tag, want_method, account_uri) + if ok: + return True, f"ok (from {candidate})" + reasons.append(reason) + return False, f"CAA at {candidate} does not permit issuance: {'; '.join(reasons)}" + + return True, "ok (no CAA record set; issuance is unrestricted)" + + +def render(records: Sequence[Record], domain: str) -> str: + width = 74 + lines = [ + "", + "=" * width, + f" DNS records required for {domain}", + "=" * width, + " This container holds no DNS credentials, so you must create these", + " records yourself. Certificate issuance waits until they are visible.", + "", + ] + for rec in records: + lines.append(f" {rec.type:<6} {rec.name}") + lines.append(f" {'':<6} -> {rec.value}") + if rec.note: + lines.append(f" {'':<6} ({rec.note})") + lines.append("") + lines.append("=" * width) + lines.append("") + return "\n".join(lines) + + +def verify_once( + records: Sequence[Record], + domain: str, + caa_tag: str, + caa_method: str, + account_uri: str, + resolvers: str, +) -> List[Tuple[str, bool, str]]: + results = [] + for rec in records: + if rec.type == "TXT": + ok, why = check_txt(rec, resolvers) + results.append((f"TXT {rec.name}", ok, why)) + elif rec.type == "CNAME": + ok, why = check_alias(rec, resolvers) + results.append((f"CNAME {rec.name}", ok, why)) + ok, why = check_caa(domain, caa_tag, caa_method, account_uri, resolvers) + results.append((f"CAA {domain}", ok, why)) + return results + + +def build_records(args: argparse.Namespace) -> List[Record]: + include = {part.strip() for part in args.include.split(",") if part.strip()} + records = [] + if "cname" in include: + records.append( + Record( + type="CNAME", + name=args.domain.lstrip("*."), + value=args.alias_target, + note="points the hostname at the dstack gateway", + ) + ) + if "txt" in include: + records.append( + Record( + type="TXT", + name=args.txt_name, + value=args.txt_value, + note="tells the gateway which instance to route this hostname to", + ) + ) + if "caa" in include and args.caa_value: + note = "optional; restricts issuance for this name to Let's Encrypt" + if args.account_uri: + note = "optional, but pins issuance to this enclave's ACME account" + records.append( + Record( + type="CAA", + name=args.caa_name, + value=f'0 {args.caa_tag} "{args.caa_value}"', + note=note, + ) + ) + return records + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--domain", required=True) + parser.add_argument("--alias-target", required=True, help="gateway domain") + parser.add_argument("--txt-name", required=True) + parser.add_argument("--txt-value", required=True) + parser.add_argument("--caa-name", default="") + parser.add_argument("--caa-tag", default="issue", choices=["issue", "issuewild"]) + parser.add_argument("--caa-value", default="") + parser.add_argument("--account-uri", default="") + parser.add_argument("--challenge", default="tls-alpn-01") + parser.add_argument( + "--mode", + default=os.environ.get("DNS_SETUP_MODE", "wait"), + choices=["wait", "print", "webhook"], + ) + parser.add_argument("--timeout", type=int, default=int(os.environ.get("DNS_SETUP_TIMEOUT", DEFAULT_TIMEOUT))) + parser.add_argument("--interval", type=int, default=int(os.environ.get("DNS_SETUP_INTERVAL", DEFAULT_INTERVAL))) + parser.add_argument( + "--resolver", + default=os.environ.get("DOH_RESOLVERS", os.environ.get("DOH_RESOLVER", DEFAULT_DOH)), + help="comma-separated DoH endpoints", + ) + parser.add_argument("--json", action="store_true", help="also emit the records as JSON") + parser.add_argument( + "--include", + default="cname,txt,caa", + help="comma-separated subset of records to handle (cname,txt,caa)", + ) + args = parser.parse_args() + + if not args.caa_name: + args.caa_name = args.domain.lstrip("*.") + + records = build_records(args) + print(render(records, args.domain), flush=True) + if args.json: + print(json.dumps([asdict(r) for r in records], indent=2), flush=True) + + if args.mode == "webhook": + # Imported lazily so `print`/`wait` do not depend on the hook module. + import dnshook + + try: + dnshook.notify(records, domain=args.domain, challenge=args.challenge) + except Exception as exc: # noqa: BLE001 - a hook failure must not be fatal + print(f"Warning: DNS webhook notification failed: {exc}", file=sys.stderr, flush=True) + print("Continuing; the records above still have to exist.", file=sys.stderr, flush=True) + + if args.mode == "print": + print("DNS_SETUP_MODE=print: not verifying, continuing immediately.", flush=True) + return 0 + + deadline = time.monotonic() + args.timeout + attempt = 0 + while True: + attempt += 1 + try: + results = verify_once( + records, args.domain.lstrip("*."), args.caa_tag, args.challenge, + args.account_uri, args.resolver, + ) + except ResolveError as exc: + print(f"[dns-check #{attempt}] resolver problem: {exc}", flush=True) + results = [] + + if results and all(ok for _, ok, _ in results): + print(f"[dns-check #{attempt}] all records verified:", flush=True) + for label, _, why in results: + print(f" OK {label}: {why}", flush=True) + return 0 + + for label, ok, why in results: + print(f"[dns-check #{attempt}] {'OK ' if ok else 'WAIT'} {label}: {why}", flush=True) + + remaining = deadline - time.monotonic() + if remaining <= 0: + print( + f"Error: DNS records were not visible within {args.timeout}s. " + f"Create the records above, or set DNS_SETUP_MODE=print to skip this check.", + file=sys.stderr, + flush=True, + ) + return 1 + time.sleep(min(args.interval, max(1, int(remaining)))) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/custom-domain/dstack-ingress/scripts/dnshook.py b/custom-domain/dstack-ingress/scripts/dnshook.py new file mode 100644 index 00000000..3dd45d8c --- /dev/null +++ b/custom-domain/dstack-ingress/scripts/dnshook.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python3 +"""Push the required DNS records to an operator-supplied webhook. + +Because the instance ID is baked into the TXT record in tls-alpn-01 mode, that +record changes every time the CVM instance is replaced. Doing that by hand means +downtime on every redeploy, so this hook exists to let the operator automate it. + +That makes the hook security-relevant: whoever receives it is being asked to +point a hostname at whatever instance the caller names. An unauthenticated hook +would hand domain hijacking to anyone who can reach the endpoint. So every +request carries: + + * an HMAC-SHA256 over the exact payload string, keyed by a shared secret; and + * optionally a TDX quote whose report_data commits to that same payload. + +Verify both on the receiving end. The HMAC alone only proves the caller knows +the secret; the quote proves the caller is the enclave you expect, so check the +app_id / measurements in it before touching DNS. + +Payload is sent as a *string* field rather than a nested object on purpose: the +receiver signs and hashes exactly the bytes it was given, with no JSON +re-canonicalisation to disagree about. +""" + +import argparse +import hashlib +import hmac +import json +import os +import subprocess +import sys +import time +from dataclasses import asdict +from typing import Any, Dict, List, Optional, Sequence + +import requests + +DEFAULT_TIMEOUT = 15 +DEFAULT_RETRIES = 3 +DSTACK_SOCKETS = ( + ("/var/run/dstack.sock", "http://localhost/GetQuote?report_data={rd}"), + ("/var/run/tappd.sock", "http://localhost/prpc/Tappd.RawQuote?report_data={rd}"), +) + + +def _canonical(payload: Dict[str, Any]) -> str: + return json.dumps(payload, sort_keys=True, separators=(",", ":")) + + +def _sign(payload_str: str, secret: str) -> str: + return hmac.new(secret.encode(), payload_str.encode(), hashlib.sha256).hexdigest() + + +def get_quote(payload_str: str) -> Optional[Dict[str, Any]]: + """Fetch a TDX quote committing to payload_str, or None if unavailable.""" + report_data = hashlib.sha256(payload_str.encode()).hexdigest() + for sock, url_tpl in DSTACK_SOCKETS: + if not os.path.exists(sock): + continue + try: + out = subprocess.run( + ["curl", "-s", "--unix-socket", sock, url_tpl.format(rd=report_data)], + capture_output=True, + text=True, + timeout=30, + ) + if out.returncode != 0 or not out.stdout.strip(): + print( + f"Warning: quote fetch via {sock} failed (rc={out.returncode})", + file=sys.stderr, + ) + continue + quote = json.loads(out.stdout) + quote["report_data"] = report_data + return quote + except (subprocess.SubprocessError, ValueError) as exc: + print(f"Warning: quote fetch via {sock} failed: {exc}", file=sys.stderr) + return None + + +def build_envelope( + records: Sequence[Any], + domain: str, + challenge: str, + with_quote: bool = True, +) -> Dict[str, Any]: + payload = { + "version": 1, + "domain": domain, + "challenge": challenge, + "app_id": os.environ.get("DSTACK_APP_ID", ""), + "instance_id": os.environ.get("DSTACK_INSTANCE_ID", ""), + "timestamp": int(time.time()), + "records": [asdict(r) if hasattr(r, "__dataclass_fields__") else dict(r) for r in records], + } + payload_str = _canonical(payload) + envelope: Dict[str, Any] = {"payload": payload_str} + + secret = os.environ.get("DNS_WEBHOOK_TOKEN", "") + if secret: + envelope["hmac_sha256"] = _sign(payload_str, secret) + else: + print( + "Warning: DNS_WEBHOOK_TOKEN is unset, so the webhook request is " + "unauthenticated. Anyone who can reach your endpoint could redirect " + "your domain. Set a secret.", + file=sys.stderr, + ) + + if with_quote: + quote = get_quote(payload_str) + if quote is not None: + envelope["attestation"] = quote + return envelope + + +def notify( + records: Sequence[Any], + domain: str, + challenge: str, + url: Optional[str] = None, + retries: int = DEFAULT_RETRIES, +) -> None: + url = url or os.environ.get("DNS_WEBHOOK_URL", "") + if not url: + raise RuntimeError("DNS_SETUP_MODE=webhook but DNS_WEBHOOK_URL is not set") + + envelope = build_envelope(records, domain, challenge) + timeout = int(os.environ.get("DNS_WEBHOOK_TIMEOUT", DEFAULT_TIMEOUT)) + last_error: Optional[str] = None + + for attempt in range(1, retries + 1): + try: + resp = requests.post( + url, + json=envelope, + timeout=timeout, + headers={"user-agent": "dstack-ingress/dnshook"}, + ) + if 200 <= resp.status_code < 300: + print(f"DNS webhook accepted (HTTP {resp.status_code})", flush=True) + return + body = resp.text[:512] + last_error = f"HTTP {resp.status_code}: {body}" + except requests.RequestException as exc: + last_error = str(exc) + + print( + f"DNS webhook attempt {attempt}/{retries} failed: {last_error}", + file=sys.stderr, + flush=True, + ) + if attempt < retries: + time.sleep(2**attempt) + + raise RuntimeError(f"DNS webhook failed after {retries} attempts: {last_error}") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--url", default="", help="override DNS_WEBHOOK_URL") + parser.add_argument("--domain", required=True) + parser.add_argument("--challenge", default="tls-alpn-01") + parser.add_argument( + "--records", + required=True, + help="JSON array of {type,name,value} objects", + ) + parser.add_argument( + "--dry-run", action="store_true", help="print the envelope instead of sending" + ) + args = parser.parse_args() + + records: List[Dict[str, str]] = json.loads(args.records) + if args.dry_run: + print(json.dumps(build_envelope(records, args.domain, args.challenge), indent=2)) + return 0 + notify(records, args.domain, args.challenge, url=args.url or None) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/custom-domain/dstack-ingress/scripts/entrypoint.sh b/custom-domain/dstack-ingress/scripts/entrypoint.sh index c1c88e34..1f564e75 100644 --- a/custom-domain/dstack-ingress/scripts/entrypoint.sh +++ b/custom-domain/dstack-ingress/scripts/entrypoint.sh @@ -1,8 +1,15 @@ #!/bin/bash +# entrypoint.sh - validate the settings both modes share, then hand over. +# +# There is no shared orchestration below this point: dns-01 and tls-alpn-01 have +# genuinely little in common (different ACME client, different on-disk layout, +# different DNS story, different proxy topology, different startup order), so +# each is its own program. What they truly share lives in the *-lib.sh files and +# is composed by the mode, not dispatched to. set -e -source "/scripts/functions.sh" +source /scripts/functions.sh PORT=${PORT:-443} TXT_PREFIX=${TXT_PREFIX:-"_dstack-app-address"} @@ -13,6 +20,28 @@ TIMEOUT_SERVER=${TIMEOUT_SERVER:-86400s} EVIDENCE_SERVER=${EVIDENCE_SERVER:-true} EVIDENCE_PORT=${EVIDENCE_PORT:-80} ALPN=${ALPN:-} +CHALLENGE_TYPE=${CHALLENGE_TYPE:-dns-01} + +# ACME account settings, normalised once for both modes. What you are +# configuring is an ACME account, not a particular client -- which client +# implements a mode is an implementation detail, and this image already changed +# it once (certbot for dns-01, lego for tls-alpn-01). So ACME_EMAIL and +# ACME_STAGING are the names; CERTBOT_EMAIL and CERTBOT_STAGING are the +# historical ones and keep working. +# +# Normalising here rather than per mode is what fixes the real bug: ACME_STAGING +# used to be read only on the tls-alpn-01 path, so setting it under dns-01 +# silently issued *production* certificates. +ACME_EMAIL=${ACME_EMAIL:-${CERTBOT_EMAIL:-}} +ACME_STAGING=${ACME_STAGING:-${CERTBOT_STAGING:-false}} + +case "$CHALLENGE_TYPE" in + dns-01|tls-alpn-01) ;; + *) + echo "Error: invalid CHALLENGE_TYPE '$CHALLENGE_TYPE' (expected dns-01 or tls-alpn-01)" >&2 + exit 1 + ;; +esac if ! PORT=$(sanitize_port "$PORT"); then exit 1 @@ -52,450 +81,21 @@ for var in CLIENT_MAX_BODY_SIZE PROXY_READ_TIMEOUT PROXY_SEND_TIMEOUT PROXY_CONN fi done -# Parse TARGET_ENDPOINT into host:port for haproxy backend -parse_target_endpoint() { - local endpoint="$1" - # Strip protocol prefix if present (http://, https://, grpc://) - local hostport="${endpoint#*://}" - # If no protocol was stripped, use as-is - if [ "$hostport" = "$endpoint" ]; then - hostport="$endpoint" - fi - # Strip any trailing path - hostport="${hostport%%/*}" - echo "$hostport" -} - -echo "Setting up certbot environment" - -setup_py_env() { - if [ ! -d /opt/app-venv ]; then - echo "Creating application virtual environment" - python3 -m venv --system-site-packages /opt/app-venv - fi - - # Activate venv for subsequent steps - # shellcheck disable=SC1091 - source /opt/app-venv/bin/activate - - if [ ! -f /.venv_bootstrapped ]; then - echo "Bootstrapping certbot dependencies" - pip install --upgrade pip - pip install certbot requests boto3 botocore - touch /.venv_bootstrapped - fi - - ln -sf /opt/app-venv/bin/certbot /usr/local/bin/certbot - echo 'source /opt/app-venv/bin/activate' > /etc/profile.d/app-venv.sh -} - -setup_certbot_env() { - # Ensure the virtual environment is active for certbot configuration - # shellcheck disable=SC1091 - source /opt/app-venv/bin/activate - - if [ "${DNS_PROVIDER}" = "route53" ]; then - mkdir -p /root/.aws - - cat </root/.aws/config -[profile certbot] -role_arn=${AWS_ROLE_ARN} -source_profile=certbot-source -region=${AWS_REGION:-us-east-1} -EOF - - cat </root/.aws/credentials -[certbot-source] -aws_access_key_id=${AWS_ACCESS_KEY_ID} -aws_secret_access_key=${AWS_SECRET_ACCESS_KEY} -EOF - - unset AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN - export AWS_PROFILE=certbot - fi - - # Use the unified certbot manager to install plugins and setup credentials - echo "Installing DNS plugins and setting up credentials" - certman.py setup - if [ $? -ne 0 ]; then - echo "Error: Failed to setup certbot environment" +# Everything from here on belongs to one mode. Exported so the mode script and +# the helpers it invokes see the sanitized values. +export PORT DOMAIN DOMAINS TARGET_ENDPOINT ROUTING_MAP TXT_PREFIX MAXCONN +export TIMEOUT_CONNECT TIMEOUT_CLIENT TIMEOUT_SERVER EVIDENCE_SERVER EVIDENCE_PORT ALPN +export ACME_EMAIL ACME_STAGING + +case "$CHALLENGE_TYPE" in + dns-01) + exec /scripts/dns01.sh "$@" + ;; + tls-alpn-01) + exec /scripts/tlsalpn.sh "$@" + ;; + *) + echo "Error: invalid CHALLENGE_TYPE '$CHALLENGE_TYPE' (expected dns-01 or tls-alpn-01)" >&2 exit 1 - fi -} - -setup_py_env - -# Emit common haproxy global/defaults/frontend preamble. -# Both single-domain and multi-domain modes share this identical config. -emit_haproxy_preamble() { - # "crt " loads all PEM files from the directory. - # ALPN is appended conditionally via ${ALPN:+ alpn ${ALPN}}. - cat </etc/haproxy/haproxy.cfg -global - log stdout format raw local0 - maxconn ${MAXCONN} - pidfile /var/run/haproxy/haproxy.pid - ssl-default-bind-ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305 - ssl-default-bind-ciphersuites TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256 - ssl-default-bind-options ssl-min-ver TLSv1.2 no-tls-tickets - ssl-default-bind-curves secp384r1 - -defaults - log global - mode tcp - option tcplog - timeout connect ${TIMEOUT_CONNECT} - timeout client ${TIMEOUT_CLIENT} - timeout server ${TIMEOUT_SERVER} - -frontend tls_in - bind :${PORT} ssl crt /etc/haproxy/certs/${ALPN:+ alpn ${ALPN}} -EOF - - if [ "$EVIDENCE_SERVER" = "true" ]; then - cat <<'EVIDENCE_BLOCK' >>/etc/haproxy/haproxy.cfg - - # Route /evidences requests to the local evidence HTTP server. - # accept fires once 16 bytes have arrived — enough for the - # longest prefix we match ("HEAD /evidences" = 16 chars). - # Using req.len with a concrete threshold is critical: the - # previous payload(0,0) (length 0 = "whole buffer") deferred - # evaluation until the full inspect-delay because HAProxy - # cannot know when a TCP stream ends. - tcp-request inspect-delay 5s - tcp-request content accept if { req.len ge 16 } - acl is_evidence payload(0,16) -m beg "GET /evidences" - acl is_evidence payload(0,16) -m beg "HEAD /evidences" - use_backend be_evidence if is_evidence -EVIDENCE_BLOCK - fi -} - -# Append the evidence backend block to haproxy.cfg -emit_evidence_backend() { - if [ "$EVIDENCE_SERVER" = "true" ]; then - cat <>/etc/haproxy/haproxy.cfg - -backend be_evidence - mode http - http-request replace-path /evidences(.*) \1 - server evidence 127.0.0.1:${EVIDENCE_PORT} -EOF - fi -} - -# Generate haproxy.cfg for single-domain mode (DOMAIN + TARGET_ENDPOINT) -setup_haproxy_cfg() { - local target_hostport - target_hostport=$(parse_target_endpoint "$TARGET_ENDPOINT") - - emit_haproxy_preamble - - cat <>/etc/haproxy/haproxy.cfg - - default_backend be_upstream - -backend be_upstream - server app1 ${target_hostport} -EOF - - emit_evidence_backend -} - -# Generate haproxy.cfg for multi-domain mode (ROUTING_MAP) -setup_haproxy_cfg_multi() { - emit_haproxy_preamble - - # Parse ROUTING_MAP and generate use_backend rules + backend sections - # Support both newline-separated and comma-separated formats - local routing_map_normalized - routing_map_normalized=$(echo "$ROUTING_MAP" | tr ',' '\n') - - local backend_rules="" - local backend_sections="" - local first_be_name="" - local domain target be_name - - while IFS= read -r line; do - [[ -n "$line" ]] || continue - [[ "$line" == \#* ]] && continue - domain="${line%%=*}" - target="${line#*=}" - domain=$(echo "$domain" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') - target=$(echo "$target" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') - [[ -n "$domain" && -n "$target" ]] || continue - - # Validate domain and target to prevent config injection - if ! domain=$(sanitize_domain "$domain"); then - echo "Error: Invalid domain in ROUTING_MAP: ${line}" >&2 - exit 1 - fi - if ! target=$(sanitize_target_endpoint "$target"); then - echo "Error: Invalid target in ROUTING_MAP: ${line}" >&2 - exit 1 - fi - - # Strip protocol prefix from target if present - target=$(parse_target_endpoint "$target") - - # Generate safe backend name from domain - be_name="be_$(echo "$domain" | sed 's/[^A-Za-z0-9]/_/g')" - - if [ -z "$first_be_name" ]; then - first_be_name="$be_name" - fi - - backend_rules="${backend_rules} - use_backend ${be_name} if { ssl_fc_sni -i ${domain} }" - backend_sections="${backend_sections} - -backend ${be_name} - server s1 ${target}" - done <<< "$routing_map_normalized" - - echo "$backend_rules" >> /etc/haproxy/haproxy.cfg - - # Default to first backend in ROUTING_MAP - if [ -n "$first_be_name" ]; then - echo "" >> /etc/haproxy/haproxy.cfg - echo " default_backend ${first_be_name}" >> /etc/haproxy/haproxy.cfg - fi - - echo "$backend_sections" >> /etc/haproxy/haproxy.cfg - - emit_evidence_backend -} - -set_alias_record() { - local domain="$1" - if [ -n "${ACME_CHALLENGE_ALIAS:-}" ]; then - # certbot validates the challenge at the base name even for a wildcard, - # so the _acme-challenge CNAME must use the base domain (strip "*."). - local base="${domain#\*.}" - echo "[challenge-delegation] Not touching ${domain}'s own zone (token is scoped to the delegated zone)." - echo "[challenge-delegation] Set these in your production zone yourself (static, one-time):" - echo " ${domain} CNAME ${GATEWAY_DOMAIN}" - echo " _acme-challenge.${base} CNAME _acme-challenge.${base}.${ACME_CHALLENGE_ALIAS}" - return - fi - echo "Setting alias record for $domain" - dnsman.py set_alias \ - --domain "$domain" \ - --content "$GATEWAY_DOMAIN" - - if [ $? -ne 0 ]; then - echo "Error: Failed to set alias record for $domain" - exit 1 - fi - echo "Alias record set for $domain" -} - -set_txt_record() { - local domain="$1" - local APP_ID - - if [[ -S /var/run/dstack.sock ]]; then - DSTACK_APP_ID=$(curl -s --unix-socket /var/run/dstack.sock http://localhost/Info | jq -j .app_id) - export DSTACK_APP_ID - else - DSTACK_APP_ID=$(curl -s --unix-socket /var/run/tappd.sock http://localhost/prpc/Tappd.Info | jq -j .app_id) - export DSTACK_APP_ID - fi - APP_ID=${APP_ID:-"$DSTACK_APP_ID"} - - local txt_domain - if [[ "$domain" == \*.* ]]; then - # Wildcard domain: *.myapp.com → _dstack-app-address-wildcard.myapp.com - txt_domain="${TXT_PREFIX}-wildcard.${domain#\*.}" - else - txt_domain="${TXT_PREFIX}.${domain}" - fi - - if [ -n "${ACME_CHALLENGE_ALIAS:-}" ]; then - echo "[challenge-delegation] Set this in your production zone yourself (static, one-time):" - echo " ${txt_domain} TXT \"$APP_ID:$PORT\"" - return - fi - - dnsman.py set_txt \ - --domain "$txt_domain" \ - --content "$APP_ID:$PORT" - - if [ $? -ne 0 ]; then - echo "Error: Failed to set TXT record for $domain" - exit 1 - fi -} - -# caa_domain_and_tag DOMAIN -> prints "caa_domain caa_tag" (strips a wildcard). -caa_domain_and_tag() { - local domain="$1" - if [[ "$domain" == \*.* ]]; then - echo "${domain#\*.} issuewild" - else - echo "$domain issue" - fi -} - -# In delegation mode we have NO token for the served domain's zone, so we cannot -# set the accounturi CAA ourselves. That CAA is the forge-prevention (only this -# enclave's ACME account may issue), so this path is deliberately NOT gated by -# SET_CAA and fails closed if the record is confirmed absent — otherwise anyone -# who can satisfy the delegated DNS-01 challenge could obtain a cert for the -# domain. Set ALLOW_MISSING_CAA=true to override (accept the risk). -verify_delegation_caa() { - local domain="$1" - local account_file account_uri caa_domain caa_tag caa_value resp status found - - if ! account_file=$(get_letsencrypt_account_file); then - echo "ERROR: cannot read the Let's Encrypt account file to determine the required accounturi CAA" >&2 - caa_fail_or_allow "$domain" - return - fi - read -r caa_domain caa_tag < <(caa_domain_and_tag "$domain") - account_uri=$(jq -j '.uri' "$account_file") - caa_value="letsencrypt.org;validationmethods=dns-01;accounturi=$account_uri" - - echo "[challenge-delegation] Set this CAA in your production zone (static, one-time):" - echo " ${caa_domain} CAA 0 ${caa_tag} \"${caa_value}\"" - - # Verify via DNS-over-HTTPS (dig is not installed in the image; curl+jq are). - resp=$(curl -s --max-time 10 "https://dns.google/resolve?name=${caa_domain}&type=257" 2>/dev/null || true) - if [ -z "$resp" ]; then - echo "WARNING: could not reach dns.google to verify the CAA (network issue) — NOT confirmed; continuing" - return - fi - status=$(echo "$resp" | jq -r '.Status // empty' 2>/dev/null || true) - if [ "$status" != "0" ]; then - echo "WARNING: CAA DoH query for ${caa_domain} returned status ${status:-unknown} — NOT confirmed; continuing" - return - fi - # Literal (grep -F) match on the CAA data fields — the account URI contains - # '/' and '.', which must not be treated as regex. - found=$(echo "$resp" | jq -r '.Answer // [] | .[] | .data' 2>/dev/null | grep -F "accounturi=$account_uri" || true) - if [ -n "$found" ]; then - echo "[challenge-delegation] Verified: accounturi CAA is present for $caa_domain" - return - fi - echo "ERROR: the accounturi CAA is NOT present for $caa_domain." >&2 - echo "ERROR: without it, anyone who can satisfy the delegated DNS-01 challenge could obtain a" >&2 - echo "ERROR: publicly-trusted certificate for this domain (forged TLS termination)." >&2 - caa_fail_or_allow "$domain" -} - -caa_fail_or_allow() { - local domain="$1" - if [ "${ALLOW_MISSING_CAA:-false}" = "true" ]; then - echo "WARNING: ALLOW_MISSING_CAA=true — continuing without a verified accounturi CAA for $domain" - return 0 - fi - echo "ERROR: refusing to continue without the accounturi CAA for $domain." >&2 - echo "ERROR: set the CAA record shown above, or ALLOW_MISSING_CAA=true to override." >&2 - exit 1 -} - -set_caa_record() { - local domain="$1" - - # Delegation mode is handled separately and is NOT gated by SET_CAA. - if [ -n "${ACME_CHALLENGE_ALIAS:-}" ]; then - verify_delegation_caa "$domain" - return - fi - - if [ "$SET_CAA" != "true" ]; then - echo "Skipping CAA record setup" - return - fi - - local account_file account_uri caa_domain caa_tag - if ! account_file=$(get_letsencrypt_account_file); then - echo "Warning: Cannot set CAA record - account file not found" - echo "This is not critical - certificates can still be issued without CAA records" - return - fi - read -r caa_domain caa_tag < <(caa_domain_and_tag "$domain") - account_uri=$(jq -j '.uri' "$account_file") - - echo "Adding CAA record ($caa_tag) for $caa_domain, accounturi=$account_uri" - dnsman.py set_caa \ - --domain "$caa_domain" \ - --caa-tag "$caa_tag" \ - --caa-value "letsencrypt.org;validationmethods=dns-01;accounturi=$account_uri" - - if [ $? -ne 0 ]; then - echo "Warning: Failed to set CAA record for $domain" - echo "This is not critical - certificates can still be issued without CAA records" - echo "Consider disabling CAA records by setting SET_CAA=false if this continues to fail" - fi -} - -process_domain() { - local domain="$1" - echo "Processing domain: $domain" - - set_alias_record "$domain" - set_txt_record "$domain" - renew-certificate.sh "$domain" || echo "First certificate renewal failed for $domain, will retry after set CAA record" - set_caa_record "$domain" - renew-certificate.sh "$domain" -} - -bootstrap() { - echo "Bootstrap: Setting up domains" - - local all_domains - all_domains=$(get-all-domains.sh) - - if [ -z "$all_domains" ]; then - echo "Error: No domains found. Set either DOMAIN or DOMAINS environment variable" - exit 1 - fi - - echo "Found domains:" - echo "$all_domains" - - while IFS= read -r domain; do - [[ -n "$domain" ]] || continue - process_domain "$domain" - done <<<"$all_domains" - - # Generate evidences after all certificates are obtained - echo "Generating evidence files for all domains..." - generate-evidences.sh - - touch /.bootstrapped -} - -# Credentials are now handled by certman.py setup - -# Setup certbot environment (venv is already created in Dockerfile) -setup_certbot_env - -# Check if it's the first time the container is started -if [ ! -f "/.bootstrapped" ]; then - bootstrap -else - echo "Certificate for $DOMAIN already exists" - generate-evidences.sh -fi - -# Build combined PEM files for haproxy -build-combined-pems.sh - -# Generate haproxy config -if [ -n "$ROUTING_MAP" ]; then - setup_haproxy_cfg_multi -elif [ -n "$DOMAIN" ] && [ -n "$TARGET_ENDPOINT" ]; then - setup_haproxy_cfg -fi - -# Start evidence HTTP server if enabled -if [ "$EVIDENCE_SERVER" = "true" ]; then - mini_httpd -d /evidences -p "${EVIDENCE_PORT}" -D -l /dev/stderr & - echo "Evidence server started on port ${EVIDENCE_PORT} (mini_httpd)" -fi - -renewal-daemon.sh & - -exec "$@" + ;; +esac diff --git a/custom-domain/dstack-ingress/scripts/evidence-lib.sh b/custom-domain/dstack-ingress/scripts/evidence-lib.sh new file mode 100644 index 00000000..644200a3 --- /dev/null +++ b/custom-domain/dstack-ingress/scripts/evidence-lib.sh @@ -0,0 +1,75 @@ +#!/bin/bash +# evidence-lib.sh - the evidence file format and its TDX quote. +# +# The modes disagree about where the ACME account document and the certificates +# live on disk, so each mode hands its own paths to evidence_collect_*. What the +# evidence *is* -- the file names, the manifest, and the report_data binding -- +# is one definition, here. + +EVIDENCE_DIR=${EVIDENCE_DIR:-/evidences} + +evidence_reset() { + mkdir -p "$EVIDENCE_DIR" +} + +# These files exist to be published, so force them world-readable. lego writes +# its account document and certificates 0600 and cp keeps that mode, which makes +# the evidence server answer 403. Neither carries a private key: the account +# document holds the account URL and status, and a certificate chain is public +# by construction. +evidence_collect_account() { + cp "$1" "$EVIDENCE_DIR/acme-account.json" + chmod 644 "$EVIDENCE_DIR/acme-account.json" +} + +evidence_collect_cert() { + local domain="$1" path="$2" + if [ ! -f "$path" ]; then + echo "Warning: Certificate not found for domain: $domain" + return 0 + fi + cp "$path" "$EVIDENCE_DIR/cert-${domain}.pem" + chmod 644 "$EVIDENCE_DIR/cert-${domain}.pem" +} + +# Hash whatever was collected and bind it into a TDX quote. +evidence_finalize() { + cd "$EVIDENCE_DIR" || return 1 +# Generate checksum for all files + sha256sum acme-account.json cert-*.pem > sha256sum.txt 2>/dev/null || { + echo "Error: No certificate files found" + return 1 + } + + QUOTED_HASH=$(sha256sum sha256sum.txt | awk '{print $1}') + + PADDED_HASH="${QUOTED_HASH}" + while [ ${#PADDED_HASH} -lt 128 ]; do + PADDED_HASH="${PADDED_HASH}0" + done + QUOTED_HASH="${PADDED_HASH}" + + if [[ -S /var/run/dstack.sock ]]; then + curl -s --unix-socket /var/run/dstack.sock "http://localhost/GetQuote?report_data=${QUOTED_HASH}" > quote.json + else + curl -s --unix-socket /var/run/tappd.sock "http://localhost/prpc/Tappd.RawQuote?report_data=${QUOTED_HASH}" > quote.json + fi + if [ $? -ne 0 ]; then + echo "Error: Failed to generate evidences" + return 1 + fi + echo "Generated evidences successfully" +} + +evidence_start_server() { + [ "${EVIDENCE_SERVER:-true}" = "true" ] || return 0 + # Bind loopback explicitly. Left to itself mini_httpd binds [::]:port first, + # and since the container has net.ipv6.bindv6only=0 that socket already + # covers IPv4, so its second bind fails with EADDRINUSE and it logs a + # permanent "bind: Address already in use". The only consumer is haproxy's + # be_evidence backend on 127.0.0.1 anyway; listening on every interface also + # exposed this to other containers on the compose network, bypassing the + # proxy. + mini_httpd -d "$EVIDENCE_DIR" -p "${EVIDENCE_PORT:-80}" -h 127.0.0.1 -D -l /dev/stderr & + echo "Evidence server started on 127.0.0.1:${EVIDENCE_PORT:-80} (mini_httpd)" +} diff --git a/custom-domain/dstack-ingress/scripts/functions.sh b/custom-domain/dstack-ingress/scripts/functions.sh index 2ff37e43..17dc396f 100644 --- a/custom-domain/dstack-ingress/scripts/functions.sh +++ b/custom-domain/dstack-ingress/scripts/functions.sh @@ -166,7 +166,8 @@ get_letsencrypt_account_path() { local base_path="/etc/letsencrypt/accounts" local api_endpoint="acme-v02.api.letsencrypt.org" - if [[ "$CERTBOT_STAGING" == "true" ]]; then + # entrypoint.sh normalises CERTBOT_STAGING into ACME_STAGING. + if [[ "$ACME_STAGING" == "true" ]]; then api_endpoint="acme-staging-v02.api.letsencrypt.org" fi @@ -187,3 +188,36 @@ get_letsencrypt_account_file() { echo "${account_files[0]}" } + +txt_record_name() { + local domain="$1" + if [[ "$domain" == \*.* ]]; then + # Wildcard domain: *.myapp.com → _dstack-app-address-wildcard.myapp.com + echo "${TXT_PREFIX}-wildcard.${domain#\*.}" + else + echo "${TXT_PREFIX}.${domain}" + fi +} + +caa_tag_for() { + if [[ "$1" == \*.* ]]; then echo "issuewild"; else echo "issue"; fi +} + +# Query the guest agent once for this app's identity. +load_dstack_identity() { + local info + if [[ -S /var/run/dstack.sock ]]; then + info=$(curl -s --unix-socket /var/run/dstack.sock http://localhost/Info) + else + info=$(curl -s --unix-socket /var/run/tappd.sock http://localhost/prpc/Tappd.Info) + fi + + DSTACK_APP_ID=$(echo "$info" | jq -j .app_id) + DSTACK_INSTANCE_ID=$(echo "$info" | jq -j .instance_id) + export DSTACK_APP_ID DSTACK_INSTANCE_ID + + if [ -z "$DSTACK_APP_ID" ] || [ "$DSTACK_APP_ID" = "null" ]; then + echo "Error: could not read app_id from the dstack guest agent" >&2 + exit 1 + fi +} diff --git a/custom-domain/dstack-ingress/scripts/generate-evidences.sh b/custom-domain/dstack-ingress/scripts/generate-evidences.sh deleted file mode 100644 index afc8bfa9..00000000 --- a/custom-domain/dstack-ingress/scripts/generate-evidences.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/bash - -set -e - -source "/scripts/functions.sh" - -if ! ACME_ACCOUNT_FILE=$(get_letsencrypt_account_file); then - echo "Error: Cannot generate evidences without Let's Encrypt account file" - exit 1 -fi - -mkdir -p /evidences -cd /evidences || exit -cp "${ACME_ACCOUNT_FILE}" acme-account.json - -# Get all domains and copy their certificates -all_domains=$(get-all-domains.sh) -if [ -z "$all_domains" ]; then - echo "Error: No domains found for evidence generation" - exit 1 -fi - -# Copy all certificate files -while IFS= read -r domain; do - [[ -n "$domain" ]] || continue - cert_file="/etc/letsencrypt/live/$(cert_dir_name "$domain")/fullchain.pem" - if [ -f "$cert_file" ]; then - cp "$cert_file" "cert-${domain}.pem" - else - echo "Warning: Certificate not found for domain: $domain" - fi -done <<< "$all_domains" - -# Generate checksum for all files -sha256sum acme-account.json cert-*.pem > sha256sum.txt 2>/dev/null || { - echo "Error: No certificate files found" - exit 1 -} - -QUOTED_HASH=$(sha256sum sha256sum.txt | awk '{print $1}') - -PADDED_HASH="${QUOTED_HASH}" -while [ ${#PADDED_HASH} -lt 128 ]; do - PADDED_HASH="${PADDED_HASH}0" -done -QUOTED_HASH="${PADDED_HASH}" - -if [[ -S /var/run/dstack.sock ]]; then - curl -s --unix-socket /var/run/dstack.sock "http://localhost/GetQuote?report_data=${QUOTED_HASH}" > quote.json -else - curl -s --unix-socket /var/run/tappd.sock "http://localhost/prpc/Tappd.RawQuote?report_data=${QUOTED_HASH}" > quote.json -fi -if [ $? -ne 0 ]; then - echo "Error: Failed to generate evidences" - exit 1 -fi -echo "Generated evidences successfully" diff --git a/custom-domain/dstack-ingress/scripts/haproxy-lib.sh b/custom-domain/dstack-ingress/scripts/haproxy-lib.sh new file mode 100644 index 00000000..8d506fce --- /dev/null +++ b/custom-domain/dstack-ingress/scripts/haproxy-lib.sh @@ -0,0 +1,187 @@ +#!/bin/bash +# haproxy-lib.sh - the haproxy config pieces that are identical in every mode. +# +# The modes differ only in their frontends: dns-01 terminates TLS on the public +# port, tls-alpn-01 puts a ClientHello-peeking frontend there and moves the TLS +# frontend to loopback. global/defaults, the evidence ACL, the backends and the +# reload are the same either way, so they live here and each mode composes them. +# Nothing in this file branches on the mode; callers pass what differs. + +# Parse TARGET_ENDPOINT into host:port for haproxy backend +parse_target_endpoint() { + local endpoint="$1" + # Strip protocol prefix if present (http://, https://, grpc://) + local hostport="${endpoint#*://}" + # If no protocol was stripped, use as-is + if [ "$hostport" = "$endpoint" ]; then + hostport="$endpoint" + fi + # Strip any trailing path + hostport="${hostport%%/*}" + echo "$hostport" +} + +# global + defaults. Always the first thing written to haproxy.cfg. +haproxy_emit_global() { + cat </etc/haproxy/haproxy.cfg +global + log stdout format raw local0 + maxconn ${MAXCONN} + pidfile /var/run/haproxy/haproxy.pid + ssl-default-bind-ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305 + ssl-default-bind-ciphersuites TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256 + ssl-default-bind-options ssl-min-ver TLSv1.2 no-tls-tickets + ssl-default-bind-curves secp384r1 + +defaults + log global + mode tcp + option tcplog + timeout connect ${TIMEOUT_CONNECT} + timeout client ${TIMEOUT_CLIENT} + timeout server ${TIMEOUT_SERVER} + +EOF +} + +# The TLS-terminating frontend. $1 is the bind spec -- the one part the modes +# disagree about. +haproxy_emit_tls_frontend() { + local bind_spec="$1" + # "crt " loads all PEM files from the directory. + # ALPN is appended conditionally via ${ALPN:+ alpn ${ALPN}}. + cat <>/etc/haproxy/haproxy.cfg +frontend tls_in + bind ${bind_spec} ssl crt /etc/haproxy/certs/${ALPN:+ alpn ${ALPN}} +EOF + + if [ "$EVIDENCE_SERVER" = "true" ]; then + cat <<'EVIDENCE_BLOCK' >>/etc/haproxy/haproxy.cfg + + # Route /evidences requests to the local evidence HTTP server. + # accept fires once 16 bytes have arrived — enough for the + # longest prefix we match ("HEAD /evidences" = 16 chars). + # Using req.len with a concrete threshold is critical: the + # previous payload(0,0) (length 0 = "whole buffer") deferred + # evaluation until the full inspect-delay because HAProxy + # cannot know when a TCP stream ends. + tcp-request inspect-delay 5s + tcp-request content accept if { req.len ge 16 } + acl is_evidence payload(0,16) -m beg "GET /evidences" + acl is_evidence payload(0,16) -m beg "HEAD /evidences" + use_backend be_evidence if is_evidence +EVIDENCE_BLOCK + fi +} + +# Append the evidence backend block to haproxy.cfg +haproxy_emit_evidence_backend() { + if [ "$EVIDENCE_SERVER" = "true" ]; then + cat <>/etc/haproxy/haproxy.cfg + +backend be_evidence + mode http + http-request replace-path /evidences(.*) \1 + server evidence 127.0.0.1:${EVIDENCE_PORT} +EOF + fi +} + +# Generate haproxy.cfg for single-domain mode (DOMAIN + TARGET_ENDPOINT) +haproxy_emit_backends_single() { + local target_hostport + target_hostport=$(parse_target_endpoint "$TARGET_ENDPOINT") + + cat <>/etc/haproxy/haproxy.cfg + + default_backend be_upstream + +backend be_upstream + server app1 ${target_hostport} +EOF +} + +# Generate haproxy.cfg for multi-domain mode (ROUTING_MAP) +haproxy_emit_backends_multi() { + # Parse ROUTING_MAP and generate use_backend rules + backend sections + # Support both newline-separated and comma-separated formats + local routing_map_normalized + routing_map_normalized=$(echo "$ROUTING_MAP" | tr ',' '\n') + + local backend_rules="" + local backend_sections="" + local first_be_name="" + local domain target be_name + + while IFS= read -r line; do + [[ -n "$line" ]] || continue + [[ "$line" == \#* ]] && continue + domain="${line%%=*}" + target="${line#*=}" + domain=$(echo "$domain" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') + target=$(echo "$target" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') + [[ -n "$domain" && -n "$target" ]] || continue + + # Validate domain and target to prevent config injection + if ! domain=$(sanitize_domain "$domain"); then + echo "Error: Invalid domain in ROUTING_MAP: ${line}" >&2 + exit 1 + fi + if ! target=$(sanitize_target_endpoint "$target"); then + echo "Error: Invalid target in ROUTING_MAP: ${line}" >&2 + exit 1 + fi + + # Strip protocol prefix from target if present + target=$(parse_target_endpoint "$target") + + # Generate safe backend name from domain + be_name="be_$(echo "$domain" | sed 's/[^A-Za-z0-9]/_/g')" + + if [ -z "$first_be_name" ]; then + first_be_name="$be_name" + fi + + backend_rules="${backend_rules} + use_backend ${be_name} if { ssl_fc_sni -i ${domain} }" + backend_sections="${backend_sections} + +backend ${be_name} + server s1 ${target}" + done <<< "$routing_map_normalized" + + echo "$backend_rules" >> /etc/haproxy/haproxy.cfg + + # Default to first backend in ROUTING_MAP + if [ -n "$first_be_name" ]; then + echo "" >> /etc/haproxy/haproxy.cfg + echo " default_backend ${first_be_name}" >> /etc/haproxy/haproxy.cfg + fi + + echo "$backend_sections" >> /etc/haproxy/haproxy.cfg +} + +# Pick the backend layout from the environment, then append the evidence backend. +haproxy_emit_backends() { + if [ -n "${ROUTING_MAP:-}" ]; then + haproxy_emit_backends_multi + elif [ -n "${DOMAIN:-}" ] && [ -n "${TARGET_ENDPOINT:-}" ]; then + haproxy_emit_backends_single + fi + haproxy_emit_evidence_backend +} + +haproxy_reload() { + if [ ! -f /var/run/haproxy/haproxy.pid ]; then + # Normal during a startup pass: haproxy has not been exec'd yet and will + # read the fresh certificates when it starts. + echo "HAProxy is not running yet; certificates will be picked up at start" + return 0 + fi + if kill -USR2 "$(cat /var/run/haproxy/haproxy.pid)"; then + echo "HAProxy reloaded with the new certificates" + else + echo "HAProxy reload failed: SIGUSR2 to PID $(cat /var/run/haproxy/haproxy.pid) failed" >&2 + return 1 + fi +} diff --git a/custom-domain/dstack-ingress/scripts/legoman.py b/custom-domain/dstack-ingress/scripts/legoman.py new file mode 100644 index 00000000..2880803f --- /dev/null +++ b/custom-domain/dstack-ingress/scripts/legoman.py @@ -0,0 +1,258 @@ +#!/usr/bin/env python3 +"""Certificate management for the tls-alpn-01 path, backed by lego. + +certbot is not usable here. Its standalone plugin is HTTP-01 only, the +maintainers declined to add tls-alpn-01 (certbot/certbot#6724), and the acme +library dropped the challenge entirely in 4.2.0 -- the last release carrying +`acme.standalone.TLSALPN01Server` is 4.1.1, where it is already marked +deprecated. Building on that would mean pinning both `certbot` and `acme` to a +deleted, unmaintained API inside a component whose whole job is TLS. + +lego implements tls-alpn-01 as a first-class challenge and ships as a single +static binary. It binds a loopback address (`--tls.address`); haproxy forwards +only `acme-tls/1` ClientHellos there, so issuance never disturbs serving +traffic. + +Targets the lego 5.x CLI, which differs from 4.x in ways that matter here: +`renew` folded into `run --renew-days`, `--tls.port` became `--tls.address`, +flags moved from global to per-command, and the account document renamed +`registration.uri` to `registration.accountURL`. 5.x also added +`accounts register`, which lets us create the account -- and therefore know its +URI -- before the first certificate exists, so the CAA record can be printed +with `accounturi` up front. + +The dns-01 path still runs certbot via certman.py; nothing here touches it. +""" + +import argparse +import glob +import json +import os +import subprocess +import sys +from typing import List, Optional, Tuple +from urllib.parse import urlparse + +LEGO_BIN = os.environ.get("LEGO_BIN", "/usr/local/bin/lego") +LEGO_PATH = os.environ.get("LEGO_PATH", "/etc/letsencrypt/lego") +ACME_PROD = "https://acme-v02.api.letsencrypt.org/directory" +ACME_STAGING = "https://acme-staging-v02.api.letsencrypt.org/directory" + +# Exit codes: 0 changed, 2 nothing to do, 1 failed. The caller reloads on 0. +EXIT_CHANGED = 0 +EXIT_ERROR = 1 +EXIT_UNCHANGED = 2 + +RUN_TIMEOUT = 600 +REGISTER_TIMEOUT = 120 + + +def acme_server() -> str: + # CERTBOT_STAGING is the historical name and still works, but there is no + # certbot on this path: the ACME client here is lego. + staging = os.environ.get("ACME_STAGING") or os.environ.get("CERTBOT_STAGING", "false") + return ACME_STAGING if staging == "true" else ACME_PROD + + +def server_dir() -> str: + """lego namespaces account storage by the CA host (plus port, if any).""" + parsed = urlparse(acme_server()) + host = parsed.hostname or "acme" + return f"{host}_{parsed.port}" if parsed.port else host + + +def account_file() -> Optional[str]: + """Find the account document by globbing rather than by building the path. + + lego names the directory after the account email, and after a placeholder of + its own choosing when there is no email. Globbing means we neither have to + know that placeholder nor track it across lego versions. + """ + pattern = os.path.join(LEGO_PATH, "accounts", server_dir(), "*", "account.json") + matches = sorted(glob.glob(pattern)) + return matches[0] if matches else None + + +def account_uri() -> Optional[str]: + path = account_file() + if not path: + return None + try: + with open(path, encoding="utf-8") as fh: + data = json.load(fh) + except (OSError, ValueError) as exc: + print(f"Warning: cannot read lego account file {path}: {exc}", file=sys.stderr) + return None + registration = data.get("registration") or {} + # lego 5.x renamed this from "uri"; accept both so a directory written by an + # older binary still resolves. + return registration.get("accountURL") or registration.get("uri") + + +def cert_paths(domain: str) -> Tuple[str, str]: + base = os.path.join(LEGO_PATH, "certificates", domain) + return f"{base}.crt", f"{base}.key" + + +def certificate_exists(domain: str) -> bool: + crt, key = cert_paths(domain) + return os.path.isfile(crt) and os.path.isfile(key) + + +def _renewed_marker(domain: str) -> str: + return os.path.join(LEGO_PATH, f".renewed-{domain}") + + +def _common_flags(email: str) -> List[str]: + flags = ["--path", LEGO_PATH, "--server", acme_server(), "--accept-tos"] + # The ACME contact address is optional (RFC 8555 §7.3), and Let's Encrypt + # stopped sending expiry notifications in 2025, so it buys little. It is + # also published: the account document is served as evidence, so an address + # set here becomes public. Omit it unless the operator wants it. + if email: + flags += ["--email", email] + return flags + + +def _run(cmd: List[str], timeout: int = RUN_TIMEOUT) -> Tuple[int, str]: + masked = [ + arg if not (i > 0 and cmd[i - 1] in ("--email", "-m")) else "" + for i, arg in enumerate(cmd) + ] + print(f"Executing: {' '.join(masked)}", flush=True) + try: + result = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout) + except subprocess.TimeoutExpired: + print(f"lego timed out after {timeout}s", file=sys.stderr) + return 124, "" + output = (result.stdout or "") + (result.stderr or "") + for line in output.strip().splitlines(): + print(f" lego| {line}", flush=True) + return result.returncode, output + + +def register(email: str) -> int: + """Create the ACME account without issuing anything.""" + if account_file(): + print("ACME account already exists") + return EXIT_UNCHANGED + + print("Registering ACME account") + code, _ = _run( + [LEGO_BIN, "accounts", "register"] + _common_flags(email), + timeout=REGISTER_TIMEOUT, + ) + if code == 0 and account_file(): + print("✓ ACME account registered") + return EXIT_CHANGED + print(f"✗ ACME account registration failed (exit code {code})", file=sys.stderr) + return EXIT_ERROR + + +def _challenge_flags() -> List[str]: + address = os.environ.get("TLSALPN_ADDRESS", "127.0.0.1") + port = os.environ.get("TLSALPN_PORT", "9443") + return ["--tls", "--tls.address", f"{address}:{port}"] + + +def run_cert(domain: str, email: str) -> int: + """`lego run` both obtains and renews, deciding by remaining lifetime.""" + if domain.startswith("*."): + print( + f"Error: cannot issue a wildcard certificate for {domain} with tls-alpn-01. " + f"RFC 8737 forbids it; use CHALLENGE_TYPE=dns-01 for wildcards.", + file=sys.stderr, + ) + return EXIT_ERROR + + # Let lego decide whether a renewal is due: it consults the CA's ARI + # endpoint (RFC 9773) as well as the local window, and the CA can ask for + # early renewal during an incident. Deciding here from notAfter would + # override that. + # + # We only need to know whether it acted, and lego says so itself: + # --deploy-hook runs "in cases where a certificate is successfully + # created/renewed" and stays silent otherwise. That beats inferring it -- + # matching log text is not stable (4.x wrote "no renewal", 5.x writes + # "Skip renewal"), and the exit code is 0 either way. + marker = _renewed_marker(domain) + if os.path.exists(marker): + os.remove(marker) + + cmd = ( + [LEGO_BIN, "run"] + + _common_flags(email) + + ["--domains", domain] + + _challenge_flags() + + ["--deploy-hook", f"touch {marker}"] + ) + renew_days = os.environ.get("RENEW_DAYS_BEFORE", "") + if renew_days: + cmd += ["--renew-days", renew_days] + + print(f"{'Renewing' if certificate_exists(domain) else 'Obtaining'} certificate for {domain} via tls-alpn-01") + code, _ = _run(cmd) + if code != 0: + print(f"✗ lego failed for {domain} (exit code {code})", file=sys.stderr) + return EXIT_ERROR + if not certificate_exists(domain): + print(f"✗ lego reported success but no certificate for {domain}", file=sys.stderr) + return EXIT_ERROR + + if not os.path.exists(marker): + print(f"Certificate for {domain} is still current; nothing to do") + return EXIT_UNCHANGED + os.remove(marker) + + print(f"✓ Certificate ready for {domain}") + return EXIT_CHANGED + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "action", + choices=["obtain", "renew", "auto", "register", "account-uri", "cert-path"], + ) + parser.add_argument("--domain", help="Domain name") + parser.add_argument("--email", help="Email for ACME registration") + args = parser.parse_args() + + email = args.email or os.environ.get("ACME_EMAIL") or os.environ.get("CERTBOT_EMAIL", "") + + if args.action == "account-uri": + uri = account_uri() + if not uri: + return EXIT_ERROR + print(uri) + return EXIT_CHANGED + + if args.action == "cert-path": + if not args.domain: + print("Error: --domain is required", file=sys.stderr) + return EXIT_ERROR + crt, key = cert_paths(args.domain) + print(f"{crt} {key}") + return EXIT_CHANGED + + if not os.path.isfile(LEGO_BIN): + print(f"Error: lego binary not found at {LEGO_BIN}", file=sys.stderr) + return EXIT_ERROR + + os.makedirs(LEGO_PATH, exist_ok=True) + + if args.action == "register": + code = register(email) + # "already registered" is success for callers that just need it to exist. + return EXIT_CHANGED if code == EXIT_UNCHANGED else code + + if not args.domain: + print("Error: --domain is required", file=sys.stderr) + return EXIT_ERROR + + # obtain / renew / auto all map onto `lego run`, which decides for itself. + return run_cert(args.domain, email) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/custom-domain/dstack-ingress/scripts/renew-certificate.sh b/custom-domain/dstack-ingress/scripts/renew-certificate.sh deleted file mode 100755 index 8c822cbd..00000000 --- a/custom-domain/dstack-ingress/scripts/renew-certificate.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -source /opt/app-venv/bin/activate - -DOMAIN=$1 - -# Use the unified certbot manager -SCRIPT_DIR="$(dirname "$(readlink -f "$0")")" -python3 "$SCRIPT_DIR/certman.py" auto --domain "$DOMAIN" --email "$CERTBOT_EMAIL" -CERT_STATUS=$? - -if [ $CERT_STATUS -eq 1 ]; then - echo "Certificate management failed" >&2 - exit 1 -elif [ $CERT_STATUS -eq 2 ]; then - echo "No certificates need renewal, skipping evidence generation" -fi - -exit 0 \ No newline at end of file diff --git a/custom-domain/dstack-ingress/scripts/renewal-daemon.sh b/custom-domain/dstack-ingress/scripts/renewal-daemon.sh deleted file mode 100755 index d20614d0..00000000 --- a/custom-domain/dstack-ingress/scripts/renewal-daemon.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash - -while true; do - echo "[$(date)] Checking for certificate renewal" - - all_domains=$(get-all-domains.sh) - - if [ -n "$all_domains" ]; then - renewal_occurred=false - while IFS= read -r domain; do - [[ -n "$domain" ]] || continue - echo "[$(date)] Checking renewal for domain: $domain" - if renew-certificate.sh "$domain"; then - renewal_occurred=true - else - echo "Certificate renewal check failed for $domain with status $?" - fi - done <<<"$all_domains" - - if [ "$renewal_occurred" = true ]; then - echo "[$(date)] Generating evidence files after renewals..." - generate-evidences.sh || echo "Evidence generation failed" - - # Rebuild combined PEM files for haproxy - build-combined-pems.sh || echo "Combined PEM build failed" - - # Graceful reload: send SIGUSR2 to haproxy master process - if [ ! -f /var/run/haproxy/haproxy.pid ]; then - echo "HAProxy reload failed: PID file /var/run/haproxy/haproxy.pid not found" >&2 - elif ! kill -USR2 "$(cat /var/run/haproxy/haproxy.pid)"; then - echo "HAProxy reload failed: SIGUSR2 to PID $(cat /var/run/haproxy/haproxy.pid) failed" >&2 - else - echo "Certificate renewed and HAProxy reloaded successfully" - fi - fi - else - echo "[$(date)] No domains configured for renewal" - fi - - echo "[$(date)] Next renewal check in 12 hours" - sleep 43200 -done diff --git a/custom-domain/dstack-ingress/scripts/tests/test_dnsguide.py b/custom-domain/dstack-ingress/scripts/tests/test_dnsguide.py new file mode 100644 index 00000000..8a801b33 --- /dev/null +++ b/custom-domain/dstack-ingress/scripts/tests/test_dnsguide.py @@ -0,0 +1,165 @@ +#!/usr/bin/env python3 +"""Unit tests for dnsguide's record parsing and CAA evaluation. + +Run: python3 scripts/tests/test_dnsguide.py +""" + +import os +import sys +import unittest + +sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")) + +import dnsguide # noqa: E402 + +# Same record, as each resolver actually returns it. Google answers in +# presentation format; Cloudflare answers in RFC 3597 generic format. +CAA_PRESENTATION = '0 issue "letsencrypt.org;validationmethods=dns-01"' +CAA_GENERIC = ( + "\\# 47 00 05 69 73 73 75 65 6c 65 74 73 65 6e 63 72 79 70 74 2e 6f 72 67 3b " + "76 61 6c 69 64 61 74 69 6f 6e 6d 65 74 68 6f 64 73 3d 64 6e 73 2d 30 31" +) + + +class TestCaaParsing(unittest.TestCase): + def test_presentation_format(self): + self.assertEqual( + dnsguide._parse_caa(CAA_PRESENTATION), + (0, "issue", "letsencrypt.org;validationmethods=dns-01"), + ) + + def test_generic_format_matches_presentation(self): + """Both resolvers must yield the same tuple, or the check is resolver-dependent.""" + self.assertEqual( + dnsguide._parse_caa(CAA_GENERIC), dnsguide._parse_caa(CAA_PRESENTATION) + ) + + def test_issuewild_tag(self): + parsed = dnsguide._parse_caa('0 issuewild "letsencrypt.org"') + self.assertEqual(parsed, (0, "issuewild", "letsencrypt.org")) + + def test_garbage_is_rejected(self): + for junk in ("", "nonsense", "\\# zz", "\\# 2"): + self.assertIsNone(dnsguide._parse_caa(junk), junk) + + +class TestCaaPermits(unittest.TestCase): + ACCOUNT = "https://acme-staging-v02.api.letsencrypt.org/acme/acct/1" + + def test_method_restriction_blocks_other_method(self): + ok, why = dnsguide._caa_permits( + "letsencrypt.org;validationmethods=dns-01", "issue", "tls-alpn-01", "" + ) + self.assertFalse(ok) + self.assertIn("tls-alpn-01", why) + + def test_method_restriction_allows_listed_method(self): + ok, _ = dnsguide._caa_permits( + "letsencrypt.org;validationmethods=dns-01,tls-alpn-01", "issue", "tls-alpn-01", "" + ) + self.assertTrue(ok) + + def test_no_parameters_allows_any_method(self): + ok, _ = dnsguide._caa_permits("letsencrypt.org", "issue", "tls-alpn-01", "") + self.assertTrue(ok) + + def test_other_issuer_blocks(self): + ok, why = dnsguide._caa_permits("digicert.com", "issue", "tls-alpn-01", "") + self.assertFalse(ok) + self.assertIn("digicert.com", why) + + def test_account_mismatch_blocks(self): + ok, why = dnsguide._caa_permits( + f"letsencrypt.org;accounturi={self.ACCOUNT}", "issue", "tls-alpn-01", + "https://acme-staging-v02.api.letsencrypt.org/acme/acct/999", + ) + self.assertFalse(ok) + self.assertIn("accounturi", why) + + def test_account_unknown_skips_the_comparison(self): + """Before our account exists we cannot compare it, so do not block on it.""" + ok, _ = dnsguide._caa_permits( + f"letsencrypt.org;accounturi={self.ACCOUNT}", "issue", "tls-alpn-01", "" + ) + self.assertTrue(ok) + + +class TestCaaCheckDeduplication(unittest.TestCase): + """One record seen twice, in two wire formats, must be reported once. + + Resolvers disagree on CAA presentation: Cloudflare hands back RFC 3597 + generic hex, Google the human-readable form. The union is taken over raw + rdata, so the same record arrives as two different strings. + """ + + PRESENTATION = '0 issue "letsencrypt.org;validationmethods=dns-01"' + # The same record, generic-encoded: flags=0, tag len 5, "issue", value. + GENERIC = ( + r"\# 44 00 05 69 73 73 75 65 6c 65 74 73 65 6e 63 72 79 70 74 2e 6f 72 67 " + r"3b 76 61 6c 69 64 61 74 69 6f 6e 6d 65 74 68 6f 64 73 3d 64 6e 73 2d 30 31" + ) + + def setUp(self): + self._saved = dnsguide.query_union + dnsguide.query_union = lambda name, rr, resolvers: ( + [self.GENERIC, self.PRESENTATION] if name == "app.example.com" else [] + ) + self.addCleanup(lambda: setattr(dnsguide, "query_union", self._saved)) + + def test_both_encodings_parse_to_the_same_record(self): + self.assertEqual( + dnsguide._parse_caa(self.GENERIC), dnsguide._parse_caa(self.PRESENTATION) + ) + + def test_reason_is_not_repeated(self): + ok, reason = dnsguide.check_caa( + "app.example.com", "issue", "tls-alpn-01", "", "r1,r2" + ) + self.assertFalse(ok) + self.assertEqual(reason.count("excludes tls-alpn-01"), 1, reason) + + +class TestTxtNormalisation(unittest.TestCase): + def test_strips_quotes(self): + self.assertEqual(dnsguide._unquote_txt('"abc:443"'), "abc:443") + + def test_joins_split_chunks(self): + self.assertEqual(dnsguide._unquote_txt('"aaa" "bbb"'), "aaabbb") + + def test_passes_through_unquoted(self): + self.assertEqual(dnsguide._unquote_txt("abc:443"), "abc:443") + + +class TestRecordBuilding(unittest.TestCase): + def _args(self, **over): + import argparse + + base = dict( + domain="app.example.com", + alias_target="gw.example.com", + txt_name="_dstack-app-address.app.example.com", + txt_value="deadbeef:443", + caa_name="app.example.com", + caa_tag="issue", + caa_value="letsencrypt.org;validationmethods=tls-alpn-01", + account_uri="", + include="cname,txt,caa", + ) + base.update(over) + return argparse.Namespace(**base) + + def test_all_records(self): + types = [r.type for r in dnsguide.build_records(self._args())] + self.assertEqual(types, ["CNAME", "TXT", "CAA"]) + + def test_include_filters(self): + types = [r.type for r in dnsguide.build_records(self._args(include="txt"))] + self.assertEqual(types, ["TXT"]) + + def test_caa_omitted_without_value(self): + types = [r.type for r in dnsguide.build_records(self._args(caa_value=""))] + self.assertEqual(types, ["CNAME", "TXT"]) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/custom-domain/dstack-ingress/scripts/tlsalpn.sh b/custom-domain/dstack-ingress/scripts/tlsalpn.sh new file mode 100644 index 00000000..485bce35 --- /dev/null +++ b/custom-domain/dstack-ingress/scripts/tlsalpn.sh @@ -0,0 +1,336 @@ +#!/bin/bash +# tlsalpn.sh - certificates via lego and the TLS-ALPN-01 challenge, with no DNS +# credentials in the container. +# +# entrypoint.sh validates the shared settings and then execs this file. There is +# no shared orchestration above it and no interface it has to implement -- this +# is simply the program that runs for tls-alpn-01, top to bottom. + +set -e + +source /scripts/functions.sh +source /scripts/haproxy-lib.sh +source /scripts/evidence-lib.sh + +# ACME_EMAIL / ACME_STAGING are normalised by entrypoint.sh and exported, so +# they are already correct here whichever name the operator used. +# +# The contact address is optional. RFC 8555 makes it optional, Let's Encrypt +# stopped sending expiry notifications in 2025, and it does not stay private: +# the account document is published at /evidences/acme-account.json, so an +# address set here is readable by anyone who fetches the evidence. + +LEGO_BIN=${LEGO_BIN:-/usr/local/bin/lego} +LEGO_PATH=${LEGO_PATH:-/etc/letsencrypt/lego} +TLSALPN_ADDRESS=${TLSALPN_ADDRESS:-127.0.0.1} +TLSALPN_PORT=${TLSALPN_PORT:-9443} +TLS_TERMINATE_PORT=${TLS_TERMINATE_PORT:-9444} +DNS_SETUP_MODE=${DNS_SETUP_MODE:-wait} +DNS_SETTLE_SECONDS=${DNS_SETTLE_SECONDS:-30} +export LEGO_BIN LEGO_PATH TLSALPN_ADDRESS TLSALPN_PORT DNS_SETUP_MODE + +case "$DNS_SETUP_MODE" in + wait|print|webhook) ;; + *) + echo "Error: invalid DNS_SETUP_MODE '$DNS_SETUP_MODE' (expected wait, print or webhook)" >&2 + exit 1 + ;; +esac + +# lego stores certificates under $LEGO_PATH/certificates/.{crt,key}; +# the .crt already holds the full chain. +cert_fullchain_path() { echo "${LEGO_PATH}/certificates/${1}.crt"; } +cert_privkey_path() { echo "${LEGO_PATH}/certificates/${1}.key"; } + +acme_account_file() { + local files=( "${LEGO_PATH}"/accounts/*/*/account.json ) + if [[ ! -f "${files[0]}" ]]; then + echo "Error: lego account file not found under ${LEGO_PATH}/accounts" >&2 + return 1 + fi + echo "${files[0]}" +} + +setup_py_env() { + if [ ! -d /opt/app-venv ]; then + echo "Creating application virtual environment" + python3 -m venv --system-site-packages /opt/app-venv + fi + # shellcheck disable=SC1091 + source /opt/app-venv/bin/activate + + if [ ! -f /.venv_bootstrapped ]; then + # lego handles ACME here, so certbot and the cloud SDKs are dead weight. + echo "Bootstrapping python dependencies (no certbot on this path)" + pip install --upgrade pip + pip install requests + touch /.venv_bootstrapped + fi + echo 'source /opt/app-venv/bin/activate' > /etc/profile.d/app-venv.sh +} + +check_lego() { + if [ ! -x "$LEGO_BIN" ]; then + echo "Error: lego binary not found at $LEGO_BIN" >&2 + exit 1 + fi + echo "Using lego for tls-alpn-01: $($LEGO_BIN --version)" +} + +require_instance_id() { + if [ -z "${DSTACK_INSTANCE_ID:-}" ] || [ "$DSTACK_INSTANCE_ID" = "null" ]; then + echo "Error: could not read instance_id from the dstack guest agent, which" >&2 + echo "tls-alpn-01 needs to pin gateway routing to this instance" >&2 + exit 1 + fi +} + +# RFC 8737 forbids tls-alpn-01 for wildcard identifiers, so a wildcard here can +# never succeed. That makes it a configuration error, and it has to be caught +# before the startup sequence below: past that point the container has generated +# a placeholder certificate for a name that will never get a real one, started +# haproxy serving it, and settled into retrying an unsatisfiable config forever. +# +# Fatal for the container rather than skipping the one domain. A mixed config +# does contain the damage -- the other domains are issued normally -- but it +# leaves a self-signed certificate served for the wildcard indefinitely and a +# permanently failing loop, which reads as a TLS bug rather than a typo. +reject_wildcards() { + local wildcards + wildcards=$(get-all-domains.sh | grep '^\*\.' || true) + [ -n "$wildcards" ] || return 0 + echo "Error: tls-alpn-01 cannot issue wildcard certificates (RFC 8737)." >&2 + echo "$wildcards" | sed 's/^/ /' >&2 + echo "Use CHALLENGE_TYPE=dns-01 for wildcards, or list the names individually." >&2 + exit 1 +} + +# What the gateway should route this hostname to. +# +# dns-01 uses the app id, so the gateway load-balances across every instance of +# the app. tls-alpn-01 cannot: the challenge is answered by whichever instance +# lego runs on, and routing by app id makes the gateway race connections across +# the app's instances while the CA validates from several vantage points at +# once, so the challenge would land on the wrong replica. Pinning to the +# instance id makes validation deterministic -- at the cost of sending all +# traffic to this one instance, so this mode is effectively single-instance. +txt_record_value() { echo "${DSTACK_INSTANCE_ID}:${PORT}"; } + +# haproxy owns the public port, but tls-alpn-01 needs the ACME responder to +# complete the TLS handshake itself. So the public port is a plain TCP frontend +# that peeks at the ClientHello and forwards only acme-tls/1 to lego; everything +# else goes to the real TLS frontend on loopback. PROXY protocol carries the +# client address across that hop so the backend still sees the real peer. +emit_peek_frontend() { + cat <>/etc/haproxy/haproxy.cfg +frontend tls_peek + bind :${PORT} + tcp-request inspect-delay 5s + tcp-request content accept if { req.ssl_hello_type 1 } + # Only the CA sends this ALPN protocol. A client that sends it anyway just + # reaches lego's responder, which refuses anything but acme-tls/1. + use_backend be_acme if { req.ssl_alpn -i acme-tls/1 } + default_backend be_tls_terminate + +backend be_acme + server lego ${TLSALPN_ADDRESS}:${TLSALPN_PORT} + +backend be_tls_terminate + server local 127.0.0.1:${TLS_TERMINATE_PORT} send-proxy-v2 + +EOF +} + +# haproxy refuses to start when the crt directory has no PEM in it, and here it +# has to be listening before the first certificate can exist -- it is what +# forwards the challenge to lego. Seed a placeholder the real cert overwrites. +ensure_placeholder_certs() { + local all_domains domain pem + all_domains=$(get-all-domains.sh) + while IFS= read -r domain; do + [[ -n "$domain" ]] || continue + pem="/etc/haproxy/certs/${domain}.pem" + [ -f "$pem" ] && continue + echo "Generating placeholder certificate for ${domain}" + openssl req -x509 -newkey rsa:2048 -nodes -days 1 \ + -keyout /tmp/placeholder.key -out /tmp/placeholder.crt \ + -subj "/CN=${domain#\*.}" >/dev/null 2>&1 + cat /tmp/placeholder.crt /tmp/placeholder.key >"$pem" + chmod 600 "$pem" + rm -f /tmp/placeholder.key /tmp/placeholder.crt + done <<<"$all_domains" +} + +build_combined_pems() { + local domain fullchain privkey combined + mkdir -p /etc/haproxy/certs + while IFS= read -r domain; do + [[ -n "$domain" ]] || continue + fullchain=$(cert_fullchain_path "$domain") + privkey=$(cert_privkey_path "$domain") + combined="/etc/haproxy/certs/${domain}.pem" + if [ -f "$fullchain" ] && [ -f "$privkey" ]; then + cat "$fullchain" "$privkey" > "$combined" + chmod 600 "$combined" + echo "Combined PEM created: ${combined}" + elif [ -f "$combined" ]; then + # Expected on a first run: ensure_placeholder_certs has already put a + # self-signed certificate here so haproxy can bind and forward + # acme-tls/1, and the real one does not exist yet. Leave it alone. + echo "No certificate for ${domain} yet; serving the placeholder" + else + echo "Warning: no certificate or placeholder for ${domain}, skipping" + fi + done <<<"$(get-all-domains.sh)" +} + +collect_evidence() { + local account domain + if ! account=$(acme_account_file); then + echo "Error: cannot generate evidence without the ACME account file" >&2 + return 1 + fi + evidence_reset + evidence_collect_account "$account" + while IFS= read -r domain; do + [[ -n "$domain" ]] || continue + evidence_collect_cert "$domain" "$(cert_fullchain_path "$domain")" + done <<<"$(get-all-domains.sh)" + evidence_finalize +} + +process_domain() { + local domain="$1" + + # No wildcard check here: reject_wildcards has already refused to start over + # the same domain list, which cannot change while the container runs. + # legoman.py checks independently, guarding the ACME call itself. + + # Register the ACME account first so the CAA record we print can already + # pin accounturi. Without this the operator would have to add CAA in a + # second pass, after the account exists. + if ! legoman.py register ${ACME_EMAIL:+--email "$ACME_EMAIL"}; then + echo "Error: failed to register the ACME account" >&2 + exit 1 + fi + + local account_uri caa_value + account_uri=$(legoman.py account-uri 2>/dev/null || true) + if [ -n "$account_uri" ]; then + caa_value="letsencrypt.org;validationmethods=tls-alpn-01;accounturi=$account_uri" + else + echo "Warning: could not read the ACME account URI; the CAA record will not pin it" >&2 + caa_value="letsencrypt.org;validationmethods=tls-alpn-01" + fi + + # Wait for the records routing depends on. An existing CAA record is also + # checked for compatibility -- not because CAA is required (an absent CAA + # record set permits every CA), but because an incompatible one makes the CA + # refuse and burns Let's Encrypt's failed-validation budget: 5 per account + # per hostname per hour. + if ! dnsguide.py \ + --domain "$domain" \ + --alias-target "$GATEWAY_DOMAIN" \ + --txt-name "$(txt_record_name "$domain")" \ + --txt-value "$(txt_record_value)" \ + --caa-name "${domain#\*.}" \ + --caa-tag "$(caa_tag_for "$domain")" \ + --caa-value "$caa_value" \ + --account-uri "$account_uri" \ + --challenge tls-alpn-01 \ + --mode "$DNS_SETUP_MODE"; then + echo "Error: required DNS records for $domain are not in place" >&2 + exit 1 + fi + + # Public DNS being correct is not the same as the gateway acting on it. The + # gateway resolves the app-address TXT itself and caches the answer for the + # record's TTL, so right after the value changes -- which is every time the + # CVM instance is replaced, since the record names the instance -- it can + # still route the challenge to the previous target. Observed exactly that: + # dnsguide passed, then the CA got "Error getting validation data" because + # the gateway was still sending it to the old instance. + # Only on the first pass. The settle wait exists because the TXT value has + # just changed -- true at startup, and after an instance replacement, which + # is a fresh container and therefore also a first pass. Renewal passes reuse + # a record the gateway resolved long ago. + if [ "$FIRST_PASS" = "true" ] && [ "${DNS_SETTLE_SECONDS}" -gt 0 ]; then + echo "Waiting ${DNS_SETTLE_SECONDS}s for the gateway's DNS cache to expire" + sleep "${DNS_SETTLE_SECONDS}" + fi + + legoman.py auto --domain "$domain" ${ACME_EMAIL:+--email "$ACME_EMAIL"} +} + +# One pass over every domain: make sure the records exist, then issue or renew. +# Idempotent, so the first pass and every later one are the same code. There is +# deliberately only one of these -- an earlier split between a one-shot +# bootstrap and a renewal daemon raced for lego's challenge port, and the loser +# still spent one of Let's Encrypt's five failed validations per hostname/hour. +FIRST_PASS=true + +run_pass() { + local domain status failed=0 changed=0 + while IFS= read -r domain; do + [[ -n "$domain" ]] || continue + # `if` context, not a bare command: process_domain returns 2 for + # "nothing to do", and under `set -e` a bare non-zero would kill the + # whole script on every quiet renewal pass. + if ( process_domain "$domain" ); then status=0; else status=$?; fi + case $status in + 0) changed=1 ;; + 2) ;; + *) echo "Certificate management failed for $domain" >&2; failed=1 ;; + esac + done <<<"$(get-all-domains.sh)" + + if [ "$changed" -eq 1 ]; then + collect_evidence || echo "Evidence generation failed" >&2 + build_combined_pems || echo "Combined PEM build failed" >&2 + haproxy_reload || true + fi + FIRST_PASS=false + [ "$failed" -eq 0 ] +} + +cert_loop() { + local attempt=0 delay + while true; do + if run_pass; then + attempt=0 + delay=${RENEW_INTERVAL:-43200} + else + # The normal state here is "waiting for an operator to create a DNS + # record", so falling back to the 12-hour interval would be a + # terrible retry rate. + attempt=$((attempt + 1)) + delay=$((60 * attempt)) + [ "$delay" -gt 1800 ] && delay=1800 + echo "[$(date)] Pass failed (attempt ${attempt}); retrying in ${delay}s" + fi + echo "[$(date)] Next certificate check in ${delay}s" + sleep "$delay" + done +} + +reject_wildcards +setup_py_env +check_lego +load_dstack_identity +require_instance_id + +# Ordering is load-bearing: haproxy has to be forwarding acme-tls/1 before the +# first certificate can exist, so start on a placeholder and let the loop +# converge behind it. +ensure_placeholder_certs +build_combined_pems + +haproxy_emit_global +emit_peek_frontend +haproxy_emit_tls_frontend "127.0.0.1:${TLS_TERMINATE_PORT} accept-proxy" +haproxy_emit_backends + +evidence_start_server +cert_loop & + +exec "$@"