From ead398fac243b0d0fb51990718f681d8cf5ad3b8 Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Wed, 22 Jul 2026 16:17:20 +0300 Subject: [PATCH 1/8] Pin HTTP startup probe and document warm capacity of 2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cloud Run's default TCP startup probe passes the moment gunicorn's master binds the port, ~161s before a worker finishes importing (the import runs post-fork; --preload is deliberately unset). Cloud Run therefore marked booting instances 'started' and routed live traffic onto them, where requests queued until the 300s timeout — the 'no available instance' 500s and 504s seen on every scale-out burst. Probe /readiness-check over HTTP instead so Cloud Run withholds traffic until the app can serve. The probe window (failureThreshold x periodSeconds) is 240s, the platform maximum; measured import is ~161s, leaving ~79s headroom. Past the window Cloud Run kills the container, so a slow import now fails the candidate deploy in CI on --no-traffic rather than degrading production. Also corrects the warm-capacity runbook, which documented the revision-level --min-instances flag for a service-level setting: --min is service-level warm capacity, --min-instances is the per-tag cost bomb. Records the new value of 2 and its cost. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/scripts/cloud_run_env.sh | 15 +++++++ .github/scripts/deploy_cloud_run_candidate.sh | 1 + .../cloud-run-http-startup-probe.fixed.md | 1 + .../cloud-run-warm-capacity-2.changed.md | 1 + docs/migration/cloud-run-operations.md | 38 +++++++++++++++-- tests/unit/test_cloud_run_deploy_scripts.py | 42 +++++++++++++++++++ 6 files changed, 95 insertions(+), 3 deletions(-) create mode 100644 changelog.d/cloud-run-http-startup-probe.fixed.md create mode 100644 changelog.d/cloud-run-warm-capacity-2.changed.md diff --git a/.github/scripts/cloud_run_env.sh b/.github/scripts/cloud_run_env.sh index f5388559a..ee8e0aae1 100755 --- a/.github/scripts/cloud_run_env.sh +++ b/.github/scripts/cloud_run_env.sh @@ -20,6 +20,20 @@ cloud_run_set_defaults() { CLOUD_RUN_CONCURRENCY="${CLOUD_RUN_CONCURRENCY:-6}" CLOUD_RUN_WEB_CONCURRENCY="${CLOUD_RUN_WEB_CONCURRENCY:-2}" CLOUD_RUN_PORT="${CLOUD_RUN_PORT:-8080}" + # HTTP startup probe on /readiness-check, pinned on every deploy. The default + # TCP probe passes the instant gunicorn's master binds the port — ~161s before + # a worker finishes importing — so Cloud Run routed live traffic onto + # instances that could not answer, and those requests queued until the 300s + # request timeout. Probing readiness over HTTP makes Cloud Run withhold + # traffic until the app can actually serve. + # Window: failureThreshold x periodSeconds = 240s, the platform maximum + # (Cloud Run shuts the container down past it). Measured import is ~161s + # post-cpu-boost, so this leaves ~79s of headroom; if import ever grows past + # the window the candidate revision fails to deploy in CI (on --no-traffic, + # before any promote) rather than degrading production. + # timeoutSeconds must not exceed periodSeconds; requests hang during import, + # so each attempt fails fast on timeout and retries. + CLOUD_RUN_STARTUP_PROBE="${CLOUD_RUN_STARTUP_PROBE:-httpGet.path=/readiness-check,httpGet.port=${CLOUD_RUN_PORT},periodSeconds=10,failureThreshold=24,timeoutSeconds=5}" CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET="${CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET:-policyengine-api-prod-db-password:latest}" CLOUD_RUN_GITHUB_MICRODATA_TOKEN_SECRET="${CLOUD_RUN_GITHUB_MICRODATA_TOKEN_SECRET:-policyengine-api-prod-github-microdata-token:latest}" CLOUD_RUN_ANTHROPIC_API_KEY_SECRET="${CLOUD_RUN_ANTHROPIC_API_KEY_SECRET:-policyengine-api-prod-anthropic-api-key:latest}" @@ -50,6 +64,7 @@ cloud_run_set_defaults() { export CLOUD_RUN_CONCURRENCY export CLOUD_RUN_WEB_CONCURRENCY export CLOUD_RUN_PORT + export CLOUD_RUN_STARTUP_PROBE export CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET export CLOUD_RUN_GITHUB_MICRODATA_TOKEN_SECRET export CLOUD_RUN_ANTHROPIC_API_KEY_SECRET diff --git a/.github/scripts/deploy_cloud_run_candidate.sh b/.github/scripts/deploy_cloud_run_candidate.sh index 312973bea..c06373987 100755 --- a/.github/scripts/deploy_cloud_run_candidate.sh +++ b/.github/scripts/deploy_cloud_run_candidate.sh @@ -54,5 +54,6 @@ cloud_run_run gcloud run deploy "${CLOUD_RUN_SERVICE}" \ --min-instances "${CLOUD_RUN_MIN_INSTANCES}" \ --max-instances "${CLOUD_RUN_MAX_INSTANCES}" \ --concurrency "${CLOUD_RUN_CONCURRENCY}" \ + --startup-probe "${CLOUD_RUN_STARTUP_PROBE}" \ --set-env-vars "${set_env_vars}" \ --set-secrets "${set_secret_vars}" diff --git a/changelog.d/cloud-run-http-startup-probe.fixed.md b/changelog.d/cloud-run-http-startup-probe.fixed.md new file mode 100644 index 000000000..9596c303e --- /dev/null +++ b/changelog.d/cloud-run-http-startup-probe.fixed.md @@ -0,0 +1 @@ +Pin an HTTP startup probe on /readiness-check so Cloud Run stops routing traffic to instances that are still importing. diff --git a/changelog.d/cloud-run-warm-capacity-2.changed.md b/changelog.d/cloud-run-warm-capacity-2.changed.md new file mode 100644 index 000000000..b539a4b50 --- /dev/null +++ b/changelog.d/cloud-run-warm-capacity-2.changed.md @@ -0,0 +1 @@ +Raise Cloud Run service-level warm capacity to 2 instances and correct the runbook flag. diff --git a/docs/migration/cloud-run-operations.md b/docs/migration/cloud-run-operations.md index 7f53c5a53..54a4d3356 100644 --- a/docs/migration/cloud-run-operations.md +++ b/docs/migration/cloud-run-operations.md @@ -71,19 +71,43 @@ Values measured and justified in gcloud inherits unspecified template fields (a mid-campaign CI deploy once shipped an inherited test concurrency), and `--concurrency default` resolves to the *platform* default (640), not the historical 80. +- **`--startup-probe httpGet.path=/readiness-check`** — pinned on every deploy. + Cloud Run's default is a *TCP* probe, which passes the instant gunicorn's master + binds the port — roughly 161s before a worker finishes importing (the worker + imports post-fork; `--preload` is deliberately not set). Cloud Run therefore + considered a booting instance "started" and routed live traffic onto it, where + requests queued until the 300s request timeout: this produced the + "no available instance" 500s and 504s during every scale-out burst. An HTTP + probe on `/readiness-check` makes Cloud Run **withhold traffic until the app can + actually serve**. Window is `failureThreshold x periodSeconds = 240s`, the + platform maximum — past it Cloud Run shuts the container down. Measured import + is ~161s post-cpu-boost, leaving ~79s of headroom; if import ever outgrows the + window the candidate revision fails to deploy in CI (on `--no-traffic`, before + any promote) rather than degrading production, so **import time is now a deploy + dependency — watch it**. Note this only stops bad routing; it does not add + capacity, so it pairs with warm capacity below. - **Scaling pins live in `push.yml` per job**: production `max-instances 4` (peak real traffic ~11 RPS, mostly cached/light; ~1–2 concurrent uncached calculates per instance), staging `min 0 / max 1`. - **Warm capacity is a service-level setting made manually, once** — CI keeps revision-level `--min-instances 0`, because revision-level minimums keep a warm 16Gi - instance alive per accumulated `stage3-*` tag: + instance alive per accumulated `stage3-*` tag. + + **The flag names differ by one word and mean opposite things:** service-level + warm capacity is `--min` (or the `run.googleapis.com/minScale` annotation on the + *service* metadata); `--min-instances` is the *revision-level* setting and is the + per-tag cost bomb. `--min` requires a recent gcloud (it does not exist in 461). ```bash gcloud run services update policyengine-api \ --project policyengine-api --region us-central1 \ - --min-instances 1 + --min 2 ``` + On older gcloud, set it by exporting the service YAML and adding + `run.googleapis.com/minScale: "2"` to the **service** `metadata.annotations` + (never under `spec.template`), then `gcloud run services replace`. + **When:** once, immediately after the Stage 3 PR merges — before evaluating the Stage 3 exit gates (the idle-readiness gate cannot pass without it) and before any public traffic. **Verify** it took, and re-verify during ramp incident response: @@ -91,10 +115,18 @@ Values measured and justified in ```bash gcloud run services describe policyengine-api \ --project policyengine-api --region us-central1 --format yaml | grep -i minscale - # expect a service-level minScale annotation of 1; a minScale under + # expect a service-level minScale annotation; a minScale under # spec.template (revision-level) would be the per-tag cost bomb — remove it. ``` + **Current value: 2** (raised from 1 on 2026-07-21). One warm instance meant every + burst beyond a single instance's capacity landed on a cold boot; two warm + instances carry the burst during the ~161s a scaled-out instance takes to become + ready. Cost is ~$131/month per warm instance at 4 vCPU / 16Gi (idle CPU + $0.0000025/vCPU-s, idle memory $0.0000025/GiB-s — memory gets no idle discount and + is ~80% of it), so trimming the memory allocation is the lever if this needs to + get cheaper. + Rationale: the user-facing scale-from-zero wake was measured at 282.8s — 17s under the 300s request timeout — so a cold start must never sit on a public request path. diff --git a/tests/unit/test_cloud_run_deploy_scripts.py b/tests/unit/test_cloud_run_deploy_scripts.py index 32895da97..36067add9 100644 --- a/tests/unit/test_cloud_run_deploy_scripts.py +++ b/tests/unit/test_cloud_run_deploy_scripts.py @@ -376,6 +376,48 @@ def test_deploy_cloud_run_candidate_pins_runtime_shape(): assert "--min-instances 0 " in result.stdout +def test_deploy_cloud_run_candidate_pins_http_startup_probe(): + """The startup probe must poll readiness over HTTP, never TCP. + + gunicorn's master binds the port ~161s before a worker finishes importing, + so a TCP probe reports "started" while the app cannot answer and Cloud Run + routes live traffic onto it. Probing /readiness-check makes Cloud Run + withhold traffic until the app can serve. + """ + result = _run_script( + ".github/scripts/deploy_cloud_run_candidate.sh", + _script_env( + **_required_runtime_env(), + CLOUD_RUN_IMAGE_URI="us-central1-docker.pkg.dev/project/repo/api:sha", + CLOUD_RUN_TAG="stage3-test", + ), + ) + + assert result.returncode == 0, result.stderr + assert "--startup-probe " in result.stdout + assert "httpGet.path=/readiness-check" in result.stdout + assert "httpGet.port=8080" in result.stdout + # A tcpSocket probe would reintroduce routing-before-ready. + assert "tcpSocket" not in result.stdout + # failureThreshold x periodSeconds must stay within Cloud Run's 240s cap: + # past it the platform shuts the container down and the deploy fails. + probe = next( + part + for part in result.stdout.split() + if "httpGet.path=/readiness-check" in part + ) + # The dry-run echoes the command shell-escaped, so commas arrive as "\,". + settings = dict( + item.split("=", 1) for item in probe.replace("\\", "").split(",") if "=" in item + ) + period = int(settings["periodSeconds"]) + threshold = int(settings["failureThreshold"]) + assert period * threshold <= 240, f"startup window {period * threshold}s > 240s cap" + # Must still comfortably cover the ~161s measured import. + assert period * threshold >= 200 + assert int(settings["timeoutSeconds"]) <= period + + def test_push_workflow_pins_cloud_run_scaling_per_job(): workflow = _push_workflow() staging_deploy = _workflow_job_block(workflow, "deploy-cloud-run-staging") From 11d85270d9fbd60b5798a2a68340ea261eea1adb Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Wed, 22 Jul 2026 16:31:54 +0300 Subject: [PATCH 2/8] Revert the HTTP startup probe: boot times exceed the platform cap MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Measured 48 real boots over 7 days (Starting gunicorn -> last Application startup complete): p50 201s, p90 371s, p95 417s, max 503s, with 11/48 (23%) over 240s. Cloud Run hard-caps startup-probe time at 240s and shuts the container down past it, so an HTTP readiness probe would kill ~23% of boots — failing that share of deploys and thrashing scale-out instances, worst under the CPU contention that triggers scale-out in the first place. Stage 2's '~161s import' measures the import step on an uncontended instance, not boot-to-ready; sizing the probe from it was the error. Documents the distribution and the prerequisite (cut p95 boot-to-ready well under 240s via lazy country loading or cached tax-benefit systems) so this is not re-attempted from config alone. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/scripts/cloud_run_env.sh | 15 ------ .github/scripts/deploy_cloud_run_candidate.sh | 1 - .../cloud-run-http-startup-probe.fixed.md | 1 - docs/migration/cloud-run-operations.md | 46 +++++++++++++------ tests/unit/test_cloud_run_deploy_scripts.py | 42 ----------------- 5 files changed, 31 insertions(+), 74 deletions(-) delete mode 100644 changelog.d/cloud-run-http-startup-probe.fixed.md diff --git a/.github/scripts/cloud_run_env.sh b/.github/scripts/cloud_run_env.sh index ee8e0aae1..f5388559a 100755 --- a/.github/scripts/cloud_run_env.sh +++ b/.github/scripts/cloud_run_env.sh @@ -20,20 +20,6 @@ cloud_run_set_defaults() { CLOUD_RUN_CONCURRENCY="${CLOUD_RUN_CONCURRENCY:-6}" CLOUD_RUN_WEB_CONCURRENCY="${CLOUD_RUN_WEB_CONCURRENCY:-2}" CLOUD_RUN_PORT="${CLOUD_RUN_PORT:-8080}" - # HTTP startup probe on /readiness-check, pinned on every deploy. The default - # TCP probe passes the instant gunicorn's master binds the port — ~161s before - # a worker finishes importing — so Cloud Run routed live traffic onto - # instances that could not answer, and those requests queued until the 300s - # request timeout. Probing readiness over HTTP makes Cloud Run withhold - # traffic until the app can actually serve. - # Window: failureThreshold x periodSeconds = 240s, the platform maximum - # (Cloud Run shuts the container down past it). Measured import is ~161s - # post-cpu-boost, so this leaves ~79s of headroom; if import ever grows past - # the window the candidate revision fails to deploy in CI (on --no-traffic, - # before any promote) rather than degrading production. - # timeoutSeconds must not exceed periodSeconds; requests hang during import, - # so each attempt fails fast on timeout and retries. - CLOUD_RUN_STARTUP_PROBE="${CLOUD_RUN_STARTUP_PROBE:-httpGet.path=/readiness-check,httpGet.port=${CLOUD_RUN_PORT},periodSeconds=10,failureThreshold=24,timeoutSeconds=5}" CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET="${CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET:-policyengine-api-prod-db-password:latest}" CLOUD_RUN_GITHUB_MICRODATA_TOKEN_SECRET="${CLOUD_RUN_GITHUB_MICRODATA_TOKEN_SECRET:-policyengine-api-prod-github-microdata-token:latest}" CLOUD_RUN_ANTHROPIC_API_KEY_SECRET="${CLOUD_RUN_ANTHROPIC_API_KEY_SECRET:-policyengine-api-prod-anthropic-api-key:latest}" @@ -64,7 +50,6 @@ cloud_run_set_defaults() { export CLOUD_RUN_CONCURRENCY export CLOUD_RUN_WEB_CONCURRENCY export CLOUD_RUN_PORT - export CLOUD_RUN_STARTUP_PROBE export CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET export CLOUD_RUN_GITHUB_MICRODATA_TOKEN_SECRET export CLOUD_RUN_ANTHROPIC_API_KEY_SECRET diff --git a/.github/scripts/deploy_cloud_run_candidate.sh b/.github/scripts/deploy_cloud_run_candidate.sh index c06373987..312973bea 100755 --- a/.github/scripts/deploy_cloud_run_candidate.sh +++ b/.github/scripts/deploy_cloud_run_candidate.sh @@ -54,6 +54,5 @@ cloud_run_run gcloud run deploy "${CLOUD_RUN_SERVICE}" \ --min-instances "${CLOUD_RUN_MIN_INSTANCES}" \ --max-instances "${CLOUD_RUN_MAX_INSTANCES}" \ --concurrency "${CLOUD_RUN_CONCURRENCY}" \ - --startup-probe "${CLOUD_RUN_STARTUP_PROBE}" \ --set-env-vars "${set_env_vars}" \ --set-secrets "${set_secret_vars}" diff --git a/changelog.d/cloud-run-http-startup-probe.fixed.md b/changelog.d/cloud-run-http-startup-probe.fixed.md deleted file mode 100644 index 9596c303e..000000000 --- a/changelog.d/cloud-run-http-startup-probe.fixed.md +++ /dev/null @@ -1 +0,0 @@ -Pin an HTTP startup probe on /readiness-check so Cloud Run stops routing traffic to instances that are still importing. diff --git a/docs/migration/cloud-run-operations.md b/docs/migration/cloud-run-operations.md index 54a4d3356..c805853bc 100644 --- a/docs/migration/cloud-run-operations.md +++ b/docs/migration/cloud-run-operations.md @@ -71,21 +71,37 @@ Values measured and justified in gcloud inherits unspecified template fields (a mid-campaign CI deploy once shipped an inherited test concurrency), and `--concurrency default` resolves to the *platform* default (640), not the historical 80. -- **`--startup-probe httpGet.path=/readiness-check`** — pinned on every deploy. - Cloud Run's default is a *TCP* probe, which passes the instant gunicorn's master - binds the port — roughly 161s before a worker finishes importing (the worker - imports post-fork; `--preload` is deliberately not set). Cloud Run therefore - considered a booting instance "started" and routed live traffic onto it, where - requests queued until the 300s request timeout: this produced the - "no available instance" 500s and 504s during every scale-out burst. An HTTP - probe on `/readiness-check` makes Cloud Run **withhold traffic until the app can - actually serve**. Window is `failureThreshold x periodSeconds = 240s`, the - platform maximum — past it Cloud Run shuts the container down. Measured import - is ~161s post-cpu-boost, leaving ~79s of headroom; if import ever outgrows the - window the candidate revision fails to deploy in CI (on `--no-traffic`, before - any promote) rather than degrading production, so **import time is now a deploy - dependency — watch it**. Note this only stops bad routing; it does not add - capacity, so it pairs with warm capacity below. +- **The startup probe stays TCP — an HTTP readiness probe is NOT viable. Do not + re-attempt it without first cutting boot time.** The default TCP probe passes the + instant gunicorn's master binds the port, which happens long before a worker + finishes importing (the import runs post-fork; `--preload` is deliberately unset). + That is why Cloud Run routes live traffic onto instances that cannot yet answer, + where requests queue until the 300s timeout — the "no available instance" 500s and + 504s seen on scale-out bursts. Probing `/readiness-check` over HTTP would fix that + routing, **but Cloud Run hard-caps total startup-probe time + (`failureThreshold x periodSeconds`) at 240s and shuts the container down past it**, + and this app's real boot times do not fit: + + | Boot-to-ready, 48 boots over 7 days (2026-07-14→21) | | + |---|---| + | p50 | 201s | + | p90 | 371s | + | p95 | 417s | + | max | 503s | + | **exceeding a 240s probe window** | **11/48 = 23%** | + + Measured from each instance's `Starting gunicorn` to its last + `Application startup complete`. A 240s HTTP probe would therefore kill ~23% of + boots — failing that share of deploys and thrashing scale-out instances. The tail + is worst precisely under CPU contention, i.e. during the traffic bursts that + trigger scale-out. (Stage 2's "~161s import" measures only the import step on an + uncontended instance; it is not boot-to-ready and must not be used to size this.) + + The prerequisite for ever enabling the probe is getting p95 boot-to-ready + comfortably under 240s — i.e. lazy per-country loading or caching the compiled + tax-benefit systems, not a config change. Until then, warm capacity below is the + mitigation: it does not stop bad routing, but it reduces how often scale-out + happens at all. - **Scaling pins live in `push.yml` per job**: production `max-instances 4` (peak real traffic ~11 RPS, mostly cached/light; ~1–2 concurrent uncached calculates per instance), staging `min 0 / max 1`. diff --git a/tests/unit/test_cloud_run_deploy_scripts.py b/tests/unit/test_cloud_run_deploy_scripts.py index 36067add9..32895da97 100644 --- a/tests/unit/test_cloud_run_deploy_scripts.py +++ b/tests/unit/test_cloud_run_deploy_scripts.py @@ -376,48 +376,6 @@ def test_deploy_cloud_run_candidate_pins_runtime_shape(): assert "--min-instances 0 " in result.stdout -def test_deploy_cloud_run_candidate_pins_http_startup_probe(): - """The startup probe must poll readiness over HTTP, never TCP. - - gunicorn's master binds the port ~161s before a worker finishes importing, - so a TCP probe reports "started" while the app cannot answer and Cloud Run - routes live traffic onto it. Probing /readiness-check makes Cloud Run - withhold traffic until the app can serve. - """ - result = _run_script( - ".github/scripts/deploy_cloud_run_candidate.sh", - _script_env( - **_required_runtime_env(), - CLOUD_RUN_IMAGE_URI="us-central1-docker.pkg.dev/project/repo/api:sha", - CLOUD_RUN_TAG="stage3-test", - ), - ) - - assert result.returncode == 0, result.stderr - assert "--startup-probe " in result.stdout - assert "httpGet.path=/readiness-check" in result.stdout - assert "httpGet.port=8080" in result.stdout - # A tcpSocket probe would reintroduce routing-before-ready. - assert "tcpSocket" not in result.stdout - # failureThreshold x periodSeconds must stay within Cloud Run's 240s cap: - # past it the platform shuts the container down and the deploy fails. - probe = next( - part - for part in result.stdout.split() - if "httpGet.path=/readiness-check" in part - ) - # The dry-run echoes the command shell-escaped, so commas arrive as "\,". - settings = dict( - item.split("=", 1) for item in probe.replace("\\", "").split(",") if "=" in item - ) - period = int(settings["periodSeconds"]) - threshold = int(settings["failureThreshold"]) - assert period * threshold <= 240, f"startup window {period * threshold}s > 240s cap" - # Must still comfortably cover the ~161s measured import. - assert period * threshold >= 200 - assert int(settings["timeoutSeconds"]) <= period - - def test_push_workflow_pins_cloud_run_scaling_per_job(): workflow = _push_workflow() staging_deploy = _workflow_job_block(workflow, "deploy-cloud-run-staging") From e9e5aa414a5778bb7b0bd6a0c287f5bc5949c3ba Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Wed, 22 Jul 2026 16:41:46 +0300 Subject: [PATCH 3/8] Document the services-replace path for service-level warm capacity Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/migration/cloud-run-operations.md | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/docs/migration/cloud-run-operations.md b/docs/migration/cloud-run-operations.md index c805853bc..2aca92df3 100644 --- a/docs/migration/cloud-run-operations.md +++ b/docs/migration/cloud-run-operations.md @@ -122,7 +122,20 @@ Values measured and justified in On older gcloud, set it by exporting the service YAML and adding `run.googleapis.com/minScale: "2"` to the **service** `metadata.annotations` - (never under `spec.template`), then `gcloud run services replace`. + (never under `spec.template`), then `gcloud run services replace`: + + ```bash + gcloud run services describe policyengine-api --project policyengine-api \ + --region us-central1 --format export > svc.yaml + # edit ONLY metadata.annotations."run.googleapis.com/minScale", then diff to + # confirm a single changed line before applying + gcloud run services replace svc.yaml --project policyengine-api --region us-central1 + ``` + + `replace` re-applies the whole spec and prints "Creating Revision", but Cloud Run + deduplicates an unchanged template — the serving revision and its tag are + preserved. Verify that: `status.latestReadyRevisionName` and the 100%-traffic + entry should be unchanged afterwards. **When:** once, immediately after the Stage 3 PR merges — before evaluating the Stage 3 exit gates (the idle-readiness gate cannot pass without it) and before any From fd45ab914f4b73628e845df8f9fabc61479f06fc Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Wed, 22 Jul 2026 19:11:21 +0300 Subject: [PATCH 4/8] Pin HTTP startup probe with an extended 360s window Cloud Run's default TCP probe passes when gunicorn's master binds the port, long before a worker finishes importing, so live traffic was routed onto instances that could not answer and queued to the 300s timeout. The platform caps failureThreshold x periodSeconds at 240s, which alone covers only 77% of our measured boots. initialDelaySeconds is additive (no probe runs during it, so no failures accumulate) and separately capped at 240s, giving a real ceiling of 480s. Using 120 + 24x10 = 360s covers ~90% of boots while keeping initialDelaySeconds below the p50 boot (201s), since it also delays availability and would otherwise slow every scale-out. ~10% of boots still exceed this and will be killed and retried: a deliberate trade, since the TCP probe served real users 500s and 504s from instances that were never ready. Cutting boot time removes the residual risk. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/scripts/cloud_run_env.sh | 22 ++++++++ .github/scripts/deploy_cloud_run_candidate.sh | 1 + .../cloud-run-http-startup-probe.fixed.md | 1 + tests/unit/test_cloud_run_deploy_scripts.py | 54 +++++++++++++++++++ 4 files changed, 78 insertions(+) create mode 100644 changelog.d/cloud-run-http-startup-probe.fixed.md diff --git a/.github/scripts/cloud_run_env.sh b/.github/scripts/cloud_run_env.sh index f5388559a..feba8fa1a 100755 --- a/.github/scripts/cloud_run_env.sh +++ b/.github/scripts/cloud_run_env.sh @@ -20,6 +20,27 @@ cloud_run_set_defaults() { CLOUD_RUN_CONCURRENCY="${CLOUD_RUN_CONCURRENCY:-6}" CLOUD_RUN_WEB_CONCURRENCY="${CLOUD_RUN_WEB_CONCURRENCY:-2}" CLOUD_RUN_PORT="${CLOUD_RUN_PORT:-8080}" + # HTTP startup probe on /readiness-check. Cloud Run's default TCP probe passes + # the instant gunicorn's master binds the port — long before a worker finishes + # importing (the import runs post-fork; --preload is deliberately unset) — so + # Cloud Run routed live traffic onto instances that could not answer and those + # requests queued to the 300s timeout. Probing readiness over HTTP makes Cloud + # Run withhold traffic until the app can actually serve. + # + # Window arithmetic (both halves are capped at 240s each by the platform): + # initialDelaySeconds 120 + failureThreshold 24 x periodSeconds 10 (=240) + # = 360s before Cloud Run shuts the container down. + # initialDelaySeconds is additive — no probe runs during it, so no failures + # accumulate — but it ALSO delays availability, since the first probe cannot + # succeed before it elapses. 120s sits below the measured p50 boot (201s), so + # it rarely delays a genuinely-ready instance while buying 120s of headroom. + # Measured boot-to-ready (48 boots, 7d): p50 201s, p90 371s, p95 417s, max + # 503s — so ~10% of boots still exceed this window and will be killed and + # retried. That is a deliberate trade: a killed-and-retried instance costs + # capacity latency, whereas the TCP probe served real users 500s and 504s + # from instances that were never ready. Cutting boot time is what removes the + # residual risk; see docs/migration/cloud-run-operations.md. + CLOUD_RUN_STARTUP_PROBE="${CLOUD_RUN_STARTUP_PROBE:-httpGet.path=/readiness-check,httpGet.port=${CLOUD_RUN_PORT},initialDelaySeconds=120,periodSeconds=10,failureThreshold=24,timeoutSeconds=5}" CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET="${CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET:-policyengine-api-prod-db-password:latest}" CLOUD_RUN_GITHUB_MICRODATA_TOKEN_SECRET="${CLOUD_RUN_GITHUB_MICRODATA_TOKEN_SECRET:-policyengine-api-prod-github-microdata-token:latest}" CLOUD_RUN_ANTHROPIC_API_KEY_SECRET="${CLOUD_RUN_ANTHROPIC_API_KEY_SECRET:-policyengine-api-prod-anthropic-api-key:latest}" @@ -50,6 +71,7 @@ cloud_run_set_defaults() { export CLOUD_RUN_CONCURRENCY export CLOUD_RUN_WEB_CONCURRENCY export CLOUD_RUN_PORT + export CLOUD_RUN_STARTUP_PROBE export CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET export CLOUD_RUN_GITHUB_MICRODATA_TOKEN_SECRET export CLOUD_RUN_ANTHROPIC_API_KEY_SECRET diff --git a/.github/scripts/deploy_cloud_run_candidate.sh b/.github/scripts/deploy_cloud_run_candidate.sh index 312973bea..c06373987 100755 --- a/.github/scripts/deploy_cloud_run_candidate.sh +++ b/.github/scripts/deploy_cloud_run_candidate.sh @@ -54,5 +54,6 @@ cloud_run_run gcloud run deploy "${CLOUD_RUN_SERVICE}" \ --min-instances "${CLOUD_RUN_MIN_INSTANCES}" \ --max-instances "${CLOUD_RUN_MAX_INSTANCES}" \ --concurrency "${CLOUD_RUN_CONCURRENCY}" \ + --startup-probe "${CLOUD_RUN_STARTUP_PROBE}" \ --set-env-vars "${set_env_vars}" \ --set-secrets "${set_secret_vars}" diff --git a/changelog.d/cloud-run-http-startup-probe.fixed.md b/changelog.d/cloud-run-http-startup-probe.fixed.md new file mode 100644 index 000000000..76006ccac --- /dev/null +++ b/changelog.d/cloud-run-http-startup-probe.fixed.md @@ -0,0 +1 @@ +Gate Cloud Run traffic on an HTTP readiness startup probe so instances that are still importing no longer receive requests. diff --git a/tests/unit/test_cloud_run_deploy_scripts.py b/tests/unit/test_cloud_run_deploy_scripts.py index 32895da97..621e30106 100644 --- a/tests/unit/test_cloud_run_deploy_scripts.py +++ b/tests/unit/test_cloud_run_deploy_scripts.py @@ -376,6 +376,60 @@ def test_deploy_cloud_run_candidate_pins_runtime_shape(): assert "--min-instances 0 " in result.stdout +def test_deploy_cloud_run_candidate_pins_http_startup_probe(): + """The startup probe must poll readiness over HTTP, never TCP. + + gunicorn's master binds the port long before a worker finishes importing, + so a TCP probe reports "started" while the app cannot answer and Cloud Run + routes live traffic onto it. Probing /readiness-check makes Cloud Run + withhold traffic until the app can actually serve. + """ + result = _run_script( + ".github/scripts/deploy_cloud_run_candidate.sh", + _script_env( + **_required_runtime_env(), + CLOUD_RUN_IMAGE_URI="us-central1-docker.pkg.dev/project/repo/api:sha", + CLOUD_RUN_TAG="stage3-test", + ), + ) + + assert result.returncode == 0, result.stderr + assert "--startup-probe " in result.stdout + assert "httpGet.path=/readiness-check" in result.stdout + assert "httpGet.port=8080" in result.stdout + # A tcpSocket probe would reintroduce routing-before-ready. + assert "tcpSocket" not in result.stdout + + probe = next( + part + for part in result.stdout.split() + if "httpGet.path=/readiness-check" in part + ) + # The dry-run echoes the command shell-escaped, so commas arrive as "\,". + settings = dict( + item.split("=", 1) + for item in probe.replace("\\", "").split(",") + if "=" in item + ) + period = int(settings["periodSeconds"]) + threshold = int(settings["failureThreshold"]) + initial_delay = int(settings["initialDelaySeconds"]) + + # Cloud Run caps EACH half at 240s and shuts the container down past the + # total, so both halves and their sum are load-bearing. + assert threshold * period <= 240, "failureThreshold x periodSeconds > 240s cap" + assert initial_delay <= 240, "initialDelaySeconds > 240s cap" + # initialDelaySeconds is additive (no probe runs during it), so the real + # deadline is the sum. Keep it wide enough to cover the measured p90 boot + # (371s at time of writing) — see cloud-run-operations.md for the data. + assert initial_delay + threshold * period >= 340 + # ...but initialDelaySeconds also delays availability, since no probe can + # succeed before it elapses. Keep it under the measured p50 boot (201s) so + # it does not needlessly slow down every scale-out. + assert initial_delay <= 200 + assert int(settings["timeoutSeconds"]) <= period + + def test_push_workflow_pins_cloud_run_scaling_per_job(): workflow = _push_workflow() staging_deploy = _workflow_job_block(workflow, "deploy-cloud-run-staging") From 6a124c46b2d457b657ff4712d6677be481ee338c Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Wed, 22 Jul 2026 19:37:35 +0300 Subject: [PATCH 5/8] Widen the startup-probe window to 420s via initialDelaySeconds=180 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit failureThreshold x periodSeconds is already at its 240s platform ceiling, so initialDelaySeconds is the only lever left. Raising it 120 -> 180 halves the share of boots killed by the window (12.5% -> 6.2%); the instances it newly delays are mostly booting at 155-179s, so they wait ~25s. 240 was rejected because it would hold 77% of instances to a full 240s, slowing every scale-out, for four further points. The costs are asymmetric: a delayed instance loses tens of seconds, a killed one loses its whole boot plus a retry (400s+) and fails the deploy if it happens in CI. Also replaces the stale docs section that declared the HTTP probe unviable — that conclusion assumed the 240s cap was the whole window, before initialDelaySeconds was found to be additive. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/scripts/cloud_run_env.sh | 38 +++++++----- docs/migration/cloud-run-operations.md | 84 ++++++++++++++++---------- 2 files changed, 77 insertions(+), 45 deletions(-) diff --git a/.github/scripts/cloud_run_env.sh b/.github/scripts/cloud_run_env.sh index feba8fa1a..beebbb888 100755 --- a/.github/scripts/cloud_run_env.sh +++ b/.github/scripts/cloud_run_env.sh @@ -27,20 +27,30 @@ cloud_run_set_defaults() { # requests queued to the 300s timeout. Probing readiness over HTTP makes Cloud # Run withhold traffic until the app can actually serve. # - # Window arithmetic (both halves are capped at 240s each by the platform): - # initialDelaySeconds 120 + failureThreshold 24 x periodSeconds 10 (=240) - # = 360s before Cloud Run shuts the container down. - # initialDelaySeconds is additive — no probe runs during it, so no failures - # accumulate — but it ALSO delays availability, since the first probe cannot - # succeed before it elapses. 120s sits below the measured p50 boot (201s), so - # it rarely delays a genuinely-ready instance while buying 120s of headroom. - # Measured boot-to-ready (48 boots, 7d): p50 201s, p90 371s, p95 417s, max - # 503s — so ~10% of boots still exceed this window and will be killed and - # retried. That is a deliberate trade: a killed-and-retried instance costs - # capacity latency, whereas the TCP probe served real users 500s and 504s - # from instances that were never ready. Cutting boot time is what removes the - # residual risk; see docs/migration/cloud-run-operations.md. - CLOUD_RUN_STARTUP_PROBE="${CLOUD_RUN_STARTUP_PROBE:-httpGet.path=/readiness-check,httpGet.port=${CLOUD_RUN_PORT},initialDelaySeconds=120,periodSeconds=10,failureThreshold=24,timeoutSeconds=5}" + # Window arithmetic (the platform caps EACH half at 240s): + # initialDelaySeconds 180 + failureThreshold 24 x periodSeconds 10 (=240) + # = 420s before Cloud Run shuts the container down. + # failureThreshold x periodSeconds is already AT its 240s ceiling, so + # initialDelaySeconds is the only way to widen the window. It is additive (no + # probe runs during it, so no failures accumulate) but it also delays + # availability: an instance ready sooner than initialDelaySeconds still waits + # for the first probe. Only faster-than-initialDelay boots pay that; slower + # ones are probed on the next tick after they become ready. + # + # Measured boot-to-ready (48 boots, 7d): p50 201s, p90 371s, p95 417s, + # max 503s. Against that distribution: + # initialDelay 120 -> 360s window: 12.5% killed, 8.3% delayed (~31s median) + # initialDelay 180 -> 420s window: 6.2% killed, 22.9% delayed (~25s median) + # initialDelay 240 -> 480s window: 2.1% killed, 77.1% delayed (~50s median) + # 180 halves the kill rate versus 120 while the newly-delayed instances are + # mostly booting at 155-179s, so they wait only seconds. 240 would hold 77% of + # instances to a full 240s, slowing every scale-out, for a further 4 points. + # The costs are asymmetric: a delayed instance loses tens of seconds, a killed + # one loses its whole boot plus a retry (400s+) and fails the deploy if it + # happens in CI. Cutting boot time is what removes the residual risk entirely + # — at a 20s boot this would be initialDelay 0 and a 240s window covering + # everything. See docs/migration/cloud-run-operations.md. + CLOUD_RUN_STARTUP_PROBE="${CLOUD_RUN_STARTUP_PROBE:-httpGet.path=/readiness-check,httpGet.port=${CLOUD_RUN_PORT},initialDelaySeconds=180,periodSeconds=10,failureThreshold=24,timeoutSeconds=5}" CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET="${CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET:-policyengine-api-prod-db-password:latest}" CLOUD_RUN_GITHUB_MICRODATA_TOKEN_SECRET="${CLOUD_RUN_GITHUB_MICRODATA_TOKEN_SECRET:-policyengine-api-prod-github-microdata-token:latest}" CLOUD_RUN_ANTHROPIC_API_KEY_SECRET="${CLOUD_RUN_ANTHROPIC_API_KEY_SECRET:-policyengine-api-prod-anthropic-api-key:latest}" diff --git a/docs/migration/cloud-run-operations.md b/docs/migration/cloud-run-operations.md index 2aca92df3..19406d1e1 100644 --- a/docs/migration/cloud-run-operations.md +++ b/docs/migration/cloud-run-operations.md @@ -71,37 +71,59 @@ Values measured and justified in gcloud inherits unspecified template fields (a mid-campaign CI deploy once shipped an inherited test concurrency), and `--concurrency default` resolves to the *platform* default (640), not the historical 80. -- **The startup probe stays TCP — an HTTP readiness probe is NOT viable. Do not - re-attempt it without first cutting boot time.** The default TCP probe passes the - instant gunicorn's master binds the port, which happens long before a worker - finishes importing (the import runs post-fork; `--preload` is deliberately unset). - That is why Cloud Run routes live traffic onto instances that cannot yet answer, - where requests queue until the 300s timeout — the "no available instance" 500s and - 504s seen on scale-out bursts. Probing `/readiness-check` over HTTP would fix that - routing, **but Cloud Run hard-caps total startup-probe time - (`failureThreshold x periodSeconds`) at 240s and shuts the container down past it**, - and this app's real boot times do not fit: - - | Boot-to-ready, 48 boots over 7 days (2026-07-14→21) | | - |---|---| - | p50 | 201s | - | p90 | 371s | - | p95 | 417s | - | max | 503s | - | **exceeding a 240s probe window** | **11/48 = 23%** | - - Measured from each instance's `Starting gunicorn` to its last - `Application startup complete`. A 240s HTTP probe would therefore kill ~23% of - boots — failing that share of deploys and thrashing scale-out instances. The tail - is worst precisely under CPU contention, i.e. during the traffic bursts that - trigger scale-out. (Stage 2's "~161s import" measures only the import step on an - uncontended instance; it is not boot-to-ready and must not be used to size this.) - - The prerequisite for ever enabling the probe is getting p95 boot-to-ready - comfortably under 240s — i.e. lazy per-country loading or caching the compiled - tax-benefit systems, not a config change. Until then, warm capacity below is the - mitigation: it does not stop bad routing, but it reduces how often scale-out - happens at all. +- **`--startup-probe httpGet.path=/readiness-check` — pinned on every deploy.** + Cloud Run's default is a *TCP* probe, which passes the instant gunicorn's master + binds the port, long before a worker finishes importing (the import runs + post-fork; `--preload` is deliberately unset). Cloud Run therefore treated a + booting instance as "started" and routed live traffic onto it, where requests + queued until the 300s timeout — the "no available instance" 500s and 504s seen on + scale-out bursts. Probing `/readiness-check` over HTTP makes Cloud Run withhold + traffic until the app can actually serve. + + **Sizing it is constrained by measured boot times.** Boot-to-ready across 48 + boots over 7 days (2026-07-14→21), measured from each instance's + `Starting gunicorn` to its last `Application startup complete`: + + | p50 | p90 | p95 | max | + |---|---|---|---| + | 201s | 371s | 417s | 503s | + + The tail is worst under CPU contention — i.e. during the very bursts that trigger + scale-out. (Stage 2's "~161s import" measures only the import step on an + uncontended instance; it is **not** boot-to-ready and must not be used to size + this.) + + Cloud Run caps `failureThreshold x periodSeconds` at 240s **and** + `initialDelaySeconds` at 240s, and shuts the container down past their sum. We run + `initialDelaySeconds 180 + 24 x 10s = 420s`. The threshold/period half is already + at its ceiling, so `initialDelaySeconds` is the only way to widen the window; it + is additive (no probe runs during it) but also delays availability for instances + that boot faster than it. Trade-off against the distribution above: + + | initialDelay | window | boots killed | boots delayed | median penalty | + |---|---|---|---|---| + | 120 | 360s | 12.5% | 8.3% | 31s | + | **180 (current)** | **420s** | **6.2%** | **22.9%** | **25s** | + | 240 (max) | 480s | 2.1% | 77.1% | 50s | + + A delayed instance loses tens of seconds; a **killed** one loses its whole boot + plus a retry (400s+) and fails the deploy if it happens in CI — so the asymmetry + favours a wider window. 240 was rejected because it holds 77% of instances to a + full 240s, slowing every scale-out. + + **Residual risk: ~6% of boots still exceed 420s and will be killed and retried**, + which can fail a deploy. That is accepted deliberately — the alternative (TCP) + served real users 5xx from instances that were never ready. Qualified 2026-07-22 + on a `--no-traffic` revision: Cloud Run stored the config exactly as specified and + the revision reached `Ready` on a 181.6s boot. + + The real fix is cutting boot time — **~82% of it is constructing the US + tax-benefit system** (`policyengine_api.country` builds all five countries at + import; US alone is ~90% of that, and `CountryTaxBenefitSystem()` is ~91% of the + per-country cost). At a 20s boot this would be `initialDelay 0` with a 240s window + covering every boot. Note that lazily deferring the build does **not** help: it + relocates the cost onto the first request, where readiness would lie and a user + would absorb it. - **Scaling pins live in `push.yml` per job**: production `max-instances 4` (peak real traffic ~11 RPS, mostly cached/light; ~1–2 concurrent uncached calculates per instance), staging `min 0 / max 1`. From ad12619bc1f77fd7b89f7089af2bd81d3953bf5b Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Wed, 22 Jul 2026 19:47:19 +0300 Subject: [PATCH 6/8] Fold in the remaining runbook details from #3744 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Records that services replace needs the Cloud Resource Manager API enabled, and that service-level warm capacity was first set on 2026-07-08 before being raised on 2026-07-22 — both via the YAML-replace fallback. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/migration/cloud-run-operations.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/migration/cloud-run-operations.md b/docs/migration/cloud-run-operations.md index 19406d1e1..4dd8b0e97 100644 --- a/docs/migration/cloud-run-operations.md +++ b/docs/migration/cloud-run-operations.md @@ -144,7 +144,8 @@ Values measured and justified in On older gcloud, set it by exporting the service YAML and adding `run.googleapis.com/minScale: "2"` to the **service** `metadata.annotations` - (never under `spec.template`), then `gcloud run services replace`: + (never under `spec.template`), then `gcloud run services replace` — which + requires the Cloud Resource Manager API enabled on the project: ```bash gcloud run services describe policyengine-api --project policyengine-api \ @@ -170,7 +171,9 @@ Values measured and justified in # spec.template (revision-level) would be the per-tag cost bomb — remove it. ``` - **Current value: 2** (raised from 1 on 2026-07-21). One warm instance meant every + **Current value: 2.** Originally set to 1 on 2026-07-08 (via the YAML-replace + fallback, since the gcloud in use lacked `--min`), raised to 2 on 2026-07-22 by + the same route. One warm instance meant every burst beyond a single instance's capacity landed on a cold boot; two warm instances carry the burst during the ~161s a scaled-out instance takes to become ready. Cost is ~$131/month per warm instance at 4 vCPU / 16Gi (idle CPU From 5424b7c258796f74e0295c76830704eafc3e6a3c Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Wed, 22 Jul 2026 20:08:59 +0300 Subject: [PATCH 7/8] Trim the startup-probe comment to three lines The sizing analysis belongs in cloud-run-operations.md, which already carries it; the env file just needs the gist and a pointer. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/scripts/cloud_run_env.sh | 33 +++----------------------------- 1 file changed, 3 insertions(+), 30 deletions(-) diff --git a/.github/scripts/cloud_run_env.sh b/.github/scripts/cloud_run_env.sh index beebbb888..17c70e127 100755 --- a/.github/scripts/cloud_run_env.sh +++ b/.github/scripts/cloud_run_env.sh @@ -20,36 +20,9 @@ cloud_run_set_defaults() { CLOUD_RUN_CONCURRENCY="${CLOUD_RUN_CONCURRENCY:-6}" CLOUD_RUN_WEB_CONCURRENCY="${CLOUD_RUN_WEB_CONCURRENCY:-2}" CLOUD_RUN_PORT="${CLOUD_RUN_PORT:-8080}" - # HTTP startup probe on /readiness-check. Cloud Run's default TCP probe passes - # the instant gunicorn's master binds the port — long before a worker finishes - # importing (the import runs post-fork; --preload is deliberately unset) — so - # Cloud Run routed live traffic onto instances that could not answer and those - # requests queued to the 300s timeout. Probing readiness over HTTP makes Cloud - # Run withhold traffic until the app can actually serve. - # - # Window arithmetic (the platform caps EACH half at 240s): - # initialDelaySeconds 180 + failureThreshold 24 x periodSeconds 10 (=240) - # = 420s before Cloud Run shuts the container down. - # failureThreshold x periodSeconds is already AT its 240s ceiling, so - # initialDelaySeconds is the only way to widen the window. It is additive (no - # probe runs during it, so no failures accumulate) but it also delays - # availability: an instance ready sooner than initialDelaySeconds still waits - # for the first probe. Only faster-than-initialDelay boots pay that; slower - # ones are probed on the next tick after they become ready. - # - # Measured boot-to-ready (48 boots, 7d): p50 201s, p90 371s, p95 417s, - # max 503s. Against that distribution: - # initialDelay 120 -> 360s window: 12.5% killed, 8.3% delayed (~31s median) - # initialDelay 180 -> 420s window: 6.2% killed, 22.9% delayed (~25s median) - # initialDelay 240 -> 480s window: 2.1% killed, 77.1% delayed (~50s median) - # 180 halves the kill rate versus 120 while the newly-delayed instances are - # mostly booting at 155-179s, so they wait only seconds. 240 would hold 77% of - # instances to a full 240s, slowing every scale-out, for a further 4 points. - # The costs are asymmetric: a delayed instance loses tens of seconds, a killed - # one loses its whole boot plus a retry (400s+) and fails the deploy if it - # happens in CI. Cutting boot time is what removes the residual risk entirely - # — at a 20s boot this would be initialDelay 0 and a 240s window covering - # everything. See docs/migration/cloud-run-operations.md. + # Readiness gate: a TCP probe passes at port-bind, minutes before the app can + # serve. Window is 180s + 24x10s = 420s; both halves are capped at 240s. + # Sizing rationale in docs/migration/cloud-run-operations.md. CLOUD_RUN_STARTUP_PROBE="${CLOUD_RUN_STARTUP_PROBE:-httpGet.path=/readiness-check,httpGet.port=${CLOUD_RUN_PORT},initialDelaySeconds=180,periodSeconds=10,failureThreshold=24,timeoutSeconds=5}" CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET="${CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET:-policyengine-api-prod-db-password:latest}" CLOUD_RUN_GITHUB_MICRODATA_TOKEN_SECRET="${CLOUD_RUN_GITHUB_MICRODATA_TOKEN_SECRET:-policyengine-api-prod-github-microdata-token:latest}" From e004986184ae4a92970db3fabe36021cd679ce6a Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Wed, 22 Jul 2026 20:12:34 +0300 Subject: [PATCH 8/8] Apply ruff format Co-Authored-By: Claude Opus 4.8 (1M context) --- tests/unit/test_cloud_run_deploy_scripts.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/unit/test_cloud_run_deploy_scripts.py b/tests/unit/test_cloud_run_deploy_scripts.py index 621e30106..d05c058d4 100644 --- a/tests/unit/test_cloud_run_deploy_scripts.py +++ b/tests/unit/test_cloud_run_deploy_scripts.py @@ -407,9 +407,7 @@ def test_deploy_cloud_run_candidate_pins_http_startup_probe(): ) # The dry-run echoes the command shell-escaped, so commas arrive as "\,". settings = dict( - item.split("=", 1) - for item in probe.replace("\\", "").split(",") - if "=" in item + item.split("=", 1) for item in probe.replace("\\", "").split(",") if "=" in item ) period = int(settings["periodSeconds"]) threshold = int(settings["failureThreshold"])