From 5adc68ea9c49c898c9a22735599b9ad3034cf186 Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Thu, 23 Jul 2026 18:54:25 +0300 Subject: [PATCH 1/4] Add explicit readiness gate to live smoke suites MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The staging smoke suites run test_live_calculate.py first, whose first test posts a household calculate directly with a 90s client timeout and no retry. Against the scale-to-zero staging Cloud Run service the candidate instance is cold, and the ~200s US tax-benefit-system import loses the race to the timeout (httpx.ReadTimeout). The failure cascades — the staging Cloud Run integration test skips promote-cloud-run-staging, which skips the production Cloud Run deploy. Add tests/integration/test_live_readiness.py, listed first in both staging smoke steps, which polls /readiness-check until the deployed revision answers 200. This proves the deploy came up and warms the single instance so every following test hits a ready service. Kept as a visible test (not a conftest fixture) so the wait is obvious in the suite output and easy to maintain. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/push.yml | 4 +- .../staging-smoke-readiness-gate.fixed.md | 1 + tests/integration/test_live_readiness.py | 69 +++++++++++++++++++ 3 files changed, 72 insertions(+), 2 deletions(-) create mode 100644 changelog.d/staging-smoke-readiness-gate.fixed.md create mode 100644 tests/integration/test_live_readiness.py diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 513417d1f..f190e6285 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -287,7 +287,7 @@ jobs: - name: Install staging test dependencies run: pip install pytest httpx - name: Run staging smoke test - run: python -m pytest tests/integration/test_live_calculate.py tests/integration/test_live_economy.py tests/integration/test_live_budget_window_cache.py -v + run: python -m pytest tests/integration/test_live_readiness.py tests/integration/test_live_calculate.py tests/integration/test_live_economy.py tests/integration/test_live_budget_window_cache.py -v env: API_BASE_URL: ${{ needs.deploy-staging.outputs.url }} STAGING_API_TEST_PROBE_ID: ${{ needs.deploy-staging.outputs.version }} @@ -340,7 +340,7 @@ jobs: - name: Install staging test dependencies run: pip install pytest httpx - name: Run staging smoke test - run: python -m pytest tests/integration/test_live_calculate.py tests/integration/test_live_economy.py tests/integration/test_live_budget_window_cache.py -v + run: python -m pytest tests/integration/test_live_readiness.py tests/integration/test_live_calculate.py tests/integration/test_live_economy.py tests/integration/test_live_budget_window_cache.py -v env: API_BASE_URL: ${{ needs.deploy-cloud-run-staging.outputs.url }} STAGING_API_TEST_PROBE_ID: cloud-run-${{ needs.deploy-cloud-run-staging.outputs.tag }} diff --git a/changelog.d/staging-smoke-readiness-gate.fixed.md b/changelog.d/staging-smoke-readiness-gate.fixed.md new file mode 100644 index 000000000..701507326 --- /dev/null +++ b/changelog.d/staging-smoke-readiness-gate.fixed.md @@ -0,0 +1 @@ +Add an explicit readiness gate that runs first in the live deployment smoke suites, polling /readiness-check until the freshly deployed instance is warm so the first calculate no longer races the cold-start import and intermittently fails (and skips the production Cloud Run deploy). diff --git a/tests/integration/test_live_readiness.py b/tests/integration/test_live_readiness.py new file mode 100644 index 000000000..aba4ad03c --- /dev/null +++ b/tests/integration/test_live_readiness.py @@ -0,0 +1,69 @@ +"""Readiness gate for the live deployment smoke suites. + +This module is listed FIRST in the "Run staging smoke test" workflow steps so it +runs before the calculate/economy/budget-window suites. The staging services are +scale-to-zero, so a freshly deployed candidate is cold when the suite starts and +needs to import the tax-benefit system (~200s for the US system) before it can +answer a household calculate. The first heavy request would otherwise race that +import against the per-request timeout and fail intermittently — which is exactly +what blocks the downstream production deploy when it does. + +Polling /readiness-check here does two things: it proves the deployed revision +actually came up, and it warms the instance so every following test hits a ready +service. It is deliberately a plain, visible test rather than a conftest fixture +so the wait is obvious in the suite output and easy to maintain. +""" + +import os +import time + +import httpx + +# Cloud Run's startup probe allows initialDelaySeconds 180 + failureThreshold 24 +# x periodSeconds 10 = 420s before it abandons a boot. Wait a little longer than +# that so a slow-but-valid boot is not reported as a failure here. +READINESS_DEADLINE_SECONDS = float( + os.environ.get("STAGING_API_READINESS_DEADLINE_SECONDS", "480") +) +READINESS_POLL_INTERVAL_SECONDS = float( + os.environ.get("STAGING_API_READINESS_POLL_INTERVAL_SECONDS", "5") +) +# Per-request cap: a request that lands mid-boot is held by Cloud Run until the +# instance is ready; bound the individual wait so we keep polling and logging +# progress rather than blocking on a single long request. +READINESS_REQUEST_TIMEOUT_SECONDS = float( + os.environ.get("STAGING_API_READINESS_REQUEST_TIMEOUT_SECONDS", "30") +) + + +def test_service_is_ready(api_base_url: str) -> None: + """Block until the deployed service answers /readiness-check with 200.""" + deadline = time.monotonic() + READINESS_DEADLINE_SECONDS + attempts = 0 + last_result = "no request completed" + + with httpx.Client( + base_url=api_base_url, + timeout=READINESS_REQUEST_TIMEOUT_SECONDS, + follow_redirects=True, + ) as client: + while True: + attempts += 1 + try: + response = client.get("/readiness-check") + except httpx.RequestError as error: + # Connection/read timeouts while the instance is still importing + # are expected; keep polling until the deadline. + last_result = f"{type(error).__name__}: {error}" + else: + if response.status_code == 200: + return + last_result = f"HTTP {response.status_code}: {response.text[:200]}" + + if time.monotonic() >= deadline: + raise AssertionError( + f"Service at {api_base_url} did not become ready within " + f"{READINESS_DEADLINE_SECONDS:.0f}s ({attempts} attempts); " + f"last result was {last_result}" + ) + time.sleep(READINESS_POLL_INTERVAL_SECONDS) From f474e3aba803f8186280a96bc0f90492199d5208 Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Thu, 23 Jul 2026 19:03:30 +0300 Subject: [PATCH 2/4] Update workflow-command assertion for the readiness gate test_push_workflow_tests_app_engine_and_cloud_run_staging_tracks pins the exact staging smoke-test command; update the expected string to include the readiness gate at the head, which also guards that it stays first. Co-Authored-By: Claude Opus 4.8 (1M context) --- tests/unit/test_cloud_run_deploy_scripts.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_cloud_run_deploy_scripts.py b/tests/unit/test_cloud_run_deploy_scripts.py index 34d8b3e76..a501eb801 100644 --- a/tests/unit/test_cloud_run_deploy_scripts.py +++ b/tests/unit/test_cloud_run_deploy_scripts.py @@ -622,8 +622,11 @@ def test_push_workflow_tests_app_engine_and_cloud_run_staging_tracks(): workflow, "ensure-production-model-version-aligns-with-sim-api", ) + # The readiness gate must run first so the cold candidate is warm before the + # calculate suite hits it; pin its position at the head of both commands. live_test_command = ( - "python -m pytest tests/integration/test_live_calculate.py " + "python -m pytest tests/integration/test_live_readiness.py " + "tests/integration/test_live_calculate.py " "tests/integration/test_live_economy.py " "tests/integration/test_live_budget_window_cache.py -v" ) From a64cd3b0d734af2c191d59775aaca5349b35d154 Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Thu, 23 Jul 2026 19:19:47 +0300 Subject: [PATCH 3/4] Make readiness a dedicated gate job before the smoke suite MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the in-suite readiness test with a separate wait-for-cloud-run-staging-ready job that polls /readiness-check (.github/scripts/wait_for_service_ready.sh) until the freshly deployed candidate answers 200. integration-tests-staging-cloud-run now needs that job, so the gate runs before all smoke tests and the first calculate never races the cold-start import. Its sole job is to determine readiness — no test assertions live in it. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/scripts/wait_for_service_ready.sh | 39 +++++++++++ .github/workflows/push.yml | 23 ++++++- .../staging-smoke-readiness-gate.fixed.md | 2 +- tests/integration/test_live_readiness.py | 69 ------------------- tests/unit/test_cloud_run_deploy_scripts.py | 26 +++++-- 5 files changed, 82 insertions(+), 77 deletions(-) create mode 100755 .github/scripts/wait_for_service_ready.sh delete mode 100644 tests/integration/test_live_readiness.py diff --git a/.github/scripts/wait_for_service_ready.sh b/.github/scripts/wait_for_service_ready.sh new file mode 100755 index 000000000..6c0a28885 --- /dev/null +++ b/.github/scripts/wait_for_service_ready.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# Poll a freshly deployed service's /readiness-check until it answers 200. +# +# The staging services are scale-to-zero, so a candidate revision is cold on its +# first request and must import the tax-benefit system (~200s for the US system) +# before it can serve a calculate. Run as a gate job ahead of the smoke suites, +# this makes the tests hit a warm instance instead of racing the cold-start +# import (which otherwise times out and blocks the production deploy). + +url="${SERVICE_URL:?SERVICE_URL is required}" +url="${url%/}" +deadline_seconds="${READINESS_DEADLINE_SECONDS:-480}" +poll_interval_seconds="${READINESS_POLL_INTERVAL_SECONDS:-5}" +request_timeout_seconds="${READINESS_REQUEST_TIMEOUT_SECONDS:-30}" + +deadline=$(( SECONDS + deadline_seconds )) +attempt=0 + +while true; do + attempt=$(( attempt + 1 )) + status="$(curl -s -o /dev/null -w '%{http_code}' \ + --max-time "${request_timeout_seconds}" "${url}/readiness-check" || true)" + + if [[ "${status}" == "200" ]]; then + echo "Ready after ${attempt} attempt(s): ${url}/readiness-check -> 200" + exit 0 + fi + + if (( SECONDS >= deadline )); then + echo "::error::${url} did not become ready within ${deadline_seconds}s (last status: ${status:-none})" >&2 + exit 1 + fi + + echo "Attempt ${attempt}: ${url}/readiness-check -> ${status:-no response}; retrying in ${poll_interval_seconds}s" + sleep "${poll_interval_seconds}" +done diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index f190e6285..d6770ca46 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -287,7 +287,7 @@ jobs: - name: Install staging test dependencies run: pip install pytest httpx - name: Run staging smoke test - run: python -m pytest tests/integration/test_live_readiness.py tests/integration/test_live_calculate.py tests/integration/test_live_economy.py tests/integration/test_live_budget_window_cache.py -v + run: python -m pytest tests/integration/test_live_calculate.py tests/integration/test_live_economy.py tests/integration/test_live_budget_window_cache.py -v env: API_BASE_URL: ${{ needs.deploy-staging.outputs.url }} STAGING_API_TEST_PROBE_ID: ${{ needs.deploy-staging.outputs.version }} @@ -323,10 +323,27 @@ jobs: env: APP_ENGINE_VERSION: ${{ needs.deploy-staging.outputs.version }} + wait-for-cloud-run-staging-ready: + name: Wait for staging Cloud Run readiness + runs-on: ubuntu-latest + needs: deploy-cloud-run-staging + if: | + (github.repository == 'PolicyEngine/policyengine-api') + && (github.event.head_commit.message == 'Update PolicyEngine API') + steps: + - name: Checkout repo + uses: actions/checkout@v4 + - name: Wait for /readiness-check to return 200 + run: bash .github/scripts/wait_for_service_ready.sh + env: + SERVICE_URL: ${{ needs.deploy-cloud-run-staging.outputs.url }} + integration-tests-staging-cloud-run: name: Run Cloud Run staging integration tests runs-on: ubuntu-latest - needs: deploy-cloud-run-staging + needs: + - deploy-cloud-run-staging + - wait-for-cloud-run-staging-ready if: | (github.repository == 'PolicyEngine/policyengine-api') && (github.event.head_commit.message == 'Update PolicyEngine API') @@ -340,7 +357,7 @@ jobs: - name: Install staging test dependencies run: pip install pytest httpx - name: Run staging smoke test - run: python -m pytest tests/integration/test_live_readiness.py tests/integration/test_live_calculate.py tests/integration/test_live_economy.py tests/integration/test_live_budget_window_cache.py -v + run: python -m pytest tests/integration/test_live_calculate.py tests/integration/test_live_economy.py tests/integration/test_live_budget_window_cache.py -v env: API_BASE_URL: ${{ needs.deploy-cloud-run-staging.outputs.url }} STAGING_API_TEST_PROBE_ID: cloud-run-${{ needs.deploy-cloud-run-staging.outputs.tag }} diff --git a/changelog.d/staging-smoke-readiness-gate.fixed.md b/changelog.d/staging-smoke-readiness-gate.fixed.md index 701507326..7ea01981e 100644 --- a/changelog.d/staging-smoke-readiness-gate.fixed.md +++ b/changelog.d/staging-smoke-readiness-gate.fixed.md @@ -1 +1 @@ -Add an explicit readiness gate that runs first in the live deployment smoke suites, polling /readiness-check until the freshly deployed instance is warm so the first calculate no longer races the cold-start import and intermittently fails (and skips the production Cloud Run deploy). +Add a dedicated readiness-gate job that polls /readiness-check until the freshly deployed staging Cloud Run revision is warm before the smoke suite runs, so the first calculate no longer races the cold-start import and intermittently fails (which skips the production Cloud Run deploy). diff --git a/tests/integration/test_live_readiness.py b/tests/integration/test_live_readiness.py deleted file mode 100644 index aba4ad03c..000000000 --- a/tests/integration/test_live_readiness.py +++ /dev/null @@ -1,69 +0,0 @@ -"""Readiness gate for the live deployment smoke suites. - -This module is listed FIRST in the "Run staging smoke test" workflow steps so it -runs before the calculate/economy/budget-window suites. The staging services are -scale-to-zero, so a freshly deployed candidate is cold when the suite starts and -needs to import the tax-benefit system (~200s for the US system) before it can -answer a household calculate. The first heavy request would otherwise race that -import against the per-request timeout and fail intermittently — which is exactly -what blocks the downstream production deploy when it does. - -Polling /readiness-check here does two things: it proves the deployed revision -actually came up, and it warms the instance so every following test hits a ready -service. It is deliberately a plain, visible test rather than a conftest fixture -so the wait is obvious in the suite output and easy to maintain. -""" - -import os -import time - -import httpx - -# Cloud Run's startup probe allows initialDelaySeconds 180 + failureThreshold 24 -# x periodSeconds 10 = 420s before it abandons a boot. Wait a little longer than -# that so a slow-but-valid boot is not reported as a failure here. -READINESS_DEADLINE_SECONDS = float( - os.environ.get("STAGING_API_READINESS_DEADLINE_SECONDS", "480") -) -READINESS_POLL_INTERVAL_SECONDS = float( - os.environ.get("STAGING_API_READINESS_POLL_INTERVAL_SECONDS", "5") -) -# Per-request cap: a request that lands mid-boot is held by Cloud Run until the -# instance is ready; bound the individual wait so we keep polling and logging -# progress rather than blocking on a single long request. -READINESS_REQUEST_TIMEOUT_SECONDS = float( - os.environ.get("STAGING_API_READINESS_REQUEST_TIMEOUT_SECONDS", "30") -) - - -def test_service_is_ready(api_base_url: str) -> None: - """Block until the deployed service answers /readiness-check with 200.""" - deadline = time.monotonic() + READINESS_DEADLINE_SECONDS - attempts = 0 - last_result = "no request completed" - - with httpx.Client( - base_url=api_base_url, - timeout=READINESS_REQUEST_TIMEOUT_SECONDS, - follow_redirects=True, - ) as client: - while True: - attempts += 1 - try: - response = client.get("/readiness-check") - except httpx.RequestError as error: - # Connection/read timeouts while the instance is still importing - # are expected; keep polling until the deadline. - last_result = f"{type(error).__name__}: {error}" - else: - if response.status_code == 200: - return - last_result = f"HTTP {response.status_code}: {response.text[:200]}" - - if time.monotonic() >= deadline: - raise AssertionError( - f"Service at {api_base_url} did not become ready within " - f"{READINESS_DEADLINE_SECONDS:.0f}s ({attempts} attempts); " - f"last result was {last_result}" - ) - time.sleep(READINESS_POLL_INTERVAL_SECONDS) diff --git a/tests/unit/test_cloud_run_deploy_scripts.py b/tests/unit/test_cloud_run_deploy_scripts.py index a501eb801..d3ab8301d 100644 --- a/tests/unit/test_cloud_run_deploy_scripts.py +++ b/tests/unit/test_cloud_run_deploy_scripts.py @@ -622,11 +622,8 @@ def test_push_workflow_tests_app_engine_and_cloud_run_staging_tracks(): workflow, "ensure-production-model-version-aligns-with-sim-api", ) - # The readiness gate must run first so the cold candidate is warm before the - # calculate suite hits it; pin its position at the head of both commands. live_test_command = ( - "python -m pytest tests/integration/test_live_readiness.py " - "tests/integration/test_live_calculate.py " + "python -m pytest tests/integration/test_live_calculate.py " "tests/integration/test_live_economy.py " "tests/integration/test_live_budget_window_cache.py -v" ) @@ -647,6 +644,27 @@ def test_push_workflow_tests_app_engine_and_cloud_run_staging_tracks(): assert "bash .github/scripts/get_cloud_run_service_url.sh" in cloud_run_promotion +def test_cloud_run_staging_tests_gate_on_a_readiness_job(): + workflow = _push_workflow() + readiness_gate = _workflow_job_block( + workflow, "wait-for-cloud-run-staging-ready" + ) + cloud_run_tests = _workflow_job_block( + workflow, "integration-tests-staging-cloud-run" + ) + + # A dedicated job proves the deployed candidate answers /readiness-check + # before any smoke test runs, so the first calculate does not race the + # cold-start import. + assert "bash .github/scripts/wait_for_service_ready.sh" in readiness_gate + assert ( + "SERVICE_URL: ${{ needs.deploy-cloud-run-staging.outputs.url }}" + in readiness_gate + ) + # The Cloud Run smoke suite must depend on that gate. + assert "- wait-for-cloud-run-staging-ready" in cloud_run_tests + + def test_push_workflow_deploys_app_engine_production_candidate_before_staging_gate(): workflow = _push_workflow() app_engine_candidate = _workflow_job_block(workflow, "deploy-production-candidate") From 66e3e2105a698ba32896da378b2f01ff2a4d03cb Mon Sep 17 00:00:00 2001 From: Anthony Volk Date: Thu, 23 Jul 2026 19:23:12 +0300 Subject: [PATCH 4/4] Apply ruff format to readiness-gate unit test Co-Authored-By: Claude Opus 4.8 (1M context) --- tests/unit/test_cloud_run_deploy_scripts.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/unit/test_cloud_run_deploy_scripts.py b/tests/unit/test_cloud_run_deploy_scripts.py index d3ab8301d..2a99493ee 100644 --- a/tests/unit/test_cloud_run_deploy_scripts.py +++ b/tests/unit/test_cloud_run_deploy_scripts.py @@ -646,9 +646,7 @@ def test_push_workflow_tests_app_engine_and_cloud_run_staging_tracks(): def test_cloud_run_staging_tests_gate_on_a_readiness_job(): workflow = _push_workflow() - readiness_gate = _workflow_job_block( - workflow, "wait-for-cloud-run-staging-ready" - ) + readiness_gate = _workflow_job_block(workflow, "wait-for-cloud-run-staging-ready") cloud_run_tests = _workflow_job_block( workflow, "integration-tests-staging-cloud-run" )