Add explicit readiness gate to live smoke suites (fix cold-start flake blocking prod Cloud Run deploy)

.github/scripts/wait_for_service_ready.sh

@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# Poll a freshly deployed service's /readiness-check until it answers 200.
+#
+# The staging services are scale-to-zero, so a candidate revision is cold on its
+# first request and must import the tax-benefit system (~200s for the US system)
+# before it can serve a calculate. Run as a gate job ahead of the smoke suites,
+# this makes the tests hit a warm instance instead of racing the cold-start
+# import (which otherwise times out and blocks the production deploy).
+
+url="${SERVICE_URL:?SERVICE_URL is required}"
+url="${url%/}"
+deadline_seconds="${READINESS_DEADLINE_SECONDS:-480}"
+poll_interval_seconds="${READINESS_POLL_INTERVAL_SECONDS:-5}"
+request_timeout_seconds="${READINESS_REQUEST_TIMEOUT_SECONDS:-30}"
+
+deadline=$(( SECONDS + deadline_seconds ))
+attempt=0
+
+while true; do
+  attempt=$(( attempt + 1 ))
+  status="$(curl -s -o /dev/null -w '%{http_code}' \
+    --max-time "${request_timeout_seconds}" "${url}/readiness-check" || true)"
+
+  if [[ "${status}" == "200" ]]; then
+    echo "Ready after ${attempt} attempt(s): ${url}/readiness-check -> 200"
+    exit 0
+  fi
+
+  if (( SECONDS >= deadline )); then
+    echo "::error::${url} did not become ready within ${deadline_seconds}s (last status: ${status:-none})" >&2
+    exit 1
+  fi
+
+  echo "Attempt ${attempt}: ${url}/readiness-check -> ${status:-no response}; retrying in ${poll_interval_seconds}s"
+  sleep "${poll_interval_seconds}"
+done

.github/workflows/push.yml

@@ -323,10 +323,27 @@ jobs:
         env:
           APP_ENGINE_VERSION: ${{ needs.deploy-staging.outputs.version }}
 
+  wait-for-cloud-run-staging-ready:
+    name: Wait for staging Cloud Run readiness
+    runs-on: ubuntu-latest
+    needs: deploy-cloud-run-staging
+    if: |
+      (github.repository == 'PolicyEngine/policyengine-api')
+      && (github.event.head_commit.message == 'Update PolicyEngine API')
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v4
+      - name: Wait for /readiness-check to return 200
+        run: bash .github/scripts/wait_for_service_ready.sh
+        env:
+          SERVICE_URL: ${{ needs.deploy-cloud-run-staging.outputs.url }}
+
   integration-tests-staging-cloud-run:
     name: Run Cloud Run staging integration tests
     runs-on: ubuntu-latest
-    needs: deploy-cloud-run-staging
+    needs:
+      - deploy-cloud-run-staging
+      - wait-for-cloud-run-staging-ready
     if: |
       (github.repository == 'PolicyEngine/policyengine-api')
       && (github.event.head_commit.message == 'Update PolicyEngine API')

changelog.d/staging-smoke-readiness-gate.fixed.md

@@ -0,0 +1 @@
+Add a dedicated readiness-gate job that polls /readiness-check until the freshly deployed staging Cloud Run revision is warm before the smoke suite runs, so the first calculate no longer races the cold-start import and intermittently fails (which skips the production Cloud Run deploy).

tests/unit/test_cloud_run_deploy_scripts.py

@@ -644,6 +644,25 @@ def test_push_workflow_tests_app_engine_and_cloud_run_staging_tracks():
     assert "bash .github/scripts/get_cloud_run_service_url.sh" in cloud_run_promotion
 
 
+def test_cloud_run_staging_tests_gate_on_a_readiness_job():
+    workflow = _push_workflow()
+    readiness_gate = _workflow_job_block(workflow, "wait-for-cloud-run-staging-ready")
+    cloud_run_tests = _workflow_job_block(
+        workflow, "integration-tests-staging-cloud-run"
+    )
+
+    # A dedicated job proves the deployed candidate answers /readiness-check
+    # before any smoke test runs, so the first calculate does not race the
+    # cold-start import.
+    assert "bash .github/scripts/wait_for_service_ready.sh" in readiness_gate
+    assert (
+        "SERVICE_URL: ${{ needs.deploy-cloud-run-staging.outputs.url }}"
+        in readiness_gate
+    )
+    # The Cloud Run smoke suite must depend on that gate.
+    assert "- wait-for-cloud-run-staging-ready" in cloud_run_tests
+
+
 def test_push_workflow_deploys_app_engine_production_candidate_before_staging_gate():
     workflow = _push_workflow()
     app_engine_candidate = _workflow_job_block(workflow, "deploy-production-candidate")