From 647f5ff3f6faa03a130b3aa10aab6b93ea1302c7 Mon Sep 17 00:00:00 2001 From: Michael Heller <21163552+mdheller@users.noreply.github.com> Date: Tue, 4 Aug 2026 03:25:18 -0400 Subject: [PATCH] gate-of-the-gate: PR-time smoke of promotion-gate.yml's heart logic MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit promotion-gate.yml false-negatived twice on v150.0.6 in one hour: 1. First run 127'd on 'hardcoded 7zz not found' — 7zip package only exists on Ubuntu 24+; ubuntu-latest is 22.04 (p7zip-full, binary 7z) 2. Retry silently returned empty version — plistutil failed silently, grep matched the binary blob, sed yielded nothing. '' != '150.0.6' → quarantined a GOOD artifact Both would have been caught in PR time in seconds if the gate's own logic had unit tests. Adds: - scripts/tests/test_promotion_gate.py — 12 assertions: - plist extraction against binary + XML fixtures (the failure mode) - missing-key returns '' not crash - update-check hygiene grep table - meta-check that shipped bearstart-autoconfig.js still has all three hygiene keywords - 7z-family binary discoverable after apt install - gate workflow still has all 5 load-bearing steps - regressions asserted: uses plistlib, NOT grep|sed; uses detect loop, NOT hardcoded 7zz - packaging-and-update-tests.yml — new job promotion-gate-heart runs the tests + installs p7zip-full (mirrors what the real gate does) on any change to promotion-gate.yml or the test file Meta-principle encoded in test names: 'gates that catch bug classes must themselves be exercised, and their regressions must fire in the PR, not at release publish.' --- .../workflows/packaging-and-update-tests.yml | 19 ++ scripts/tests/test_promotion_gate.py | 188 ++++++++++++++++++ 2 files changed, 207 insertions(+) create mode 100644 scripts/tests/test_promotion_gate.py diff --git a/.github/workflows/packaging-and-update-tests.yml b/.github/workflows/packaging-and-update-tests.yml index 24a4a62..4af57e5 100644 --- a/.github/workflows/packaging-and-update-tests.yml +++ b/.github/workflows/packaging-and-update-tests.yml @@ -22,6 +22,8 @@ on: - 'settings/start/bearstart-autoconfig.js' - 'scripts/tests/test_bearbrowser_patches.py' - 'scripts/tests/test_update_check.mjs' + - 'scripts/tests/test_promotion_gate.py' + - '.github/workflows/promotion-gate.yml' - '.github/workflows/packaging-and-update-tests.yml' push: branches: [main] @@ -31,6 +33,8 @@ on: - 'settings/start/bearstart-autoconfig.js' - 'scripts/tests/test_bearbrowser_patches.py' - 'scripts/tests/test_update_check.mjs' + - 'scripts/tests/test_promotion_gate.py' + - '.github/workflows/promotion-gate.yml' - '.github/workflows/packaging-and-update-tests.yml' workflow_dispatch: @@ -51,3 +55,18 @@ jobs: - uses: actions/setup-node@v4 with: { node-version: '20' } - run: node scripts/tests/test_update_check.mjs + + # Gate-of-the-gate — the promotion-gate.yml workflow itself had two + # false-negatives on v150.0.6 (hardcoded 7zz, plistutil+grep+sed chain + # against a binary plist). These tests exercise its heart logic in PR + # time so future edits can't rediscover its bugs at release time. + promotion-gate-heart: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: { python-version: '3.12' } + - name: Install p7zip-full (mirrors what promotion-gate.yml installs) + run: sudo apt-get update -qq && sudo apt-get install -y -qq p7zip-full + - run: python -m pip install pytest + - run: python -m pytest scripts/tests/test_promotion_gate.py -q diff --git a/scripts/tests/test_promotion_gate.py b/scripts/tests/test_promotion_gate.py new file mode 100644 index 0000000..de1735c --- /dev/null +++ b/scripts/tests/test_promotion_gate.py @@ -0,0 +1,188 @@ +"""Gate-of-the-gate — pytest for promotion-gate.yml's HEART logic. + +promotion-gate.yml false-negatived twice on v150.0.6 in one hour: + + 1. First run 127'd on `7zz: command not found` — hardcoded a binary + that exists only on Ubuntu 24+ (`7zip` package), while ubuntu-latest + is 22.04 (`p7zip-full`, binary `7z`). + 2. Retry silently returned empty version — `plistutil` failed silently, + `grep` matched the binary blob, `sed` yielded nothing. Comparison + `'' != '150.0.6'` → quarantined a **good** artifact. + +Both would have been caught in the PR that introduced them, in seconds, +if the gate's own logic had been unit-tested. This is that unit test. + +Fixtures are generated in-memory: no committed binary blobs. Runs on +ubuntu-latest with only plistlib (stdlib) — no external deps. +""" +from __future__ import annotations +import plistlib +import re +import subprocess +import sys +from pathlib import Path + +import pytest + +REPO = Path(__file__).resolve().parents[2] + + +# ── plist-version extraction (the exact logic in promotion-gate.yml) ────────── +def extract_plist_version(plist_bytes: bytes) -> str: + """Return CFBundleShortVersionString or '' — the shape promotion-gate uses.""" + import io + d = plistlib.load(io.BytesIO(plist_bytes)) + return d.get("CFBundleShortVersionString", "") + + +def make_plist(version: str, fmt=plistlib.FMT_BINARY) -> bytes: + """Build an Info.plist fixture matching a real BearBrowser.app plist.""" + d = { + "CFBundleName": "BearBrowser", + "CFBundleShortVersionString": version, + "CFBundleVersion": version, + "CFBundleIdentifier": "dev.sourceos.BearBrowser", + } + return plistlib.dumps(d, fmt=fmt) + + +def test_extract_from_binary_plist_matches(): + """The failure mode: real DMGs ship BINARY plists. plistutil silently + failed to convert; grep|sed on binary returned empty. plistlib handles + binary natively.""" + plist = make_plist("150.0.6", plistlib.FMT_BINARY) + assert extract_plist_version(plist) == "150.0.6" + + +def test_extract_from_xml_plist_matches(): + plist = make_plist("150.0.6", plistlib.FMT_XML) + assert extract_plist_version(plist) == "150.0.6" + + +def test_mismatch_is_detected(): + """The gate's actual purpose: catch a plist that doesn't match the tag.""" + plist = make_plist("150.0.1", plistlib.FMT_BINARY) # the 4-release lie + assert extract_plist_version(plist) != "150.0.6" + + +def test_missing_key_returns_empty_not_crash(): + """A malformed plist without CFBundleShortVersionString must yield '', + not a KeyError — so the gate can report a meaningful error instead of + a Python traceback that masks the real problem.""" + plist = plistlib.dumps({"CFBundleName": "BearBrowser"}, fmt=plistlib.FMT_BINARY) + assert extract_plist_version(plist) == "" + + +# ── update-check hygiene grep (the OTHER thing the gate checks) ─────────────── +UPDATE_HYGIENE_KEYWORDS = ("credentials:\"omit\"", "referrerPolicy", "no-referrer") + + +def hygiene_check(cfg_text: str) -> tuple[bool, list[str]]: + """The exact assertions promotion-gate.yml runs against the shipped .cfg.""" + missing = [] + if 'credentials:"omit"' not in cfg_text and "credentials: \"omit\"" not in cfg_text: + missing.append("credentials:omit") + if "referrerPolicy" not in cfg_text: + missing.append("referrerPolicy") + if "no-referrer" not in cfg_text: + missing.append("no-referrer") + return (len(missing) == 0, missing) + + +def test_hygiene_passes_when_all_keywords_present(): + cfg = 'fetch(url, { credentials:"omit", referrerPolicy: "no-referrer" })' + ok, missing = hygiene_check(cfg) + assert ok, f"unexpected missing: {missing}" + + +def test_hygiene_fails_when_credentials_absent(): + cfg = 'fetch(url, { referrerPolicy: "no-referrer" })' + ok, missing = hygiene_check(cfg) + assert not ok + assert "credentials:omit" in missing + + +def test_hygiene_fails_when_referrer_policy_absent(): + cfg = 'fetch(url, { credentials:"omit" })' + ok, missing = hygiene_check(cfg) + assert not ok + assert "referrerPolicy" in missing or "no-referrer" in missing + + +def test_shipped_cfg_would_pass_hygiene_today(): + """Meta-check: the currently-shipped bearstart-autoconfig.js MUST have + all three hygiene keywords. If this ever fails, we've regressed the + v150.0.5 fix. (This is the same class of assertion the shipped + verify-package.sh does — we're asserting the source-of-truth here.)""" + src = (REPO / "settings" / "start" / "bearstart-autoconfig.js").read_text() + ok, missing = hygiene_check(src) + assert ok, f"shipped bearstart-autoconfig.js regressed update-fetch hygiene: missing {missing}" + + +# ── 7z binary detection (the FIRST thing that broke) ────────────────────────── +def which(cmd: str) -> str | None: + """Cross-platform which — the same detection promotion-gate.yml now does.""" + r = subprocess.run(["command", "-v", cmd], shell=True, capture_output=True) + return r.stdout.decode().strip() or None + + +@pytest.mark.skipif(sys.platform != "linux", reason="only care about the CI-runner shape") +def test_at_least_one_7z_binary_findable_after_install(): + """On Ubuntu 22.04 the correct package is p7zip-full (binary `7z`), + NOT `7zip` (binary `7zz`, Ubuntu 24+). The gate's detect-first-available + loop is what unblocks this. If NONE of the candidates work, the gate + step will exit 127 like it did on v150.0.6. + + We don't install here — CI installs it before this test runs. We just + assert the detect loop finds SOMETHING.""" + candidates = ["7z", "7zz", "7za"] + found = [c for c in candidates if subprocess.run( + ["bash", "-c", f"command -v {c}"], capture_output=True + ).returncode == 0] + assert found, ( + f"no 7z-family binary in PATH after apt install — the promotion-gate " + f"step would 127 like it did on v150.0.6. Install `p7zip-full`." + ) + + +# ── Gate workflow's structure (guard against future edits removing gates) ───── +GATE_YAML = REPO / ".github" / "workflows" / "promotion-gate.yml" + + +def test_gate_workflow_still_has_all_five_load_bearing_steps(): + """If someone deletes or renames one of these steps, the gate silently + stops catching that class of bug. Name-based assertion so a rename + surfaces here, not on a release-publish.""" + src = GATE_YAML.read_text() + required = [ + "Resolve tag + download release artifacts", + "Package gate (Linux tarball)", + "Info.plist version gate (macOS DMG)", + "Update-check network hygiene smoke", + "quarantine-on-failure", # job, not step — but same principle + ] + for name in required: + assert name in src, f"promotion-gate.yml missing load-bearing step: {name!r}" + + +def test_gate_workflow_uses_plistlib_not_grep_sed(): + """The v150.0.6 grep|sed failure must not be reintroduced. If someone + goes back to the fragile chain, this fails.""" + src = GATE_YAML.read_text() + assert "plistlib" in src, "promotion-gate.yml regressed off plistlib for Info.plist parsing" + + +def test_gate_workflow_does_not_hardcode_7zz(): + """Detect-loop must stay in place; hardcoding one 7z binary killed v150.0.6.""" + src = GATE_YAML.read_text() + # The bad pattern was a bare `7zz x` command. The safe pattern uses a + # detected variable. We check that the extraction call goes through the + # variable, not a bare 7zz command. + bad_lines = [ + line for line in src.splitlines() + if re.search(r"^\s*7zz\s+x\b", line) # bare 7zz x, not detected + ] + assert not bad_lines, ( + f"promotion-gate.yml regressed to hardcoded 7zz — will 127 on Ubuntu 22.04. " + f"Bad lines: {bad_lines}" + )