diff --git a/.github/dependabot.yml b/.github/dependabot.yml index fe105d09..22b207af 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,5 +1,11 @@ # Dependabot configuration # +# Notes: +# - Updates are grouped into a single weekly PR per ecosystem to reduce +# review noise and avoid changelog merge conflicts between bot PRs. +# - Bot PRs get an automated changelog entry commit and are auto-merged +# once approved (see .github/workflows/bot-pr-automation.yml). +# # References: # - https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file # @@ -8,10 +14,16 @@ updates: - package-ecosystem: "github-actions" directory: "/" schedule: - interval: "daily" + interval: "weekly" labels: [ "github_actions" ] + groups: + github-actions: + patterns: [ "*" ] - package-ecosystem: "pip" directory: "/" schedule: - interval: "daily" + interval: "weekly" labels: [ "dependencies" ] + groups: + pip: + patterns: [ "*" ] diff --git a/.github/required-status-checks.json b/.github/required-status-checks.json new file mode 100644 index 00000000..ef769af4 --- /dev/null +++ b/.github/required-status-checks.json @@ -0,0 +1,17 @@ +{ + "strict": true, + "checks": [ + { + "context": "All CI checks passed" + }, + { + "context": "Check for entry in Changelog" + }, + { + "context": "Build distributions" + }, + { + "context": "Analyze (Python)" + } + ] +} diff --git a/.github/workflows/bot-pr-automation.yml b/.github/workflows/bot-pr-automation.yml new file mode 100644 index 00000000..ea08fc8f --- /dev/null +++ b/.github/workflows/bot-pr-automation.yml @@ -0,0 +1,254 @@ +# Automation for PRs opened by trusted bots (dependabot and pre-commit.ci). +# +# For every bot PR, this workflow: +# 1. Commits a changelog entry for the PR (generated from the PR's title) +# directly to the PR branch, so that the "Check for entry in Changelog" +# requirement is satisfied and the entry can be reviewed as part of the +# PR itself. This step is idempotent and self-healing: if a bot +# force-pushes its branch (wiping our commit), the resulting +# `synchronize` event simply re-adds the entry. +# 2. Enables GitHub's native auto-merge (squash). The PR will then merge +# automatically as soon as the branch protection requirements are met +# (i.e., an approving review plus all required status checks passing). +# +# Additionally, on every push to main, the `heal` job brings all open bot +# PRs whose branches no longer contain the tip of main back into a +# mergeable state by merging main into them: +# - PRs that merely fell behind main are updated so that they keep +# satisfying the "require branches to be up to date before merging" +# branch protection setting (which would otherwise leave auto-merge +# waiting forever, since nothing else updates bot PR branches). +# - PRs that became conflicted (e.g., two bot PRs whose changelog +# entries collide once the first one merges) are additionally +# resolved by taking main's version of the changelog and +# regenerating the PR's own changelog entry on top of it. +# +# The only human action left is the review itself. +# +# Setup requirements: +# - An AUTO_MERGE_PAT repository secret: a fine-grained personal access +# token scoped to this repository only, with read/write permissions +# for "Contents" and "Pull requests". The default GITHUB_TOKEN is +# deliberately *not* used as a fallback: events caused by pushes and +# merges performed with GITHUB_TOKEN do not trigger other workflows +# (GitHub's recursive-workflow protection), which would deadlock this +# automation - CI would never run on the changelog commits pushed to +# bot PRs, so the required status checks would stay pending forever +# and auto-merge would never fire. Merges to main performed with +# GITHUB_TOKEN would similarly skip the post-merge workflows on main +# (CI run, TestPyPI publish). Every job therefore fails fast with a +# clear error if the secret is not configured. +# - "Allow auto-merge" enabled in the repository settings +# (Settings > General > Pull Requests). +# - Branch protection on main requiring an approving review and the +# required status checks defined in +# .github/required-status-checks.json (the config-as-code source of +# truth; apply it with ./cicd_utils/apply-branch-protection.sh). +# These are stable, first-party contexts only - the ci.yml +# "All CI checks passed" gate job covers the whole test matrix, so +# Python version changes never require touching the list. External +# services (codecov, pre-commit.ci, Read the Docs, etc.) are +# advisory: they stay visible on PRs, but merges don't depend on +# their uptime. (Auto-merge only waits for *required* checks.) +# +# Security: this workflow runs on pull_request_target with write +# permissions, but only ever acts on same-repository PRs opened by +# trusted bots, and never executes code from the PR branches: both jobs +# run the changelog helper script from a checkout of the main branch +# (which is also where this workflow definition itself is taken from), +# only ever treating the PR branches' content as data. +# +name: Bot PR automation + +on: + pull_request_target: + types: [ opened, reopened, synchronize ] + push: + branches: [ main ] + +permissions: {} + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: false + +jobs: + changelog: + name: Add changelog entry + if: >- + github.event_name == 'pull_request_target' && + github.event.pull_request.head.repo.full_name == github.repository && + (github.event.pull_request.user.login == 'dependabot[bot]' || + github.event.pull_request.user.login == 'pre-commit-ci[bot]') + runs-on: ubuntu-latest + timeout-minutes: 3 + permissions: + contents: write + steps: + - name: Ensure AUTO_MERGE_PAT is configured + env: + AUTO_MERGE_PAT: ${{ secrets.AUTO_MERGE_PAT }} + run: | + if [ -z "$AUTO_MERGE_PAT" ]; then + echo "::error::The AUTO_MERGE_PAT secret is not configured." \ + "Falling back to GITHUB_TOKEN would deadlock this automation" \ + "(its pushes/merges do not trigger other workflows, so" \ + "required status checks would never run). See the setup" \ + "requirements at the top of .github/workflows/bot-pr-automation.yml." + exit 1 + fi + + # Note: this checks out the *base* branch (main), not the PR branch, + # so that the changelog helper script executed below always comes + # from trusted code and never from the PR branch. + - uses: actions/checkout@v7 + with: + token: ${{ secrets.AUTO_MERGE_PAT }} + + - name: Add changelog entry and push + env: + PR_NUMBER: ${{ github.event.pull_request.number }} + PR_TITLE: ${{ github.event.pull_request.title }} + HEAD_REF: ${{ github.event.pull_request.head.ref }} + run: | + python3 -m pip install --quiet markdown-it-py + + # Apply the (trusted, from main) helper script to the PR + # branch's version of the changelog + git fetch origin "$HEAD_REF" + git show FETCH_HEAD:docs/reference/changelog.md > "$RUNNER_TEMP/changelog.md" + cp "$RUNNER_TEMP/changelog.md" "$RUNNER_TEMP/changelog.orig.md" + python3 cicd_utils/cicd/scripts/add_changelog_entry.py "$PR_NUMBER" "$PR_TITLE" \ + --changelog "$RUNNER_TEMP/changelog.md" + if cmp -s "$RUNNER_TEMP/changelog.md" "$RUNNER_TEMP/changelog.orig.md"; then + echo "Changelog entry already present; nothing to do." + exit 0 + fi + + # Commit the updated changelog on top of the PR branch. This + # materializes the bot's tree in the working directory, but no + # code from it is ever executed (do not add steps below that + # run anything from the working tree!). + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git checkout -B "$HEAD_REF" FETCH_HEAD + cp "$RUNNER_TEMP/changelog.md" docs/reference/changelog.md + git add docs/reference/changelog.md + git commit -m "Add changelog entry for #${PR_NUMBER}" + git push origin "HEAD:$HEAD_REF" + + automerge: + name: Enable auto-merge + if: >- + github.event_name == 'pull_request_target' && + github.event.action != 'synchronize' && + github.event.pull_request.head.repo.full_name == github.repository && + (github.event.pull_request.user.login == 'dependabot[bot]' || + github.event.pull_request.user.login == 'pre-commit-ci[bot]') + runs-on: ubuntu-latest + timeout-minutes: 2 + permissions: + contents: write + pull-requests: write + steps: + - name: Ensure AUTO_MERGE_PAT is configured + env: + AUTO_MERGE_PAT: ${{ secrets.AUTO_MERGE_PAT }} + run: | + if [ -z "$AUTO_MERGE_PAT" ]; then + echo "::error::The AUTO_MERGE_PAT secret is not configured." \ + "Falling back to GITHUB_TOKEN would deadlock this automation" \ + "(its pushes/merges do not trigger other workflows, so" \ + "required status checks would never run). See the setup" \ + "requirements at the top of .github/workflows/bot-pr-automation.yml." + exit 1 + fi + + - name: Enable auto-merge (squash) + run: gh pr merge --auto --squash "$PR_URL" + env: + PR_URL: ${{ github.event.pull_request.html_url }} + GH_TOKEN: ${{ secrets.AUTO_MERGE_PAT }} + + heal: + name: Heal out-of-date bot PRs + if: github.event_name == 'push' + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: write + pull-requests: read + steps: + - name: Ensure AUTO_MERGE_PAT is configured + env: + AUTO_MERGE_PAT: ${{ secrets.AUTO_MERGE_PAT }} + run: | + if [ -z "$AUTO_MERGE_PAT" ]; then + echo "::error::The AUTO_MERGE_PAT secret is not configured." \ + "Falling back to GITHUB_TOKEN would deadlock this automation" \ + "(its pushes/merges do not trigger other workflows, so" \ + "required status checks would never run). See the setup" \ + "requirements at the top of .github/workflows/bot-pr-automation.yml." + exit 1 + fi + + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + token: ${{ secrets.AUTO_MERGE_PAT }} + + - name: Merge main into out-of-date or conflicted bot PRs + env: + GH_TOKEN: ${{ secrets.AUTO_MERGE_PAT }} + run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + python3 -m pip install --quiet markdown-it-py + + # Stash a trusted copy of the helper script (we're on main here) + # so that we never execute code from the PR branches that get + # checked out in the loop below + cp cicd_utils/cicd/scripts/add_changelog_entry.py "$RUNNER_TEMP/add_changelog_entry.py" + + prs=$( + gh pr list --author 'app/dependabot' --json number,headRefName,title --jq '.[] | [.number, .headRefName, .title] | @tsv' + gh pr list --author 'app/pre-commit-ci' --json number,headRefName,title --jq '.[] | [.number, .headRefName, .title] | @tsv' + ) + [ -n "$prs" ] || { echo "No open bot PRs."; exit 0; } + + while IFS=$'\t' read -r number head_ref title; do + [ -n "$number" ] || continue + + # Determine staleness locally (instead of polling GitHub's + # asynchronous mergeability computation): a PR branch needs + # healing whenever the tip of main is not part of its history + git fetch origin "$head_ref" < /dev/null + if git merge-base --is-ancestor origin/main "origin/$head_ref" < /dev/null; then + echo "PR #${number} (${head_ref}) is up to date with main." + continue + fi + echo "PR #${number} (${head_ref}) is behind main; merging main into it..." + + git checkout -B "$head_ref" "origin/$head_ref" < /dev/null + + if git merge --no-edit origin/main < /dev/null; then + git push origin "HEAD:$head_ref" < /dev/null + continue + fi + + conflicts=$(git diff --name-only --diff-filter=U) + if [ "$conflicts" != "docs/reference/changelog.md" ]; then + git merge --abort < /dev/null + echo "::warning::PR #${number} has conflicts beyond the changelog; skipping." + continue + fi + + # Resolve the changelog conflict by taking main's version + # and re-generating this PR's changelog entry on top of it + git checkout --theirs docs/reference/changelog.md < /dev/null + python3 "$RUNNER_TEMP/add_changelog_entry.py" "$number" "$title" \ + --changelog docs/reference/changelog.md + git add docs/reference/changelog.md + git commit --no-edit < /dev/null + git push origin "HEAD:$head_ref" < /dev/null + done <<< "$prs" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3df39def..67368556 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -116,3 +116,33 @@ jobs: env_vars: OS,PYTHON fail_ci_if_error: true verbose: true + + # Single, stable "gate" context for branch protection, following the + # pattern used by e.g. CPython, pydantic, uv, and FastAPI (see + # https://github.com/marketplace/actions/alls-green#why): branch + # protection requires only this one context instead of enumerating + # every matrix job, so adding or dropping Python versions never + # requires updating the repository settings. The full list of + # required contexts lives in .github/required-status-checks.json. + all-green: + name: All CI checks passed + # `always()` is load-bearing: without it, a failure in a needed job + # would leave this job "skipped", and GitHub treats skipped required + # checks as passing! + if: always() + needs: [ static-checks, software-tests ] + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Fail unless all needed jobs succeeded + env: + NEEDS_JSON: ${{ toJSON(needs) }} + run: | + echo "$NEEDS_JSON" | jq . + failing=$(echo "$NEEDS_JSON" | jq -r \ + 'to_entries[] | select(.value.result != "success") | "\(.key): \(.value.result)"') + if [ -n "$failing" ]; then + echo "The following required jobs did not succeed:" + echo "$failing" + exit 1 + fi diff --git a/cicd_utils/apply-branch-protection.sh b/cicd_utils/apply-branch-protection.sh new file mode 100755 index 00000000..97c6378a --- /dev/null +++ b/cicd_utils/apply-branch-protection.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash + +# Apply the repository's required-status-checks configuration to main +# +# The status checks that must pass before merging into main are kept as +# config-as-code in .github/required-status-checks.json (the single +# source of truth; GitHub's branch protection settings UI is just a +# mirror of it). This script pushes that configuration to GitHub. It +# shows the current and desired configurations and asks for +# confirmation before applying (pass --yes to skip the prompt). +# +# Notes on the configuration itself: +# * "checks" should only ever contain stable, first-party contexts. In +# particular, the "All CI checks passed" gate job in ci.yml covers +# the whole test matrix, so adding/dropping Python versions never +# requires updating this list. External services (codecov, +# pre-commit.ci, Read the Docs, etc.) are advisory by design: they +# stay visible on PRs, but merges don't depend on their uptime. +# * "strict" means PR branches must be up to date with main before +# merging (bot PRs are kept up to date automatically by the heal job +# in .github/workflows/bot-pr-automation.yml). +# +# Usage: ./cicd_utils/apply-branch-protection.sh [--yes] +# +# Requirements: +# * GitHub CLI (gh) must be installed and authenticated. +# * jq must be installed for JSON parsing. +# * The authenticated user must have admin access to the repository +# (branch protection can only be read and written by admins). +# * The script should be run from the root of the repository. + +set -euo pipefail + +CONFIG_FILE=".github/required-status-checks.json" +BRANCH="main" + +if [[ ! -f "$CONFIG_FILE" ]]; then + echo "Error: Config file not found at $CONFIG_FILE (run this script from the repo root)" + exit 1 +fi + +if ! command -v gh &> /dev/null; then + echo "Error: GitHub CLI (gh) is not installed" + exit 1 +fi + +repo=$(gh repo view --json nameWithOwner --jq .nameWithOwner) +endpoint="repos/${repo}/branches/${BRANCH}/protection/required_status_checks" +normalize='{strict: .strict, checks: [.checks[] | {context: .context}]}' + +echo "🔍 Current configuration (${repo}, branch: ${BRANCH}):" +if current=$(gh api "$endpoint" --jq "$normalize" 2>/dev/null); then + echo "$current" | jq . +else + current="" + echo "(no required status checks are currently configured)" +fi +echo + +echo "📄 Desired configuration (from ${CONFIG_FILE}):" +desired=$(jq "$normalize" "$CONFIG_FILE") +echo "$desired" | jq . +echo + +if [[ -n "$current" && "$(echo "$current" | jq -S .)" == "$(echo "$desired" | jq -S .)" ]]; then + echo "✅ Branch protection is already in sync with ${CONFIG_FILE}; nothing to do." + exit 0 +fi + +if [[ "${1:-}" != "--yes" ]]; then + read -r -p "Apply the desired configuration? [y/N] " answer || answer="" + if [[ "$answer" != "y" && "$answer" != "Y" ]]; then + echo "Aborted." + exit 1 + fi +fi + +gh api -X PATCH "$endpoint" --input "$CONFIG_FILE" > /dev/null +echo "🚀 Applied. New configuration:" +gh api "$endpoint" --jq "$normalize" | jq . diff --git a/cicd_utils/cicd/scripts/add_changelog_entry.py b/cicd_utils/cicd/scripts/add_changelog_entry.py new file mode 100755 index 00000000..34e03d7d --- /dev/null +++ b/cicd_utils/cicd/scripts/add_changelog_entry.py @@ -0,0 +1,250 @@ +#!/usr/bin/env python +"""Add a changelog entry for a given pull request. + +Inserts a ``- ({gh-pr}`<number>`)`` entry into the ``### Dependencies`` +subsection of the ``Unreleased changes`` section of the changelog, creating +the subsection (or the whole section) if it doesn't exist yet. The +subsection's entries are kept sorted alphabetically: the whole list is +re-sorted on every insertion, which also repairs any pre-existing ordering +violations (e.g., from manual edits). + +The changelog's structure is discovered with ``markdown-it-py`` (using the +tokens' source line maps), while the actual edit is a surgical line splice. +This keeps the rest of the file byte-for-byte untouched (as opposed to, +e.g., re-rendering the whole document with ``mdformat``, which would +restyle it). + +This script is idempotent: if the changelog already references the given +PR number, the file is left unchanged. + +Used by the ``.github/workflows/bot-pr-automation.yml`` workflow to automatically add +changelog entries to pull requests opened by trusted bots (e.g., dependabot +and pre-commit.ci). + +Usage: + python cicd_utils/cicd/scripts/add_changelog_entry.py <pr-number> <pr-title> +""" + +from __future__ import annotations + +import argparse +from pathlib import Path +from typing import TYPE_CHECKING + +from markdown_it import MarkdownIt + +if TYPE_CHECKING: + from markdown_it.token import Token + +PATH_ROOT_DIR = Path(__file__).parents[3] +PATH_TO_CHANGELOG = PATH_ROOT_DIR.joinpath("docs/reference/changelog.md") + +UNRELEASED_HEADING = "Unreleased changes" +DEPS_HEADING = "Dependencies" + +# Known bot prefixes that should be stripped from PR titles to +# match the changelog's entry conventions (e.g., the convention +# for pre-commit.ci PRs is simply "pre-commit autoupdate"). +STRIP_TITLE_PREFIXES = ("[pre-commit.ci] ",) + + +def format_entry(pr_number: int, pr_title: str) -> str: + """Format a changelog entry for the given PR number and title.""" + title = pr_title.strip() + for prefix in STRIP_TITLE_PREFIXES: + title = title.removeprefix(prefix) + return f"- {title} ({{gh-pr}}`{pr_number}`)" + + +def _token_lines(token: Token) -> tuple[int, int]: + """Return a block token's source line range as an ``(start, end)`` tuple.""" + if token.map is None: + raise AssertionError("Block-level tokens always carry a source line map") + return token.map[0], token.map[1] + + +def _is_heading(tokens: list[Token], i: int, tag: str, text: str | None = None) -> bool: + """Whether ``tokens[i]`` opens a heading with the given tag (and text).""" + token = tokens[i] + if token.type != "heading_open" or token.tag != tag: + return False + return text is None or tokens[i + 1].content == text + + +def _find_unreleased_section(tokens: list[Token]) -> tuple[int, int] | None: + """Find the (start, end) token index range of the 'Unreleased changes' section. + + ``start`` points at the first token after the section's heading and + ``end`` at the next section's ``heading_open`` token (or one past the + last token). Returns :data:`None` if the section doesn't exist. + """ + start = None + for i in range(len(tokens)): + if not _is_heading(tokens, i, tag="h2"): + continue + if start is not None: + return start, i + if _is_heading(tokens, i, tag="h2", text=UNRELEASED_HEADING): + start = i + 3 # skip the heading_open, inline, and heading_close tokens + if start is None: + return None + return start, len(tokens) + + +def _new_unreleased_section(entry: str) -> list[str]: + return [ + UNRELEASED_HEADING, + "-" * len(UNRELEASED_HEADING), + "", + f"### {DEPS_HEADING}", + "", + entry, + "", + "---", + "", + ] + + +def _insert_unreleased_section(lines: list[str], tokens: list[Token], entry: str) -> list[str]: + """Insert a whole new 'Unreleased changes' section before the first section.""" + for i in range(len(tokens)): + if _is_heading(tokens, i, tag="h2"): + at = _token_lines(tokens[i])[0] + return [*lines[:at], *_new_unreleased_section(entry), *lines[at:]] + # No sections yet (e.g., an empty changelog): append at the end, making + # sure that the new section's heading is preceded by a blank line + # (otherwise the preceding paragraph would absorb the setext heading) + if lines and lines[-1].strip(): + lines = [*lines, ""] + return [*lines, *_new_unreleased_section(entry)] + + +def _find_deps_subsection(tokens: list[Token], start: int, end: int) -> tuple[int, int] | None: + """Find the (start, end) token index range of the '### Dependencies' subsection. + + ``start`` points at the subsection's ``heading_open`` token and ``end`` + at the next subsection's ``heading_open`` token, the section's trailing + thematic break, or the end of the section. Returns :data:`None` if the + subsection doesn't exist. + """ + sub_start = None + for i in range(start, end): + token = tokens[i] + if token.type == "heading_open": + if sub_start is not None: + return sub_start, i + if _is_heading(tokens, i, tag="h3", text=DEPS_HEADING): + sub_start = i + elif sub_start is not None and token.type == "hr": + return sub_start, i + if sub_start is None: + return None + return sub_start, end + + +def _insert_entry_into_subsection( + lines: list[str], tokens: list[Token], sub_start: int, sub_end: int, entry: str +) -> list[str]: + """Insert the entry into the subsection's bullet list, keeping it sorted. + + All of the subsection's list items are re-sorted alphabetically as a + whole, which also repairs any pre-existing ordering violations. Each + item's source lines are moved as one block, so that multi-line entries + (continuation lines, nested lists, etc.) are preserved intact. + """ + blocks: list[list[str]] = [] + span_start = span_end = None + for i in range(sub_start, sub_end): + token = tokens[i] + if token.type != "list_item_open" or token.level != 1: + continue # only consider items of top-level bullet lists + item_start, item_end = _token_lines(token) + block = lines[item_start:item_end] + # An item's line map may extend over trailing blank lines + # (e.g., in loose lists); strip them so that the re-assembled + # list is a tight, contiguous block of entries + while block and not block[-1].strip(): + block.pop() + blocks.append(block) + if span_start is None: + span_start = item_start + span_end = item_end + if span_start is None or span_end is None: + # No bullet list yet: insert right after the subsection's heading, + # keeping a blank line between the heading and the first entry + insert_at = _token_lines(tokens[sub_start])[1] + return [*lines[:insert_at], "", entry, *lines[insert_at:]] + # Walk back over any trailing blank lines covered by the last item's map + while span_end > span_start and not lines[span_end - 1].strip(): + span_end -= 1 + blocks.append([entry]) + blocks.sort(key=lambda block: block[0].casefold()) + sorted_lines = [line for block in blocks for line in block] + return [*lines[:span_start], *sorted_lines, *lines[span_end:]] + + +def _find_subsection_insertion(tokens: list[Token], start: int, end: int, n_lines: int) -> int: + """Find the source line at which to insert a new subsection at the end of the section.""" + # Insert before the section's trailing thematic break (if any) + for i in range(start, end): + if tokens[i].type == "hr": + return _token_lines(tokens[i])[0] + if end < len(tokens): + return _token_lines(tokens[end])[0] + return n_lines + + +def add_changelog_entry(changelog: Path, pr_number: int, pr_title: str) -> bool: + """Add a changelog entry for the given PR. Returns whether the file was changed.""" + text = changelog.read_text() + if f"{{gh-pr}}`{pr_number}`" in text: + print(f"Changelog already references PR #{pr_number}; nothing to do.") + return False + + entry = format_entry(pr_number, pr_title) + lines = text.splitlines() + tokens = MarkdownIt().parse(text) + + section = _find_unreleased_section(tokens) + if section is None: + lines = _insert_unreleased_section(lines, tokens, entry) + else: + start, end = section + subsection = _find_deps_subsection(tokens, start, end) + if subsection is None: + insert_at = _find_subsection_insertion(tokens, start, end, len(lines)) + # Walk back over any blank lines + while insert_at > 0 and not lines[insert_at - 1].strip(): + insert_at -= 1 + lines[insert_at:insert_at] = ["", f"### {DEPS_HEADING}", "", entry] + else: + sub_start, sub_end = subsection + lines = _insert_entry_into_subsection(lines, tokens, sub_start, sub_end, entry) + + while lines and not lines[-1].strip(): + lines.pop() + changelog.write_text("\n".join(lines) + "\n") + print(f"Added changelog entry: {entry}") + return True + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("pr_number", type=int, help="The pull request number.") + parser.add_argument("pr_title", type=str, help="The pull request title.") + parser.add_argument( + "--changelog", + type=Path, + default=PATH_TO_CHANGELOG, + help="Path to the changelog file.", + ) + args = parser.parse_args() + add_changelog_entry( + changelog=args.changelog, + pr_number=args.pr_number, + pr_title=args.pr_title, + ) + + +if __name__ == "__main__": + main() diff --git a/cicd_utils/find-unmentioned-prs.sh b/cicd_utils/find-unmentioned-prs.sh index 4f54ab3d..1c4df0ee 100755 --- a/cicd_utils/find-unmentioned-prs.sh +++ b/cicd_utils/find-unmentioned-prs.sh @@ -68,6 +68,7 @@ else echo "📋 PRs not mentioned in changelog (${#unmentioned_prs[@]} total):" echo + suggested_entries=() for pr in "${unmentioned_prs[@]}"; do # Get PR title and URL for better readability pr_info=$(gh pr view "$pr" --json title,url --jq '{title: .title, url: .url}') @@ -77,5 +78,13 @@ else echo " #$pr: $pr_title" echo " $pr_url" echo + + suggested_entries+=("- ${pr_title} ({gh-pr}\`${pr}\`)") + done + + echo "📝 Suggested changelog entries (review and paste under 'Unreleased changes'):" + echo + for entry in "${suggested_entries[@]}"; do + echo "$entry" done fi diff --git a/docs/development/release_process.md b/docs/development/release_process.md index 2d2f4927..4a4c6e3c 100644 --- a/docs/development/release_process.md +++ b/docs/development/release_process.md @@ -6,7 +6,7 @@ You need to have push-access to the project's repository to make releases. Therefore, the following release steps are intended to be used as a reference for maintainers or [collaborators](https://docs.github.com/en/account-and-profile/setting-up-and-managing-your-personal-account-on-github/managing-user-account-settings/permission-levels-for-a-personal-account-repository#collaborator-access-for-a-repository-owned-by-a-personal-account) with push-access to the repository. ::: -1. Review the **`## Unreleased changes`** section at the top of the {repo-file}`docs/reference/changelog.md` file and, if necessary, group and/or split entries into relevant subsections (e.g., _Features_, _Docs_, _Bugfixes_, _Security_, etc.). Take a look at previous release notes for guidance and try to keep the format consistent. You can also use the `./cicd_utils/find-unmentioned-prs.sh` helper script to find merged PRs that were not mentioned in the changelog yet. +1. Review the **`## Unreleased changes`** section at the top of the {repo-file}`docs/reference/changelog.md` file and, if necessary, group and/or split entries into relevant subsections (e.g., _Features_, _Docs_, _Bugfixes_, _Security_, etc.). Take a look at previous release notes for guidance and try to keep the format consistent. You can also use the `./cicd_utils/find-unmentioned-prs.sh` helper script to find merged PRs that were not mentioned in the changelog yet (if any are found, it prints ready-to-paste changelog entries for them). Note that PRs opened by trusted bots (e.g., dependabot and pre-commit.ci) get an automated changelog entry commit and are auto-merged once approved (see {repo-file}`.github/workflows/bot-pr-automation.yml`), so they should already be covered in the changelog. 2. [Review](https://github.com/tpvasconcelos/ridgeplot/compare) new usages of `.. versionadded::`, `.. versionchanged::`, and `.. deprecated::` directives that were added to the documentation since the last release. If necessary, update the version numbers in these directives to reflect the new release version. * You can determine the latest release version by running `git describe --tags --abbrev=0` on the `main` branch. Based on this, you can determine the next release version by incrementing the relevant _MAJOR_, _MINOR_, or _PATCH_ numbers. 3. **IMPORTANT:** Remember to switch to the `main` branch and pull the latest changes before proceeding. diff --git a/docs/reference/changelog.md b/docs/reference/changelog.md index c08ef295..ac64e32b 100644 --- a/docs/reference/changelog.md +++ b/docs/reference/changelog.md @@ -22,21 +22,25 @@ Unreleased changes - Review the coverage configuration in light of `covdefaults`, adopting its `assert_never` exclusion and `skip_covered` report setting, and raising all package coverage gates to 100% ({gh-pr}`390`) - Adopt pytest's strict mode, following the recommendations from pytest's "Good Integration Practices" guide ({gh-pr}`387`) - Make the e2e path sanity check independent of the checkout directory's name, so tests can run from git worktrees ({gh-pr}`391`) +- Fix the test suite's compatibility with the latest pytest release ({gh-pr}`384`) +- Automate bot PR maintenance: weekly grouped dependabot updates, automated changelog entries, and auto-merge once approved ({gh-pr}`385`) +- Use the official `astral-sh/setup-uv` action to install and cache `uv` in CI ({gh-pr}`386`) +- Let `uv` manage the Python interpreter and virtual environment in CI ({gh-pr}`393`) +- Remove the stale `requirements/*.txt` glob from the CI cache key, left over from the migration to PEP 735 dependency groups ({gh-pr}`394`) + +### Dependencies + +- Bump actions/checkout from 6 to 7 ({gh-pr}`383`) - Bump actions/download-artifact from 7 to 8 ({gh-pr}`368`) +- Bump actions/github-script from 8 to 9 ({gh-pr}`373`) - Bump actions/upload-artifact from 6 to 7 ({gh-pr}`369`) -- Bump sigstore/gh-action-sigstore-python from 3.2.0 to 3.3.0 ({gh-pr}`370`) - Bump codecov/codecov-action from 5 to 6 ({gh-pr}`371`) -- Bump actions/github-script from 8 to 9 ({gh-pr}`373`) -- pre-commit autoupdate ({gh-pr}`374`) -- Bump softprops/action-gh-release from 2 to 3 ({gh-pr}`380`) - Bump codecov/codecov-action from 6 to 7 ({gh-pr}`381`) +- Bump sigstore/gh-action-sigstore-python from 3.2.0 to 3.3.0 ({gh-pr}`370`) - Bump sigstore/gh-action-sigstore-python from 3.3.0 to 3.4.0 ({gh-pr}`382`) -- Bump actions/checkout from 6 to 7 ({gh-pr}`383`) +- Bump softprops/action-gh-release from 2 to 3 ({gh-pr}`380`) +- pre-commit autoupdate ({gh-pr}`374`) - pre-commit autoupdate ({gh-pr}`379`) -- Fix the test suite's compatibility with the latest pytest release ({gh-pr}`384`) -- Use the official `astral-sh/setup-uv` action to install and cache `uv` in CI ({gh-pr}`386`) -- Let `uv` manage the Python interpreter and virtual environment in CI ({gh-pr}`393`) -- Remove the stale `requirements/*.txt` glob from the CI cache key, left over from the migration to PEP 735 dependency groups ({gh-pr}`394`) --- diff --git a/pyproject.toml b/pyproject.toml index 8030b2ee..1e9bcc2a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,6 +65,7 @@ cicd_utils = [ # (multi-platform) resolution with `uv sync`/`uv lock` "kaleido<0.4,!=0.2.1.post1", # ./cicd_utils/cicd/scripts/extract_latest_release_notes.py + # ./cicd_utils/cicd/scripts/add_changelog_entry.py (markdown-it-py only) "markdown-it-py", "mdit-py-plugins", "mdformat", diff --git a/tests/cicd_utils/test_scripts/test_add_changelog_entry.py b/tests/cicd_utils/test_scripts/test_add_changelog_entry.py new file mode 100644 index 00000000..a64344ed --- /dev/null +++ b/tests/cicd_utils/test_scripts/test_add_changelog_entry.py @@ -0,0 +1,519 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest + +from cicd.scripts.add_changelog_entry import ( + PATH_TO_CHANGELOG, + add_changelog_entry, + format_entry, + main, +) + +if TYPE_CHECKING: + from pathlib import Path + + +def test_path_to_changelog_exists() -> None: + assert PATH_TO_CHANGELOG.exists() + assert PATH_TO_CHANGELOG.is_file() + + +@pytest.mark.parametrize( + ("pr_number", "pr_title", "expected"), + [ + ( + 383, + "Bump actions/checkout from 6 to 7", + "- Bump actions/checkout from 6 to 7 ({gh-pr}`383`)", + ), + (379, "[pre-commit.ci] pre-commit autoupdate", "- pre-commit autoupdate ({gh-pr}`379`)"), + (42, " Padded title ", "- Padded title ({gh-pr}`42`)"), + ], +) +def test_format_entry(pr_number: int, pr_title: str, expected: str) -> None: + assert format_entry(pr_number, pr_title) == expected + + +CHANGELOG_WITH_DEPS_SUBSECTION = """\ +# Release Notes + +Intro paragraph... + +Unreleased changes +------------------ + +### Dependencies + +- Old entry ({gh-pr}`100`) + +--- + +0.1.0 +----- + +- Old release change ({gh-pr}`99`) +""" + +EXPECTED_WITH_DEPS_SUBSECTION = """\ +# Release Notes + +Intro paragraph... + +Unreleased changes +------------------ + +### Dependencies + +- Bump foo from 1 to 2 ({gh-pr}`123`) +- Old entry ({gh-pr}`100`) + +--- + +0.1.0 +----- + +- Old release change ({gh-pr}`99`) +""" + +CHANGELOG_WITHOUT_DEPS_SUBSECTION = """\ +# Release Notes + +Unreleased changes +------------------ + +### Bug fixes + +- Fix something ({gh-pr}`101`) + +--- + +0.1.0 +----- + +- Old release change ({gh-pr}`99`) +""" + +EXPECTED_WITHOUT_DEPS_SUBSECTION = """\ +# Release Notes + +Unreleased changes +------------------ + +### Bug fixes + +- Fix something ({gh-pr}`101`) + +### Dependencies + +- Bump foo from 1 to 2 ({gh-pr}`123`) + +--- + +0.1.0 +----- + +- Old release change ({gh-pr}`99`) +""" + +CHANGELOG_WITHOUT_UNRELEASED_SECTION = """\ +# Release Notes + +Intro paragraph... + +0.1.0 +----- + +- Old release change ({gh-pr}`99`) +""" + +EXPECTED_WITHOUT_UNRELEASED_SECTION = """\ +# Release Notes + +Intro paragraph... + +Unreleased changes +------------------ + +### Dependencies + +- Bump foo from 1 to 2 ({gh-pr}`123`) + +--- + +0.1.0 +----- + +- Old release change ({gh-pr}`99`) +""" + +CHANGELOG_WITHOUT_ANY_SECTIONS = """\ +# Release Notes + +Intro paragraph... +""" + +EXPECTED_WITHOUT_ANY_SECTIONS = """\ +# Release Notes + +Intro paragraph... + +Unreleased changes +------------------ + +### Dependencies + +- Bump foo from 1 to 2 ({gh-pr}`123`) + +--- +""" + +CHANGELOG_WITH_TRAILING_SUBSECTION = """\ +# Release Notes + +Unreleased changes +------------------ + +### Dependencies + +- Old entry ({gh-pr}`100`) + +### Documentation + +- Documentation change ({gh-pr}`101`) +""" + +EXPECTED_WITH_TRAILING_SUBSECTION = """\ +# Release Notes + +Unreleased changes +------------------ + +### Dependencies + +- Bump foo from 1 to 2 ({gh-pr}`123`) +- Old entry ({gh-pr}`100`) + +### Documentation + +- Documentation change ({gh-pr}`101`) +""" + +CHANGELOG_WITH_LOOSE_ENTRIES = """\ +# Release Notes + +Unreleased changes +------------------ + +- Loose entry ({gh-pr}`100`) +""" + +EXPECTED_WITH_LOOSE_ENTRIES = """\ +# Release Notes + +Unreleased changes +------------------ + +- Loose entry ({gh-pr}`100`) + +### Dependencies + +- Bump foo from 1 to 2 ({gh-pr}`123`) +""" + +CHANGELOG_WITH_ATX_HEADINGS = """\ +# Release Notes + +## Unreleased changes + +### Dependencies + +- Old entry ({gh-pr}`100`) + +--- + +## 0.1.0 + +- Old release change ({gh-pr}`99`) +""" + +EXPECTED_WITH_ATX_HEADINGS = """\ +# Release Notes + +## Unreleased changes + +### Dependencies + +- Bump foo from 1 to 2 ({gh-pr}`123`) +- Old entry ({gh-pr}`100`) + +--- + +## 0.1.0 + +- Old release change ({gh-pr}`99`) +""" + +CHANGELOG_WITH_EMPTY_DEPS_SUBSECTION = """\ +# Release Notes + +Unreleased changes +------------------ + +### Dependencies + +--- + +0.1.0 +----- + +- Old release change ({gh-pr}`99`) +""" + +EXPECTED_WITH_EMPTY_DEPS_SUBSECTION = """\ +# Release Notes + +Unreleased changes +------------------ + +### Dependencies + +- Bump foo from 1 to 2 ({gh-pr}`123`) + +--- + +0.1.0 +----- + +- Old release change ({gh-pr}`99`) +""" + +CHANGELOG_WITHOUT_THEMATIC_BREAK = """\ +# Release Notes + +Unreleased changes +------------------ + +### Bug fixes + +- Fix something ({gh-pr}`101`) + +0.1.0 +----- + +- Old release change ({gh-pr}`99`) +""" + +EXPECTED_WITHOUT_THEMATIC_BREAK = """\ +# Release Notes + +Unreleased changes +------------------ + +### Bug fixes + +- Fix something ({gh-pr}`101`) + +### Dependencies + +- Bump foo from 1 to 2 ({gh-pr}`123`) + +0.1.0 +----- + +- Old release change ({gh-pr}`99`) +""" + +CHANGELOG_WITH_UNSORTED_ENTRIES = """\ +# Release Notes + +Unreleased changes +------------------ + +### Dependencies + +- pre-commit autoupdate ({gh-pr}`102`) +- Bump zebra from 3 to 4 ({gh-pr}`101`) +- Bump apple from 1 to 2 ({gh-pr}`100`) + +--- +""" + +EXPECTED_WITH_UNSORTED_ENTRIES = """\ +# Release Notes + +Unreleased changes +------------------ + +### Dependencies + +- Bump apple from 1 to 2 ({gh-pr}`100`) +- Bump foo from 1 to 2 ({gh-pr}`123`) +- Bump zebra from 3 to 4 ({gh-pr}`101`) +- pre-commit autoupdate ({gh-pr}`102`) + +--- +""" + +CHANGELOG_WITH_MULTILINE_ITEM = """\ +# Release Notes + +Unreleased changes +------------------ + +### Dependencies + +- Bump zebra from 3 to 4 ({gh-pr}`101`) + (this bump includes breaking changes) +- Bump apple from 1 to 2 ({gh-pr}`100`) + +--- +""" + +EXPECTED_WITH_MULTILINE_ITEM = """\ +# Release Notes + +Unreleased changes +------------------ + +### Dependencies + +- Bump apple from 1 to 2 ({gh-pr}`100`) +- Bump foo from 1 to 2 ({gh-pr}`123`) +- Bump zebra from 3 to 4 ({gh-pr}`101`) + (this bump includes breaking changes) + +--- +""" + +CHANGELOG_WITH_LOOSE_LIST = """\ +# Release Notes + +Unreleased changes +------------------ + +### Dependencies + +- Bump zebra from 3 to 4 ({gh-pr}`101`) + +- Bump apple from 1 to 2 ({gh-pr}`100`) + +--- +""" + +EXPECTED_WITH_LOOSE_LIST = """\ +# Release Notes + +Unreleased changes +------------------ + +### Dependencies + +- Bump apple from 1 to 2 ({gh-pr}`100`) +- Bump foo from 1 to 2 ({gh-pr}`123`) +- Bump zebra from 3 to 4 ({gh-pr}`101`) + +--- +""" + + +@pytest.mark.parametrize( + ("changelog_content", "expected_content"), + [ + (CHANGELOG_WITH_DEPS_SUBSECTION, EXPECTED_WITH_DEPS_SUBSECTION), + (CHANGELOG_WITHOUT_DEPS_SUBSECTION, EXPECTED_WITHOUT_DEPS_SUBSECTION), + (CHANGELOG_WITHOUT_UNRELEASED_SECTION, EXPECTED_WITHOUT_UNRELEASED_SECTION), + (CHANGELOG_WITHOUT_ANY_SECTIONS, EXPECTED_WITHOUT_ANY_SECTIONS), + (CHANGELOG_WITH_TRAILING_SUBSECTION, EXPECTED_WITH_TRAILING_SUBSECTION), + (CHANGELOG_WITH_LOOSE_ENTRIES, EXPECTED_WITH_LOOSE_ENTRIES), + (CHANGELOG_WITH_ATX_HEADINGS, EXPECTED_WITH_ATX_HEADINGS), + (CHANGELOG_WITH_EMPTY_DEPS_SUBSECTION, EXPECTED_WITH_EMPTY_DEPS_SUBSECTION), + (CHANGELOG_WITHOUT_THEMATIC_BREAK, EXPECTED_WITHOUT_THEMATIC_BREAK), + (CHANGELOG_WITH_UNSORTED_ENTRIES, EXPECTED_WITH_UNSORTED_ENTRIES), + (CHANGELOG_WITH_MULTILINE_ITEM, EXPECTED_WITH_MULTILINE_ITEM), + (CHANGELOG_WITH_LOOSE_LIST, EXPECTED_WITH_LOOSE_LIST), + ], + ids=[ + "existing-deps-subsection", + "missing-deps-subsection", + "missing-unreleased-section", + "missing-any-sections", + "trailing-subsection", + "loose-entries", + "atx-headings", + "empty-deps-subsection", + "missing-thematic-break", + "unsorted-existing-entries", + "multi-line-item", + "loose-list-items", + ], +) +def test_add_changelog_entry(changelog_content: str, expected_content: str, tmp_path: Path) -> None: + changelog_path = tmp_path / "changelog.md" + changelog_path.write_text(changelog_content) + changed = add_changelog_entry( + changelog=changelog_path, pr_number=123, pr_title="Bump foo from 1 to 2" + ) + assert changed is True + assert changelog_path.read_text() == expected_content + + +def test_add_changelog_entry_sorts_case_insensitively(tmp_path: Path) -> None: + # With a case-sensitive sort, "Update ..." (uppercase "U") would + # wrongly sort before "pre-commit ..." (lowercase "p") + changelog_path = tmp_path / "changelog.md" + changelog_path.write_text( + CHANGELOG_WITH_DEPS_SUBSECTION.replace("Old entry", "pre-commit autoupdate") + ) + changed = add_changelog_entry( + changelog=changelog_path, pr_number=123, pr_title="Update foo to v2" + ) + assert changed is True + text = changelog_path.read_text() + pre_commit_entry = "- pre-commit autoupdate ({gh-pr}`100`)" + update_entry = "- Update foo to v2 ({gh-pr}`123`)" + assert text.index(pre_commit_entry) < text.index(update_entry) + + +def test_add_changelog_entry_to_real_changelog(tmp_path: Path) -> None: + changelog_path = tmp_path / "changelog.md" + changelog_path.write_text(PATH_TO_CHANGELOG.read_text()) + changed = add_changelog_entry( + changelog=changelog_path, pr_number=999999, pr_title="Bump foo from 1 to 2" + ) + assert changed is True + text = changelog_path.read_text() + entry = "- Bump foo from 1 to 2 ({gh-pr}`999999`)" + assert entry in text + # The new entry should land in the unreleased section (i.e., before + # the first released version's section) + first_release_at = text.index("\n0.") + assert text.index(entry) < first_release_at + + +def test_add_changelog_entry_is_idempotent(tmp_path: Path) -> None: + changelog_path = tmp_path / "changelog.md" + changelog_path.write_text(CHANGELOG_WITH_DEPS_SUBSECTION) + changed = add_changelog_entry( + changelog=changelog_path, pr_number=100, pr_title="Some already mentioned PR" + ) + assert changed is False + assert changelog_path.read_text() == CHANGELOG_WITH_DEPS_SUBSECTION + + +def test_main(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + changelog_path = tmp_path / "changelog.md" + changelog_path.write_text(CHANGELOG_WITH_DEPS_SUBSECTION) + monkeypatch.setattr( + "sys.argv", + [ + "add_changelog_entry.py", + "123", + "Bump foo from 1 to 2", + "--changelog", + str(changelog_path), + ], + ) + main() + assert changelog_path.read_text() == EXPECTED_WITH_DEPS_SUBSECTION