From d6fdb1fad6c5113765dbb4837f97195b4beb612f Mon Sep 17 00:00:00 2001 From: Trevor Gamblin Date: Wed, 15 Jul 2026 21:28:27 -0400 Subject: [PATCH 1/5] ci_scripts: update_doc.py: add Add a new version ofthe update_doc.py script from wheel_builder, converted using Claude. Compared to the old one, we just generate, parse, and update Markdown files directly for each package, and do it in docs/packages/.md instead of the longer docs/source/packages/.yaml. AI-Generated: Uses Claude Code Sonnet 5 Signed-off-by: Trevor Gamblin --- ci_scripts/update_doc.py | 304 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 304 insertions(+) create mode 100644 ci_scripts/update_doc.py diff --git a/ci_scripts/update_doc.py b/ci_scripts/update_doc.py new file mode 100644 index 0000000..6b298c2 --- /dev/null +++ b/ci_scripts/update_doc.py @@ -0,0 +1,304 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2025 BayLibre, SAS +# SPDX-FileCopyrightText: 2026 The RISE Project +# SPDX-License-Identifier: MIT +""" +Extract metadata from a just-built riscv64 wheel, add or update the +corresponding docs/packages/.md page with the new version, and open a +pull request with the change. + +Based on the script at: +https://gitlab.com/riseproject/python/wheel_builder/-/blob/main/ci_scripts/update_doc.py + +That script wrote a small YAML file per package, which a separate stage later +rendered into a Sphinx page. Since python-wheels' docs/packages/*.md pages are +hand-off Markdown with no such intermediate source, this script edits the +rendered page directly instead of re-introducing a YAML layer. +""" + +import os +import re +import string +import subprocess +import sys +import zipfile +from email.message import Message +from email.parser import Parser +from pathlib import Path + +REPO = "riseproject-dev/python-wheels" +REGISTRY_URL = "https://pypi.riseproject.dev/simple/" +DOCS_DIR = Path("docs/packages") +PACKAGES_FILE = Path("ci_scripts/packages.txt") +ARTIFACTS_PATH = os.environ.get("ARTIFACTS_PATH", "dist") + + +def find_wheel_file(path): + for file in Path(path).glob("*.whl"): + return file + return None + + +def normalize_name(name): + """ + https://packaging.python.org/en/latest/specifications/name-normalization/#name-normalization + """ + return re.sub(r"[-_.]+", "-", name).lower() + + +def normalize_label(label): + """ + https://packaging.python.org/en/latest/specifications/well-known-project-urls/#label-normalization + """ + chars_to_remove = string.punctuation + string.whitespace + removal_map = str.maketrans("", "", chars_to_remove) + return label.translate(removal_map).lower() + + +def extract_license(message): + license = message.get("License-Expression") + if not license: + license = message.get("License", "Unknown") + return license + + +def extract_source_code_url(message): + # Collect all "Project-URL" lines + project_urls = message.get_all("Project-URL", []) + well_known_labels = ["source", "repository", "sourcecode", "github"] + + for entry in project_urls: + try: + label, url = map(str.strip, entry.split(",", 1)) + if normalize_label(label) in well_known_labels: + return url + except ValueError: + continue # skip malformed lines + + # A lot of projects use homepage as source code url. Done in a second + # loop so a homepage entry appearing before a well-known source label + # doesn't win by accident. + for entry in project_urls: + try: + label, url = map(str.strip, entry.split(",", 1)) + if normalize_label(label) == "homepage": + return url + except ValueError: + continue + + return message.get("Home-page") # deprecated fallback, may be None + + +def extract_metadata_from_whl(whl_path): + """ + Extract metadata according to https://packaging.python.org/en/latest/specifications/core-metadata/ + """ + with zipfile.ZipFile(whl_path, "r") as z: + metadata_file = next(f for f in z.namelist() if f.endswith("METADATA")) + content = z.read(metadata_file).decode() + message: Message = Parser().parsestr(content) + return { + "name": message.get("Name"), + "version": message.get("Version"), + "license": extract_license(message), + "source_code": extract_source_code_url(message), + } + + +def find_patch_dir(slug, version): + """ + Look for a `patches//` directory as described in + docs/development.md, trying both a `v`-prefixed and bare version tag. + """ + for tag in (f"v{version}", version): + candidate = Path("patches") / slug / tag + if candidate.exists(): + return candidate + return None + + +def render_version_block(slug, version, license, patch_dir, *, latest): + label = f"{version} (latest)" if latest else version + open_attr = " open" if latest else "" + install_target = slug if latest else f"{slug}=={version}" + + lines = [ + f'
', + f"{label}", + "", + "```bash", + f"pip install {install_target} --index-url {REGISTRY_URL}", + "```", + "", + f"- **License:** {license}", + ] + + if patch_dir is not None: + patch_url = f"https://github.com/{REPO}/tree/main/{patch_dir}" + lines.append(f"- **Patch applied for this version:** [{patch_url}]({patch_url})") + + lines += ["
", ""] + return "\n".join(lines) + + +def render_new_page(slug, display_name, version, license, source_code, patch_dir): + lines = [ + "---", + f"title: {display_name}", + "layout: default", + "parent: Supported Packages", + "---", + "", + "", + "", + f"# {display_name}", + "", + ] + if source_code: + lines.append(f"- **Source Code:** [{source_code}]({source_code})") + lines += ["- **Supported versions:**", ""] + lines.append(render_version_block(slug, version, license, patch_dir, latest=True)) + return "\n".join(lines).rstrip() + "\n" + + +LATEST_DETAILS_RE = re.compile( + r'
\n([^<]+?)' +) + + +def insert_new_version(content, slug, version, license, patch_dir): + """ + Insert a new version block ahead of the current latest block, demoting the + previous latest block to a plain (non-open, non-"(latest)") entry. + + Returns None if this exact version is already documented. + """ + already_documented = re.search( + rf'{re.escape(version)}( \(latest\))?', content + ) + if already_documented: + return None + + match = LATEST_DETAILS_RE.search(content) + if match is None: + raise ValueError(f"No existing version blocks found for {slug}") + + demoted_label = match.group(1).removesuffix(" (latest)") + demoted_header = f'
\n{demoted_label}' + new_block = render_version_block(slug, version, license, patch_dir, latest=True) + + # The demoted block's install command was rendered unpinned (it used to be + # latest); pin it to its own version now that it no longer is. + rest = content[match.end() :] + old_install = f"pip install {slug} --index-url {REGISTRY_URL}" + new_install = f"pip install {slug}=={demoted_label} --index-url {REGISTRY_URL}" + rest = rest.replace(old_install, new_install, 1) + + return content[: match.start()] + new_block + "\n" + demoted_header + rest + + +def add_index_entry(slug): + """Add a new package to the alphabetically sorted list in docs/packages/index.md.""" + index_path = DOCS_DIR / "index.md" + lines = index_path.read_text().splitlines() + header_end = next(i for i, line in enumerate(lines) if line.startswith("- [")) + header, entries = lines[:header_end], lines[header_end:] + entries.append(f"- [{slug}]({slug}.html)") + entries = sorted(set(entries), key=lambda line: line.split("[", 1)[1].split("]", 1)[0].lower()) + index_path.write_text("\n".join(header + entries) + "\n") + + +def add_to_packages_file(slug): + lines = PACKAGES_FILE.read_text().splitlines() + header_end = next(i for i, line in enumerate(lines) if line and not line.startswith("#")) + header, entries = lines[:header_end], [line for line in lines[header_end:] if line] + entries = sorted(set(entries) | {slug}, key=str.casefold) + PACKAGES_FILE.write_text("\n".join(header + entries) + "\n") + + +def git_run(*args): + subprocess.run(["git", *args], check=True) + + +def configure_git_identity(): + git_run("config", "user.name", "github-actions[bot]") + git_run("config", "user.email", "41898282+github-actions[bot]@users.noreply.github.com") + + +def extract_pr_url(stdout): + for line in stdout.split("\n"): + line = line.strip() + if "github.com" in line and "/pull/" in line: + return line + return None + + +def main(): + whl_file = find_wheel_file(ARTIFACTS_PATH) + if not whl_file: + print(f"No .whl file found in {ARTIFACTS_PATH}") + sys.exit(1) + + metadata = extract_metadata_from_whl(whl_file) + display_name = metadata["name"] + version = metadata["version"] + license = metadata["license"] + source_code = metadata["source_code"] + + if not display_name or not version: + print("Name or version could not be extracted") + sys.exit(1) + + slug = normalize_name(display_name) + patch_dir = find_patch_dir(slug, version) + page_path = DOCS_DIR / f"{slug}.md" + is_new = not page_path.exists() + + if is_new: + page_path.write_text( + render_new_page(slug, display_name, version, license, source_code, patch_dir) + ) + else: + updated = insert_new_version(page_path.read_text(), slug, version, license, patch_dir) + if updated is None: + print(f"{slug} {version} is already documented; nothing to do") + return + page_path.write_text(updated) + + configure_git_identity() + + branch = f"github-actions/{'add' if is_new else 'update'}-doc-for-{slug}" + git_run("switch", "-c", branch) + git_run("add", str(page_path)) + + if is_new: + add_index_entry(slug) + add_to_packages_file(slug) + git_run("add", str(DOCS_DIR / "index.md"), str(PACKAGES_FILE)) + git_run("commit", "-s", "-m", f"docs: add {slug}\n\nAdd version {version}") + else: + git_run("commit", "-s", "-m", f"docs: update {slug}\n\nAdd version {version}") + + git_run("push", "origin", branch) + + result = subprocess.run( + [ + "gh", "pr", "create", "--draft", + "--repo", REPO, + "--base", "main", + "--head", branch, + "--reviewer", "threexc,justeph", + "--title", f"docs: {'add' if is_new else 'update'} {slug}", + "--body", + "Automatically generated PR to document a newly published wheel. " + "Please review it carefully before merging.\n\n" + "If necessary, force-push this branch.", + ], + capture_output=True, text=True, check=True, + ) + pr_url = extract_pr_url(result.stdout) + print(f"[+] Opened PR: {pr_url or '(URL not found in output)'}") + + +if __name__ == "__main__": + main() From bd7864b28db966cc8aaa3f96c68ae649a06c9c03 Mon Sep 17 00:00:00 2001 From: Trevor Gamblin Date: Wed, 15 Jul 2026 21:37:20 -0400 Subject: [PATCH 2/5] build-numpy.yml: add doc update step Signed-off-by: Trevor Gamblin --- .github/workflows/build-numpy.yml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-numpy.yml b/.github/workflows/build-numpy.yml index 974a15f..d590d41 100644 --- a/.github/workflows/build-numpy.yml +++ b/.github/workflows/build-numpy.yml @@ -103,9 +103,15 @@ jobs: if: github.ref == 'refs/heads/main' runs-on: ubuntu-latest permissions: - contents: read + contents: write + pull-requests: write steps: + - name: Checkout python-wheels + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Download wheels uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: @@ -121,3 +127,8 @@ jobs: gitlab-project-id: ${{ vars.GITLAB_PROJECT_ID }} files: | dist/*.whl + + - name: Open docs update PR + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: python3 ci_scripts/update_doc.py From bff4df55643195dd47eab4f83dfc0e20835b74fd Mon Sep 17 00:00:00 2001 From: Trevor Gamblin Date: Wed, 15 Jul 2026 21:40:54 -0400 Subject: [PATCH 3/5] actions: publish-wheels: add composite publishing action Signed-off-by: Trevor Gamblin --- actions/publish-wheels/.action.yml.swp | Bin 0 -> 12288 bytes actions/publish-wheels/action.yml | 91 +++++++++++++++++++++++++ 2 files changed, 91 insertions(+) create mode 100644 actions/publish-wheels/.action.yml.swp create mode 100644 actions/publish-wheels/action.yml diff --git a/actions/publish-wheels/.action.yml.swp b/actions/publish-wheels/.action.yml.swp new file mode 100644 index 0000000000000000000000000000000000000000..9795b470f20821ce27cadbe6650c608730deac2c GIT binary patch literal 12288 zcmeI2&u<{1Qj)UP;ZYQ7D#7zTXvx|LMTOP6=72f7)>ni%)GlZ?d;4r?>pUY z5HK;`G#*qm(W4%WiN7v*F&fXFB=HX*>cxxkAJFgn&dheFO92x(KxV?HyPfy<_kEt{ zeH$LDc6#1CbC>1CbC>1CbC>1CbC>1CbC>1CbC>1CbxD6Fxj$zdC$NzVC zWAga_|NQU&pC2-e?|}p+xcZ=Bdwd%d3VZ@S2JeE`!Cvs^eTMNlxCA}`KW;&P;7hOwPJ_c>3QT}2n^6uv z1D}HTz#KRTc7dJXG4SiXhVc{l7JLJ~0GGf!;BDZ6S#SV64R(Mn;OkB37fA3tFu^D= zz~znT2S_jtHiO^q!5H8oxBy-S$G{YL9Q+2&mF}07_T^J5@E;T){lrNyNomr|9FlUEXY_}WT`%C~GLHo^ehOn1N>&FX5=&lW@)e{Vc|p6-q);;0q~-Zs6sqbsgH_s| z^qFNwak$7KhvyWk50ze{saED%t=33Jl=YSf9;XrwY{;6j)p5F>baScE63bDUQ~7)y zzI=luDjAw---kL3sa=F8-=SujXGJokqnr_yVc=wo12})mtN#! zYh|49{#GgrDlC6p+d@r=rLS9A6<%-Qd48A^b^||Tj+u{iXb*3+>J4k$Zdv=s_KnvX zb*o;R*vINMKEZ2NgEi`V$M`;*Xge)WRBDy6f)Gct4k=AUTU~gIOWQRa9;qePvLz4i zNIo&7#o0??A`dOs3;j=L?=#oU65G@;Lh?wc?M;NX>JdBaM4|8`A2HzUDAFA{yv3mY zB#n&d-(&t-;>8@s+G4&)_rl?6G|&#!WI}L^n5S&ZC-o3I#h(V?~*b8K+M$IeeK&_?OPFLT5 z1OE_URk|eev)|LVx8u)L(^Z=u2roACeI8RM5t1U}k3e&*ES^!(bT#8c07?ogSn3$j@dgc4znDF@+nHfw*D>@bjgMUta;%I}Sm(2is1 zo>Y*WNh~;%Itmes(Oj94fPbMIebiYo3v8EI;83W}zkWe;E}c-pOAn&s{A|t4{u(M2 znhYE3jT}2WS@IJDC)~Pv|34$82jVRrq<@I6I}WAMyzfC`l(e;M3zAOMGsi29%9y#^ z?63eC$w9KRfOFMe-do?6TTU}QuC(2yd!5Jm+)w)lkTL|peYiJv+!JVGFR?Zs5!!MJ zvZ_5FbZUJfszvnudM*p@cYt4~w@qzuP{@LlB+7SnOpf5!nNCXa#n2)vbb2IQghGd` z9&y|cdt@b^@0ibIRS&8>a8mi?EkEok|3pM^LN0<4Nde~$x>7#G5isD9NrwmOzNj5B z;9bhDoQSX5VXIlUqFSLdF^hz*n8kl|m3`y-eJ4Lh2X9NwbgOADyETpWogXnB?R_8J z=nH@#Tn&m>fKf~$4f(oqZZ&$%a>#y-HHMU4B1f9pH7>JXI-|4c^)!xT3Sf3VX9Aas Fe*rEz=hXlJ literal 0 HcmV?d00001 diff --git a/actions/publish-wheels/action.yml b/actions/publish-wheels/action.yml new file mode 100644 index 0000000..16418ad --- /dev/null +++ b/actions/publish-wheels/action.yml @@ -0,0 +1,91 @@ +name: 'Publish riscv64 Wheels and Document Release' +description: > + Checks out python-wheels, downloads a package's built wheel artifacts, + publishes them to the GitLab PyPI Package Registry, and opens a pull + request documenting the new version in docs/packages/. Wraps the publish + job body shared by every build-.yml workflow so it doesn't need + to be duplicated per package. + +inputs: + + # ── Required ──────────────────────────────────────────────────────────────── + + artifact-pattern: + description: > + Pattern passed to actions/download-artifact to select this package's + wheel artifacts, e.g. "numpy-2.5.1-*-manylinux_riscv64". + required: true + + gitlab-username: + description: Passed through to the publish-to-gitlab action. + required: true + + gitlab-token: + description: Passed through to the publish-to-gitlab action. + required: true + + gitlab-project-id: + description: Passed through to the publish-to-gitlab action. + required: true + + gh-token: + description: > + GitHub token used to push the docs branch and open the docs PR. + Composite actions cannot read the `secrets` context directly, so the + caller must pass it explicitly (e.g. secrets.GITHUB_TOKEN). + required: true + + # ── Optional ──────────────────────────────────────────────────────────────── + + artifact-path: + description: Directory the wheel artifacts are downloaded into. + required: false + default: 'dist' + + files: + description: Newline-separated glob(s) of files to publish. Passed through to publish-to-gitlab. + required: false + default: 'dist/*.whl' + + skip-existing: + description: Passed through to the publish-to-gitlab action. + required: false + default: 'false' + + twine-version: + description: Passed through to the publish-to-gitlab action. + required: false + default: '' + +runs: + using: 'composite' + steps: + + - name: Checkout python-wheels + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Download wheels + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: ${{ inputs.artifact-pattern }} + path: ${{ inputs.artifact-path }} + merge-multiple: true + + - name: Publish to GitLab PyPI registry + uses: riseproject-dev/python-wheels/actions/publish-to-gitlab@main + with: + gitlab-username: ${{ inputs.gitlab-username }} + gitlab-token: ${{ inputs.gitlab-token }} + gitlab-project-id: ${{ inputs.gitlab-project-id }} + files: ${{ inputs.files }} + skip-existing: ${{ inputs.skip-existing }} + twine-version: ${{ inputs.twine-version }} + + - name: Open docs update PR + shell: bash + env: + GH_TOKEN: ${{ inputs.gh-token }} + ARTIFACTS_PATH: ${{ inputs.artifact-path }} + run: python3 ci_scripts/update_doc.py From 60a32bb4e8060508c317f991d588030e93ef1dcb Mon Sep 17 00:00:00 2001 From: Trevor Gamblin Date: Wed, 15 Jul 2026 21:41:25 -0400 Subject: [PATCH 4/5] workflows: build-numpy.yml: use publish-wheels Signed-off-by: Trevor Gamblin --- .github/workflows/build-numpy.yml | 27 ++++++--------------------- 1 file changed, 6 insertions(+), 21 deletions(-) diff --git a/.github/workflows/build-numpy.yml b/.github/workflows/build-numpy.yml index d590d41..d6960bc 100644 --- a/.github/workflows/build-numpy.yml +++ b/.github/workflows/build-numpy.yml @@ -12,6 +12,8 @@ on: paths: - '.github/workflows/build-numpy.yml' - 'actions/publish-to-gitlab/**' + - 'actions/publish-wheels/**' + - 'ci_scripts/update_doc.py' concurrency: group: ${{ github.workflow }}-${{ inputs.version || '2.5.1' }}-${{ github.head_ref || github.run_id }} @@ -107,28 +109,11 @@ jobs: pull-requests: write steps: - - name: Checkout python-wheels - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Download wheels - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - pattern: numpy-${{ env.NUMPY_VERSION }}-*-manylinux_riscv64 - path: dist - merge-multiple: true - - - name: Publish to GitLab PyPI registry - uses: riseproject-dev/python-wheels/actions/publish-to-gitlab@main + - name: Publish wheels and open docs PR + uses: riseproject-dev/python-wheels/actions/publish-wheels@main with: + artifact-pattern: numpy-${{ env.NUMPY_VERSION }}-*-manylinux_riscv64 gitlab-username: ${{ vars.GITLAB_DEPLOY_USER }} gitlab-token: ${{ secrets.GITLAB_DEPLOY_TOKEN }} gitlab-project-id: ${{ vars.GITLAB_PROJECT_ID }} - files: | - dist/*.whl - - - name: Open docs update PR - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: python3 ci_scripts/update_doc.py + gh-token: ${{ secrets.GITHUB_TOKEN }} From 0d6ac1c4127c8b5e2dab0eb0ec5170cc962b8c6e Mon Sep 17 00:00:00 2001 From: Trevor Gamblin Date: Wed, 15 Jul 2026 21:46:34 -0400 Subject: [PATCH 5/5] docs: development.md: update publish example Signed-off-by: Trevor Gamblin --- docs/development.md | 41 +++++++++++++++++++++++------------------ 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/docs/development.md b/docs/development.md index 0c76159..e030e31 100644 --- a/docs/development.md +++ b/docs/development.md @@ -129,9 +129,15 @@ when invoked. ### Using the python-wheels Repository in Workflows The `python-wheels` repository contains some custom Actions we require, and -patch files to apply for certain projects. The most critical example is the -`publish-to-gitlab` Action. With it in place, the `build-numpy.yml` script's -`publish` job looks like this: +patch files to apply for certain projects. The one every `build-.yml` +workflow needs is `publish-wheels`, which performs the following steps: + +1. Downloads the built wheel(s) from the previous job +2. Uploads them to the GitLab PyPI registry (via the lower-level + `publish-to-gitlab` Action) +3. Opens a PR against `docs/packages/.md` documenting the new version (via + `ci_scripts/update_doc.py`). With it in place, the `build-numpy.yml` script's + `publish` job looks like this: ``` publish: @@ -143,29 +149,28 @@ publish: if: github.ref == 'refs/heads/main' runs-on: ubuntu-latest permissions: - contents: read + contents: write + pull-requests: write steps: - - name: Download wheels - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - pattern: numpy-${{ env.NUMPY_VERSION }}-*-manylinux_riscv64 - path: dist - merge-multiple: true - - - name: Publish to GitLab PyPI registry - uses: riseproject-dev/python-wheels/actions/publish-to-gitlab@main + - name: Publish wheels and open docs PR + uses: riseproject-dev/python-wheels/actions/publish-wheels@main with: + artifact-pattern: numpy-${{ env.NUMPY_VERSION }}-*-manylinux_riscv64 gitlab-username: ${{ vars.GITLAB_DEPLOY_USER }} gitlab-token: ${{ secrets.GITLAB_DEPLOY_TOKEN }} gitlab-project-id: ${{ vars.GITLAB_PROJECT_ID }} - files: | - dist/*.whl + gh-token: ${{ secrets.GITHUB_TOKEN }} ``` -Other workflows need to follow a similar process - checkout the `python-wheels` -repo, and run the `publish-to-gitlab` action to upload built wheels to the RISE -Python registry. +`permissions` needs `contents: write` and `pull-requests: write` here (not just +`contents: read`) since the docs step pushes a branch and opens a PR with the +default `GITHUB_TOKEN`. + +Other workflows need to follow the same process, modifying `artifact-pattern` to +match their own artifact naming scheme and otherwise reusing `publish-wheels` +like the example. The `publish-to-gitlab` Action should only be used directly if +a workflow needs the upload step without the docs PR side effect. ## Testing a New Workflow