From 254f6e80673f1a74cdf0335f65320f1f8b840f65 Mon Sep 17 00:00:00 2001 From: Benjamin Leggett Date: Fri, 17 Jul 2026 18:53:26 -0400 Subject: [PATCH 1/2] fix(ci): shfmt check in CI --- .github/workflows/lint.yml | 35 ++++++ hack/build/build.sh | 30 +++--- hack/build/common.sh | 22 ++-- hack/build/firmware.sh | 40 +++---- hack/build/generate-clean-flavor-config.sh | 42 ++++---- hack/build/generate-clean-variant-config.sh | 14 +-- hack/build/generate-docker-script.py | 111 ++++++++++++++------ hack/build/generate-kfragment.sh | 6 +- hack/build/generate-merge-script.py | 8 +- hack/build/generate-sbom.py | 1 + hack/build/matrix.py | 40 +++++-- hack/build/nvidiagpu-common.sh | 22 ++-- hack/build/patchlist.py | 2 +- hack/build/record-digest.py | 1 + hack/build/refresh-nvidia-versions.py | 7 +- hack/build/util.py | 13 ++- hack/code/format.sh | 28 ++++- 17 files changed, 279 insertions(+), 143 deletions(-) create mode 100644 .github/workflows/lint.yml diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000..f7516a9 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,35 @@ +name: Lint +on: + pull_request: + push: + branches: + - main +permissions: + contents: read +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + format: + runs-on: ubuntu-latest + env: + SHFMT_VERSION: "3.12.0" + BLACK_VERSION: "26.5.1" + steps: + - name: Checkout + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v4 + # Pin tool versions to match the local formatter; a version skew would + # reformat differently and flip the gate red on already-formatted code. + - name: Install formatters + run: | + mkdir -p "${HOME}/.local/bin" + echo "${HOME}/.local/bin" >> "${GITHUB_PATH}" + curl -sSfL "https://github.com/mvdan/sh/releases/download/v${SHFMT_VERSION}/shfmt_v${SHFMT_VERSION}_linux_amd64" -o "${HOME}/.local/bin/shfmt" + chmod +x "${HOME}/.local/bin/shfmt" + pipx install "black==${BLACK_VERSION}" + - name: Show versions + run: | + shfmt --version + black --version + - name: Check formatting + run: ./hack/code/format.sh --check diff --git a/hack/build/build.sh b/hack/build/build.sh index 7b30d30..04c8449 100755 --- a/hack/build/build.sh +++ b/hack/build/build.sh @@ -36,27 +36,27 @@ SQUASH_SIZE=$(stat -c %s "${ADDONS_SQUASHFS_PATH}") # Host kernel size matters less, but we still don't want it to massively balloon all of a sudden. case "$KERNEL_FLAVOR" in - host*) - if [ "${SQUASH_SIZE}" -gt 419430400 ]; then - echo "ERROR: squashfs is >400MB in size (${SQUASH_SIZE} bytes) which is undesirable for the 'host' kernel, validate kconfig options!" >&2 - exit 1 - fi - ;; +host*) + if [ "${SQUASH_SIZE}" -gt 419430400 ]; then + echo "ERROR: squashfs is >400MB in size (${SQUASH_SIZE} bytes) which is undesirable for the 'host' kernel, validate kconfig options!" >&2 + exit 1 + fi + ;; esac # Generally we want to keep zone kernels small, because large kernels -> longer pull times -> increased zone cold boot times. # We don't really care how big the host kernel is. # Nvidia kernels have chonker firmwares, even compressed (like 200MB total size), so not much we can really do there. case "$KERNEL_FLAVOR" in - zone-nvidiagpu) - # Firmware means this is unavoidably quite large - ;; - zone*) - if [ "${SQUASH_SIZE}" -gt 52428800 ]; then - echo "ERROR: squashfs is >50MB in size (${SQUASH_SIZE} bytes) which is undesirable for the 'zone' kernel, validate kconfig options!" >&2 - exit 1 - fi - ;; +zone-nvidiagpu) + # Firmware means this is unavoidably quite large + ;; +zone*) + if [ "${SQUASH_SIZE}" -gt 52428800 ]; then + echo "ERROR: squashfs is >50MB in size (${SQUASH_SIZE} bytes) which is undesirable for the 'zone' kernel, validate kconfig options!" >&2 + exit 1 + fi + ;; esac if [ "${TARGET_ARCH_STANDARD}" = "x86_64" ]; then diff --git a/hack/build/common.sh b/hack/build/common.sh index ee8c55a..9d33950 100644 --- a/hack/build/common.sh +++ b/hack/build/common.sh @@ -98,15 +98,15 @@ KERNEL_ARCH_STANDARD=$TARGET_ARCH_STANDARD # HACK: kconfig paths use different arch keywords, so we have to get cute and munge case "${TARGET_ARCH_STANDARD}" in - x86_64) - KERNEL_ARCH_STANDARD="x86" - ;; - aarch64) - KERNEL_ARCH_STANDARD="arm64" - ;; - *) - KERNEL_ARCH_STANDARD="${TARGET_ARCH_STANDARD}" - ;; +x86_64) + KERNEL_ARCH_STANDARD="x86" + ;; +aarch64) + KERNEL_ARCH_STANDARD="arm64" + ;; +*) + KERNEL_ARCH_STANDARD="${TARGET_ARCH_STANDARD}" + ;; esac KCONFIG_FRAGMENT_DEST="${KERNEL_SRC}/arch/${KERNEL_ARCH_STANDARD}/configs/" @@ -114,7 +114,7 @@ KCONFIG_FRAGMENT_DEST="${KERNEL_SRC}/arch/${KERNEL_ARCH_STANDARD}/configs/" # Copy out our custom kconfig - if we are building for a -, merge the variant fragment with the flavor baseconfig # by copying the fragment into the kernel src tree and letting the kernel's `make` merge them case "${KERNEL_FLAVOR}" in - *-*) +*-*) # Looks like we are dealing with -.config, versus .config, so we have 2 fragments FLAVOR=$(echo "${KERNEL_FLAVOR}" | cut -d'-' -f1) VARIANT=$(echo "${KERNEL_FLAVOR}" | cut -d'-' -f2) @@ -141,7 +141,7 @@ case "${KERNEL_FLAVOR}" in # NOTE `make` craps the bed if you pass a leading space in front of the fragment here. MAKE_CONFIG_FRAGMENTS="${FLAVOR}.config ${FLAVOR}-${VARIANT}.fragment.config" ;; - *) +*) # Looks like we are dealing with just one .config fragment BASE_FLAVOR_CONFIG="${KERNEL_DIR}/configs/${TARGET_ARCH_STANDARD}/${KERNEL_FLAVOR}.config" diff --git a/hack/build/firmware.sh b/hack/build/firmware.sh index d975998..5368112 100644 --- a/hack/build/firmware.sh +++ b/hack/build/firmware.sh @@ -28,12 +28,14 @@ if [ -n "${FIRMWARE_SIG_URL}" ]; then unxz "$FIRMWARE_URL" # We've uncompressed it, update the env var so later stuff points at the right file FIRMWARE_URL="${FIRMWARE_URL%.xz}" - gpg --verify "$FIRMWARE_SIG_URL" || { echo "ERROR: signature ${FIRMWARE_SIG_URL} cannot validate ${FIRMWARE_URL}"; exit 1; } + gpg --verify "$FIRMWARE_SIG_URL" || { + echo "ERROR: signature ${FIRMWARE_SIG_URL} cannot validate ${FIRMWARE_URL}" + exit 1 + } else echo "No firmware signature defined, no validation will be performed" fi - # Note that this assumes the archive is a .tar file, and has already been validated elsewhere. if [ -n "${FIRMWARE_URL}" ]; then echo "untarring firmware at $FIRMWARE_URL" @@ -125,8 +127,8 @@ if [ "${KERNEL_FLAVOR}" = "zone-nvidiagpu" ] && [ "${TARGET_ARCH_STANDARD}" = "x -mindepth 1 \ -path "./${triplet}/*" -prune -o \ -path "./${triplet}" -prune -o \ - -type d -print0 \ - | while IFS= read -r -d '' rel_dir; do + -type d -print0 | + while IFS= read -r -d '' rel_dir; do # Strip leading "./" rel_dir="${rel_dir#./}" mkdir -p "${dst}/${rel_dir}" @@ -156,8 +158,8 @@ if [ "${KERNEL_FLAVOR}" = "zone-nvidiagpu" ] && [ "${TARGET_ARCH_STANDARD}" = "x find . \ -path "./${triplet}/*" -prune -o \ \( -type f -o -type l \) \ - -print0 \ - | while IFS= read -r -d '' rel_item; do + -print0 | + while IFS= read -r -d '' rel_item; do rel_item="${rel_item#./}" # Filter: only files and symlinks that resolve to files if ! should_mirror "$rel_item"; then @@ -260,19 +262,19 @@ if [ "${KERNEL_FLAVOR}" = "zone-nvidiagpu" ] && [ "${TARGET_ARCH_STANDARD}" = "x multiarch_symlink_mirror "$NVIDIA_BOOTSTRAP_OVERLAY_PATH" x86_64-linux-gnu mkdir -p "$ADDONS_OUTPUT_PATH/hooks" - cat > "$ADDONS_OUTPUT_PATH/hooks/nvidia-persist.toml" <<-EOF -[[hooks.setup]] -overlay = "nvidia-bootstrap" -modules = ["nvidia", "nvidia_drm", "nvidia_uvm"] -execute = ["/usr/bin/nvidia-smi", "-pm", "1"] -ignore-failure = true - -[[hooks.hotplug]] -overlay = "nvidia-bootstrap" -modules = ["nvidia", "nvidia_drm", "nvidia_uvm"] -execute = ["/usr/bin/nvidia-smi", "-pm", "1"] -ignore-failure = true -EOF + cat >"$ADDONS_OUTPUT_PATH/hooks/nvidia-persist.toml" <<-EOF + [[hooks.setup]] + overlay = "nvidia-bootstrap" + modules = ["nvidia", "nvidia_drm", "nvidia_uvm"] + execute = ["/usr/bin/nvidia-smi", "-pm", "1"] + ignore-failure = true + + [[hooks.hotplug]] + overlay = "nvidia-bootstrap" + modules = ["nvidia", "nvidia_drm", "nvidia_uvm"] + execute = ["/usr/bin/nvidia-smi", "-pm", "1"] + ignore-failure = true + EOF cd "$OLDDIR" fi diff --git a/hack/build/generate-clean-flavor-config.sh b/hack/build/generate-clean-flavor-config.sh index ab2892d..639dd9c 100755 --- a/hack/build/generate-clean-flavor-config.sh +++ b/hack/build/generate-clean-flavor-config.sh @@ -1,7 +1,7 @@ #!/bin/sh if [ $# -ne 4 ]; then - cat << EOF + cat < Fetch an arch-specific kernel default configuration for a specific stable upstream kernel version and compare it @@ -24,7 +24,7 @@ Notes: - Only lines that were added or changed in are saved - must end in '.config' or kernel make will complain. EOF - exit 1 + exit 1 fi UPSTREAM_KVER="$1" @@ -40,47 +40,47 @@ KERNEL_URL="https://cdn.kernel.org/pub/linux/kernel/v$(echo "$UPSTREAM_KVER" | c TARBALL_PATH="$TEMP_DIR/linux-${UPSTREAM_KVER}.tar.xz" if [ ! -f "$FULL_EDERA_FLAVOR_CONFIG" ]; then - echo "Error: Complete Edera kernel config file does not exist at $FULL_EDERA_FLAVOR_CONFIG!" - exit 1 + echo "Error: Complete Edera kernel config file does not exist at $FULL_EDERA_FLAVOR_CONFIG!" + exit 1 fi echo "Fetching kernel.org stable kernel default config for version $UPSTREAM_KVER (arch: $ARCH)..." # Map architecture names to kernel arch names and config paths case "$ARCH" in - x86_64) - CONFIG_SNIP="arch/x86/configs/x86_64_defconfig" - ;; - arm64|aarch64) - CONFIG_SNIP="arch/arm64/configs/defconfig" - ;; - *) - echo "Error: Unsupported architecture: $ARCH" - exit 1 - ;; +x86_64) + CONFIG_SNIP="arch/x86/configs/x86_64_defconfig" + ;; +arm64 | aarch64) + CONFIG_SNIP="arch/arm64/configs/defconfig" + ;; +*) + echo "Error: Unsupported architecture: $ARCH" + exit 1 + ;; esac # Extract only the config file we need mkdir -p "$TEMP_DIR/linux" echo "Downloading released kernel $UPSTREAM_KVER from $KERNEL_URL to $TARBALL_PATH" if ! curl -sSL "$KERNEL_URL" -o "$TARBALL_PATH"; then - echo "Error: Failed to download kernel version $UPSTREAM_KVER." - exit 1 + echo "Error: Failed to download kernel version $UPSTREAM_KVER." + exit 1 fi if ! tar -xf "$TARBALL_PATH" --strip-components=1 -C "$TEMP_DIR/linux" "linux-${UPSTREAM_KVER}/$CONFIG_SNIP"; then - echo "Error: Failed to extract config file from the tarball." - exit 1 + echo "Error: Failed to extract config file from the tarball." + exit 1 fi CONFIG_PATH="$TEMP_DIR/linux/$CONFIG_SNIP" if [ -f "$CONFIG_PATH" ]; then - echo "Generating a trimmed delta flavor config between kernel $UPSTREAM_KVER default config for $ARCH and flavor config $FULL_EDERA_FLAVOR_CONFIG" + echo "Generating a trimmed delta flavor config between kernel $UPSTREAM_KVER default config for $ARCH and flavor config $FULL_EDERA_FLAVOR_CONFIG" ./hack/build/generate-kfragment.sh "$CONFIG_PATH" "$FULL_EDERA_FLAVOR_CONFIG" "$DELTA_EDERA_FLAVOR_CONFIG" else - echo "Error: Config file not found at $CONFIG_PATH" - exit 1 + echo "Error: Config file not found at $CONFIG_PATH" + exit 1 fi echo "trimmed flavor delta config saved to $DELTA_EDERA_FLAVOR_CONFIG" diff --git a/hack/build/generate-clean-variant-config.sh b/hack/build/generate-clean-variant-config.sh index 0a229d1..91dc9ac 100755 --- a/hack/build/generate-clean-variant-config.sh +++ b/hack/build/generate-clean-variant-config.sh @@ -1,7 +1,7 @@ #!/bin/sh if [ $# -ne 5 ]; then - cat << EOF + cat < Fetch an arch-specific Edera base flavor kernel configuration and compare it with another (local) config, @@ -23,7 +23,7 @@ Notes: - Only lines that were added or changed in are saved in - must end in '.config' or kernel make will complain. EOF - exit 1 + exit 1 fi FLAVOR="$1" @@ -40,8 +40,8 @@ trap 'rm -rf "$TEMP_DIR"; echo "Cleaning up temporary files..."; exit' INT TERM crane export "ghcr.io/edera-dev/$FLAVOR-kernel:$VERSION" - --platform=linux/"$ARCH" | tar --keep-directory-symlink -xf - -C "$TEMP_DIR" if [ ! -f "$TEMP_DIR/kernel/config.gz" ]; then - echo "Error: Exported Edera flavor config file does not exist at $TEMP_DIR/kernel/config.gz!" - exit 1 + echo "Error: Exported Edera flavor config file does not exist at $TEMP_DIR/kernel/config.gz!" + exit 1 fi gunzip "$TEMP_DIR/kernel/config.gz" @@ -49,11 +49,11 @@ gunzip "$TEMP_DIR/kernel/config.gz" EXTRACTED_CONFIG="$TEMP_DIR/kernel/config" if [ -f "$EXTRACTED_CONFIG" ]; then - echo "Generating a trimmed delta variant config between latest edera $FLAVOR kernel config for $ARCH and $CUSTOMIZED_VARIANT_CONFIG" + echo "Generating a trimmed delta variant config between latest edera $FLAVOR kernel config for $ARCH and $CUSTOMIZED_VARIANT_CONFIG" ./hack/build/generate-kfragment.sh "$EXTRACTED_CONFIG" "$CUSTOMIZED_VARIANT_CONFIG" "$DELTA_VARIANT_CONFIG" else - echo "Error: Config file not found at $EXTRACTED_CONFIG" - exit 1 + echo "Error: Config file not found at $EXTRACTED_CONFIG" + exit 1 fi echo "trimmed delta variant config saved to $DELTA_VARIANT_CONFIG" diff --git a/hack/build/generate-docker-script.py b/hack/build/generate-docker-script.py index 7d7d55e..836140a 100644 --- a/hack/build/generate-docker-script.py +++ b/hack/build/generate-docker-script.py @@ -7,7 +7,13 @@ from packaging.version import parse, Version from matrix import CONFIG -from util import format_image_name, maybe, smart_script_split, parse_text_bool, get_branch_tag_suffix +from util import ( + format_image_name, + maybe, + smart_script_split, + parse_text_bool, + get_branch_tag_suffix, +) # Targets packaged from artifacts compiled out-of-band (the docker run compile # phase) and imported into the image build through the `prebuilt` context. @@ -35,7 +41,7 @@ def quoted(text: str) -> str: def dockerify_version(version_string: str) -> str: # "+" is valid for both python versions and semver, # but docker rejects it for tags, so sanitize - return version_string.replace('+', '-') + return version_string.replace("+", "-") def arch_to_platform(arch: str) -> str: @@ -70,8 +76,15 @@ def docker_build_staged( has_firmware = flavor == "zone-amdgpu" has_nvidia = flavor == "zone-nvidiagpu" if has_nvidia: - nv_version = version.split("+nvidia-")[1] if "+nvidia-" in version else version.split("-nvidia-")[1] - nv_modules_url = "https://github.com/NVIDIA/open-gpu-kernel-modules/archive/refs/tags/%s.tar.gz" % nv_version + nv_version = ( + version.split("+nvidia-")[1] + if "+nvidia-" in version + else version.split("-nvidia-")[1] + ) + nv_modules_url = ( + "https://github.com/NVIDIA/open-gpu-kernel-modules/archive/refs/tags/%s.tar.gz" + % nv_version + ) version = dockerify_version(version) if has_nvidia: target = "build-staged-nvidiagpu" @@ -81,26 +94,35 @@ def docker_build_staged( target = "build-staged" iidfile = "image-id-%s-%s-%s" % (version, flavor, target) command = [ - "docker", "buildx", "build", - "--builder", "edera", + "docker", + "buildx", + "build", + "--builder", + "edera", "--load", - "-f", quoted("Dockerfile"), - "--target", quoted(target), - "--iidfile", quoted(iidfile), + "-f", + quoted("Dockerfile"), + "--target", + quoted(target), + "--iidfile", + quoted(iidfile), ] for platform in docker_platforms(archs): command += ["--platform", quoted(platform)] command += ["--build-arg", quoted("KERNEL_SRC_URL=%s" % src_url)] if has_firmware: command += [ - "--build-arg", quoted("FIRMWARE_URL=%s" % firmware_url), - "--build-arg", quoted("FIRMWARE_SIG_URL=%s" % firmware_sig_url), + "--build-arg", + quoted("FIRMWARE_URL=%s" % firmware_url), + "--build-arg", + quoted("FIRMWARE_SIG_URL=%s" % firmware_sig_url), ] if has_nvidia: command += ["--build-arg", quoted("NV_MODULES_TARBALL_URL=%s" % nv_modules_url)] command += ["."] return [""] + smart_script_split( - command, "stage=stage flavor=%s version=%s arch=%s" % (flavor, version, ",".join(archs)) + command, + "stage=stage flavor=%s version=%s arch=%s" % (flavor, version, ",".join(archs)), ) @@ -125,7 +147,9 @@ def docker_compile( staged_iidfile = "image-id-%s-%s-%s" % (version, flavor, stage_target) lines += ["", "rm -rf target && mkdir -p target && chmod a+rwX target"] - lines += ['mkdir -p "${HOME}/.cache/kernel-sccache" && chmod -R a+rwX "${HOME}/.cache/kernel-sccache"'] + lines += [ + 'mkdir -p "${HOME}/.cache/kernel-sccache" && chmod -R a+rwX "${HOME}/.cache/kernel-sccache"' + ] # The extracted kernel source and object trees are the dominant disk # consumers of a build. By default they live in the container's writable @@ -149,10 +173,14 @@ def docker_compile( "docker", "run", "--rm", - "--platform", quoted(platform), - "-e", quoted("KERNEL_VERSION=%s" % version), - "-e", quoted("KERNEL_FLAVOR=%s" % flavor), - "-e", quoted("KERNEL_SRC_URL=/build/override-kernel-src.tar.xz"), + "--platform", + quoted(platform), + "-e", + quoted("KERNEL_VERSION=%s" % version), + "-e", + quoted("KERNEL_FLAVOR=%s" % flavor), + "-e", + quoted("KERNEL_SRC_URL=/build/override-kernel-src.tar.xz"), # The Azure sccache env is passed through by name only (no values), # so the generated script stays free of secrets; docker omits any # that are unset on the host. Without Azure config sccache falls @@ -160,27 +188,39 @@ def docker_compile( # must match what the "configure sccache" step in # .github/workflows/matrix.yml exports; a name missing here # silently never reaches the build container. - "-e", quoted("SCCACHE_AZURE_CONNECTION_STRING"), - "-e", quoted("SCCACHE_AZURE_BLOB_CONTAINER"), - "-e", quoted("SCCACHE_AZURE_KEY_PREFIX"), - "-e", quoted("SCCACHE_AZURE_RW_MODE"), - "-e", quoted("SCCACHE_DIR=/home/build/.cache/sccache"), - "-v", quoted("${HOME}/.cache/kernel-sccache:/home/build/.cache/sccache"), - "-v", quoted("${PWD}/target:/build/target"), + "-e", + quoted("SCCACHE_AZURE_CONNECTION_STRING"), + "-e", + quoted("SCCACHE_AZURE_BLOB_CONTAINER"), + "-e", + quoted("SCCACHE_AZURE_KEY_PREFIX"), + "-e", + quoted("SCCACHE_AZURE_RW_MODE"), + "-e", + quoted("SCCACHE_DIR=/home/build/.cache/sccache"), + "-v", + quoted("${HOME}/.cache/kernel-sccache:/home/build/.cache/sccache"), + "-v", + quoted("${PWD}/target:/build/target"), ] if scratch_dir: compile_command += [ - "-v", quoted("%s/src:/build/src" % scratch_dir), - "-v", quoted("%s/obj:/build/obj" % scratch_dir), + "-v", + quoted("%s/src:/build/src" % scratch_dir), + "-v", + quoted("%s/obj:/build/obj" % scratch_dir), ] if has_firmware: compile_command += [ - "-e", quoted("FIRMWARE_URL=/build/override-firmware.tar.xz"), - "-e", quoted("FIRMWARE_SIG_URL=/build/override-firmware.tar.sign"), + "-e", + quoted("FIRMWARE_URL=/build/override-firmware.tar.xz"), + "-e", + quoted("FIRMWARE_SIG_URL=/build/override-firmware.tar.sign"), ] if has_nvidia: compile_command += [ - "-e", quoted("NVIDIA_MODULES_PATH=/build/override-nvidia-modules.tar.gz"), + "-e", + quoted("NVIDIA_MODULES_PATH=/build/override-nvidia-modules.tar.gz"), ] compile_command += [ '"$(cat %s)"' % staged_iidfile, @@ -194,7 +234,6 @@ def docker_compile( return lines - def docker_build( target: str, name: str, @@ -274,7 +313,8 @@ def docker_build( if use_push_by_digest: metadata_file = metadata_path(image_root, actual_target) image_build_command += [ - "--metadata-file", quoted(metadata_file), + "--metadata-file", + quoted(metadata_file), "--output", quoted( "type=image,name=%s,push-by-digest=true,name-canonical=true,push=true" @@ -287,14 +327,16 @@ def docker_build( image_build_command, "stage=build image=%s arch=%s" % (image_root, archs[0]) ) record_command = [ - "python3", "hack/build/record-digest.py", + "python3", + "hack/build/record-digest.py", quoted(image_root), quoted(metadata_file), quoted(DIGESTS_FILE), ] lines += [""] lines += smart_script_split( - record_command, "stage=record-digest image=%s arch=%s" % (image_root, archs[0]) + record_command, + "stage=record-digest image=%s arch=%s" % (image_root, archs[0]), ) return lines @@ -322,7 +364,8 @@ def docker_build( image_build_command += ["."] lines += [""] lines += smart_script_split( - image_build_command, "stage=build image=%s arch=%s" % (image_root, ",".join(archs)) + image_build_command, + "stage=build image=%s arch=%s" % (image_root, ",".join(archs)), ) if publish: diff --git a/hack/build/generate-kfragment.sh b/hack/build/generate-kfragment.sh index b7ad909..e78ef77 100755 --- a/hack/build/generate-kfragment.sh +++ b/hack/build/generate-kfragment.sh @@ -1,7 +1,7 @@ #!/usr/bin/env sh if [ $# -ne 3 ]; then - cat << EOF + cat < Diff two kernel configuration files and output a "fragment" containing @@ -31,7 +31,7 @@ if [ ! -f "$2" ]; then echo "Error: Updated config file '$2' does not exist." fi -cat > "$3"<< EOF +cat >"$3" < "$3"<< EOF # EOF -diff -u "$1" "$2" | grep '^+' | grep -v '^+++' | sed 's/^+//' | grep -v '^[[:space:]]*#' >> "$3" +diff -u "$1" "$2" | grep '^+' | grep -v '^+++' | sed 's/^+//' | grep -v '^[[:space:]]*#' >>"$3" diff --git a/hack/build/generate-merge-script.py b/hack/build/generate-merge-script.py index 15a19d2..96c5fa0 100644 --- a/hack/build/generate-merge-script.py +++ b/hack/build/generate-merge-script.py @@ -19,6 +19,7 @@ The merge matrix entry already contains the canonical produces list and tags, so we don't need to re-derive them here. """ + import json import os import stat @@ -93,16 +94,15 @@ def main() -> None: create_command += [quoted("%s@%s" % (image_name, digest))] lines += [""] lines += smart_script_split( - create_command, "stage=merge image=%s archs=%d" % (image_name, len(digests)) + create_command, + "stage=merge image=%s archs=%d" % (image_name, len(digests)), ) for tag in tags: ref = "%s:%s" % (image_name, tag) sign_command = ["cosign", "sign", "--yes", quoted(ref)] lines += [""] - lines += smart_script_split( - sign_command, "stage=sign image=%s" % ref - ) + lines += smart_script_split(sign_command, "stage=sign image=%s" % ref) with open("merge.sh", "w") as out: out.write("\n".join(lines)) diff --git a/hack/build/generate-sbom.py b/hack/build/generate-sbom.py index 4fbbc8e..cfe9b5c 100644 --- a/hack/build/generate-sbom.py +++ b/hack/build/generate-sbom.py @@ -26,6 +26,7 @@ This was created with Claude. """ + import json import os import re diff --git a/hack/build/matrix.py b/hack/build/matrix.py index ac9020c..8cdeecb 100644 --- a/hack/build/matrix.py +++ b/hack/build/matrix.py @@ -64,6 +64,7 @@ def get_all_kernel_releases() -> list[str]: releases.append(kernel_version) return releases + @cache def get_all_firmware_releases() -> list[str]: # Snapshot tags are pure YYYYMMDD and map 1:1 to the published @@ -80,6 +81,7 @@ def get_all_firmware_releases() -> list[str]: snapshots.reverse() return snapshots + def merge_matrix(matrix_list: list[list[dict[str, any]]]) -> list[dict[str, any]]: all_builds = OrderedDict() # type: dict[str, dict[str, any]] for builds in matrix_list: @@ -281,20 +283,28 @@ def generate_matrix(tags: dict[str, str]) -> list[dict[str, any]]: produces = [] local_version_tags = [] for tag in version_tags: - local_append = tag+"-"+local_tag + local_append = tag + "-" + local_tag local_version_tags.append(local_append) kernel_output = format_image_name( - image_name_format, flavor, version_info, "[flavor]-kernel", local_append + image_name_format, + flavor, + version_info, + "[flavor]-kernel", + local_append, ) kernel_sdk_output = format_image_name( - image_name_format, flavor, version_info, "[flavor]-kernel-sdk", local_append + image_name_format, + flavor, + version_info, + "[flavor]-kernel-sdk", + local_append, ) produces.append(kernel_output) produces.append(kernel_sdk_output) for arch in architectures: version_builds.append( { - "version": version+"+"+local_tag, + "version": version + "+" + local_tag, "firmware_url": firmware_url, "firmware_sig_url": firmware_sig_url, "tags": local_version_tags, @@ -311,7 +321,11 @@ def generate_matrix(tags: dict[str, str]) -> list[dict[str, any]]: image_name_format, flavor, version_info, "[flavor]-kernel", tag ) kernel_sdk_output = format_image_name( - image_name_format, flavor, version_info, "[flavor]-kernel-sdk", tag + image_name_format, + flavor, + version_info, + "[flavor]-kernel-sdk", + tag, ) produces.append(kernel_output) produces.append(kernel_sdk_output) @@ -362,7 +376,7 @@ def build_release_tags(versions: list[str]) -> dict[str, str]: parsed_ver = parse(raw_version) # Hardcode skip of pre-5.x.x kernels if parsed_ver.major < 5: - print(f'skipping {raw_version}, too old to support') + print(f"skipping {raw_version}, too old to support") continue major = str(parsed_ver.major) major_minor = "%s.%s" % (parsed_ver.major, parsed_ver.minor) @@ -378,8 +392,11 @@ def build_release_tags(versions: list[str]) -> dict[str, str]: def generate_stable_matrix() -> list[dict[str, any]]: current_kernel_releases = get_current_kernel_releases() latest_stable = current_kernel_releases["latest_stable"]["version"] - versions = [r["version"] for r in current_kernel_releases["releases"] - if r["moniker"] in ["stable", "longterm"]] + versions = [ + r["version"] + for r in current_kernel_releases["releases"] + if r["moniker"] in ["stable", "longterm"] + ] tags = build_release_tags(versions) tags["stable"] = latest_stable tags["latest"] = latest_stable @@ -388,8 +405,11 @@ def generate_stable_matrix() -> list[dict[str, any]]: def generate_lts_matrix() -> list[dict[str, any]]: current_kernel_releases = get_current_kernel_releases() - versions = [r["version"] for r in current_kernel_releases["releases"] - if r["moniker"] == "longterm"] + versions = [ + r["version"] + for r in current_kernel_releases["releases"] + if r["moniker"] == "longterm" + ] return generate_matrix(build_release_tags(versions)) diff --git a/hack/build/nvidiagpu-common.sh b/hack/build/nvidiagpu-common.sh index 7a9c735..40299db 100644 --- a/hack/build/nvidiagpu-common.sh +++ b/hack/build/nvidiagpu-common.sh @@ -26,16 +26,16 @@ NV_WORKDIR="$(mktemp -d)/nvidia-modules/${NV_VERSION}" mkdir -p "$NV_WORKDIR" if [ -n "${NVIDIA_MODULES_PATH}" ] && [ -f "${NVIDIA_MODULES_PATH}" ]; then - ARCHIVE="${NVIDIA_MODULES_PATH}" + ARCHIVE="${NVIDIA_MODULES_PATH}" else - RELEASE_JSON=$(curl -s --retry 5 --retry-delay 2 --retry-max-time 30 --retry-all-errors "https://api.github.com/repos/${NV_KMOD_REPO_OWNER}/${NV_KMOD_REPO_NAME}/releases/tags/${NV_VERSION}") - TARBALL_URL=$(echo "$RELEASE_JSON" | grep -o '"tarball_url": *"[^"]*"' | sed 's/"tarball_url": *"\(.*\)"/\1/') - if [ -z "$TARBALL_URL" ]; then - echo "Failed to fetch release information for version $NV_VERSION" - exit 1 - fi - ARCHIVE="$NV_WORKDIR/driver-src.tar.gz" - curl -L -o "$ARCHIVE" "$TARBALL_URL" + RELEASE_JSON=$(curl -s --retry 5 --retry-delay 2 --retry-max-time 30 --retry-all-errors "https://api.github.com/repos/${NV_KMOD_REPO_OWNER}/${NV_KMOD_REPO_NAME}/releases/tags/${NV_VERSION}") + TARBALL_URL=$(echo "$RELEASE_JSON" | grep -o '"tarball_url": *"[^"]*"' | sed 's/"tarball_url": *"\(.*\)"/\1/') + if [ -z "$TARBALL_URL" ]; then + echo "Failed to fetch release information for version $NV_VERSION" + exit 1 + fi + ARCHIVE="$NV_WORKDIR/driver-src.tar.gz" + curl -L -o "$ARCHIVE" "$TARBALL_URL" fi echo "Building NVIDIA driver version: $NV_VERSION" @@ -46,7 +46,7 @@ OLDPWD=$(pwd) cd "$(find "$NV_WORKDIR" -mindepth 1 -maxdepth 1 -type d | head -1)" # Apply nvidia hackpatches if we have them -for patch in "${OLDPWD}"/patches-nvidia/*.patch; do patch -p1 < "$patch"; done +for patch in "${OLDPWD}"/patches-nvidia/*.patch; do patch -p1 <"$patch"; done if [ "${TARGET_ARCH_STANDARD}" = "aarch64" ]; then CROSS_ENV="env CC=aarch64-linux-gnu-gcc LD=aarch64-linux-gnu-ld AR=aarch64-linux-gnu-ar CXX=aarch64-linux-gnu-g++ OBJCOPY=aarch64-linux-gnu-objcopy" @@ -54,7 +54,7 @@ else CROSS_ENV="" fi -${CROSS_ENV} make -C . TARGET_ARCH="${TARGET_ARCH_STANDARD}" SYSSRC="${KERNEL_SRC}" SYSOUT="${KERNEL_OBJ}" -j"${KERNEL_BUILD_JOBS}" "${CROSS_COMPILE_MAKE}" modules +${CROSS_ENV} make -C . TARGET_ARCH="${TARGET_ARCH_STANDARD}" SYSSRC="${KERNEL_SRC}" SYSOUT="${KERNEL_OBJ}" -j"${KERNEL_BUILD_JOBS}" "${CROSS_COMPILE_MAKE}" modules echo "Nvidia $NV_VERSION build done" diff --git a/hack/build/patchlist.py b/hack/build/patchlist.py index f617be7..98a1697 100644 --- a/hack/build/patchlist.py +++ b/hack/build/patchlist.py @@ -12,7 +12,7 @@ try: target_version = parse(sys.argv[1]) except Exception: - target_version = parse(sys.argv[1].split('-')[0]) + target_version = parse(sys.argv[1].split("-")[0]) kernel_flavor = sys.argv[2] series = "%s.%s" % (target_version.major, target_version.minor) diff --git a/hack/build/record-digest.py b/hack/build/record-digest.py index ddcb5a6..7da16fa 100644 --- a/hack/build/record-digest.py +++ b/hack/build/record-digest.py @@ -6,6 +6,7 @@ The merge job consumes this file (one per (version, flavor, arch)) to create the final manifest list with `docker buildx imagetools create`. """ + import json import os import sys diff --git a/hack/build/refresh-nvidia-versions.py b/hack/build/refresh-nvidia-versions.py index e453bc2..9aa7ee3 100755 --- a/hack/build/refresh-nvidia-versions.py +++ b/hack/build/refresh-nvidia-versions.py @@ -6,6 +6,7 @@ the three matching lines change. If no new versions found, should not update the file. Currently only supports amd64 drivers. A human must review the PR opened by the GH Action that runs this. """ + import re import sys import urllib.request @@ -38,7 +39,8 @@ def fetch_latest_versions(url: str = NVIDIA_URL) -> dict[str, str]: block_match = LINUX_X86_64_BLOCK.search(html) if not block_match: raise RuntimeError( - "Could not locate the Linux x86_64 block on %s — page layout may have changed." % url + "Could not locate the Linux x86_64 block on %s — page layout may have changed." + % url ) body = block_match.group("body") @@ -100,7 +102,8 @@ def rewrite_config(versions: dict[str, str], path: Path = CONFIG_PATH) -> bool: missing = set(versions) - seen_labels if missing: raise RuntimeError( - "config.yaml is missing local_tags lines for: %s" % ", ".join(sorted(missing)) + "config.yaml is missing local_tags lines for: %s" + % ", ".join(sorted(missing)) ) if changed: diff --git a/hack/build/util.py b/hack/build/util.py index d09f574..2070558 100644 --- a/hack/build/util.py +++ b/hack/build/util.py @@ -13,7 +13,7 @@ def get_branch_tag_suffix() -> Optional[str]: ref_name = os.getenv("GITHUB_REF_NAME", "") if not ref_name or ref_name == "main": return None - return re.sub(r'[^a-zA-Z0-9._-]', '_', ref_name) + return re.sub(r"[^a-zA-Z0-9._-]", "_", ref_name) def format_image_name( @@ -38,6 +38,7 @@ def maybe(m: dict[str, any], k: str, default_value: any = None) -> any: else: return default_value + def matches_constraints( version: Version, flavor: str, @@ -157,7 +158,7 @@ def list_remote_git_tags(url: str, attempts: int = 6) -> list[str]: parts = line.decode("utf-8").strip().split("\t") if len(parts) != 2 or not parts[1].startswith("refs/tags/"): continue - tags.append(parts[1][len("refs/tags/"):]) + tags.append(parts[1][len("refs/tags/") :]) return tags @@ -178,7 +179,13 @@ def parse_text_constraint(text: str) -> dict[str, any]: constraint[key] = parse_text_bool(value) elif key == "lower" or key == "upper": constraint[key] = value - elif key == "flavors" or key == "flavor" or key == "series" or key == "exact" or key == "arch": + elif ( + key == "flavors" + or key == "flavor" + or key == "series" + or key == "exact" + or key == "arch" + ): if key == "flavor": key = "flavors" constraint[key] = value.split(",") diff --git a/hack/code/format.sh b/hack/code/format.sh index 175f07a..1727fb4 100755 --- a/hack/code/format.sh +++ b/hack/code/format.sh @@ -4,5 +4,29 @@ set -e REAL_SCRIPT="$(realpath "${0}")" cd "$(dirname "${REAL_SCRIPT}")/../.." -shfmt -w hack/**/*.sh -black hack/**/*.py +# --check reports diffs and exits non-zero without writing; used to gate CI. +CHECK="" +if [ "${1:-}" = "--check" ]; then + CHECK="1" +fi + +# find, not a `**` glob: POSIX sh does not recurse `**`, so a glob would +# silently miss nested files and let unformatted code slip through the gate. +SH_FILES="$(find hack -type f -name '*.sh')" +PY_FILES="$(find hack -type f -name '*.py')" + +if [ -n "${CHECK}" ]; then + # Run both so a contributor sees every offending file in one pass, + # rather than fixing shfmt only to trip black on the next run. + RC=0 + # shellcheck disable=SC2086 # word-splitting the file list is intended + shfmt -d ${SH_FILES} || RC=1 + # shellcheck disable=SC2086 + black --check ${PY_FILES} || RC=1 + exit "${RC}" +else + # shellcheck disable=SC2086 + shfmt -w ${SH_FILES} + # shellcheck disable=SC2086 + black ${PY_FILES} +fi From 1ba221acbe413f045409395ae9843ad213532913 Mon Sep 17 00:00:00 2001 From: Benjamin Leggett Date: Fri, 17 Jul 2026 19:02:08 -0400 Subject: [PATCH 2/2] fix(ci): shellcheck CI --- .github/workflows/lint.yml | 6 +++++- hack/build/activate-env.sh | 1 + hack/build/firmware.sh | 17 +++++++++-------- hack/code/format.sh | 15 ++++++++++----- 4 files changed, 25 insertions(+), 14 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index f7516a9..5599a74 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -15,21 +15,25 @@ jobs: env: SHFMT_VERSION: "3.12.0" BLACK_VERSION: "26.5.1" + SHELLCHECK_VERSION: "0.11.0" steps: - name: Checkout uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v4 # Pin tool versions to match the local formatter; a version skew would # reformat differently and flip the gate red on already-formatted code. - - name: Install formatters + - name: Install tools run: | mkdir -p "${HOME}/.local/bin" echo "${HOME}/.local/bin" >> "${GITHUB_PATH}" curl -sSfL "https://github.com/mvdan/sh/releases/download/v${SHFMT_VERSION}/shfmt_v${SHFMT_VERSION}_linux_amd64" -o "${HOME}/.local/bin/shfmt" chmod +x "${HOME}/.local/bin/shfmt" + curl -sSfL "https://github.com/koalaman/shellcheck/releases/download/v${SHELLCHECK_VERSION}/shellcheck-v${SHELLCHECK_VERSION}.linux.x86_64.tar.xz" | tar -xJ -C /tmp + install -m755 "/tmp/shellcheck-v${SHELLCHECK_VERSION}/shellcheck" "${HOME}/.local/bin/shellcheck" pipx install "black==${BLACK_VERSION}" - name: Show versions run: | shfmt --version black --version + shellcheck --version - name: Check formatting run: ./hack/code/format.sh --check diff --git a/hack/build/activate-env.sh b/hack/build/activate-env.sh index 0eb8c04..282b519 100755 --- a/hack/build/activate-env.sh +++ b/hack/build/activate-env.sh @@ -10,6 +10,7 @@ activate_env() { python3 -m venv venv pip install --upgrade pip fi + # shellcheck source=/dev/null # venv is created at runtime, not present at lint time . venv/bin/activate pip3 install -qq -r requirements.txt } diff --git a/hack/build/firmware.sh b/hack/build/firmware.sh index 5368112..7ef44c5 100644 --- a/hack/build/firmware.sh +++ b/hack/build/firmware.sh @@ -104,11 +104,11 @@ if [ "${KERNEL_FLAVOR}" = "zone-nvidiagpu" ] && [ "${TARGET_ARCH_STANDARD}" = "x create_links() { local base_path="$1" # create soname links - find "$base_path" -type f -name '*.so*' ! -path '*xorg/*' -print0 | while read -d $'\0' _lib; do + find "$base_path" -type f -name '*.so*' ! -path '*xorg/*' -print0 | while read -r -d $'\0' _lib; do _soname=$(dirname "${_lib}")/$(readelf -d "${_lib}" | grep -Po 'SONAME.*: \[\K[^]]*' || true) - _base=$(echo ${_soname} | sed -r 's/(.*)\.so.*/\1.so/') - [[ -e "${_soname}" ]] || ln -s $(basename "${_lib}") "${_soname}" - [[ -e "${_base}" ]] || ln -s $(basename "${_soname}") "${_base}" + _base=$(echo "${_soname}" | sed -r 's/(.*)\.so.*/\1.so/') + [[ -e "${_soname}" ]] || ln -s "$(basename "${_lib}")" "${_soname}" + [[ -e "${_base}" ]] || ln -s "$(basename "${_soname}")" "${_base}" done } @@ -192,15 +192,15 @@ if [ "${KERNEL_FLAVOR}" = "zone-nvidiagpu" ] && [ "${TARGET_ARCH_STANDARD}" = "x # Wayland/GBM mkdir -p "$WORKLOAD_OVERLAY_PATH/usr/lib/gbm" - ln -s ../libnvidia-allocator.so.$NV_VERSION "$WORKLOAD_OVERLAY_PATH/usr/lib/gbm/nvidia-drm_gbm.so" + ln -s "../libnvidia-allocator.so.${NV_VERSION}" "$WORKLOAD_OVERLAY_PATH/usr/lib/gbm/nvidia-drm_gbm.so" # DRI driver install -Dm755 "$NV_EXTRACT_PATH/out/"nvidia_drv.so "$WORKLOAD_OVERLAY_PATH/usr/lib/xorg/modules/drivers/nvidia_drv.so" # GLX extensions - install -Dm755 "$NV_EXTRACT_PATH/out/"libglxserver_nvidia.so.$NV_VERSION "$WORKLOAD_OVERLAY_PATH/usr/lib/nvidia/xorg/libglxserver_nvidia.so.$NV_VERSION" - ln -s libglxserver_nvidia.so.$NV_VERSION "$WORKLOAD_OVERLAY_PATH/usr/lib/nvidia/xorg/libglxserver_nvidia.so.1" - ln -s libglxserver_nvidia.so.$NV_VERSION "$WORKLOAD_OVERLAY_PATH/usr/lib/nvidia/xorg/libglxserver_nvidia.so" + install -Dm755 "$NV_EXTRACT_PATH/out/libglxserver_nvidia.so.${NV_VERSION}" "$WORKLOAD_OVERLAY_PATH/usr/lib/nvidia/xorg/libglxserver_nvidia.so.${NV_VERSION}" + ln -s "libglxserver_nvidia.so.${NV_VERSION}" "$WORKLOAD_OVERLAY_PATH/usr/lib/nvidia/xorg/libglxserver_nvidia.so.1" + ln -s "libglxserver_nvidia.so.${NV_VERSION}" "$WORKLOAD_OVERLAY_PATH/usr/lib/nvidia/xorg/libglxserver_nvidia.so" # Remove already-installed libs, so we don't accidentally include them in # filters below @@ -245,6 +245,7 @@ if [ "${KERNEL_FLAVOR}" = "zone-nvidiagpu" ] && [ "${TARGET_ARCH_STANDARD}" = "x mkdir -p "$NVIDIA_BOOTSTRAP_OVERLAY_PATH" + # shellcheck disable=SC2043 # one entry today; loop mirrors the sibling install blocks for BINARY in "$NV_EXTRACT_PATH/out/"nvidia-smi; do BN="$(basename "$BINARY")" install -Dm755 "$BINARY" "$NVIDIA_BOOTSTRAP_OVERLAY_PATH/usr/bin/$BN" diff --git a/hack/code/format.sh b/hack/code/format.sh index 1727fb4..a46f1bb 100755 --- a/hack/code/format.sh +++ b/hack/code/format.sh @@ -4,7 +4,7 @@ set -e REAL_SCRIPT="$(realpath "${0}")" cd "$(dirname "${REAL_SCRIPT}")/../.." -# --check reports diffs and exits non-zero without writing; used to gate CI. +# --check reports problems and exits non-zero without writing; used to gate CI. CHECK="" if [ "${1:-}" = "--check" ]; then CHECK="1" @@ -15,18 +15,23 @@ fi SH_FILES="$(find hack -type f -name '*.sh')" PY_FILES="$(find hack -type f -name '*.py')" +RC=0 if [ -n "${CHECK}" ]; then - # Run both so a contributor sees every offending file in one pass, - # rather than fixing shfmt only to trip black on the next run. - RC=0 + # Run every tool so a contributor sees all offenders in one pass, rather + # than fixing one only to trip the next on the following run. # shellcheck disable=SC2086 # word-splitting the file list is intended shfmt -d ${SH_FILES} || RC=1 # shellcheck disable=SC2086 black --check ${PY_FILES} || RC=1 - exit "${RC}" else # shellcheck disable=SC2086 shfmt -w ${SH_FILES} # shellcheck disable=SC2086 black ${PY_FILES} fi + +# The linter has no autofix, so it runs as a check in both modes. +# shellcheck disable=SC2086 +shellcheck ${SH_FILES} || RC=1 + +exit "${RC}"