Clone specific software-layer-commit and implement CI to check merged status

.github/workflows/check_software_layer_scripts_commit.yml

@@ -0,0 +1,70 @@
+# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions
+#
+# Our goal is to ensure that the bot/software_layer_scripts_commit in software-layer/main always
+# points to the latest software-layer-scripts/main.
+#
+# This workflow checks if the commit SHA in bot/software_layer_scripts_commit
+# matches the latest commit on the main branch of EESSI/software-layer-scripts.
+# It only runs when the 'bot:deploy' label is present on the PR AND if the bot/software_layer_scripts_commit
+# has been changed. This ensures that:
+# - Contributors that don't know about bot/software_layer_scripts_commit (i.e.: haven't updated it) don't get
+# a CI that complains about the version not being the latest. That's fine, we will update that file in other PRs
+# and if users don't change the file in their PR, there's no risk of overwriting with an older version
+# - Contributors that have changed bot/software_layer_scripts_commit can build with a fixed version (commit)
+# of EESSI/software-layer-scripts, ensuring that builds for all targets are done in a coherent manner.
+# After all builds are deployed, the CI is run, checking that this is the latest commit. If not, contributors
+# will have to update that before merging. This ensures that merging their PR does not cause software-layer/main's
+# bot/software_layer_scripts_commit to start pointing to an _older_ commit.
+name: Check software-layer-scripts commit is up to date
+on:
+  pull_request:
+    types: [opened, synchronize, reopened, labeled]
+    paths:
+      - 'bot/software_layer_scripts_commit'
+permissions:
+  contents: read # to fetch code (actions/checkout)
+jobs:
+  check_latest_commit:
+    if: >
+      (
+        contains(github.event.pull_request.labels.*.name, 'bot:deploy')
+        || contains(github.event.pull_request.labels.*.name, 'force-sw-layer-commit-ci')
+      )
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Check out software-layer repository (shallow)
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+        with:
+          fetch-depth: 1
+
+      - name: Read commit SHA from bot/software_layer_scripts_commit
+        id: read_sha
+        run: |
+          if [[ ! -f bot/software_layer_scripts_commit ]]; then
+            echo "ERROR: bot/software_layer_scripts_commit not found!"
+            exit 1
+          fi
+          SHA=$(cat bot/software_layer_scripts_commit | tr -d '[:space:]')
+          echo "sha=$SHA" >> $GITHUB_OUTPUT
+          echo "Found SHA: $SHA"
+
+      - name: Get latest commit on main of EESSI/software-layer-scripts
+        id: latest_sha
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          SHA=$(gh api repos/EESSI/software-layer-scripts/commits/main --jq .sha)
+          echo "sha=$SHA" >> $GITHUB_OUTPUT
+          echo "Latest commit on main: $SHA"
+
+      - name: Compare SHAs
+        run: |
+          if [[ "${{ steps.read_sha.outputs.sha }}" == "${{ steps.latest_sha.outputs.sha }}" ]]; then
+            echo "OK: bot/software_layer_scripts_commit matches the latest commit on main."
+          else
+            echo "ERROR: bot/software_layer_scripts_commit does not match the latest commit on main of EESSI/software-layer-scripts."
+            echo "Stored commit:  ${{ steps.read_sha.outputs.sha }}"
+            echo "Latest commit:  ${{ steps.latest_sha.outputs.sha }}"
+            echo "Please update bot/software_layer_scripts_commit to ${{ steps.latest_sha.outputs.sha }} and push the change."
+            exit 1
+          fi

.github/workflows/test_software_layer_scripts.yml

@@ -0,0 +1,126 @@
+# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions
+#
+# This workflow verifies that the correct version of software-layer-scripts is used.
+# 
+# First, check_bot_build_checksums checks if the bot/build.sh code that clones software-layer-scripts is untouched,
+# as this normally shouldn't change, so let's be really cautious here. Having this CI means that a change in bot/build.sh should at least be accompanied by
+# a change in this CI, making it stand out to reviewers and increasing the likelihood of this being caught.
+# 
+# Second, check-software_layer_scripts_commit checks if the commit used in bot/software_layer_scripts_commit is a merge-commit for a
+# merge into the default branch of software-layer-scripts. This guarantees that everything that is associated with
+# that commit was approved by a reviewer (and deployed, if needed)
+name: Verify software-layer-scripts
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+  workflow_dispatch:
+permissions:
+  contents: read # to fetch code (actions/checkout)
+jobs:
+  check_bot_build_checksum:
+    runs-on: ubuntu-24.04
+    steps:
+        - name: Check out software-layer repository (shallow)
+          uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+          with:
+              fetch-depth: 1  # We only need the current revision to read bot/software_layer_scripts_commit
+
+        - name: Compute bot/build.sh checksum and verify it
+          run: |
+            # Print clear error if file doesn't exist at all
+            if [[ ! -f bot/build.sh ]]; then
+              echo "ERROR: File bot/build.sh not found!"
+              exit 1
+            fi
+
+            # Reference checksum
+            # UPDATE THIS CHECKSUM IF AND ONLY IF WE ACTUALLY WANT TO CHANGE bot/build.sh
+            EXPECTED_CHECKSUM="622e0b4d39543aaf70ee912edd0660c3746d7a1ecb176e0d41df598155e50e21"
+
+            # Compute checksum
+            COMPUTED_CHECKSUM=$(sha256sum bot/build.sh | awk '{print $1}')
+            echo "Computed checksum: $COMPUTED_CHECKSUM"
+            echo "Reference checksum: $EXPECTED_CHECKSUM"
+
+            # Compare checksums
+            if [[ "$COMPUTED_CHECKSUM" != "$EXPECTED_CHECKSUM" ]]; then
+              echo "ERROR: Checksum mismatch! The file bot/build.sh has been modified."
+              exit 1
+            else
+              echo "Checksum for bot/build.sh matches the reference value"
+            fi
+  check_software_layer_scripts_commit:
+    runs-on: ubuntu-24.04
+    steps:
+        - name: Check out software-layer repository (shallow)
+          uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+          with:
+              fetch-depth: 1  # We only need the current revision to read bot/software_layer_scripts_commit
+        - name: Checkout software-layer-scripts (full history)
+          uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+          with:
+            repository: EESSI/software-layer-scripts
+            path: EESSI-software-layer-scripts
+            fetch-depth: 0 # full history → required for ancestry checks
+
+        - name: Read commit SHA
+          id: read_sha
+          run: |
+            SHA=$(cat bot/software_layer_scripts_commit | tr -d '[:space:]')
+            echo "sha=$SHA" >> $GITHUB_OUTPUT
+            echo "Found SHA: $SHA"
+
+        - name: Verify SHA exists in software‑layer‑scripts
+          working-directory: EESSI-software-layer-scripts
+          run: |
+            SHA="${{ steps.read_sha.outputs.sha }}"
+
+            echo "Checking out commit ${SHA} from software-layer-scripts"
+            git fetch --depth=1 origin ${SHA}
+            git checkout --detach ${SHA}
+
+            # Validate that this object is _actually_ a commit
+            if ! git cat-file -e "${SHA}^{commit}" 2>/dev/null; then
+              echo "Commit $SHA not found in software‑layer‑scripts."
+              exit 1
+            fi
+            echo "Commit $SHA exists in software‑layer‑scripts."
+
+        - name: Check that SHA is merged into the default branch
+          working-directory: EESSI-software-layer-scripts
+          run: |
+            SHA="${{ steps.read_sha.outputs.sha }}"
+
+            # git merge‑base --is‑ancestor returns 0 if $SHA is an ancestor of origin/main
+            if git merge-base --is-ancestor "$SHA" origin/main; then
+              echo "Commit $SHA is merged into origin/main."
+            else
+              echo "Commit $SHA is NOT merged into origin/main."
+              exit 1
+            fi
+
+        - name: Verify commit is signed by GitHub’s web‑flow key
+          working-directory: EESSI-software-layer-scripts
+          env:
+            GIT_TRACE: 1   # extra debug output if something goes wrong
+          run: |
+            SHA="${{ steps.read_sha.outputs.sha }}"
+
+            # Import the public key that GitHub uses for UI‑generated merges
+            echo "Importing GitHub web‑flow GPG key…"
+            curl -sSfL https://github.com/web-flow.gpg | gpg --dearmor > web-flow.gpg
+            gpg --import web-flow.gpg
+            # (optional) show the fingerprint for debugging
+            echo "Fingerprint of the web-flow GPG key:"
+            gpg --list-keys --fingerprint | grep -i "web-flow" -A1
+
+            # Verify the commit’s GPG signature
+            echo "Verifying the signature of commit $SHA…"
+            if git verify-commit "$SHA"; then
+              echo "Commit $SHA is signed and the signature validates with the web‑flow key."
+              echo "All verification steps succeeded."
+            else
+              echo "Commit $SHA is either unsigned or not signed by the web‑flow key."
+              exit 1
+            fi

bot/build.sh

@@ -3,7 +3,19 @@
 # give up as soon as any error occurs
 set -e
 
-git clone https://github.com/EESSI/software-layer-scripts
+TOPDIR=$(dirname $(realpath $0))
+
+# Clone the commit from software-layer-scripts that corresponds to `bot/software_layer_scripts_commit`
+commit_sha=$(cat ${TOPDIR}/software_layer_scripts_commit)
+
+# Get a shallow clone first
+git clone --depth 1 --filter=blob:none --no-checkout https://github.com/EESSI/software-layer-scripts
+
+# Fetch the relevant commit & check it out
+cd software-layer-scripts
+git fetch --depth=1 origin ${commit_sha}
+git checkout --detach ${commit_sha}
+cd ..
 
 # symlink everything, except for:
 # - common files like LICENSE and README.md