From 33231933c36c8895e76972e37be3bca1914c9368 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 28 Aug 2026 02:08:23 -0700 Subject: [PATCH] improvement(ci): collect idle node_modules sticky disks at 3 days MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit node_modules sticky disks are keyed on hashFiles('bun.lock') by design, so a lockfile bump orphans the old disk. Blacksmith already evicts after 7 days of inactivity, so this is not a leak — it is a retention window far too generous for a key that churns ~4.7 disks/day. Measured across the 39 live disks, the median one is USED for 0.16 days and then billed for another 7, so the retention tail is almost the whole cost. Collecting at 3 days takes the family from ~236 GB to ~111 GB steady-state. Age-based, never PR-triggered. The key holds no PR identifier, so every open PR whose checkout has the same bun.lock mounts the same disk — with ~180 open PRs over roughly 20 distinct lockfile hashes, sharing is the common case, and a delete on PR close would destroy a disk dozens of other open PRs are using. Two independent guards on what may be deleted, because the blast radius of a wrong key is a cache every CI job depends on: a server-side --search, then a local regex re-proving the full key shape. The event segment is [a-z_]+ rather than an enumerated push|pull_request — the key interpolates github.event_name, and a workflow_dispatch disk already exists that an enumerated list would have skipped forever. Verified against the live account: matches all 39 node_modules disks and none of the 19 bun/turbo/Docker disks. Runs on a GitHub-hosted runner so collection still works during a Blacksmith outage or a CI_PROVIDER break-glass switch, which is exactly when disks idle and still bill. The CLI is pinned by version and SHA256 rather than piped from a remote installer, since the job holds an org-wide token, and auto-update is disabled so the pin holds. Deletes fail the job rather than continue-on-error, so a revoked token cannot silently revert us to 7-day billing. Requires a BLACKSMITH_CLI_TOKEN repository secret; run once with dry_run first. --- .github/workflows/stickydisk-gc.yml | 147 ++++++++++++++++++++++++++++ 1 file changed, 147 insertions(+) create mode 100644 .github/workflows/stickydisk-gc.yml diff --git a/.github/workflows/stickydisk-gc.yml b/.github/workflows/stickydisk-gc.yml new file mode 100644 index 00000000000..2305c7dc434 --- /dev/null +++ b/.github/workflows/stickydisk-gc.yml @@ -0,0 +1,147 @@ +name: Sticky Disk GC + +# node_modules sticky disks are keyed on hashFiles('bun.lock') by design — see the +# "Mount node_modules" comment in test-build.yml. A sticky disk is a mutable volume +# and `bun install --frozen-lockfile` adds what the lockfile needs without pruning +# what it dropped, so branches on different lockfiles must not share one. That +# design is correct and is preserved here; its cost is a new 6-16 GB disk per +# lockfile bump, created at ~4.7/day. +# +# Blacksmith already evicts any sticky disk after 7 days of inactivity, so this is +# not a leak — it is a retention window that is far too generous for this key. The +# median disk is only USED for ~0.16 days and then billed for another 7, so the +# retention tail is almost the entire cost. This collects at 3 days instead. +# +# Age-based on purpose. The key contains NO PR identifier, so every open PR whose +# checkout has the same bun.lock mounts the SAME disk — with ~180 open PRs sharing +# on the order of 20 distinct lockfile hashes, the shared case is the common case. +# Deleting on PR close would therefore destroy a disk that dozens of other open PRs +# are actively using. Never add a pull_request or pull_request_target trigger here. + +on: + schedule: + - cron: '17 9 * * *' + workflow_dispatch: + inputs: + retention_days: + description: Delete node_modules disks unused for more than this many days. + required: false + default: '3' + dry_run: + description: List what would be deleted without deleting it. + type: boolean + required: false + default: false + +# Nothing in this job reads the repository. +permissions: {} + +concurrency: + group: stickydisk-gc + cancel-in-progress: false + +jobs: + gc: + name: Reclaim idle node_modules sticky disks + # GitHub-hosted on purpose, not a Blacksmith runner: the CLI is a pure API + # client, and GC has to keep working during a Blacksmith outage or a + # CI_PROVIDER break-glass switch — exactly when disks sit idle and still bill. + runs-on: ubuntu-latest + timeout-minutes: 10 + env: + # Pinned binary + checksum rather than `curl https://get.blacksmith.sh | sh`: + # this job holds an org-wide token, so it must not execute unpinned remote + # shell. Bump both values together; the vendor publishes a .sha256 sidecar + # next to the binary to check against. + BLACKSMITH_CLI_VERSION: v0.4.57 + BLACKSMITH_CLI_SHA256: 7f60f3b9f8d4d7644d9743f5d962acb3b3dbf675f51676702e5f292e02060bca + # The CLI self-updates in the background on every invocation, which would + # silently defeat the pin above. + BLACKSMITH_DISABLE_AUTO_UPDATE: '1' + BLACKSMITH_ORG: simstudioai + TARGET_REPO: ${{ github.repository }} + RETENTION_DAYS: ${{ inputs.retention_days || '3' }} + DRY_RUN: ${{ inputs.dry_run || 'false' }} + + steps: + - name: Install Blacksmith CLI + run: | + set -euo pipefail + url="https://clireleases.blacksmith.sh/cli/${BLACKSMITH_CLI_VERSION}/linux/amd64/blacksmith" + curl -fsSL "$url" -o /usr/local/bin/blacksmith + echo "${BLACKSMITH_CLI_SHA256} /usr/local/bin/blacksmith" | sha256sum -c - + chmod +x /usr/local/bin/blacksmith + + - name: Authenticate + env: + BLACKSMITH_CLI_TOKEN: ${{ secrets.BLACKSMITH_CLI_TOKEN }} + run: | + set -euo pipefail + printf '%s' "$BLACKSMITH_CLI_TOKEN" \ + | blacksmith auth login --api-token - --non-interactive --organization "$BLACKSMITH_ORG" + + - name: Delete node_modules disks idle beyond the retention window + run: | + set -euo pipefail + + blacksmith stickydisk list \ + --repo "$TARGET_REPO" \ + --search '-node-modules-' \ + --per-page 100 \ + --format json > disks.json + + # Two independent guards, because the blast radius of a wrong key is a + # cache every CI job depends on: + # 1. --search narrows server-side to the node_modules family. + # 2. The regex re-proves each key's full shape locally. The event + # segment is [a-z_]+ rather than an enumerated list — the key + # interpolates ${{ github.event_name }}, and a workflow_dispatch + # disk already exists that an enumerated push|pull_request would + # have silently skipped forever. + # Verified against the live account: this matches all 39 node_modules + # disks and none of the 19 bun/turbo/Docker disks, which are mounted + # every run, never idle, and must survive. + # + # Grouped by key before the staleness test because `delete` without + # --arch removes every architecture variant, so a key may only go when + # its NEWEST variant is stale. + jq -r --arg repo "$TARGET_REPO" --argjson days "$RETENTION_DAYS" ' + (now - ($days * 86400)) as $cutoff + | .entries + | map(select(.type == "stickydisk")) + | map(select(.key | test("^" + ($repo | gsub("/"; "\\/")) + "-node-modules-[a-z_]+-[0-9a-f]{64}$"))) + | group_by(.key) + | map({ + key: .[0].key, + gb: (map(.size_bytes) | add / 1000000000 * 100 | round / 100), + last_used: (map(.last_used_at | sub("\\.[0-9]+Z$"; "Z") | fromdateiso8601) | max) + }) + | map(select(.last_used < $cutoff)) + | .[] | "\(.key)\t\(.gb)" + ' disks.json > stale.tsv + + count=$(wc -l < stale.tsv | tr -d ' ') + reclaimed=$(awk -F'\t' '{s+=$2} END {printf "%.1f", s+0}' stale.tsv) + { + echo "### Sticky disk GC" + echo "Retention: ${RETENTION_DAYS}d · dry run: ${DRY_RUN} · candidates: ${count} (${reclaimed} GB)" + } >> "$GITHUB_STEP_SUMMARY" + + failed=0 + while IFS=$'\t' read -r key gb; do + [ -n "$key" ] || continue + if [ "$DRY_RUN" = "true" ]; then + echo "- would delete \`${key}\` (${gb} GB)" >> "$GITHUB_STEP_SUMMARY" + continue + fi + if blacksmith stickydisk delete --repo "$TARGET_REPO" --key "$key" --yes; then + echo "- deleted \`${key}\` (${gb} GB)" >> "$GITHUB_STEP_SUMMARY" + else + echo "- FAILED \`${key}\`" >> "$GITHUB_STEP_SUMMARY" + failed=1 + fi + done < stale.tsv + + # Fail loudly rather than continue-on-error: a revoked token or a changed + # CLI JSON shape would otherwise silently revert us to 7-day billing. + exit "$failed"