diff --git a/.github/scripts/select-checks.sh b/.github/scripts/select-checks.sh index 92860958..34662b5a 100755 --- a/.github/scripts/select-checks.sh +++ b/.github/scripts/select-checks.sh @@ -15,31 +15,24 @@ # EVERY suite (GitLab: the `base_changes` anchor). # * Otherwise only the suites whose own directory changed run. # * GitLab's `splunk_base_build_rule` -- a change to splunk_base also runs the -# other three splunk suites, which import its test helpers -- is not ported -# here because no splunk suite runs yet. It lands with them in phase 2 -# (STAC-25531). +# other three splunk suites, which import its test helpers. # * push / workflow_dispatch run everything (GitLab: `master_branch`, # `release_branch`). # -# Writes three arrays to $GITHUB_OUTPUT for `fromJson()` in a matrix: -# checks -- suites that need no credentials -# private_checks -- suites that install from the private GitLab PyPI -# index, and are cleared to run on this event -# deferred_private_checks -- private-index suites withheld from this event -# (always empty outside pull requests) +# Writes two arrays to $GITHUB_OUTPUT for `fromJson()` in a matrix: +# checks -- suites that run in the shared BCI container +# docker_checks -- suites that need a live Docker daemon and so run directly +# on the runner (STAC-25531) # -# The split is a security boundary, not a convenience. The credential-free suites -# run with no secrets in scope at all. The private-index suites need a registry -# password, so they are kept in a separate job -- and, on pull requests, are not -# run at all (STAC-25540, second review pass). -# -# That last part is the whole point, so it is worth stating plainly: a -# `pull_request` run executes the pull request's own copy of the workflow and of -# every script it calls. Hardening the job cannot keep a determined pull request -# away from a secret the run is holding -- it can always edit the thing that holds -# it. The only run that cannot leak the credential is a run that never receives -# it, so these suites are deferred to push, tag and workflow_dispatch events, -# whose contents are reviewed before they reach the release branch. +# Every suite here is credential-free, and that is worth keeping. Until +# STAC-25544 `vsphere` resolved only against a private package registry, which +# meant withholding the credential from pull requests and therefore not running +# the suite on them at all -- a real coverage gap, because a `pull_request` run +# executes the pull request's own copy of the workflow and of every script it +# calls, so a run holding a secret cannot be hardened against the pull request +# that edits it. Modernising the VMware pin onto public PyPI removed the secret +# and with it the gap. If a suite ever appears to need a registry credential +# again, removing that need is the fix; splitting the matrix is not. set -euo pipefail @@ -100,22 +93,6 @@ SPLUNK_DEPENDENTS=( splunk_topology ) -# Suites whose requirements resolve only against the private GitLab PyPI index. -# `vsphere` pins vsphere-automation-sdk, which VMware never published to public -# PyPI (the name is squatted there by an unrelated 0.0.1 placeholder), so it is -# mirrored into the StackVista package registry and needs authentication. -# -# Everything not listed here is credential-free and must stay that way: adding a -# suite to this list stops it running on pull requests altogether, and removing -# the need for the private index is always the better fix. For vsphere that fix -# looks reachable -- VMware now publishes the SDK to public PyPI under renamed -# packages (vmware-vapi-runtime, vmware-vapi-common-client, pyvmomi) and ships -# the NSX/VMC wheels from its own public index -- so this list should shrink to -# nothing once the pin is modernised. -PRIVATE_INDEX_CHECKS=( - vsphere -) - # A change anywhere here invalidates every suite: the base classes and the test # helpers are imported by all of them, and the setup scripts build the venv the # suites run in. @@ -136,14 +113,6 @@ to_json() { fi } -is_private_index() { - local candidate=$1 check - for check in "${PRIVATE_INDEX_CHECKS[@]}"; do - [ "${candidate}" = "${check}" ] && return 0 - done - return 1 -} - is_docker() { local candidate=$1 check for check in "${DOCKER_CHECKS[@]}"; do @@ -154,48 +123,27 @@ is_docker() { emit() { local -a selected=("$@") - local -a public=() docker=() private=() deferred=() + local -a public=() docker=() local check for check in ${selected[@]+"${selected[@]}"}; do - if is_private_index "${check}"; then - private+=("${check}") - elif is_docker "${check}"; then + if is_docker "${check}"; then docker+=("${check}") else public+=("${check}") fi done - # Pull requests do not run the private-index suites at all (STAC-25540, second - # review pass). See the security-boundary note at the top of this file: a - # `pull_request` run executes the pull request's own copy of the workflow and - # scripts, so the credential can only be protected by withholding it. These - # suites run on the release branch instead, where the code has been reviewed. - if [ "${EVENT_NAME}" = "pull_request" ] && [ "${#private[@]}" -gt 0 ]; then - deferred=("${private[@]}") - private=() - fi - - local public_json docker_json private_json deferred_json + local public_json docker_json public_json=$(to_json ${public[@]+"${public[@]}"}) docker_json=$(to_json ${docker[@]+"${docker[@]}"}) - private_json=$(to_json ${private[@]+"${private[@]}"}) - deferred_json=$(to_json ${deferred[@]+"${deferred[@]}"}) { echo "checks=${public_json}" echo "docker_checks=${docker_json}" - echo "private_checks=${private_json}" - echo "deferred_private_checks=${deferred_json}" } >>"${GITHUB_OUTPUT}" echo "Selected credential-free suites: ${public_json}" echo "Selected docker-daemon suites: ${docker_json}" - echo "Selected private-index suites: ${private_json}" - if [ "${deferred_json}" != "[]" ]; then - echo "Deferred private-index suites: ${deferred_json}" - echo "::notice title=Private-index suites do not run on pull requests::${deferred_json} resolve only against the private package registry. Pull requests are deliberately given no credential to reach it, so these suites run on ${BASE_REF:-the release branch} after merge." - fi } # Anything that is not a pull request is a full run. On the release branch the diff --git a/.github/workflows/checks-tests.yml b/.github/workflows/checks-tests.yml index 39640da2..70feab23 100644 --- a/.github/workflows/checks-tests.yml +++ b/.github/workflows/checks-tests.yml @@ -1,16 +1,21 @@ name: Check tests # Ported from .gitlab-ci.yml as part of the GitLab -> GitHub migration -# (STAC-25142), phase 1 (STAC-25463) and phase 2 (STAC-25531). +# (STAC-25142), phase 1 (STAC-25463), phase 2 (STAC-25531) and the private-index +# removal (STAC-25544). # # WHAT MOVED # linux_deps + the `test_` job family -> the `check-tests` matrix below, # plus `check-tests-docker` for the five -# suites needing a Docker daemon (STAC-25531) -# and `check-tests-private-index` for vsphere. +# suites needing a Docker daemon (STAC-25531). # splunk_base_build_rule -> SPLUNK_DEPENDENTS in select-checks.sh. # .linux_splunk_test's docker pull + COMPOSE_HTTP_TIMEOUT # -> steps on `check-tests-docker`. +# setup_artifact_registry.sh -> not ported. It configured pip against the +# private GitLab PyPI index for vsphere. +# STAC-25544 moved that pin to the packages +# VMware publishes on public PyPI, so there +# is no private index left to configure. # setup_artifactory_docker.sh -> not ported. It logged docker in to the # SUSE Private Registry so compose could # pull Splunk and Vault through the proxy. @@ -50,69 +55,56 @@ name: Check tests # file and .github/workflows/cerberus-notify.yml. # # CREDENTIALS -# The container image is SUSE BCI from registry.suse.com, which is public, so -# these jobs need no registry credentials at all. That is deliberate: this is a -# PUBLIC repository, and every job here executes PR-authored workflow, setup and -# test code. Any secret exposed to that code is exposed to whoever can open a -# branch. The earlier design pulled a private runner image with -# vars.REGISTRY_USER / secrets.REGISTRY_PASSWORD; dropping it removes the -# registry password from the PR path entirely and, as a side effect, lets -# Dependabot PRs run -- they receive no Actions secrets, so the image pull -# could never have succeeded for them. +# There are none, and that is the design. This is a PUBLIC repository and every +# job here executes PR-authored workflow, setup and test code, so any secret in +# scope is a secret available to whoever can open a branch. A `pull_request` run +# executes the pull request's own copy of this workflow and of every script it +# calls, which means a run holding a secret can always be made to disclose it -- +# by editing the script that fetches it, reordering steps, or adding one. A +# repository secret and pull-request-controlled code do not compose into a +# security boundary, however carefully the code in between is written. The only +# run that cannot leak a credential is a run that never receives one. # -# One credential remains: the read-only pull from the private PyPI index, for -# pins that public PyPI does not serve (currently vsphere-automation-sdk). That -# is vars.GITLAB_PACKAGE_REGISTRY_PYPI_SIMPLE_URL and -# secrets.GITLAB_PACKAGE_REGISTRY_USER, granted to this repo in pulumi-infra -# (StackVista/pulumi-infra#263), alongside the already-org-wide -# secrets.GITLAB_PACKAGE_REGISTRY_READONLY_PASSWORD. It is pull-only and -# least-privilege by construction; this repo's *publishing* role is still -# deferred, per the note above. +# Two credentials were removed to get here. The private runner image needed +# vars.REGISTRY_USER / secrets.REGISTRY_PASSWORD; switching to public SUSE BCI +# dropped it, and as a side effect let Dependabot PRs run at all, since they +# receive no Actions secrets and so could never have pulled that image. # -# No pull request ever receives it. `pull_request` runs execute the pull -# request's own copy of this workflow and of every script it calls, so a run -# that holds a secret can always be made to disclose it -- by editing the -# fetching script, reordering steps, or adding one. A repository secret and -# pull-request-controlled code cannot be arranged into a boundary. The suites -# that need this credential therefore do not run on pull requests at all; they -# run on push, tag and workflow_dispatch events, whose contents are reviewed -# before reaching the release branch. See `check-tests-private-index`. +# The second was the read-only pull from the private GitLab PyPI index, which +# existed solely because vsphere pinned `vsphere-automation-sdk` -- a package +# VMware never published to public PyPI, where the name is squatted by an +# unrelated 0.0.1 placeholder. Withholding that credential from pull requests +# was the only sound way to hold it, which meant vsphere was verified after +# merge rather than on the pull request that changed it. STAC-25544 removed the +# need instead: VMware publishes the same SDK to public PyPI under renamed +# packages (pyvmomi, vmware-vcenter, vmware-vapi-runtime, +# vmware-vapi-common-client), so the pin, the credential, the separate job and +# the coverage gap went together. vsphere is now an ordinary suite in the main +# matrix and runs on pull requests like every other one. # -# Within those runs the credential is still confined to a single step -# (STAC-25540): the script writes ~/.netrc, downloads one fixed package set into -# a local wheelhouse, deletes the netrc, and points pip at the wheelhouse, so -# the suite and its dependency tree install with nothing to authenticate -# against. The predecessor left the netrc readable for the rest of the job. That -# is defence in depth, not the boundary -- the boundary is the event condition -# above. -# -# Two earlier answers to the same review finding are recorded here so they are -# not re-proposed. A `private-package-index` GitHub Environment with required +# Three earlier answers to that same review finding are recorded so they are not +# re-proposed. A `private-package-index` GitHub Environment with required # reviewers did gate the credential, but SHARED_PATHS covers the CI files, so it # fired on roughly one commit in six and blocked authors on their own pull # requests -- while only ever constraining people who already have write access. -# Prefetching the wheelhouse on a trusted event and passing it to pull requests +# Prefetching a wheelhouse on a trusted event and handing it to pull requests # through the Actions cache also works, but any pull request can read a cache, -# and a pull request can only restore one from its base branch. +# and can only restore one from its base branch. Confining the credential to a +# single step was worth doing and was done, but it is hardening, not a boundary. # -# The durable fix is to stop needing the index: VMware now publishes this SDK to -# public PyPI under renamed packages (vmware-vapi-runtime, -# vmware-vapi-common-client, pyvmomi) and serves the NSX/VMC wheels from its own -# public index, so modernising the pin removes the credential, this job and the -# pull-request coverage gap in one change. +# If a dependency ever appears to need a private index again, removing that need +# is the fix. Reintroducing a credential to this workflow is not. # # RUNNERS -# Everything runs on GitHub-hosted runners. The suites are pure-Python and need -# no Docker daemon, so the self-hosted docker-public pool bought nothing while -# costing real isolation: fork PRs had to be excluded from it, which in turn -# meant a fork could never produce a CI verdict. On hosted runners forks run -# exactly the same matrix as any other pull request -- the private-index job is -# off the pull-request path entirely, so no fork-specific guard is needed for -# it any more. This also removes the question of -# pulling upstream images across the self-hosted NAT: the BCI reference is -# direct, from a public registry, on infrastructure that is meant to reach it. -# The phase-2 Docker suites will need a runner with a daemon; that decision -# belongs with them, not here. +# Everything runs on GitHub-hosted runners. The self-hosted docker-public pool +# bought nothing while costing real isolation: fork PRs had to be excluded from +# it, which in turn meant a fork could never produce a CI verdict. On hosted +# runners forks run exactly the same matrix as any other pull request, and no +# job holds a secret, so no fork-specific guard is needed anywhere. This also +# removes the question of pulling upstream images across the self-hosted NAT: +# the BCI reference is direct, from a public registry, on infrastructure meant +# to reach it. The Docker suites need a daemon rather than a container, so they +# run directly on the hosted runner -- see `check-tests-docker`. on: pull_request: @@ -156,8 +148,8 @@ env: # # BCI publishes 3.13.13, one patch behind the 3.13.14 the agent embeds and the # GitLab image pinned. CPython patch releases are bugfix-only, and the full - # phase-1 matrix (including vsphere against the private index) was verified - # green on 3.13.13 before this switch. Worth realigning when BCI ships .14. + # phase-1 matrix was verified green on 3.13.13 before this switch. Worth + # realigning when BCI ships .14. BCI_PYTHON_IMAGE: registry.suse.com/bci/python:3.13@sha256:7d36dd3ba6596fb690e31d956952059fd010604ad6309f06462c02c4c9c01461 # 3.13.13 # Packages the BCI image does not ship but the toolchain build needs: cython and @@ -167,18 +159,14 @@ env: jobs: select-checks: name: Select check suites to run - # Runs for forks too. Every job that runs on a pull request does so with no - # secrets in scope at all -- the one job that uses a credential does not run - # on pull requests (see `check-tests-private-index`). There is therefore - # nothing a fork branch can reach here, and blocking forks outright would + # Runs for forks too. No job in this workflow has a secret in scope, so there + # is nothing a fork branch can reach here, and blocking forks outright would # leave them unable to satisfy branch protection at all (STAC-25463 review). runs-on: ubuntu-latest timeout-minutes: 10 outputs: checks: ${{ steps.select.outputs.checks }} docker_checks: ${{ steps.select.outputs.docker_checks }} - private_checks: ${{ steps.select.outputs.private_checks }} - deferred_private_checks: ${{ steps.select.outputs.deferred_private_checks }} image: ${{ steps.image.outputs.ref }} steps: - name: Check out repository @@ -314,116 +302,6 @@ jobs: source venv/bin/activate checksdev test "${CHECK}" --bench - check-tests-private-index: - name: Check tests, private index (${{ matrix.check }}) - # Isolated from `check-tests` because this is the only job that handles a - # credential at all: vsphere pins a package that resolves solely from the - # private GitLab Package Registry. - # - # This job does not run on pull requests (STAC-25540, second review pass). - # - # An earlier revision ran it on pull requests with the credential confined to - # a single step, and claimed that reaching it would require editing this - # workflow. That claim was wrong, and the review was right to call it: a - # `pull_request` run executes the pull request's own copy of the workflow AND - # of every script it calls, so a pull request could rewrite the fetch script, - # reorder these steps, or simply add a step of its own. Repository secrets - # plus pull-request-controlled code do not make a security boundary, however - # carefully the code in between is written. - # - # Confining the credential to one step is still worth doing and is still done - # -- it keeps the password away from the suite's dependency tree, which needed - # no malice at all to read it -- but it is hardening, not a boundary. The - # boundary is this condition: the run simply never receives the secret. - # - # The cost is that vsphere is verified on the release branch rather than on - # the pull request that changes it. That is a real gap, accepted knowingly: - # the suite's own directory changes a handful of times a year, and the - # alternatives all cost more than they return right now. Sharing a prefetched - # wheelhouse through the Actions cache would work, but a pull request run can - # only restore caches from its base branch, and any pull request can read - # them. The durable fix is to stop needing the private index at all: VMware - # now publishes this SDK to public PyPI under renamed packages, so modernising - # the pin deletes this job, its credential and this trade-off together. - if: >- - ${{ github.event_name != 'pull_request' - && needs.select-checks.outputs.private_checks != '[]' }} - needs: select-checks - runs-on: ubuntu-latest - timeout-minutes: 45 - strategy: - fail-fast: false - matrix: - check: ${{ fromJson(needs.select-checks.outputs.private_checks) }} - container: - # Digest-pinned at workflow level (BCI_PYTHON_IMAGE); the ignore is only - # because Zizmor cannot follow the pin through a job output. - image: ${{ needs.select-checks.outputs.image }} # zizmor: ignore[unpinned-images] - steps: - - name: Check out repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - - - name: Install toolchain build dependencies - run: | - set -eo pipefail - zypper --non-interactive --gpg-auto-import-keys refresh - # shellcheck disable=SC2086 # deliberately word-split into package args - zypper --non-interactive install ${BCI_BUILD_PACKAGES} - - - name: Build the toolchain virtualenv - run: | - set -eo pipefail - git config --global --add safe.directory '*' - source .setup-scripts/setup_env.sh - - - name: Fetch private-index wheels and revoke the credential - env: - GITLAB_PACKAGE_REGISTRY_PYPI_SIMPLE_URL: ${{ vars.GITLAB_PACKAGE_REGISTRY_PYPI_SIMPLE_URL }} - GITLAB_PACKAGE_REGISTRY_USER: ${{ secrets.GITLAB_PACKAGE_REGISTRY_USER }} - GITLAB_PACKAGE_REGISTRY_READONLY_PASSWORD: ${{ secrets.GITLAB_PACKAGE_REGISTRY_READONLY_PASSWORD }} - # The only step in this workflow with a secret in scope. The script writes - # ~/.netrc, downloads one fixed package set, deletes the netrc, and leaves - # ~/.pip/pip.conf pointing at a local wheelhouse. Everything after it -- - # checksdev, tox, the suite's tests and their dependency tree -- runs with - # no credential on disk and no authenticated index configured. - # - # It replaces setup_artifact_registry.sh here, which left the netrc in - # place for the rest of the job (STAC-25463 review P1, STAC-25540). That - # script is untouched and still serves the GitLab pipeline definitions. - # - # The wheelhouse lives in RUNNER_TEMP rather than the workspace so it - # cannot be mistaken for repository content or swept into a build. - # - # NOTE: pip.conf is read from $HOME, so tox must still pass HOME into the - # testenv. tox drops every variable absent from `passenv`, and pip then - # resolves `~` from the passwd database rather than the environment -- - # which points at the wrong home in a container job, where HOME is - # /github/home. Without it the suite silently falls back to public PyPI - # and installs the 0.0.1 placeholder. See vsphere/tox.ini. - run: | - set -eo pipefail - .setup-scripts/fetch_private_wheels.sh "${RUNNER_TEMP}/private-wheels" - - - name: checksdev test ${{ matrix.check }} - env: - CHECK: ${{ matrix.check }} - run: | - set -eo pipefail - source venv/bin/activate - checksdev test --cov "${CHECK}" - - - name: checksdev benchmarks ${{ matrix.check }} - env: - CHECK: ${{ matrix.check }} - # Non-blocking, matching GitLab's `|| true`. - continue-on-error: true - run: | - set -eo pipefail - source venv/bin/activate - checksdev test "${CHECK}" --bench - workflow-security: name: Workflow security scan (Zizmor) # Credential-free and read-only, and it runs on GitHub-hosted infrastructure, @@ -550,7 +428,6 @@ jobs: - validate - check-tests - check-tests-docker - - check-tests-private-index - workflow-security runs-on: ubuntu-latest timeout-minutes: 5 @@ -562,11 +439,8 @@ jobs: WORKFLOW_SECURITY: ${{ needs.workflow-security.result }} CHECK_TESTS: ${{ needs.check-tests.result }} CHECK_TESTS_DOCKER: ${{ needs.check-tests-docker.result }} - CHECK_TESTS_PRIVATE: ${{ needs.check-tests-private-index.result }} SELECTED_CHECKS: ${{ needs.select-checks.outputs.checks }} SELECTED_DOCKER_CHECKS: ${{ needs.select-checks.outputs.docker_checks }} - SELECTED_PRIVATE_CHECKS: ${{ needs.select-checks.outputs.private_checks }} - DEFERRED_PRIVATE_CHECKS: ${{ needs.select-checks.outputs.deferred_private_checks }} run: | set -euo pipefail status=0 @@ -603,8 +477,8 @@ jobs: ;; esac - # Same rule as check-tests: these carry no credentials and run on every - # event, so the only legitimate skip is an empty selection. + # Same rule as check-tests: no job in this workflow carries a + # credential, so the only legitimate skip is an empty selection. printf ' %-24s %s (selected: %s)\n' "check-tests-docker" "${CHECK_TESTS_DOCKER}" "${SELECTED_DOCKER_CHECKS}" case "${CHECK_TESTS_DOCKER}" in success) ;; @@ -620,31 +494,6 @@ jobs: ;; esac - # The private-index matrix legitimately skips whenever the selector - # chose nothing for this event. On pull requests that is always: those - # suites are deferred rather than selected, because the run holds no - # credential to reach the private index with. The deferral is reported - # so a green pull request never quietly implies vsphere was covered. - printf ' %-24s %s (selected: %s, deferred: %s)\n' \ - "check-tests-private" "${CHECK_TESTS_PRIVATE}" "${SELECTED_PRIVATE_CHECKS}" "${DEFERRED_PRIVATE_CHECKS}" - case "${CHECK_TESTS_PRIVATE}" in - success) ;; - skipped) - if [ "${SELECTED_PRIVATE_CHECKS}" != "[]" ]; then - echo "::error title=Selected suites never ran::check-tests-private-index was skipped while ${SELECTED_PRIVATE_CHECKS} was selected." - status=1 - fi - ;; - *) - echo "::error title=Private-index tests did not succeed::check-tests-private-index reported '${CHECK_TESTS_PRIVATE}'." - status=1 - ;; - esac - - if [ "${DEFERRED_PRIVATE_CHECKS}" != "[]" ]; then - echo "::notice title=Not covered by this run::${DEFERRED_PRIVATE_CHECKS} need the private package registry and do not run on pull requests. They run on the release branch after merge." - fi - if [ "${status}" -ne 0 ]; then exit 1 fi diff --git a/.setup-scripts/fetch_private_wheels.sh b/.setup-scripts/fetch_private_wheels.sh deleted file mode 100755 index f138ce66..00000000 --- a/.setup-scripts/fetch_private_wheels.sh +++ /dev/null @@ -1,156 +0,0 @@ -#!/usr/bin/env bash -# Makes the packages that exist only in the private GitLab Package Registry -# available to a local wheelhouse, and destroys the credential before returning. -# -# Why this exists (STAC-25540): vsphere pins vsphere-automation-sdk==1.82.0, an -# unmodified upstream VMware wheel that VMware withdrew from public PyPI. We -# self-host it in the GitLab Package Registry only because that org was private; -# public PyPI now serves a 0.0.1 placeholder squatting the name. -# -# This script only ever runs on events whose contents have been reviewed -- push, -# tag and workflow_dispatch. It does NOT run on pull requests, and the guard below -# enforces that independently of the workflow, because a pull request can edit the -# workflow as freely as it can edit this file. That is the actual protection for -# the credential; everything else here is defence in depth (STAC-25540, second -# review pass). -# -# The defence in depth still matters. The predecessor, setup_artifact_registry.sh, -# left a 0600 ~/.netrc in place for the remainder of the job, so every later step -# -- the tox environment, the suite's own tests, their transitive dependencies -- -# could read the password. That needed no malice from anyone. Here the credential -# exists only for the duration of one pip invocation whose package set is fixed -# below, and pip is then pointed at the resulting wheelhouse so the rest of the -# job resolves offline with nothing to authenticate against. -set -euo pipefail - -# A pull request must never reach the registry password, and must not be able to -# arrange for this script to fetch it one. The workflow already declines to run -# the job on pull requests; this is the same rule stated where it cannot be -# removed by editing a YAML condition. -if [ "${GITHUB_EVENT_NAME:-}" = "pull_request" ]; then - echo "::error title=Refusing to fetch on a pull request::${0##*/} handles the private registry credential and must not run on pull_request events; the private-index suites run on the release branch instead." - exit 1 -fi - -WHEELHOUSE_ARG="${1:-}" -if [ -z "${WHEELHOUSE_ARG}" ]; then - echo "usage: ${0##*/} " >&2 - exit 2 -fi - -# Absolute: pip.conf's find-links is resolved against the working directory of -# whichever process reads it, and tox runs pip from the suite directory. -mkdir -p "${WHEELHOUSE_ARG}" -WHEELHOUSE="$(cd "${WHEELHOUSE_ARG}" && pwd)" - -# Hardcoded on purpose, and deliberately not read from the working tree. While -# the credential is on disk, a pull request must not be able to redirect pip at a -# package of its choosing. -PRIVATE_REQUIREMENTS=( - "vsphere-automation-sdk==1.82.0" -) - -for var in GITLAB_PACKAGE_REGISTRY_PYPI_SIMPLE_URL GITLAB_PACKAGE_REGISTRY_USER GITLAB_PACKAGE_REGISTRY_READONLY_PASSWORD; do - if [ -z "${!var:-}" ]; then - echo "::error title=Private PyPI index not configured::${var} is not available to this job, but this suite cannot resolve without the private index." - exit 1 - fi -done - -NETRC="${HOME}/.netrc" -PIP_CONF_DIR="${HOME}/.pip" - -revoke_credential() { - rm -f "${NETRC}" -} -# Covers the error paths too: a failed download must not leave the password on a -# disk that PR-authored test code goes on to run against. -trap revoke_credential EXIT - -# Hostname only; the simple URL carries a path after the first '/'. -NETRC_HOST="${GITLAB_PACKAGE_REGISTRY_PYPI_SIMPLE_URL%%/*}" - -umask 077 -cat > "${NETRC}" </dev/null || true)"; do - if [ -n "${candidate}" ] && [ -x "${candidate}" ]; then - PYTHON="${candidate}" - break - fi -done -if [ -z "${PYTHON}" ]; then - echo "::error title=No system interpreter::Could not locate a python3 to download with." - exit 1 -fi -if [ -n "${GITHUB_WORKSPACE:-}" ]; then - PYTHON_DIR="$(cd "$(dirname "${PYTHON}")" && pwd)" - case "${PYTHON_DIR}/" in - "${GITHUB_WORKSPACE%/}/"*) - echo "::error title=Refusing a workspace interpreter::Resolved python3 at ${PYTHON}, which is inside the checkout and therefore PR-controlled." - exit 1 - ;; - esac -fi - -echo "→ Downloading private-index packages into ${WHEELHOUSE}" -printf ' %s\n' "${PRIVATE_REQUIREMENTS[@]}" -echo " using ${PYTHON}" - -# --only-binary=:all: matters as much as the interpreter choice. Downloading an -# sdist executes its setup.py, so allowing one would hand arbitrary upstream code -# a process with the registry password readable at ~/.netrc. -"${PYTHON}" -m pip download \ - --disable-pip-version-check \ - --no-cache-dir \ - --only-binary=:all: \ - --extra-index-url "https://${GITLAB_PACKAGE_REGISTRY_PYPI_SIMPLE_URL}" \ - --dest "${WHEELHOUSE}" \ - "${PRIVATE_REQUIREMENTS[@]}" - -revoke_credential -trap - EXIT - -if [ -f "${NETRC}" ]; then - echo "::error title=Credential not revoked::${NETRC} still exists after download; refusing to continue." - exit 1 -fi - -# A silent miss here would fall through to public PyPI and install the 0.0.1 -# placeholder, which fails much later and far less legibly. -if ! find "${WHEELHOUSE}" -maxdepth 1 -iname 'vsphere_automation_sdk-*.whl' | grep -q .; then - echo "::error title=Private wheel missing::vsphere-automation-sdk was not downloaded into ${WHEELHOUSE}." - exit 1 -fi - -# Replaces the extra-index-url that setup_artifact_registry.sh used to write. -# Nothing after this point authenticates anywhere: the private packages resolve -# from the local wheelhouse, and everything else still comes from public PyPI. -mkdir -p "${PIP_CONF_DIR}" -cat > "${PIP_CONF_DIR}/pip.conf" <