ty-ecosystem-analyzer.yaml

name: ty ecosystem-analyzer

permissions: {}

on:
  pull_request:
    paths:
      - "crates/ty*/**"
      - "!crates/ty_ide/**"
      - "!crates/ty_server/**"
      - "!crates/ty_test/**"
      - "!crates/ty_completion_eval/**"
      - "!crates/ty_wasm/**"
      - "crates/ruff_db/**"
      - "crates/ruff_python_ast/**"
      - "crates/ruff_python_parser/**"
      - "crates/ruff_python_stdlib/**"
      - ".github/workflows/ty-ecosystem-analyzer.yaml"
      - ".github/ty-ecosystem.toml"
      - "crates/ty_python_semantic/resources/primer/**"
      - "Cargo.lock"
      - "pyproject.toml"
      - "uv.lock"
      - "!**.md"
      - "!**.snap"
      # It's tempting to skip all Python files in every directory,
      # but changes to Python files in `ty_vendored` can affect the output of ecosystem analysis,
      # so we apply a narrow exemption for all files in the corpus directory instead.
      - "!crates/ty_python_semantic/resources/corpus/**"

concurrency:
  group: ty-ecosystem-analyzer-${{ github.ref_name }}-${{ github.event.pull_request.number || github.sha }}
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}

defaults:
  run:
    shell: bash

env:
  CARGO_INCREMENTAL: 0
  CARGO_NET_RETRY: 10
  CARGO_TERM_COLOR: always
  RUSTUP_MAX_RETRIES: 10
  RUST_BACKTRACE: 1
  # Line-tables-only debug info: faster builds, backtraces still work.
  CARGO_PROFILE_PROFILING_DEBUG: line-tables-only

jobs:
  build-ty:
    name: Build ty (${{ matrix.revision }})
    strategy:
      matrix:
        revision: [base, pr]
    runs-on: ${{ github.repository == 'astral-sh/ruff' && 'depot-ubuntu-22.04-32' || 'ubuntu-latest' }}
    timeout-minutes: ${{ github.repository == 'astral-sh/ruff' && 5 || 10 }}
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false
          ref: ${{ github.sha }}

      - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
        with:
          lookup-only: false

      - name: Install Rust toolchain
        run: rustup show

      - name: "Install mold"
        uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1

      - name: Build ty
        env:
          REVISION: ${{ matrix.revision }}
        run: |
          if [[ "${REVISION}" == "base" ]]; then
            # Faster to do this separately than to use `fetch-depth: 0` with `actions/checkout`
            git fetch --no-tags --filter=blob:none --unshallow origin

            # The first parent contains any preceding layers in a stacked pull request.
            MERGE_BASE="$(git rev-parse "${GITHUB_SHA}^1")"
            echo "${MERGE_BASE}" > merge-base.txt
            echo "Merge base: ${MERGE_BASE}"
            echo "PR commit: ${GITHUB_SHA}"

            git checkout "${MERGE_BASE}"
          fi

          echo "Building ${REVISION} revision $(git rev-parse HEAD)"
          cargo build --package ty --profile profiling
          cp target/profiling/ty "ty-${REVISION}"

          if [[ "${REVISION}" == "pr" ]]; then
            cp crates/ty_python_semantic/resources/primer/flaky.txt projects_flaky.txt
            cp .github/ty-ecosystem.toml ty-ecosystem.toml
          fi

      - name: Upload base ty binary
        if: matrix.revision == 'base'
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: ty-build-base
          compression-level: 1
          path: |
            ty-base
            merge-base.txt

      - name: Upload PR ty binary, flaky project list, and config
        if: matrix.revision == 'pr'
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: ty-build-pr
          compression-level: 1
          path: |
            ty-pr
            projects_flaky.txt
            ty-ecosystem.toml

      - name: Upload ecosystem-analyzer dependency metadata
        if: matrix.revision == 'pr'
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: ty-build-dependencies
          path: |
            pyproject.toml
            uv.lock

  analyze-shards:
    needs: [build-ty]
    strategy:
      matrix:
        shard: [0, 1, 2, 3]
    runs-on: ${{ github.repository == 'astral-sh/ruff' && 'depot-ubuntu-22.04-32' || 'ubuntu-latest' }}
    timeout-minutes: 10
    env:
      EXCLUDE_NEWER: ${{ github.event.pull_request.updated_at }}
      # Enable experimental support for installing PEP 723 script dependencies.
      TY_UV: scripts
    steps:
      - name: Install the latest version of uv
        uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
        with:
          version: "0.12.6"
          ignore-empty-workdir: true
          # The default of `enable-cache: auto` leads to warnings from setup-uv in this job,
          # since its cache-invalidation keys aren't available until the build artifacts
          # are downloaded. But the cache doesn't help us much here anyway, since the
          # uv binary, Python binary, and all pre-built wheels are excluded from setup-uv's cache.
          # The cache would only save us from building two Git dependencies which are both
          # small, pure-Python projects.
          enable-cache: false

      - name: Download ty binaries, flaky project list, config, and dependency metadata
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          pattern: ty-build-*
          merge-multiple: true

      - name: Analyze shard ${{ matrix.shard }}
        env:
          SHARD: ${{ matrix.shard }}
        run: |
          mkdir -p ~/.config/ty
          cp ty-ecosystem.toml ~/.config/ty/ty.toml

          chmod +x ty-base ty-pr
          MERGE_BASE="$(< merge-base.txt)"
          echo "MERGE_BASE: ${MERGE_BASE}"

          uv run \
            --locked \
            --only-group=ty-ecosystem \
            ecosystem-analyzer \
            --flaky-runs 10 \
            diff \
            --projects-flaky projects_flaky.txt \
            --ty-binary-old ty-base \
            --ty-binary-new ty-pr \
            --old "${MERGE_BASE}" \
            --new "${GITHUB_SHA}" \
            --exclude-newer "${EXCLUDE_NEWER}" \
            --shard "${SHARD}" \
            --num-shards 4 \
            --output-old "diagnostics-base-${SHARD}.json" \
            --output-new "diagnostics-PR-${SHARD}.json"

      - name: Upload diagnostics
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: diagnostics-shard-${{ matrix.shard }}
          path: |
            diagnostics-base-${{ matrix.shard }}.json
            diagnostics-PR-${{ matrix.shard }}.json

  generate-report:
    name: Generate diagnostic diff report
    needs: [analyze-shards]
    runs-on: ubuntu-latest
    timeout-minutes: 5
    steps:
      - name: Install the latest version of uv
        uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
        with:
          version: "0.12.6"
          ignore-empty-workdir: true
          # The default of `enable-cache: auto` leads to warnings from setup-uv in this job,
          # since its cache-invalidation keys aren't available until the build artifacts
          # are downloaded. But the cache doesn't help us much here anyway, since the
          # uv binary, Python binary, and all pre-built wheels are excluded from setup-uv's cache.
          # The cache would only save us from building two Git dependencies which are both
          # small, pure-Python projects.
          enable-cache: false

      - name: Download ecosystem-analyzer dependency metadata
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: ty-build-dependencies

      - name: Download shard diagnostics
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          pattern: diagnostics-shard-*
          merge-multiple: true

      - name: Generate reports
        id: generate-reports
        env:
          REF_NAME: ${{ github.ref_name }}
          BASE_REF_NAME: ${{ github.event.pull_request.base.ref }}
        run: |
          # Merge shard diagnostics
          jq -s '{ outputs: [.[].outputs[]] }' diagnostics-base-*.json > diagnostics-base.json
          jq -s '{ outputs: [.[].outputs[]] }' diagnostics-PR-*.json   > diagnostics-PR.json

          mkdir dist

          # Upload a structured JSON report alongside the HTML report to make
          # ecosystem changes easier for agents to analyze.
          uv run \
            --locked \
            --only-group=ty-ecosystem \
            ecosystem-analyzer \
            generate-diff \
            diagnostics-base.json \
            diagnostics-PR.json \
            --old-name "${BASE_REF_NAME} (merge base)" \
            --new-name "$REF_NAME" \
            --output-html dist/diff.html \
            --output-json dist/diff.json

          set +e
          uv run \
            --locked \
            --only-group=ty-ecosystem \
            ecosystem-analyzer \
            generate-diff-statistics \
            diagnostics-base.json \
            diagnostics-PR.json \
            --fail-on-new-abnormal-exits \
            --old-name "${BASE_REF_NAME} (merge base)" \
            --new-name "$REF_NAME" \
            --output diff-statistics.md
          DIFF_STATISTICS_EXIT_CODE=$?
          set -e

          uv run \
            --locked \
            --only-group=ty-ecosystem \
            ecosystem-analyzer \
            generate-timing-diff \
            diagnostics-base.json \
            diagnostics-PR.json \
            --old-name "${BASE_REF_NAME} (merge base)" \
            --new-name "$REF_NAME" \
            --output-html dist/timing.html

          cat diff-statistics.md >> comment.md

          cat diff-statistics.md >> "$GITHUB_STEP_SUMMARY"

          echo "diff_statistics_exit_code=$DIFF_STATISTICS_EXIT_CODE" >> "$GITHUB_OUTPUT"

      # NOTE: astral-bot deploys every file in this artifact and uses the
      # deployed HTML URLs for the report links in its PR comment.
      # Make sure to update the bot if you rename the artifact.
      - name: "Upload full report"
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: full-report
          path: dist/

      # NOTE: astral-sh-bot uses this artifact to post comments on PRs.
      # Make sure to update the bot if you rename the artifact.
      - name: Upload comment
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: comment.md
          path: comment.md

      - name: Fail on new abnormal exits
        if: steps.generate-reports.outputs.diff_statistics_exit_code != '0'
        run: exit 1