benchmark.yml

name: Benchmark PR vs main

on:
  workflow_dispatch:
  pull_request:
    branches:
      - main
    paths:
    - '**.swift'
    - '**.yml'

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}-benchmark
  cancel-in-progress: true

env:
  ENABLE_HB_BENCHMARKS: true
jobs:
  benchmark-delta:

    runs-on: ${{ matrix.os }}
    timeout-minutes: 15
    continue-on-error: true
    permissions:
      issues: write
      pull-requests: write
      contents: read

    strategy:
      matrix:
        os: [ubuntu-latest]
        image: ["swift:latest"]

    container:
      image: ${{ matrix.image }}

    steps:
      - uses: actions/checkout@v6
        with:
          fetch-depth: 0

      - name: Install libjemalloc-dev
        run: |
          apt-get -q update
          apt-get install -y libjemalloc-dev

        # https://github.com/actions/checkout/issues/766
      - name: Mark the workspace as safe
        run: git config --global --add safe.directory ${GITHUB_WORKSPACE}

      - name: Miscellaneous
        run: |
          [ -d Benchmarks ] && echo "hasBenchmark=1" >> $GITHUB_ENV
          echo "/opt/homebrew/bin:/usr/local/bin" >> $GITHUB_PATH
      - name: Run benchmarks for PR branch
        if: ${{ env.hasBenchmark == '1' }}
        run: |
          swift package --allow-writing-to-directory .benchmarkBaselines/ benchmark baseline update pull_request
      - name: Checkout main
        run: |
          git checkout main
      - name: Run benchmarks for branch 'main'
        if: ${{ env.hasBenchmark == '1' }}
        run: |
          swift package --allow-writing-to-directory .benchmarkBaselines/ benchmark baseline update main
      - name: Compare PR and main
        if: ${{ env.hasBenchmark == '1' }}
        id: benchmark
        run: |
          set +e
          # if we had access to fd 3 we could do this in one call eg { var=$(cmd 3>&2 2>&1 1>&3); }  2>&1
          # but unfortunately we don't so we have to run the baseline check twice once to extract stdout
          # and once to extract stderr
          BENCHMARK_STDERR=$(swift package benchmark baseline check main pull_request 2>&1)
          echo "exit-status=$?" >> $GITHUB_OUTPUT
          echo "benchmark-error=$(echo -e "$BENCHMARK_STDERR" | grep -e "^error: .*" | tail -n 1 | cut -c 8-)" >> $GITHUB_OUTPUT
          set -e
      - name: Pull request comment text
        id: benchmark-comment
        run: |
          echo 'PRTEST<<EOF' >> $GITHUB_ENV

          EXIT_CODE='${{steps.benchmark.outputs.exit-status}}'
          EXIT_STATUS=0
          case "${EXIT_CODE}" in
            0)
              echo "_✅ Pull request no significant performance differences ✅_" >> $GITHUB_ENV
              ;;
            *)
              # Get error string from benchmark output
              BENCHMARK_ERROR='${{steps.benchmark.outputs.benchmark-error}}'
              case "${BENCHMARK_ERROR}" in
                "benchmarkThresholdRegression")
                  echo "_❌ Pull request has performance regressions ❌_" >> $GITHUB_ENV
                  EXIT_STATUS=1
                  ;;
                "benchmarkThresholdImprovement")
                  echo "_✅ Pull request has performance improvements ✅_" >> $GITHUB_ENV
                  ;;
                *)
                  echo "_❌ Benchmark comparison failed with error $BENCHMARK_ERROR ❌_" >> $GITHUB_ENV
                  EXIT_STATUS=1
                  ;;
              esac
              ;;
          esac

          set +e
          echo "<details><summary>Summary</summary>" >> $GITHUB_ENV
          swift package benchmark baseline check main pull_request --format markdown >> $GITHUB_ENV
          echo "</details><details><summary>Full Benchmark Comparison</summary>" >> $GITHUB_ENV
          swift package benchmark baseline compare main pull_request --no-progress --quiet --format markdown >> $GITHUB_ENV
          echo "</details>" >> $GITHUB_ENV
          set -e

          echo 'EOF' >> $GITHUB_ENV
          echo "exitStatus=$EXIT_STATUS" >> $GITHUB_ENV
      - name: Comment PR
        if: ${{ env.hasBenchmark == '1' }}
        uses: thollander/actions-comment-pull-request@v3
        with:
          github-token: ${{ secrets.GITHUB_TOKEN }}
          message: ${{ env.PRTEST }}
          comment-tag: benchmark
      - name: Exit with correct status
        run: |
          exit ${{env.exitStatus}}