compilation-benchmark.yaml

name: Benchmark
on:
  pull_request:
    paths-ignore:
      - 'docs/**'
      - '*.md'
    branches:
      - master

concurrency:
  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
  cancel-in-progress: true

jobs:
  benchmark:
    name: ${{ matrix.package }}
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        package:
          - CairoMakie
          - GLMakie
          - WGLMakie
    steps:
      - name: Checkout
        uses: actions/checkout@v6
      - name: Install xvfb
        run: sudo apt-get update && sudo apt-get install -y xorg-dev mesa-utils xvfb libgl1 freeglut3-dev libxrandr-dev libxinerama-dev libxcursor-dev libxi-dev libxext-dev
      - uses: julia-actions/setup-julia@v2
        with:
          version: '1.11'
          arch: x64
      - uses: julia-actions/cache@v2
      - name: Benchmark
        env:
          GITHUB_TOKEN: ${{ secrets.BENCHMARK_KEY }}
          PR_NUMBER: ${{ github.event.number }}
        run: >
          DISPLAY=:0 xvfb-run -s '-screen 0 1024x768x24' julia --project=./metrics/ttfp/ ./metrics/ttfp/run-benchmark.jl ${{ matrix.package }} 20 ${{ github.event.pull_request.base.ref }}
      - name: Upload plots as artifact
        uses: actions/upload-artifact@v7
        with:
          name: ${{ matrix.package }}
          path: ./benchmark_results
  post-gist:
    name: Post Benchmark Gist
    needs: benchmark  # Wait for all benchmark jobs to complete
    runs-on: ubuntu-latest
    permissions:
      statuses: write       # Permission to post workflow status
    steps:
      - name: Check token validity
        id: check_token
        env:
          BENCHMARK_KEY: ${{ secrets.BENCHMARK_KEY }}
        run: |
          if [ -z "$BENCHMARK_KEY" ]; then
            echo "Benchmark key missing — skipping benchmark upload."
            echo "NOTE: Benchmark key is not accessible for outside contributors"
            echo "has_token=false" >> $GITHUB_OUTPUT
          else
            echo "has_token=true" >> $GITHUB_OUTPUT
          fi

      - name: Download artifacts
        if: steps.check_token.outputs.has_token == 'true'
        uses: actions/download-artifact@v8
        with:
          path: ./images
          merge-multiple: true

      - name: Create Gist with images
        if: steps.check_token.outputs.has_token == 'true'
        env:
          GH_TOKEN: ${{ secrets.BENCHMARK_KEY }}
        run: |
          # Create a gist with the three images
          gist_url=$(gh gist create ./images/CairoMakie.svg ./images/GLMakie.svg ./images/WGLMakie.svg | grep -Eo 'https://gist.github.com[/a-zA-Z0-9]+')
          echo "Gist created: $gist_url"

          # Save the gist URL for later steps
          echo "GIST_URL=$gist_url" >> $GITHUB_ENV
          echo "GIST_URL_USERCONTENT=$(echo $gist_url | sed 's|github|githubusercontent|')" >> $GITHUB_ENV

      - name: Post workflow status with gist link
        if: steps.check_token.outputs.has_token == 'true'
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          gist_url=$GIST_URL
          gh api \
            --method POST \
            -H "Accept: application/vnd.github+json" \
            -H "X-GitHub-Api-Version: 2022-11-28" \
            /repos/${{ github.repository }}/statuses/${{ github.event.pull_request.head.sha }} \
            -f "state=success" \
            -f "context=Benchmark Results" \
            -f "description=Plots are available under Details" \
            -f "target_url=$gist_url"

      - name: Post comment
        if: steps.check_token.outputs.has_token == 'true'
        uses: thollander/actions-comment-pull-request@v3
        with:
          github-token: ${{ secrets.BENCHMARK_KEY }}
          comment-tag: benchmark # this allows to update the same post with new data
          message: |
            # Benchmark Results

            SHA: [${{ github.event.pull_request.head.sha }}](https://github.com/${{ github.repository }}/commit/${{ github.event.pull_request.head.sha }})

            > [!WARNING]
            > These results are subject to substantial noise because GitHub's CI runs on shared machines that are not ideally suited for benchmarking.

            ![GLMakie](${{ env.GIST_URL_USERCONTENT }}/raw/GLMakie.svg)
            ![CairoMakie](${{ env.GIST_URL_USERCONTENT }}/raw/CairoMakie.svg)
            ![WGLMakie](${{ env.GIST_URL_USERCONTENT }}/raw/WGLMakie.svg)