bench.yml

name: bench

# Build-engine benchmark. Runs on changes under `bench/` and on demand.
#
# WHY IT IS PATH-SCOPED RATHER THAN ON EVERY PUSH:
#
#   * it is heavy — a full matrix compiles the same fixture six ways per platform
#   * it is noisy — cloud runners are shared, and the CPU model changes under you
#   * it asserts nothing — no threshold, no pass/fail on timings
#
# So it fires when the SUITE itself changes, where the question "did I break the
# harness / did this shift the numbers" is actually being asked, and stays off
# every unrelated PR. A timing threshold on a shared runner would turn normal
# variance into red crosses people learn to ignore, so there is none: results
# are uploaded as artifacts and comparing them is a human act.
#
# The matrix runs platforms in parallel and `fail-fast: false`, because one
# platform missing an engine must not cancel the data from the others.
#
# See bench/README.md for the measurement contract before quoting any number.

on:
  # Changes to the suite itself — including its own tests and the project
  # descriptions it measures. Not `paths: ['**']`: the point is to fire where
  # the numbers can move, not on every commit.
  push:
    paths:
      - 'bench/**'
      - '.github/workflows/bench.yml'
  pull_request:
    paths:
      - 'bench/**'
      - '.github/workflows/bench.yml'
  workflow_dispatch:
    inputs:
      engines:
        description: 'comma-separated: mcpp,mcpp-opt,cmake,xmake,meson,bazel'
        required: false
        default: 'mcpp,mcpp-opt,cmake,xmake'
      variants:
        description: 'comma-separated: headers,modules,modules-impl'
        required: false
        default: 'headers,modules,modules-impl'
      scenarios:
        description: 'comma-separated: cold,noop,touch-hub,touch-leaf,edit-body,edit-comment'
        required: false
        # All of them. A scenario left out of the default is a scenario nobody
        # ever runs — `touch-leaf` was defined, documented and advertised, and
        # had never appeared in a single result file.
        default: 'cold,noop,touch-hub,touch-leaf,edit-body,edit-comment'
      preset:
        description: 'named fixture size: smoke | standard | large (overridden by units/fanin/weight below)'
        required: false
        default: 'standard'
      units:
        description: 'fixture translation units (0 = use the preset)'
        required: false
        default: '0'
      fanin:
        description: 'dependencies per unit (controls graph depth)'
        required: false
        default: '3'
      runs:
        description: 'repetitions per cell (0 = per-scenario default)'
        required: false
        default: '0'
      profile:
        description: 'release | debug'
        required: false
        default: 'release'
      platforms:
        description: 'comma-separated: linux,macos,windows'
        required: false
        default: 'linux,macos,windows'

concurrency:
  group: bench-${{ github.ref }}
  cancel-in-progress: true

jobs:
  # The matrix is computed rather than written out, so `platforms: linux` runs
  # ONE job instead of three jobs where two are skipped — a skipped job still
  # queues a runner and still reports a check.
  plan:
    runs-on: ubuntu-latest
    outputs:
      matrix: ${{ steps.plan.outputs.matrix }}
    steps:
      - id: plan
        shell: bash
        run: |
          set -euo pipefail
          # `inputs.*` is empty on a push/pull_request trigger, so every input needs
          # a fallback here — an empty `platforms` would otherwise plan an empty
          # matrix and the job would silently do nothing.
          want="${{ inputs.platforms || 'linux,macos,windows' }}"
          entries=()
          case ",$want," in *,linux,*)   entries+=('{"os":"ubuntu-24.04","name":"linux"}');;  esac
          case ",$want," in *,macos,*)   entries+=('{"os":"macos-14","name":"macos"}');;      esac
          case ",$want," in *,windows,*) entries+=('{"os":"windows-2022","name":"windows"}');; esac
          if [ ${#entries[@]} -eq 0 ]; then
            echo "no platform selected from '$want'" >&2
            exit 1
          fi
          printf 'matrix={"include":[%s]}\n' "$(IFS=,; echo "${entries[*]}")" >> "$GITHUB_OUTPUT"

  bench:
    needs: plan
    strategy:
      fail-fast: false          # one platform's engine gap must not cancel the rest
      matrix: ${{ fromJSON(needs.plan.outputs.matrix) }}
    runs-on: ${{ matrix.os }}
    timeout-minutes: 120
    name: bench (${{ matrix.name }})

    steps:
      - uses: actions/checkout@v4

      - uses: ./.github/actions/bootstrap-mcpp

      - name: Build the harness
        shell: bash
        run: |
          set -euo pipefail
          cd bench
          "$MCPP" build --release
          # Resolve the produced binary once; the fingerprint directory name is
          # not predictable from here.
          BIN=$(find target -type f -name 'bench' -o -type f -name 'bench.exe' | head -1)
          [ -n "$BIN" ] || { echo "harness binary not found under bench/target" >&2; exit 1; }
          echo "BENCH=$PWD/$BIN" >> "$GITHUB_ENV"

      # Engines beyond mcpp are optional by design: a missing one is reported as
      # `unavailable` with a reason, never as a slow or broken engine. Installing
      # them is therefore best-effort and never fails the job.
      - name: Install comparison engines (best effort)
        shell: bash
        continue-on-error: true
        run: |
          set -uo pipefail
          xlings install bazel -y || echo "bazel unavailable on this runner"
          xlings install xmake -y || echo "xmake unavailable on this runner"
          python3 -m pip install --quiet meson || echo "meson unavailable on this runner"
          cmake --version || true
          ninja --version || true

      - name: Report engine availability
        shell: bash
        run: |
          "$BENCH" --list

      - name: Run benchmark
        shell: bash
        run: |
          set -euo pipefail
          # The preset names the size; units/fanin override it only when set to a
          # positive number. Passing raw numbers unconditionally would make every
          # run's size an accident of this file rather than a named, comparable
          # workload — and --preset must come first so the overrides still win.
          # Every `inputs.*` needs a fallback: on a push/pull_request trigger
          # they are all EMPTY, and an empty --engines would run nothing while
          # still reporting success.
          args=( --preset "${{ inputs.preset || 'smoke' }}" )
          [ "${{ inputs.units || 0 }}" -gt 0 ] 2>/dev/null && args+=( --units "${{ inputs.units }}" )
          [ "${{ inputs.fanin || 0 }}" -gt 0 ] 2>/dev/null && args+=( --fanin "${{ inputs.fanin }}" )
          "$BENCH" \
            --engines   '${{ inputs.engines || 'mcpp,cmake,xmake,meson,bazel' }}' \
            --variants  '${{ inputs.variants || 'headers,modules,modules-impl' }}' \
            --scenarios '${{ inputs.scenarios || 'cold,noop,touch-hub,touch-leaf,edit-body,edit-comment' }}' \
            --profile   '${{ inputs.profile || 'release' }}' \
            "${args[@]}" \
            --runs      '${{ inputs.runs || 0 }}' \
            --work      "$RUNNER_TEMP/bench-work" \
            --out       "bench-${{ matrix.name }}.json"

      - name: Upload report
        uses: actions/upload-artifact@v4
        with:
          name: bench-${{ matrix.name }}
          path: bench-${{ matrix.name }}.json
          if-no-files-found: error