CodSpeed Benchmarks #315
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CodSpeed Benchmarks | |
| on: | |
| push: | |
| branches: | |
| - master | |
| pull_request: | |
| workflow_dispatch: | |
| schedule: | |
| # Keep the master-scoped caches warm: GitHub evicts caches unused for | |
| # 7 days, and PRs can only restore caches from their own ref or master. | |
| - cron: "0 6 * * 1,4" | |
| jobs: | |
| benchmarks: | |
| runs-on: codspeed-macro | |
| # Bumped from 15m: the walltime benchmarks now run a minimum number of | |
| # rounds (see the "Generate CodSpeed config" step) instead of a single | |
| # sample, which increases the benchmarking phase's duration. With no | |
| # per-benchmark max-time, this timeout is what bounds the bench phase. | |
| timeout-minutes: 30 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| valgrind: | |
| - "3.26.0" | |
| - "3.25.1" | |
| - "local" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| lfs: true | |
| submodules: recursive | |
| - uses: extractions/setup-just@v3 | |
| # Skip installing package docs to avoid wasting time when installing build dependencies | |
| # See: https://github.com/actions/runner-images/issues/10977#issuecomment-2810713336 | |
| - name: Skip installing package docs | |
| if: runner.os == 'Linux' | |
| run: | | |
| sudo tee /etc/dpkg/dpkg.cfg.d/01_nodoc > /dev/null << 'EOF' | |
| path-exclude /usr/share/doc/* | |
| path-exclude /usr/share/man/* | |
| path-exclude /usr/share/info/* | |
| EOF | |
| - name: Cache Valgrind build | |
| uses: actions/cache@v4 | |
| id: valgrind-cache | |
| with: | |
| # `.capstone` is the cycle-estimation decoder prefix; its absolute path | |
| # is baked into the Makefile's -I flags, so `make install` (which may | |
| # recompile cycledecode.o) needs it present. It is built outside | |
| # /tmp/valgrind-build, so cache it too or a cache hit breaks install. | |
| path: | | |
| /tmp/valgrind-build | |
| .capstone | |
| key: valgrind-${{ matrix.valgrind }}-${{ runner.os }}-v2-${{ matrix.valgrind == 'local' && hashFiles('coregrind/**', 'include/**', 'VEX/**', 'cachegrind/**', 'callgrind/**', 'dhat/**', 'drd/**', 'helgrind/**', 'lackey/**', 'massif/**', 'memcheck/**', 'none/**', 'exp-bbv/**', 'auxprogs/**', '*.ac', '*.am', '*.in', 'autogen.sh', 'configure*') || 'build' }} | |
| # Build and install Valgrind | |
| - name: Update apt-get cache | |
| run: | | |
| sudo apt-get update | |
| # Remove existing Valgrind installation | |
| sudo apt-get remove -y valgrind || true | |
| - name: Install build dependencies | |
| if: steps.valgrind-cache.outputs.cache-hit != 'true' | |
| run: | | |
| sudo apt-get install -y \ | |
| build-essential \ | |
| automake \ | |
| autoconf \ | |
| cmake \ | |
| gdb \ | |
| docbook \ | |
| docbook-xsl \ | |
| docbook-xml \ | |
| xsltproc | |
| - name: Build Capstone (cycle-estimation decoder) | |
| if: steps.valgrind-cache.outputs.cache-hit != 'true' && matrix.valgrind == 'local' | |
| uses: ./.github/actions/build-capstone | |
| - name: Build Valgrind (${{ matrix.valgrind }}) | |
| if: steps.valgrind-cache.outputs.cache-hit != 'true' | |
| run: just build ${{ matrix.valgrind }} | |
| - name: Install Valgrind (${{ matrix.valgrind }}) | |
| run: | | |
| just install ${{ matrix.valgrind }} | |
| # Ensure libc6-dev is installed for Valgrind to work properly | |
| sudo apt-get install -y libc6-dev stress-ng | |
| - name: Verify Valgrind build | |
| run: /usr/local/bin/valgrind --version | |
| # Setup benchmarks and run them | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v5 | |
| # Compiled locally rather than checked in (unlike the take_strings-* | |
| # binaries) since it has no arch-specific dependencies beyond libc. | |
| - name: Build bench fixtures | |
| working-directory: bench/testdata | |
| run: cc -O2 -o llsc_tzconvert_bench llsc_tzconvert_bench.c | |
| # Generate the codspeed.yml for this Valgrind version. The script derives | |
| # the version label from `valgrind --version`, so each matrix job emits its | |
| # own config (e.g. valgrind.codspeed / valgrind-3.26.0 / valgrind-3.25.1). | |
| # | |
| # The walltime sampling knobs are passed explicitly so the | |
| # stability/duration trade-off is visible and tunable here. `min-rounds` | |
| # gives each benchmark several samples for a stable estimate; the runner | |
| # rejects combining it with `max-time`, so the bench phase is bounded by | |
| # `timeout-minutes` instead. | |
| - name: Generate CodSpeed config | |
| working-directory: bench | |
| run: >- | |
| ./generate_config.py | |
| --valgrind /usr/local/bin/valgrind | |
| --output codspeed.yml | |
| --warmup-time 1s | |
| --min-rounds 5 | |
| - name: Run the benchmarks | |
| uses: CodSpeedHQ/action@main | |
| with: | |
| mode: walltime | |
| config: bench/codspeed.yml |