Skip to content
Build and Test libc++

[libc++] Add a job that runs the benchmarks as part of the CI #13706

Sign in to view logs

[libc++] Add a job that runs the benchmarks as part of the CI

[libc++] Add a job that runs the benchmarks as part of the CI #13706

Workflow file for this run

.github/workflows/libcxx-build-and-test.yaml at 9742fe0
# This file defines pre-commit CI for libc++, libc++abi, and libunwind (on Github).
#
# We split the configurations in multiple stages with the intent of saving compute time
# when a job fails early in the pipeline. This is why the jobs are marked as `continue-on-error: false`.
# We try to run the CI configurations with the most signal in the first stage.
#
# Stages 1 & 2 are meant to be "smoke tests", and are meant to catch most build/test failures quickly and without using
# too many resources.
# Stage 3 is "everything else", and is meant to catch breakages on more niche or unique configurations.
#
# Therefore, we "fail-fast" for any failures during stages 1 & 2, meaning any job failing cancels all other running jobs,
# under the assumption that if the "smoke tests" fail, then the other configurations will likely fail in the same way.
# However, stage 3 does not fail fast, as it's more likely that any one job failing is a flake or a configuration-specific
#
name: Build and Test libc++
on:
pull_request:
paths:
- 'libcxx/**'
- 'libcxxabi/**'
- 'libunwind/**'
- 'runtimes/**'
- 'cmake/**'
- '.github/workflows/libcxx-build-and-test.yaml'
schedule:
# Run nightly at 08:00 UTC (aka 00:00 Pacific, aka 03:00 Eastern)
- cron: '0 8 * * *'
permissions:
contents: read # Default everything to read-only
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
cancel-in-progress: true
jobs:
# stage1:
# if: github.repository_owner == 'llvm'
# runs-on: llvm-premerge-libcxx-next-runners
# continue-on-error: false
# strategy:
# fail-fast: false
# matrix:
# config: [
# 'frozen-cxx03-headers',
# 'generic-cxx03',
# 'generic-cxx26',
# 'generic-modules'
# ]
# cc: [ 'clang-22' ]
# cxx: [ 'clang++-22' ]
# include:
# - config: 'generic-gcc'
# cc: 'gcc-15'
# cxx: 'g++-15'
# steps:
# - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
# - name: ${{ matrix.config }}.${{ matrix.cxx }}
# run: libcxx/utils/ci/run-buildbot ${{ matrix.config }}
# env:
# CC: ${{ matrix.cc }}
# CXX: ${{ matrix.cxx }}
# - uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 # v4.3.0
# if: always()
# with:
# name: ${{ matrix.config }}-${{ matrix.cxx }}-results
# path: |
# **/test-results.xml
# **/*.abilist
# **/CMakeConfigureLog.yaml
# **/CMakeError.log
# **/CMakeOutput.log
# **/crash_diagnostics/*
# stage2:
# if: github.repository_owner == 'llvm'
# runs-on: llvm-premerge-libcxx-next-runners
# needs: [ stage1 ]
# continue-on-error: false
# strategy:
# fail-fast: false
# matrix:
# config: [
# 'generic-cxx11',
# 'generic-cxx14',
# 'generic-cxx17',
# 'generic-cxx20',
# 'generic-cxx23'
# ]
# cc: [ 'clang-22' ]
# cxx: [ 'clang++-22' ]
# include:
# - config: 'generic-gcc-cxx11'
# cc: 'gcc-15'
# cxx: 'g++-15'
# - config: 'generic-cxx26'
# cc: 'clang-21'
# cxx: 'clang++-21'
# - config: 'generic-cxx26'
# cc: 'clang-20'
# cxx: 'clang++-20'
# steps:
# - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
# - name: ${{ matrix.config }}
# run: libcxx/utils/ci/run-buildbot ${{ matrix.config }}
# env:
# CC: ${{ matrix.cc }}
# CXX: ${{ matrix.cxx }}
# - uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 # v4.3.0
# if: always() # Upload artifacts even if the build or test suite fails
# with:
# name: ${{ matrix.config }}-${{ matrix.cxx }}-results
# path: |
# **/test-results.xml
# **/*.abilist
# **/CMakeConfigureLog.yaml
# **/CMakeError.log
# **/CMakeOutput.log
# **/crash_diagnostics/*
# stage3:
# if: github.repository_owner == 'llvm'
# needs: [ stage2 ]
# continue-on-error: false
# strategy:
# fail-fast: false
# max-parallel: 8
# matrix:
# config: [
# 'generic-abi-unstable',
# 'generic-hardening-mode-debug',
# 'generic-hardening-mode-extensive',
# 'generic-hardening-mode-extensive-observe-semantic',
# 'generic-hardening-mode-fast',
# 'generic-hardening-mode-fast-with-abi-breaks',
# 'generic-merged',
# 'generic-modules-cxx17-lsv',
# 'generic-no-exceptions',
# 'generic-no-experimental',
# 'generic-no-filesystem',
# 'generic-no-localization',
# 'generic-no-terminal',
# 'generic-no-random_device',
# 'generic-no-threads',
# 'generic-no-tzdb',
# 'generic-no-unicode',
# 'generic-no-wide-characters',
# 'generic-no-rtti',
# 'generic-optimized-speed',
# 'generic-static',
# 'bootstrapping-build'
# ]
# machine: [ 'llvm-premerge-libcxx-next-runners' ]
# include:
# - config: 'generic-cxx26'
# machine: llvm-premerge-libcxx-next-runners
# - config: 'generic-asan'
# machine: llvm-premerge-libcxx-next-runners
# - config: 'generic-tsan'
# machine: llvm-premerge-libcxx-next-runners
# - config: 'generic-ubsan'
# machine: llvm-premerge-libcxx-next-runners
# # Use a larger machine for MSAN to avoid timeout and memory allocation issues.
# - config: 'generic-msan'
# machine: llvm-premerge-libcxx-next-runners
# runs-on: ${{ matrix.machine }}
# steps:
# - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
# - name: ${{ matrix.config }}
# run: libcxx/utils/ci/run-buildbot ${{ matrix.config }}
# env:
# CC: clang-22
# CXX: clang++-22
# - uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 # v4.3.0
# if: always()
# with:
# name: ${{ matrix.config }}-results
# path: |
# **/test-results.xml
# **/*.abilist
# **/CMakeConfigureLog.yaml
# **/CMakeError.log
# **/CMakeOutput.log
# **/crash_diagnostics/*
# macos:
# needs: [ stage2 ]
# strategy:
# fail-fast: false
# matrix:
# include:
# - config: generic-cxx03
# os: macos-15
# - config: generic-cxx23
# os: macos-15
# - config: generic-modules
# os: macos-15
# - config: apple-configuration
# os: macos-15
# # TODO: These jobs are intended to test back-deployment (building against ToT libc++ but running against an
# # older system-provided libc++.dylib). Doing this properly would require building the test suite on a
# # recent macOS using a recent Clang (hence recent Xcode), and then running the actual test suite on an
# # older mac. We could do that by e.g. sharing artifacts between the two jobs.
# #
# # However, our Lit configuration currently doesn't provide a good way to do that in a batch, so our only
# # alternative is to actually build on the same host that we're going to run on. Sadly, that doesn't work
# # since older macOSes don't support newer Xcodes. For now, we run the "backdeployment" jobs on recent
# # macOS versions as a way to avoid rotting that configuration, but it doesn't provide a lot of additional
# # coverage.
# - config: apple-system
# os: macos-15
# - config: apple-system-hardened
# os: macos-15
# runs-on: ${{ matrix.os }}
# steps:
# - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
# - uses: maxim-lobanov/setup-xcode@60606e260d2fc5762a71e64e74b2174e8ea3c8bd # v1.6.0
# with:
# # https://github.com/actions/runner-images/blob/main/images/macos/macos-15-Readme.md
# xcode-version: '16.3'
# - uses: seanmiddleditch/gha-setup-ninja@3b1f8f94a2f8254bd26914c4ab9474d4f0015f67 # v6
# - name: Build and test
# run: |
# python3 -m venv .venv
# source .venv/bin/activate
# python -m pip install psutil
# bash libcxx/utils/ci/run-buildbot ${{ matrix.config }}
# - uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 # v4.3.0
# if: always() # Upload artifacts even if the build or test suite fails
# with:
# name: macos-${{ matrix.config }}-results
# path: |
# **/test-results.xml
# **/*.abilist
# **/CMakeConfigureLog.yaml
# **/CMakeError.log
# **/CMakeOutput.log
# **/crash_diagnostics/*
# windows:
# runs-on: windows-2022
# needs: [ stage2 ]
# strategy:
# fail-fast: false
# matrix:
# include:
# - { config: clang-cl-dll, mingw: false }
# - { config: clang-cl-static, mingw: false }
# - { config: clang-cl-no-vcruntime, mingw: false }
# - { config: clang-cl-debug, mingw: false }
# - { config: clang-cl-static-crt, mingw: false }
# - { config: mingw-dll, mingw: true }
# - { config: mingw-static, mingw: true }
# - { config: mingw-dll-i686, mingw: true }
# - { config: mingw-incomplete-sysroot, mingw: true }
# steps:
# - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
# - name: Install dependencies
# run: |
# choco install -y ninja
# pip install psutil
# - name: Install a current LLVM
# if: ${{ matrix.mingw != true }}
# run: |
# choco install -y llvm --version=20.1.8 --allow-downgrade
# - name: Install llvm-mingw
# if: ${{ matrix.mingw == true }}
# run: |
# curl -LO https://github.com/mstorsjo/llvm-mingw/releases/download/20250709/llvm-mingw-20250709-ucrt-x86_64.zip
# powershell Expand-Archive llvm-mingw*.zip -DestinationPath .
# del llvm-mingw*.zip
# mv llvm-mingw* c:\llvm-mingw
# echo "c:\llvm-mingw\bin" | Out-File -FilePath $Env:GITHUB_PATH -Encoding utf8 -Append
# - name: Simulate a from-scratch build of llvm-mingw
# if: ${{ matrix.config == 'mingw-incomplete-sysroot' }}
# run: |
# rm -r c:\llvm-mingw\include\c++
# rm -r c:\llvm-mingw\*-w64-mingw32\lib\libc++*
# rm -r c:\llvm-mingw\*-w64-mingw32\lib\libunwind*
# - name: Add Git Bash to the path
# run: |
# echo "c:\Program Files\Git\usr\bin" | Out-File -FilePath $Env:GITHUB_PATH -Encoding utf8 -Append
# - name: Set up the MSVC dev environment
# if: ${{ matrix.mingw != true }}
# uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1.13.0
# - name: Build and test
# run: |
# bash libcxx/utils/ci/run-buildbot ${{ matrix.config }}
benchmarks:
runs-on: llvm-premerge-libcxx-next-runners # TODO: This should run on a dedicated set of machines
# needs: [ stage3 ] # TODO: Run this at the end of the pipeline once we're done testing the job
steps:
- uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
# with:
# fetch-depth: 0 # This job requires access to all the Git branches so it can diff against (usually) main
# fetch-tags: true
- name: benchmarks
# TODO: python3-virtualenv should be taken from the Docker image instead
# libcxx/utils/ci/run-buildbot benchmark-comparison
run: |
python3 -m pip install virtualenv
python3 -m venv .venv
source .venv/bin/activate
python -m pip install -r libcxx/utils/requirements.txt
env:
CC: clang-22
CXX: clang++-22
- uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 # v4.3.0
if: always()
with:
name: benchmarks-results
path: |
**/test-results.xml
**/*.abilist
**/CMakeConfigureLog.yaml
**/CMakeError.log
**/CMakeOutput.log
**/crash_diagnostics/*