Merge branch 'main' into per-runner-loops

Author: mdboom

.github/workflows/pypi.yml

@@ -0,0 +1,97 @@
+name: Publish to PyPI
+
+on:
+  push:
+    branches:
+      - main
+    tags:
+      - 'v*.*.*'
+
+jobs:
+  build:
+    name: Build distribution 📦
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        persist-credentials: false
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: "3.x"
+    - name: Install pypa/build
+      run: |
+        python3 -m pip install build --user
+    - name: Build a binary wheel and a source tarball
+      run: |
+        python3 -m build
+    - name: Store the distribution packages
+      uses: actions/upload-artifact@v4
+      with:
+        name: python-package-distributions
+        path: dist/
+
+  publish-to-pypi:
+    name: Publish to PyPI
+    if: startsWith(github.ref, 'refs/tags/')  # only publish to PyPI on tag pushes
+    needs:
+    - build
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/bench_runner
+    permissions:
+      id-token: write
+
+    steps:
+    - name: Download all the dists
+      uses: actions/download-artifact@v4
+      with:
+        name: python-package-distributions
+        path: dist/
+    - name: Publish distribution 📦 to PyPI
+      uses: pypa/gh-action-pypi-publish@release/v1
+
+  github-release:
+    name: >-
+        Sign the Python 🐍 distribution 📦 with Sigstore
+        and upload them to GitHub Release
+    needs:
+    - publish-to-pypi
+    runs-on: ubuntu-latest
+
+    permissions:
+      contents: write  # IMPORTANT: mandatory for making GitHub Releases
+      id-token: write  # IMPORTANT: mandatory for sigstore
+
+    steps:
+    - name: Download all the dists
+      uses: actions/download-artifact@v4
+      with:
+        name: python-package-distributions
+        path: dist/
+    - name: Sign the dists with Sigstore
+      uses: sigstore/[email protected]
+      with:
+        inputs: >-
+          ./dist/*.tar.gz
+          ./dist/*.whl
+    - name: Create GitHub Release
+      env:
+        GITHUB_TOKEN: ${{ github.token }}
+      run: >-
+        gh release create
+        "$GITHUB_REF_NAME"
+        --repo "$GITHUB_REPOSITORY"
+        --notes ""
+    - name: Upload artifact signatures to GitHub Release
+      env:
+        GITHUB_TOKEN: ${{ github.token }}
+        # Upload to GitHub Release using the `gh` CLI.
+        # `dist/` contains the built packages, and the
+        # sigstore-produced signatures and certificates.
+      run: >-
+        gh release upload
+        "$GITHUB_REF_NAME" dist/**
+        --repo "$GITHUB_REPOSITORY"

PROVISIONING.md

@@ -19,6 +19,8 @@ sudo apt install python3 build-essential ccache gdb lcov pkg-config \
       lzma lzma-dev tk-dev uuid-dev zlib1g-dev linux-perf
 ```
 
+The self-hosted runner needs to have `python` be Python 3.9 or later.
+
 ### Enable ccache
 
 ```bash session

README.md

@@ -10,10 +10,10 @@ For example, you can see [the Faster CPython team's benchmarking results](https:
 
 Create a new empty repository on Github and clone it locally.
 
-Add bench_runner to your `requirements.txt`. Since there are no PyPI releases (yet), you can install it from a tag in the git repo:
+Add bench_runner to your `requirements.txt`.
 
 ```text
-git+https://github.com/faster-cpython/bench_runner@{VERSION}#egg=bench_runner
+bench_runner=={VERSION}
 ```
 
 Replace the {VERSION} above with the latest version tag of `bench_runner`.

bench_runner/gh.py

@@ -74,6 +74,10 @@ def send_notification(body):
     conf = config.get_bench_runner_config()
     notification_issue = conf.get("notify", {}).get("notification_issue", 0)
 
+    if notification_issue == 0:
+        print("Not sending Github notification.")
+        return
+
     print("Sending Github notification:")
     print("---")
     print(body)

bench_runner/plot.py

@@ -65,7 +65,9 @@ class Options:
     plt.close("all")
 
     if output_filename.suffix == ".svg":
-        with tempfile.NamedTemporaryFile(delete=False) as tmp:
+        with tempfile.NamedTemporaryFile(
+            dir=output_filename.parent, delete=False
+        ) as tmp:
             with open(output_filename) as fd:
                 scour.start(Options(), fd, tmp)
             output_filename.unlink()

bench_runner/scripts/run_benchmarks.py

@@ -86,6 +86,9 @@ def run_benchmarks(
     if loops_file := os.environ.get(LOOPS_FILE_ENV_VAR):
         extra_args.append("--same-loops")
         extra_args.append(loops_file)
+        
+    if affinity := os.environ.get("CPU_AFFINITY"):
+        extra_args.append(f"--affinity={affinity}")
 
     args = [
         sys.executable,

bench_runner/templates/_benchmark.src.yml

@@ -112,7 +112,7 @@ jobs:
         shell: cmd
         run: |
           cd cpython
-          PCbuild\build.bat %BUILD_FLAGS% ${{ (inputs.pgo == true) && '--pgo' || '' }} ${{ inputs.jit == true && '--experimental-jit' || '' }} ${{ inputs.tier2 == true && '--experimental-jit-interpreter' || '' }} ${{ inputs.nogil == true && '--disable-gil' || '' }} -c Release ${{ inputs.clang == true && '"/p:PlatformToolset=clangcl"' || '' }} ${{ inputs.clang == true && '"/p:LLVMInstallDir=C:\Program Files\LLVM"' || '' }} ${{ inputs.clang == true && '"/p:LLVMToolsVersion=19.1.6"' || '' }} ${{ inputs.clang == true && '--tail-call-interp' || '' }}
+          PCbuild\build.bat %BUILD_FLAGS% ${{ (inputs.pgo == true) && '--pgo' || '' }} ${{ inputs.clang == true && '--tail-call-interp' || '' }} ${{ inputs.jit == true && '--experimental-jit' || '' }} ${{ inputs.tier2 == true && '--experimental-jit-interpreter' || '' }} ${{ inputs.nogil == true && '--disable-gil' || '' }} -c Release ${{ inputs.clang == true && '"/p:PlatformToolset=clangcl"' || '' }} ${{ inputs.clang == true && '"/p:LLVMInstallDir=C:\Program Files\LLVM"' || '' }} ${{ inputs.clang == true && '"/p:LLVMToolsVersion=19.1.6"' || '' }}
       - name: Copy Python to different location
         if: ${{ steps.should_run.outputs.should_run != 'false' }}
         run: |
@@ -161,11 +161,6 @@ jobs:
         run: |
           git gc
       - uses: fregante/setup-git-user@v2
-      - name: Setup system Python
-        if: ${{ runner.arch == 'X64' }}
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.11"
       - name: Checkout CPython
         uses: actions/checkout@v4
         with:
@@ -211,7 +206,7 @@ jobs:
         if: ${{ steps.should_run.outputs.should_run != 'false' }}
         run: |
           cd cpython
-          ./configure ${{ inputs.pgo == true && '--enable-optimizations --with-lto=full' || '' }} ${{ inputs.tier2 == true && '--enable-experimental-jit=interpreter' || '' }} ${{ inputs.jit == true && '--enable-experimental-jit=yes' || '' }} ${{ inputs.nogil == true && '--disable-gil' || '' }} ${{ inputs.clang == true && '--with-tail-call-interp' || '' }}
+          ./configure --enable-option-checking=fatal ${{ inputs.pgo == true && '--enable-optimizations --with-lto=full' || '' }} ${{ inputs.tier2 == true && '--enable-experimental-jit=interpreter' || '' }} ${{ inputs.jit == true && '--enable-experimental-jit=yes' || '' }} ${{ inputs.nogil == true && '--disable-gil' || '' }} ${{ inputs.clang == true && '--with-tail-call-interp' || '' }} ${PYTHON_CONFIGURE_FLAGS:-}
           make ${{ runner.arch == 'ARM64' && '-j' || '-j4' }}
           ./python -VV
       - name: Install pyperformance
@@ -221,7 +216,7 @@ jobs:
       - name: Tune system
         if: ${{ steps.should_run.outputs.should_run != 'false' }}
         run: |
-          sudo LD_LIBRARY_PATH=$LD_LIBRARY_PATH venv/bin/python -m pyperf system ${{ inputs.perf && 'reset' || 'tune' }}
+          sudo LD_LIBRARY_PATH=$LD_LIBRARY_PATH venv/bin/python -m pyperf system ${{ inputs.perf && 'reset' || 'tune ${CPU_AFFINITY:+--affinity="$CPU_AFFINITY"}' }}
       - name: Tune for (Linux) perf
         if: ${{ steps.should_run.outputs.should_run != 'false' && inputs.perf }}
         run: |
@@ -232,6 +227,10 @@ jobs:
         run: |
           rm -rf ~/.debug/*
           venv/bin/python -m bench_runner run_benchmarks ${{ inputs.perf && 'perf' || 'benchmark' }} cpython/python ${{ inputs.fork }} ${{ inputs.ref }} ${{ inputs.benchmarks || 'all' }} ${{ env.flags }} --run_id ${{ github.run_id }}
+      - name: Untune system
+        if: ${{ steps.should_run.outputs.should_run != 'false' }}
+        run: |
+          sudo LD_LIBRARY_PATH=$LD_LIBRARY_PATH venv/bin/python -m pyperf system reset
       # Pull again, since another job may have committed results in the meantime
       - name: Pull benchmarking
         if: ${{ steps.should_run.outputs.should_run != 'false' && !inputs.perf }}
@@ -321,11 +320,9 @@ jobs:
         if: ${{ steps.should_run.outputs.should_run != 'false' }}
         run: |
           cd cpython
-          ./configure ${{ inputs.pgo == true && '--enable-optimizations --with-lto=full' || '' }} ${{ inputs.tier2 == true && '--enable-experimental-jit=interpreter' || '' }} ${{ inputs.jit == true && '--enable-experimental-jit=yes' || '' }} ${{ inputs.nogil == true && '--disable-gil' || '' }} ${{ inputs.clang == true && '--with-tail-call-interp' || '' }}
+          ./configure --enable-option-checking=fatal ${{ inputs.pgo == true && '--enable-optimizations --with-lto=full' || '' }} ${{ inputs.tier2 == true && '--enable-experimental-jit=interpreter' || '' }} ${{ inputs.jit == true && '--enable-experimental-jit=yes' || '' }} ${{ inputs.nogil == true && '--disable-gil' || '' }} ${{ inputs.clang == true && '--with-tail-call-interp' || '' }} ${PYTHON_CONFIGURE_FLAGS:-}
           make -j4
           ./python.exe -VV
-      # On macos ARM64, actions/setup-python isn't available, so we rely on a
-      # pre-installed homebrew one, used through a venv
       - name: Install pyperformance
         if: ${{ steps.should_run.outputs.should_run != 'false' }}
         run: |

bench_runner/templates/_pystats.src.yml

@@ -100,7 +100,7 @@ jobs:
         if: ${{ steps.should_run.outputs.should_run != 'false' }}
         run: |
           cd cpython
-          ./configure --enable-pystats --prefix=$PWD/install ${{ inputs.tier2 == true && '--enable-experimental-jit=interpreter' || '' }} ${{ inputs.jit == true && '--enable-experimental-jit=yes' || '' }} ${{ inputs.nogil == true && '--disable-gil' || '' }}
+          ./configure --enable-option-checking=fatal --enable-pystats --prefix=$PWD/install ${{ inputs.tier2 == true && '--enable-experimental-jit=interpreter' || '' }} ${{ inputs.jit == true && '--enable-experimental-jit=yes' || '' }} ${{ inputs.nogil == true && '--disable-gil' || '' }}
           make -j4
           make install
       - name: Install pyperformance into the system python

bench_runner/templates/bench_runner.toml

@@ -24,7 +24,24 @@ notification_issue = 182
 #   socket.gethostname()).
 # available: True if the machine is available (we need to keep the
 #   metadata around for old machines for the generated results). (Default: True)
-
+# include_in_all: Whether to include the runner when choosing to run a
+#   benchmark on all runners.
+# env: a table of extra environment variables to pass to the runs.
+#
+# A few notable environment variables for runners:
+# CC, CXX: used by Python's configure to determine the compiler to use.
+# LLVM_AR, LLVM_PROFDATA: used for PGO optimization passes with clang, when
+#   using a version of clang that isn't the default. These must be full
+#   paths for configure to handle them correctly.
+# LLVM_BOLT, MERGE_FDATA: like AR/PROFDATA, but for BOLT.
+#   LLVM_BOLT may need to point to, e.g., /usr/lib/llvm-19/bin/llvm-bolt
+#   rather than /usr/bin/llvm-bolt-19 (see
+#   https://github.com/python/cpython/issues/127047).
+# PYTHON_CONFIGURE_FLAGS: extra flags to pass to Python's configure. These
+#   can override all earlier flags, which by default include
+#   --enable-optimizations and --with-lto=full.
+# BUILD_DEST: (windows only) where to make build.bat install. This is
+#   required for the Windows build to succeed.
 [[runners]]
 
 [runners.linux]
@@ -33,6 +50,13 @@ arch = "x86_64"
 hostname = "pyperf"
 available = true
 
+[runners.windows]
+os = "windows"
+arch = "x86_64"
+hostname = "winperf"
+available = true
+env.BUILD_DEST = "PCbuild/amd64"
+
 [plot]
 bases = ["3.11.0"]
 versions = [[3, 12]]

pyproject.toml

@@ -17,7 +17,7 @@ classifiers = [
 dependencies = [
     "matplotlib==3.10.1",
     "pyperf==2.9.0",
-    "rich==13.9.4",
+    "rich==14.0.0",
     "rich-argparse==1.7.0",
     "ruamel.yaml==0.18.10",
     "scour==0.38.2",
@@ -30,8 +30,8 @@ dynamic = ["version"]
 test = [
     "black==25.1.0",
     "filelock==3.18.0",
-    "flake8==7.1.2",
-    "pyright==1.1.397",
+    "flake8==7.2.0",
+    "pyright==1.1.398",
     "pytest==8.3.5",
     "pytest-xdist==3.6.1",
 ]