Merge pull request #5443 from matrix-org/poljar/ci/benchmarks

Enable benchmarks on the CI

Author: poljar

.github/workflows/benchmarks.yml

@@ -1,54 +1,37 @@
 name: Benchmarks
 
 on:
+  push:
+    branches:
+      - "main"
+  pull_request:
   workflow_dispatch:
 
 jobs:
   benchmarks:
     name: Run Benchmarks
     runs-on: ubuntu-latest
-    environment: matrix-rust-bot
-    if: github.event_name == 'push'
+    strategy:
+      matrix:
+        benchmark:
+          - crypto_bench
+          - event_cache
 
     steps:
-    - name: Checkout the repo
-      uses: actions/checkout@v4
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
 
-    - name: Install Rust
-      uses: dtolnay/rust-toolchain@master
-      with:
-        toolchain: nightly-2025-06-27
-        components: rustfmt
+      - name: Setup rust toolchain, cache and cargo-codspeed binary
+        uses: moonrepo/setup-rust@ede6de059f8046a5e236c94046823e2af11ca670
+        with:
+          channel: stable
+          cache-target: release
+          bins: cargo-codspeed
 
-    - name: Run Benchmarks
-      run: cargo bench | tee benchmark-output.txt
+      - name: Build the benchmark target(s)
+        run: cargo codspeed build -p benchmarks ${{ matrix.benchmark }} --features codspeed
 
-    - name: Check benchmark result for PR
-      if: github.event_name == 'pull_request'
-      uses: benchmark-action/github-action-benchmark@v1
-      with:
-        name: Rust Benchmark
-        tool: 'cargo'
-        output-file-path: benchmark-output.txt
-        auto-push: false
-        # comment to alert the user this has gone bad
-        github-token: ${{ secrets.MRB_ACCESS_TOKEN }}
-        alert-threshold: '120%'
-        comment-on-alert: true
-        fail-threshold: '150%'
-        fail-on-alert: true
-
-    - name: Store benchmark result
-      if: github.event_name != 'pull_request'
-      uses: benchmark-action/github-action-benchmark@v1
-      with:
-        name: Rust Benchmark
-        tool: 'cargo'
-        output-file-path: benchmark-output.txt
-        github-token: ${{ secrets.GITHUB_TOKEN }}
-        auto-push: true
-        # Show alert with commit comment on detecting possible performance regression
-        alert-threshold: '150%'
-        comment-on-alert: true
-        fail-on-alert: true
-        alert-comment-cc-users: '@gnunicornBen,@jplatte,@poljar'
+      - name: Run the benchmarks
+        uses: CodSpeedHQ/action@0b6e7a3d96c9d2a6057e7bcea6b45aaf2f7ce60b
+        with:
+          run: cargo codspeed run
+          token: ${{ secrets.CODSPEED_TOKEN }}