add codspeed benchmark runner

anonrig · anonrig · commit f17e8f19d10b · 2025-07-02T17:31:25.000-04:00
do not build bench-tools on windows

use local repository for google benchmark

revert ci.bazelrc changes
diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml
@@ -0,0 +1,68 @@
+name: CodSpeed
+
+on:
+  pull_request:
+    paths-ignore:
+      - 'docs/**'
+      - 'justfile'
+      - '.devcontainer'
+      - '**/*.md'
+      - '.gitignore'
+  merge_group:
+  push:
+    branches:
+      - main
+
+concurrency:
+  group: codspeed.yml-${{ github.event.pull_request.number || github.run_id }}
+  cancel-in-progress: true
+
+permissions:
+  # Write cache
+  contents: write
+
+jobs:
+  benchmarks:
+    name: Run benchmarks
+    runs-on: ubuntu-22.04
+    env:
+      BAZEL_ARGS: --config=benchmark --@codspeed//core:codspeed_mode=instrumentation --remote_cache=https://bazel:${{ secrets.BAZEL_CACHE_KEY }}@bazel-remote-cache.devprod.cloudflare.dev
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          show-progress: false
+
+      - name: Cache
+        id: cache
+        uses: actions/cache@v4
+        with:
+          path: ~/bazel-disk-cache
+          key: bazel-disk-cache-benchmarks-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.bazelversion', '.bazelrc', 'WORKSPACE') }}
+
+      - name: Setup Runner
+        uses: ./.github/actions/setup-runner
+
+      - name: Build benchmarks
+        run: |
+          bazel build ${{ env.BAZEL_ARGS }} //src/workerd/tests:all_benchmarks
+
+      - name: Generate benchmark script
+        run: |
+          echo '#!/bin/bash' > run_benchmarks.sh
+          echo 'set -e' >> run_benchmarks.sh
+          targets=$(bazel query 'deps(//src/workerd/tests:all_benchmarks, 1)' --output=label 2>/dev/null | grep -E '^//src/workerd/tests:bench-' | grep -v '@')
+          for target in $targets; do
+            # Get the binary path for each benchmark target
+            binary_path=$(bazel cquery ${{ env.BAZEL_ARGS }} --output=files "$target")
+            echo "echo 'Running benchmark: $target'" >> run_benchmarks.sh
+            echo "$binary_path" >> run_benchmarks.sh
+          done
+          chmod +x run_benchmarks.sh
+
+      - name: Run benchmarks
+        uses: CodSpeedHQ/action@v3
+        with:
+          run: ./run_benchmarks.sh
+
+      - name: Bazel shutdown
+        run: bazel shutdown
diff --git a/WORKSPACE b/WORKSPACE
@@ -232,6 +232,24 @@ new_local_repository(
     path = "empty",
 )
 
+# Tell workerd code where to find google-benchmark with CodSpeed.
+#
+# We indirect through `@workerd-google-benchmark` to allow dependents to override how and where
+# google-benchmark is built, similar to the v8 setup above.
+new_local_repository(
+    name = "workerd-google-benchmark",
+    build_file_content = """cc_library(
+        name = "benchmark",
+        deps = [ "@codspeed//google_benchmark:benchmark" ],
+        visibility = ["//visibility:public"])
+
+cc_library(
+        name = "benchmark_main",
+        deps = [ "@codspeed//google_benchmark:benchmark_main" ],
+        visibility = ["//visibility:public"])""",
+    path = "empty",
+)
+
 # rust-based lolhtml dependency, including the API header.
 # Presented as a separate repository to allow overrides.
 new_local_repository(
diff --git a/build/ci.bazelrc b/build/ci.bazelrc
@@ -43,7 +43,7 @@ build:ci-test --config=v8-codegen-opt
 # LLVM produces a bit more debug info on macOS by default to facilitate debugging with
 # LLDB. This is not needed for this use case, so disable it using -fno-standalone-debug –
 # this is already the default for Linux/Windows.
-build:ci-limit-storage --build_tag_filters=-off-by-default,-benchmark
+build:ci-limit-storage --build_tag_filters=-off-by-default,-workerd-benchmark
 build:ci-limit-storage --copt="-g1"
 build:ci-limit-storage --copt="-fno-standalone-debug"
 
diff --git a/build/deps/build_deps.jsonc b/build/deps/build_deps.jsonc
@@ -125,9 +125,9 @@
     // microbenchmark
     {
       "type": "github_release",
-      "name": "com_google_benchmark",
-      "owner": "google",
-      "repo": "benchmark"
+      "name": "codspeed",
+      "owner": "CodSpeedHQ",
+      "repo": "codspeed-cpp"
     }
   ]
 }
diff --git a/build/deps/gen/build_deps.bzl b/build/deps/gen/build_deps.bzl
@@ -11,7 +11,7 @@ load("@//build/deps:gen/dep_cargo_bazel_win_x64.bzl", "dep_cargo_bazel_win_x64")
 load("@//build/deps:gen/dep_clang_format_darwin_arm64.bzl", "dep_clang_format_darwin_arm64")
 load("@//build/deps:gen/dep_clang_format_linux_amd64.bzl", "dep_clang_format_linux_amd64")
 load("@//build/deps:gen/dep_clang_format_linux_arm64.bzl", "dep_clang_format_linux_arm64")
-load("@//build/deps:gen/dep_com_google_benchmark.bzl", "dep_com_google_benchmark")
+load("@//build/deps:gen/dep_codspeed.bzl", "dep_codspeed")
 load("@//build/deps:gen/dep_rules_nodejs.bzl", "dep_rules_nodejs")
 load("@//build/deps:gen/dep_rules_python.bzl", "dep_rules_python")
 load("@//build/deps:gen/dep_rules_rust.bzl", "dep_rules_rust")
@@ -33,4 +33,4 @@ def deps_gen():
     dep_clang_format_linux_amd64()
     dep_clang_format_linux_arm64()
     dep_clang_format_darwin_arm64()
-    dep_com_google_benchmark()
+    dep_codspeed()
diff --git a/build/deps/gen/dep_codspeed.bzl b/build/deps/gen/dep_codspeed.bzl
@@ -0,0 +1,18 @@
+# WARNING: THIS FILE IS AUTOGENERATED BY update-deps.py DO NOT EDIT
+
+load("@//:build/http.bzl", "http_archive")
+
+TAG_NAME = "v1.2.0"
+URL = "https://api.github.com/repos/CodSpeedHQ/codspeed-cpp/tarball/v1.2.0"
+STRIP_PREFIX = "CodSpeedHQ-codspeed-cpp-719c41f"
+SHA256 = "1ced2c2e813313a574f41de9a218f38b53a4114cf7e72ee286801eaba6d8b240"
+TYPE = "tgz"
+
+def dep_codspeed():
+    http_archive(
+        name = "codspeed",
+        url = URL,
+        strip_prefix = STRIP_PREFIX,
+        type = TYPE,
+        sha256 = SHA256,
+    )
diff --git a/build/deps/gen/dep_com_google_benchmark.bzl b/build/deps/gen/dep_com_google_benchmark.bzl
diff --git a/build/wd_cc_benchmark.bzl b/build/wd_cc_benchmark.bzl
@@ -25,12 +25,12 @@ def wd_cc_benchmark(
         }),
         visibility = visibility,
         deps = deps + [
-            "@com_google_benchmark//:benchmark_main",
+            "@workerd-google-benchmark//:benchmark_main",
             "//src/workerd/tests:bench-tools",
         ],
         # use the same malloc we use for server
         malloc = "//src/workerd/server:malloc",
-        tags = ["benchmark"],
+        tags = ["workerd-benchmark"],
         **kwargs
     )
 
@@ -40,5 +40,5 @@ def wd_cc_benchmark(
         outs = [name + ".benchmark.csv"],
         srcs = [name],
         cmd = "./$(location {}) --benchmark_format=csv > \"$@\"".format(name),
-        tags = ["off-by-default", "benchmark_report"],
+        tags = ["off-by-default", "benchmark_report", "workerd-benchmark"],
     )
diff --git a/compile_flags.txt b/compile_flags.txt
@@ -9,7 +9,7 @@
 -Iexternal/abseil-cpp/
 -Iexternal/simdutf/
 -Iexternal/nbytes/include/
--Iexternal/com_google_benchmark/include/
+-Iexternal/codspeed/google_benchmark/include/
 -Iexternal/perfetto/include/
 -Iexternal/perfetto/include/perfetto/base/build_configs/bazel/
 -Iexternal/ssl/include
diff --git a/doxyfile b/doxyfile
@@ -1211,7 +1211,7 @@ CLANG_OPTIONS          = -std=c++20     \
                          -Wunused-lambda-capture \
                          -Wunused-variable \
                          -no-canonical-prefixes \
-                         -Iexternal/com_google_benchmark/include/ \
+                         -Iexternal/codspeed/google_benchmark/include/ \
                          -Iexternal/ada-url/ \
                          -Iexternal/simdutf/ \
                          -Iexternal/nbytes/ \
diff --git a/justfile b/justfile
@@ -99,3 +99,6 @@ generate-types:
 
 create-external:
   tools/unix/create-external.sh
+
+bench-all:
+  bazel query 'deps(//src/workerd/tests:all_benchmarks, 1)' --output=label | grep -v 'all_benchmarks' | xargs -I {} bazel run --config=benchmark {}
diff --git a/src/workerd/tests/BUILD.bazel b/src/workerd/tests/BUILD.bazel
@@ -6,10 +6,11 @@ load("//:build/wd_test.bzl", "wd_test")
 wd_cc_library(
     name = "bench-tools",
     hdrs = ["bench-tools.h"],
+    tags = ["workerd-benchmark"],
     visibility = ["//visibility:public"],
     deps = [
         "@capnp-cpp//src/kj:kj-test",
-        "@com_google_benchmark//:benchmark",
+        "@workerd-google-benchmark//:benchmark",
     ],
 )
 
@@ -91,3 +92,17 @@ wd_test(
     args = ["--experimental"],
     data = ["unknown-import-assertions-test.js"],
 )
+
+filegroup(
+    name = "all_benchmarks",
+    srcs = [
+        ":bench-api-headers",
+        ":bench-fast-api",
+        ":bench-global-scope",
+        ":bench-json",
+        ":bench-kj-headers",
+        ":bench-mimetype",
+        ":bench-regex",
+    ],
+    visibility = ["//visibility:public"],
+)

Original file line number	Diff line number	Diff line change
`@@ -125,9 +125,9 @@`
`125`	`125`	`// microbenchmark`
`126`	`126`	`{`
`127`	`127`	`"type": "github_release",`
`128`		`- "name": "com_google_benchmark",`
`129`		`- "owner": "google",`
`130`		`- "repo": "benchmark"`
	`128`	`+ "name": "codspeed",`
	`129`	`+ "owner": "CodSpeedHQ",`
	`130`	`+ "repo": "codspeed-cpp"`
`131`	`131`	`}`
`132`	`132`	`]`
`133`	`133`	`}`