Try again

[only benchmarks]

Author: christiangnrd

.buildkite/pipeline.yml

@@ -124,28 +124,23 @@ steps:
 
   - group: ":racehorse: Benchmarks"
     steps:
-      # benchmarks outside of the master branch don't submit their results,
-      # so they can run on any system in the juliagpu queue.
-      - label: "Benchmarks (dry run)"
+      - label: "Benchmarks"
         plugins:
           - JuliaCI/julia#v1:
               version: "1.10"
         command: |
-          julia --project -e '
+          julia --project=perf -e '
             using Pkg
 
             println("--- :julia: Instantiating project")
-            Pkg.resolve()
-            Pkg.instantiate()
-            Pkg.activate("perf")
-            Pkg.resolve()
+            Pkg.develop([PackageSpec(path=pwd())])
             Pkg.instantiate()
             push!(LOAD_PATH, @__DIR__)
 
             println("+++ :julia: Benchmarking")
             include("perf/runbenchmarks.jl")'
         artifact_paths:
-          - "results.json"
+          - "benchmarkresults.json"
         agents:
           queue: "juliaecosystem"
           os: "macos"
@@ -155,6 +150,35 @@ steps:
           build.message !~ /\[only/ && !build.pull_request.draft &&
             build.message !~ /\[skip benchmarks\]/
         timeout_in_minutes: 30
+      # - label: "Benchmarks (dry run)"
+      #   plugins:
+      #     - JuliaCI/julia#v1:
+      #         version: "1.10"
+      #   command: |
+      #     julia --project -e '
+      #       using Pkg
+
+      #       println("--- :julia: Instantiating project")
+      #       Pkg.resolve()
+      #       Pkg.instantiate()
+      #       Pkg.activate("perf")
+      #       Pkg.resolve()
+      #       Pkg.instantiate()
+      #       push!(LOAD_PATH, @__DIR__)
+
+      #       println("+++ :julia: Benchmarking")
+      #       include("perf/runbenchmarks.jl")'
+      #   artifact_paths:
+      #     - "results.json"
+      #   agents:
+      #     queue: "juliaecosystem"
+      #     os: "macos"
+      #     arch: "aarch64"
+      #   if: |
+      #     build.message =~ /\[only benchmarks\]/ ||
+      #     build.message !~ /\[only/ && !build.pull_request.draft &&
+      #       build.message !~ /\[skip benchmarks\]/
+      #   timeout_in_minutes: 30
 
       # if we will submit results, use the benchmark queue so that we will
       # be running on the same system each time

.github/workflows/Benchmark.yml

@@ -0,0 +1,63 @@
+name: Benchmarks
+permissions:
+  contents: write # contents permission to update benchmark contents in gh-pages branch
+  statuses: read
+  deployments: write # deployments permission to deploy GitHub pages website
+  pull-requests: write
+
+on:
+  pull_request:
+    branches:
+      - main
+    paths:
+      - "src/**/*"
+      - "ext/**/*"
+      - "perf/**/*"
+      - ".buildkite/**/*"
+      - "Project.toml"
+      - ".github/workflows/Benchmark.yml"
+  push:
+    branches:
+      - main
+    paths:
+      - "src/**/*"
+      - "ext/**/*"
+      - "benchmarks/**/*"
+      - ".buildkite/**/*"
+      - "Project.toml"
+      - ".github/workflows/Benchmark.yml"
+
+jobs:
+  benchmark:
+    if: ${{ !contains(github.event.head_commit.message, '[skip benchmarks]') }}
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Download Buildkite Artifacts
+        id: download
+        uses: EnricoMi/download-buildkite-artifact-action@v1
+        with:
+          buildkite_token: ${{ secrets.BUILDKITE_TOKEN }}
+          ignore_build_states: blocked,canceled,skipped,not_run,failed
+          ignore_job_states: timed_out,failed
+          output_path: artifacts
+
+      - name: Locate Benchmarks Artifact
+        id: locate
+        if: ${{ steps.download.outputs.download-state == 'success' }}
+        run: echo "path=$(find artifacts -type f -name benchmarkresults.json 2>/dev/null)" >> $GITHUB_OUTPUT
+
+      - name: Upload Benchmark Results
+        if: ${{ steps.locate.outputs.path != '' }}
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          name: Metal Benchmarks
+          tool: "julia"
+          output-file-path: ${{ steps.locate.outputs.path }}
+          benchmark-data-dir-path: ""
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          comment-always: true
+          summary-always: true
+          alert-threshold: "150%"
+          fail-on-alert: false
+          auto-push: ${{ github.event_name != 'pull_request' }}

perf/Project.toml

@@ -2,5 +2,6 @@
 BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
 HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3"
 JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
+Metal = "dde4c033-4e86-420c-a63e-0dd931031962"
 StableRNGs = "860ef19b-820b-49d6-a774-d7a799459cd3"
 StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"

perf/runbenchmarks.jl

@@ -72,7 +72,7 @@ println(results)
 ## comparison
 
 # write out the results
-BenchmarkTools.save("results.json", results)
+BenchmarkTools.save("benchmarkresults.json", results)
 
 # compare against previous results
 # TODO: store these results so that we can compare when benchmarking PRs