tools: add benchmark runner to test-shared

Author: aduh95

.github/workflows/test-shared.yml

@@ -1,4 +1,4 @@
-name: Test Shared librairies
+name: Benchmark and shared librairies
 
 on:
   pull_request:
@@ -19,6 +19,30 @@ on:
       - README.md
       - .github/**
       - '!.github/workflows/test-shared.yml'
+  workflow_dispatch:
+    inputs:
+      repo:
+        type: string
+        description: 'GitHub repository to fetch from (default to the current repo)'
+      pr_id:
+        type: number
+        required: true
+        description: 'The PR to test'
+      commit:
+        required: true
+        type: string
+        description: 'The expect HEAD of the PR'
+      category:
+        required: true
+        type: string
+        description: The category (or categories) of tests to run, for example buffers, cluster etc. Maps to a folders in node/benchmark
+      filter:
+        type: string
+        description: A substring to restrict the benchmarks to run in a category. e.g. `net-c2c`
+      runs:
+        type: number
+        default: 30
+        description: How many times to repeat each benchmark
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
@@ -44,12 +68,28 @@ jobs:
             system: aarch64-linux
           - runner: macos-13
             system: x86_64-darwin
-          - runner: macos-14
+          - runner: macos-latest
             system: aarch64-darwin
     name: '${{ matrix.system }}: with shared libraries'
     runs-on: ${{ matrix.runner }}
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        if: ${{ github.event_name != 'workflow_dispatch' }}
+
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        if: ${{ github.event_name == 'workflow_dispatch' }}
+        with:
+          repository: ${{ inputs.repo || github.repository }}
+          ref: refs/pull/${{ inputs.pr_id }}/merge
+          fetch-depth: 2
+
+      - name: Validate PR head and roll back to base commit
+        if: ${{ github.event_name == 'workflow_dispatch' }}
+        run: |
+          [ "$(git rev-parse HEAD^2)" = "$EXPECTED_SHA" ]
+          git reset HEAD^ --hard
+        env:
+          EXPECTED_SHA: ${{ inputs.commit }}
 
       - uses: cachix/install-nix-action@f0fe604f8a612776892427721526b4c7cfb23aba # v31
         with:
@@ -71,10 +111,15 @@ jobs:
             core.exportVariable('ACTIONS_RESULTS_URL', process.env.ACTIONS_RESULTS_URL || '');
             core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || '');
 
-      - name: Patch npm gyp
+      - name: Patch npm gyp (to be able to build addons for the tests)
+        if: ${{ github.event_name != 'workflow_dispatch' }}
         run: curl -L https://github.com/npm/cli/pull/8531.diff | git apply --directory=deps/npm --exclude=deps/npm/package-lock.json
+      
+      - name: Load shell.nix
+        if: ${{ github.event_name == 'workflow_dispatch' }}
+        run: curl -fsLSO https://github.com/${GITHUB_REPOSITORY}/raw/${GITHUB_SHA}/shell.nix
 
-      - name: Build Node.js and run tests
+      - name: Build Node.js ${{ github.event_name == 'workflow_dispatch' && 'on the base commit' || 'and run tests' }}
         run: |
           nix-shell \
             --pure --keep FLAKY_TESTS \
@@ -84,5 +129,46 @@ jobs:
             --arg devTools '[]' \
             --arg benchmarkTools '[]' \
             --run '
-                make run-ci -j4 V=1 TEST_CI_ARGS="-p actions --measure-flakiness 9 --skip-tests=$CI_SKIP_TESTS"
+                make ${{ github.event_name == 'workflow_dispatch' && 'build' || 'run' }}-ci -j4 V=1 TEST_CI_ARGS="-p actions --measure-flakiness 9 --skip-tests=$CI_SKIP_TESTS"
+            '
+
+      - name: Re-build Node.js on the merge commit
+        # ccache is disabled here to avoid polluting the cache. Local build outputs should make this build relatively quick anyway.
+        if: ${{ github.event_name == 'workflow_dispatch' }}
+        run: |
+          mv out/Release/node base_node
+          git reset FETCH_HEAD --hard
+          nix-shell \
+            --pure \
+            --arg loadJSBuiltinsDynamically false \
+            --arg ccache 'null' \
+            --arg devTools '[]' \
+            --arg benchmarkTools '[]' \
+            --run '
+                make -j4 V=1
+            '
+
+      - name: Run benchmark
+        if: ${{ github.event_name == 'workflow_dispatch' }}
+        run: |
+          nix-shell \
+            --pure --keep FILTER \
+            --arg loadJSBuiltinsDynamically false \
+            --arg ccache 'null' \
+            --arg devTools '[]' \
+            --option download-buffer-size "$NIX_DOWNLOAD_BUFFER_SIZE" \
+            --run '
+                set -exo pipefail
+                ./base_node benchmark/compare.js \
+                  --filter "$FILTER" \
+                  --runs ${{ inputs.runs }} \
+                  --old ./base_node --new ./node \
+                  -- ${{ inputs.category }} \
+                | tee /dev/stderr \
+                | Rscript benchmark/compare.R
             '
+        env:
+          FILTER: ${{ inputs.filter }}
+          NIX_DOWNLOAD_BUFFER_SIZE: 536870912 # 512 MiB
+      
+      

shell.nix

@@ -91,7 +91,6 @@ pkgs.mkShell {
   shellHook = if (ccache != null) then ''
     export CC="${pkgs.lib.getExe ccache} $CC"
     export CXX="${pkgs.lib.getExe ccache} $CXX"
-    ls "${pkgs.lib.getLib sharedLibDeps.uvwasi}"
   '' else "";
 
   BUILD_WITH = if (ninja != null) then "ninja" else "make";