tools: add benchmark runner to `test-shared` #1598

Workflow file for this run

.github/workflows/test-shared.yml at 8fc7899

	# This action uses the following secrets:
	# CACHIX_AUTH_TOKEN: Write access to nodejs.cachix.org – without it, the cache is read-only.
	name: Benchmark and shared libraries

	on:
	pull_request:
	paths-ignore:
	- '**.md'
	- eslint.config.mjs
	- '**/eslint.config_partial.mjs'
	- android-configure
	- android-configure.py
	- android-patches/**
	- benchmarks/**
	- codecov.yml
	- deps/ada/**
	- deps/brotli/**
	- deps/cares/**
	- deps/corepack/**
	- deps/icu-small/**
	- deps/icu-tmp/**
	- deps/llhttp/**
	- deps/nghttp2/**
	- deps/ngtcp2/**
	- deps/openssl//*
	- deps/simdjson/**
	- deps/sqlite/**
	- deps/temporal/**
	- deps/uv/**
	- deps/uvwasi/**
	- deps/zlib/**
	- deps/zstd/**
	- doc/**
	- pyproject.yml
	- tsconfig.json
	- test/internet/**
	- tools/**
	- '!tools/gyp/**'
	- '!tools/nix/**'
	- '!tools/v8/**'
	- '!tools/v8_gypfiles/**'
	- typings/**
	- vcbuild.bat
	- .**
	- '!.github/workflows/test-shared.yml'
	types: [opened, synchronize, reopened, ready_for_review]
	push:
	branches:
	- main
	- canary
	- v[0-9]+.x-staging
	- v[0-9]+.x
	paths-ignore:
	- '**.md'
	- eslint.config.mjs
	- '**/eslint.config_partial.mjs'
	- android-configure
	- android-configure.py
	- android-patches/**
	- benchmarks/**
	- codecov.yml
	- deps/ada/**
	- deps/brotli/**
	- deps/cares/**
	- deps/corepack/**
	- deps/icu-small/**
	- deps/icu-tmp/**
	- deps/llhttp/**
	- deps/nghttp2/**
	- deps/ngtcp2/**
	- deps/openssl//*
	- deps/simdjson/**
	- deps/sqlite/**
	- deps/temporal/**
	- deps/uv/**
	- deps/uvwasi/**
	- deps/zlib/**
	- deps/zstd/**
	- doc/**
	- pyproject.yml
	- tsconfig.json
	- test/internet/**
	- tools/**
	- '!tools/gyp/**'
	- '!tools/nix/**'
	- '!tools/v8/**'
	- '!tools/v8_gypfiles/**'
	- typings/**
	- vcbuild.bat
	- .**
	- '!.github/workflows/test-shared.yml'
	workflow_dispatch:
	inputs:
	repo:
	type: string
	description: GitHub repository to fetch from (default to the current repo)
	pr_id:
	type: number
	required: true
	description: The PR to test
	commit:
	required: true
	type: string
	description: The expect HEAD of the PR
	category:
	required: true
	type: string
	description: The category (or categories) of tests to run, for example buffers, cluster etc. Maps to a folders in node/benchmark
	filter:
	type: string
	description: A substring to restrict the benchmarks to run in a category. e.g. `net-c2c`
	runs:
	type: number
	default: 30
	description: How many times to repeat each benchmark

	concurrency:
	group: ${{ github.workflow }}-${{ github.head_ref \|\| github.run_id }}
	cancel-in-progress: true

	env:
	FLAKY_TESTS: keep_retrying

	permissions:
	contents: read

	jobs:
	build-tarball:
	if: github.event.pull_request.draft == false
	name: ${{ github.event_name == 'workflow_dispatch' && 'Skipped job' \|\| 'Build slim tarball' }}
	runs-on: ubuntu-24.04-arm
	steps:
	- uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0
	if: ${{ github.event_name != 'workflow_dispatch' }}
	with:
	persist-credentials: false

	- name: Make tarball
	if: ${{ github.event_name != 'workflow_dispatch' }}
	run: \|
	export DATESTRING=$(date "+%Y-%m-%d")
	export COMMIT=$(git rev-parse --short=10 "$GITHUB_SHA")
	./configure && make tar -j4 SKIP_XZ=1 SKIP_SHARED_DEPS=1
	env:
	DISTTYPE: nightly

	- name: Upload tarball artifact
	if: ${{ github.event_name != 'workflow_dispatch' }}
	uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0
	with:
	name: tarballs
	path: '*.tar.gz'
	compression-level: 0

	build:
	needs: build-tarball
	strategy:
	fail-fast: false
	matrix:
	include:
	- runner: ubuntu-24.04
	system: x86_64-linux
	- runner: ubuntu-24.04-arm
	system: aarch64-linux
	- runner: macos-15-intel
	system: x86_64-darwin
	- runner: macos-latest
	system: aarch64-darwin
	name: '${{ matrix.system }}: with shared libraries'
	runs-on: ${{ matrix.runner }}
	steps:
	- uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0
	if: ${{ github.event_name != 'workflow_dispatch' }}
	with:
	name: tarballs
	path: tarballs

	- name: Extract tarball
	if: ${{ github.event_name != 'workflow_dispatch' }}
	run: \|
	tar xzf tarballs/*.tar.gz -C "$RUNNER_TEMP"
	echo "TAR_DIR=$RUNNER_TEMP/$(basename tarballs/*.tar.gz .tar.gz)" >> "$GITHUB_ENV"

	- uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
	if: ${{ github.event_name == 'workflow_dispatch' }}
	with:
	repository: ${{ inputs.repo \|\| github.repository }}
	ref: refs/pull/${{ inputs.pr_id }}/merge
	persist-credentials: false
	fetch-depth: 2

	- name: Validate PR head and roll back to base commit
	if: ${{ github.event_name == 'workflow_dispatch' }}
	run: \|
	[ "$(git rev-parse HEAD^2)" = "$EXPECTED_SHA" ]
	git reset HEAD^ --hard
	env:
	EXPECTED_SHA: ${{ inputs.commit }}

	- uses: cachix/install-nix-action@0b0e072294b088b73964f1d72dfdac0951439dbd # v31.8.4
	with:
	extra_nix_config: sandbox = true

	- uses: cachix/cachix-action@0fc020193b5a1fa3ac4575aa3a7d3aa6a35435ad # v16
	with:
	name: nodejs
	authToken: ${{ secrets.CACHIX_AUTH_TOKEN }}

	- name: Configure sccache
	uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
	with:
	script: \|
	core.exportVariable('SCCACHE_GHA_VERSION', 'on');
	core.exportVariable('ACTIONS_CACHE_SERVICE_V2', 'on');
	core.exportVariable('ACTIONS_RESULTS_URL', process.env.ACTIONS_RESULTS_URL \|\| '');
	core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN \|\| '');

	- name: Load shell.nix
	if: github.event_name != 'workflow_dispatch'
	run: \|
	mv "$TAR_DIR"/*.nix .
	mkdir tools
	mv "$TAR_DIR"/tools/nix tools/.

	- name: Build Node.js ${{ github.event_name == 'workflow_dispatch' && 'on the base commit' \|\| 'and run tests' }}
	run: \|
	nix-shell \
	-I nixpkgs=./tools/nix/pkgs.nix \
	--pure --keep TAR_DIR --keep FLAKY_TESTS \
	--keep SCCACHE_GHA_VERSION --keep ACTIONS_CACHE_SERVICE_V2 --keep ACTIONS_RESULTS_URL --keep ACTIONS_RUNTIME_TOKEN \
	--arg loadJSBuiltinsDynamically false \
	--arg ccache '(import <nixpkgs> {}).sccache' \
	--arg devTools '[]' \
	--arg benchmarkTools '[]' \
	${{ endsWith(matrix.system, '-darwin') && '--arg withAmaro false --arg withSQLite false --arg extraConfigFlags ''["--without-inspector" "--without-node-options"]'' \' \|\| '\' }}
	--run '
	make ${{ github.event_name == 'workflow_dispatch' && 'build' \|\| '-C "$TAR_DIR" run' }}-ci -j4 V=1 TEST_CI_ARGS="-p actions --measure-flakiness 9 --skip-tests=$CI_SKIP_TESTS"
	'

	- name: Re-build Node.js on the merge commit
	# ccache is disabled here to avoid polluting the cache. Local build outputs should make this build relatively quick anyway.
	if: ${{ github.event_name == 'workflow_dispatch' }}
	run: \|
	mv out/Release/node base_node
	git reset FETCH_HEAD --hard
	nix-shell \
	-I nixpkgs=./tools/nix/pkgs.nix \
	--pure \
	--arg loadJSBuiltinsDynamically false \
	--arg ccache 'null' \
	--arg devTools '[]' \
	--arg benchmarkTools '[]' \
	--run '
	make -j4 V=1
	'

	- name: Run benchmark
	if: ${{ github.event_name == 'workflow_dispatch' }}
	run: \|
	nix-shell \
	-I nixpkgs=./tools/nix/pkgs.nix \
	--pure --keep FILTER --keep LC_ALL --keep LANG \
	--arg loadJSBuiltinsDynamically false \
	--arg ccache 'null' \
	--arg icu 'null' \
	--arg sharedLibDeps '{}' \
	--arg devTools '[]' \
	--run '
	set -o pipefail
	./base_node benchmark/compare.js \
	--filter "$FILTER" \
	--runs ${{ inputs.runs }} \
	--old ./base_node --new ./node \
	-- ${{ inputs.category }} \
	\| tee /dev/stderr \
	> ${{ matrix.system }}.csv
	echo "Warning: do not take GHA benchmark results as face value, always confirm them"
	echo "using a dedicated machine, e.g. Jenkins CI."
	echo
	echo "Benchmark results:"
	echo
	echo '"'"'```'"'"'
	Rscript benchmark/compare.R < ${{ matrix.system }}.csv
	echo '"'"'```'"'"'
	echo
	echo "Warning: do not take GHA benchmark results as face value, always confirm them"
	echo "using a dedicated machine, e.g. Jenkins CI."
	' \| tee /dev/stderr >> "$GITHUB_STEP_SUMMARY"
	env:
	FILTER: ${{ inputs.filter }}

	- name: Upload raw benchmark results
	if: ${{ github.event_name == 'workflow_dispatch' }}
	uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
	with:
	name: csv-${{ matrix.system }}
	path: ${{ matrix.system }}.csv

	aggregate-benchmark-results:
	needs: build
	name: Aggregate benchmark results
	if: ${{ github.event_name == 'workflow_dispatch' }}
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
	with:
	persist-credentials: false
	sparse-checkout: \|
	benchmark/*.R
	tools/nix/*.nix
	*.nix
	sparse-checkout-cone-mode: false

	- name: Download benchmark raw results
	uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0
	with:
	pattern: csv-*
	merge-multiple: true
	path: raw-results

	- uses: cachix/install-nix-action@7be5dee1421f63d07e71ce6e0a9f8a4b07c2a487 # v31.6.1
	with:
	extra_nix_config: sandbox = true

	- name: Benchmark results
	run: \|
	nix-shell \
	-I nixpkgs=./tools/nix/pkgs.nix \
	--pure --keep LC_ALL --keep LANG \
	--arg loadJSBuiltinsDynamically false \
	--arg ccache 'null' \
	--arg icu 'null' \
	--arg sharedLibDeps '{}' \
	--arg devTools '[]' \
	--run '
	echo "Warning: do not take GHA benchmark results as face value, always confirm them"
	echo "using a dedicated machine, e.g. Jenkins CI."
	echo
	echo "Benchmark results:"
	echo
	echo '"'"'```'"'"'
	awk "FNR==1 && NR!=1{next;}{print}" raw-results/*.csv \| Rscript benchmark/compare.R
	echo '"'"'```'"'"'
	echo
	echo "Warning: do not take GHA benchmark results as face value, always confirm them"
	echo "using a dedicated machine, e.g. Jenkins CI."
	' \| tee /dev/stderr >> "$GITHUB_STEP_SUMMARY"

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Uh oh!

Uh oh!

tools: add benchmark runner to `test-shared` #1598

Workflow file

tools: add benchmark runner to `test-shared` #1598

Uh oh!

Workflow file for this run

Uh oh!

tools: add benchmark runner to test-shared #1598

Workflow file

Workflow file for this run

tools: add benchmark runner to `test-shared` #1598