diff --git a/.github/workflows/linters.yml b/.github/workflows/linters.yml index c1b0a2234e1078..5e43284edcbe3a 100644 --- a/.github/workflows/linters.yml +++ b/.github/workflows/linters.yml @@ -114,6 +114,22 @@ jobs: NODE=$(command -v node) make lint-md env: NODE_RELEASED_VERSIONS: ${{ steps.get-released-versions.outputs.NODE_RELEASED_VERSIONS }} + lint-nix: + if: github.event.pull_request.draft == false + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + persist-credentials: false + sparse-checkout: '*.nix' + sparse-checkout-cone-mode: false + - uses: cachix/install-nix-action@7be5dee1421f63d07e71ce6e0a9f8a4b07c2a487 # v31.6.1 + - name: Lint Nix files + run: nix-shell -I nixpkgs=./nixpkgs.nix -p 'nixfmt-tree' --run 'treefmt --quiet --fail-on-change' + - if: ${{ failure() }} + name: Show diff + run: git --no-pager diff + lint-py: if: github.event.pull_request.draft == false runs-on: ubuntu-latest diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index f74e912ece0fb8..ed1d2060911f4e 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -5,6 +5,7 @@ on: paths-ignore: - .mailmap - README.md + - '*.nix' - .github/** - '!.github/workflows/test-linux.yml' types: [opened, synchronize, reopened, ready_for_review] @@ -17,6 +18,7 @@ on: paths-ignore: - .mailmap - README.md + - '*.nix' - .github/** - '!.github/workflows/test-linux.yml' diff --git a/.github/workflows/test-macos.yml b/.github/workflows/test-macos.yml index 72b1e8d3de8116..673869303a4dcd 100644 --- a/.github/workflows/test-macos.yml +++ b/.github/workflows/test-macos.yml @@ -6,6 +6,7 @@ on: paths-ignore: - .mailmap - '**.md' + - '*.nix' - AUTHORS - doc/** - .github/** @@ -19,6 +20,7 @@ on: paths-ignore: - .mailmap - '**.md' + - '*.nix' - AUTHORS - doc/** - .github/** diff --git a/.github/workflows/test-shared.yml b/.github/workflows/test-shared.yml new file mode 100644 index 00000000000000..53ae9c8e164de8 --- /dev/null +++ b/.github/workflows/test-shared.yml @@ -0,0 +1,83 @@ +name: Test Shared librairies + +on: + pull_request: + paths-ignore: + - .mailmap + - '**.md' + - AUTHORS + - doc/** + - .github/** + - '!.github/workflows/test-shared.yml' + types: [opened, synchronize, reopened, ready_for_review] + push: + branches: + - main + - canary + - v[0-9]+.x-staging + - v[0-9]+.x + paths-ignore: + - .mailmap + - '**.md' + - AUTHORS + - doc/** + - .github/** + - '!.github/workflows/test-shared.yml' + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +env: + FLAKY_TESTS: keep_retrying + +permissions: + contents: read + +jobs: + build: + strategy: + fail-fast: false + matrix: + include: + - runner: ubuntu-24.04 + system: x86_64-linux + - runner: ubuntu-24.04-arm + system: aarch64-linux + - runner: macos-13 + system: x86_64-darwin + - runner: macos-latest + system: aarch64-darwin + name: '${{ matrix.system }}: with shared libraries' + runs-on: ${{ matrix.runner }} + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false + + - uses: cachix/install-nix-action@7be5dee1421f63d07e71ce6e0a9f8a4b07c2a487 # v31.6.1 + with: + extra_nix_config: sandbox = true + + - name: Configure sccache + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + with: + script: | + core.exportVariable('SCCACHE_GHA_VERSION', 'on'); + core.exportVariable('ACTIONS_CACHE_SERVICE_V2', 'on'); + core.exportVariable('ACTIONS_RESULTS_URL', process.env.ACTIONS_RESULTS_URL || ''); + core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || ''); + + - name: Build Node.js and run tests + run: | + nix-shell \ + -I nixpkgs=./nixpkgs.nix \ + --pure --keep FLAKY_TESTS \ + --keep SCCACHE_GHA_VERSION --keep ACTIONS_CACHE_SERVICE_V2 --keep ACTIONS_RESULTS_URL --keep ACTIONS_RUNTIME_TOKEN \ + --arg loadJSBuiltinsDynamically false \ + --arg ccache '(import {}).sccache' \ + --arg devTools '[]' \ + --arg benchmarkTools '[]' \ + --run ' + make run-ci -j4 V=1 TEST_CI_ARGS="-p actions --measure-flakiness 9 --skip-tests=$CI_SKIP_TESTS" + ' diff --git a/.github/workflows/tools.yml b/.github/workflows/tools.yml index 5ee73cd0cfb93f..822398a8009768 100644 --- a/.github/workflows/tools.yml +++ b/.github/workflows/tools.yml @@ -29,6 +29,7 @@ on: - llhttp - minimatch - nbytes + - nixpkgs-unstable - nghttp2 - nghttp3 - ngtcp2 @@ -181,6 +182,14 @@ jobs: cat temp-output tail -n1 temp-output | grep "NEW_VERSION=" >> "$GITHUB_ENV" || true rm temp-output + - id: nixpkgs-unstable + subsystem: tools + label: tools + run: | + ./tools/dep_updaters/update-nixpkgs-pin.sh > temp-output + cat temp-output + tail -n1 temp-output | grep "NEW_VERSION=" >> "$GITHUB_ENV" || true + rm temp-output - id: nghttp2 subsystem: deps label: dependencies @@ -280,6 +289,9 @@ jobs: uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: ${{ env.PYTHON_VERSION }} + - name: Set up Nix + if: matrix.id == 'nixpkgs-unstable' && (github.event_name == 'schedule' || inputs.id == 'all' || inputs.id == matrix.id) + uses: cachix/install-nix-action@7be5dee1421f63d07e71ce6e0a9f8a4b07c2a487 # v31.6.1 - run: ${{ matrix.run }} if: github.event_name == 'schedule' || inputs.id == 'all' || inputs.id == matrix.id env: diff --git a/BUILDING.md b/BUILDING.md index 84182f8fa05793..626446906a2677 100644 --- a/BUILDING.md +++ b/BUILDING.md @@ -239,6 +239,7 @@ Consult previous versions of this document for older versions of Node.js: Installation via Linux package manager can be achieved with: +* Nix, NixOS: `nix-shell` * Ubuntu, Debian: `sudo apt-get install python3 g++-12 gcc-12 make python3-pip` * Fedora: `sudo dnf install python3 gcc-c++ make python3-pip` * CentOS and RHEL: `sudo yum install python3 gcc-c++ make python3-pip` @@ -259,6 +260,75 @@ installed, you can find them under the menu `Xcode -> Open Developer Tool -> More Developer Tools...`. This step will install `clang`, `clang++`, and `make`. +#### Nix integration + +If you are using Nix and direnv, you can use the following to get started: + +```bash +echo 'use_nix --arg sharedLibDeps {} --argstr icu small' > .envrc +direnv allow . +make build-ci -j12 +``` + +The use of `make build-ci` is to ensure you are using the `CONFIG_FLAGS` +environment variable. You can also specify it manually: + +```bash +./configure $CONFIG_FLAGS +make -j12 +``` + +Passing the `--arg sharedLibDeps {}` instructs direnv and Nix to generate an +environment that uses the vendored-in native dependencies. Using the vendored-in +dependencies result in a result closer to the official binaries, the tradeoff +being the build will take longer to complete as you'd have to build those +dependencies instead of using the cached ones from the Nix cache. You can omit +that flag to use all the shared dependencies, or specify only some dependencies: + +```bash +cat -> .envrc <<'EOF' +use nix --arg sharedLibDeps '{ + inherit (import {}) + openssl + zlib + ; +}' +EOF +``` + +Passing the `--argstr icu small` instructs direnv and Nix to pass `--with-intl=small` in +the `CONFIG_FLAGS` environment variable. If you omit this, the prebuilt ICU from Nix cache +will be used, which should speed up greatly compilation time. + +The use of `direnv` is completely optional, you can also use `nix-shell` directly, +e.g. here's a command you can use to build a binary for benchmarking purposes: + +```bash +# Passing `--arg loadJSBuiltinsDynamically false` to instruct the compiler to +# embed the JS core files so it is no longer affected by local changes +# (necessary for getting useful benchmark results). +# Passing `--arg devTools '[]' --arg benchmarkTools '[]'` since we don't need +# those to build node. +nix-shell \ + --arg loadJSBuiltinsDynamically false \ + --arg devTools '[]' --arg benchmarkTools '[]' \ + --run 'make build-ci -j12' + +mv out/Release/node ./node_old + +# ... +# Make your local changes, and re-build node + +nix-shell \ + --arg loadJSBuiltinsDynamically false \ + --arg devTools '[]' --arg benchmarkTools '[]' \ + --run 'make build-ci -j12' + +nix-shell --pure --run './node benchmark/compare.js --old ./node_old --new ./node http | Rscript benchmark/compare.R' +``` + +There are additional attributes you can pass, see `shell.nix` file for more details. + #### Building Node.js If the path to your build directory contains a space, the build will likely @@ -267,7 +337,6 @@ fail. To build Node.js: ```bash -export CXX=g++-12 ./configure make -j4 ``` diff --git a/doc/contributing/writing-and-running-benchmarks.md b/doc/contributing/writing-and-running-benchmarks.md index a60a06b695f242..1dcb2a62a952bb 100644 --- a/doc/contributing/writing-and-running-benchmarks.md +++ b/doc/contributing/writing-and-running-benchmarks.md @@ -26,6 +26,10 @@ Basic Unix tools are required for some benchmarks. [Git for Windows][git-for-windows] includes Git Bash and the necessary tools, which need to be included in the global Windows `PATH`. +If you are using Nix, all the required tools are already listed in the +`benchmarkTools` argument of the `shell.nix` file, so you can skip those +prerequesites. + ### HTTP benchmark requirements Most of the HTTP benchmarks require a benchmarker to be installed. This can be diff --git a/nixpkgs.nix b/nixpkgs.nix new file mode 100644 index 00000000000000..3251e57538a71e --- /dev/null +++ b/nixpkgs.nix @@ -0,0 +1,10 @@ +arg: +let + repo = "https://github.com/NixOS/nixpkgs"; + rev = "ca77296380960cd497a765102eeb1356eb80fed0"; + nixpkgs = import (builtins.fetchTarball { + url = "${repo}/archive/${rev}.tar.gz"; + sha256 = "1airrw6l87iyny1a3mb29l28na4s4llifprlgpll2na461jd40iy"; + }) arg; +in +nixpkgs diff --git a/shell.nix b/shell.nix new file mode 100644 index 00000000000000..d21ceb3a635f97 --- /dev/null +++ b/shell.nix @@ -0,0 +1,120 @@ +{ + pkgs ? import "${./nixpkgs.nix}" { }, + loadJSBuiltinsDynamically ? true, # Load `lib/**.js` from disk instead of embedding + ncu-path ? null, # Provide this if you want to use a local version of NCU + icu ? pkgs.icu, + sharedLibDeps ? { + inherit (pkgs) + ada + brotli + c-ares + libuv + nghttp2 + nghttp3 + ngtcp2 + openssl + simdjson + simdutf + sqlite + uvwasi + zlib + zstd + ; + http-parser = pkgs.llhttp; + }, + ccache ? pkgs.ccache, + ninja ? pkgs.ninja, + devTools ? [ + pkgs.curl + pkgs.gh + pkgs.git + pkgs.jq + pkgs.shellcheck + ] + ++ ( + if (ncu-path == null) then + [ pkgs.node-core-utils ] + else + [ + (pkgs.writeShellScriptBin "git-node" "exec \"${ncu-path}/bin/git-node.js\" \"$@\"") + (pkgs.writeShellScriptBin "ncu-ci" "exec \"${ncu-path}/bin/ncu-ci.js\" \"$@\"") + (pkgs.writeShellScriptBin "ncu-config" "exec \"${ncu-path}/bin/ncu-config.js\" \"$@\"") + ] + ), + benchmarkTools ? [ + pkgs.R + pkgs.rPackages.ggplot2 + pkgs.rPackages.plyr + pkgs.wrk + ], + extraConfigFlags ? [ + "--without-npm" + "--debug-node" + ], +}: + +let + useSharedICU = if builtins.isString icu then icu == "system" else icu != null; + useSharedAda = builtins.hasAttr "ada" sharedLibDeps; + useSharedOpenSSL = builtins.hasAttr "openssl" sharedLibDeps; +in +pkgs.mkShell { + inherit (pkgs.nodejs_latest) nativeBuildInputs; + + buildInputs = builtins.attrValues sharedLibDeps ++ pkgs.lib.optionals useSharedICU [ icu ]; + + packages = [ + ccache + ] + ++ devTools + ++ benchmarkTools; + + shellHook = + if (ccache != null) then + '' + export CC="${pkgs.lib.getExe ccache} $CC" + export CXX="${pkgs.lib.getExe ccache} $CXX" + '' + else + ""; + + BUILD_WITH = if (ninja != null) then "ninja" else "make"; + NINJA = if (ninja != null) then "${pkgs.lib.getExe ninja}" else ""; + CI_SKIP_TESTS = pkgs.lib.concatStringsSep "," ( + [ ] + ++ pkgs.lib.optionals useSharedAda [ + # Different versions of Ada affect the WPT tests + "test-url" + ] + ++ pkgs.lib.optionals useSharedOpenSSL [ + # Path to the openssl.cnf is different from the expected one + "test-strace-openat-openssl" + ] + ); + CONFIG_FLAGS = builtins.toString ( + [ + ( + if icu == null then + "--without-intl" + else + "--with-intl=${if useSharedICU then "system" else icu}-icu" + ) + ] + ++ extraConfigFlags + ++ pkgs.lib.optionals (ninja != null) [ + "--ninja" + ] + ++ pkgs.lib.optionals loadJSBuiltinsDynamically [ + "--node-builtin-modules-path=${builtins.toString ./.}" + ] + ++ pkgs.lib.concatMap (name: [ + "--shared-${builtins.replaceStrings [ "c-ares" ] [ "cares" ] name}" + "--shared-${builtins.replaceStrings [ "c-ares" ] [ "cares" ] name}-libpath=${ + pkgs.lib.getLib sharedLibDeps.${name} + }/lib" + "--shared-${builtins.replaceStrings [ "c-ares" ] [ "cares" ] name}-include=${ + pkgs.lib.getInclude sharedLibDeps.${name} + }/include" + ]) (builtins.attrNames sharedLibDeps) + ); +} diff --git a/tools/dep_updaters/update-nixpkgs-pin.sh b/tools/dep_updaters/update-nixpkgs-pin.sh new file mode 100755 index 00000000000000..1cfb836eb2311b --- /dev/null +++ b/tools/dep_updaters/update-nixpkgs-pin.sh @@ -0,0 +1,40 @@ +#!/bin/sh +set -ex +# Shell script to update Nixpkgs pin in the source tree to the most recent +# version on the unstable channel. + +BASE_DIR=$(cd "$(dirname "$0")/../.." && pwd) +NIXPKGS_PIN_FILE="$BASE_DIR/nixpkgs.nix" + +NIXPKGS_REPO=$(grep 'repo =' "$NIXPKGS_PIN_FILE" | awk -F'"' '{ print $2 }') +CURRENT_VERSION_SHA1=$(grep 'rev =' "$NIXPKGS_PIN_FILE" | awk -F'"' '{ print $2 }') +CURRENT_VERSION=$(echo "$CURRENT_VERSION_SHA1" | head -c 7) + +NEW_UPSTREAM_SHA1=$(git ls-remote "$NIXPKGS_REPO.git" nixpkgs-unstable | awk '{print $1}') +NEW_VERSION=$(echo "$NEW_UPSTREAM_SHA1" | head -c 7) + + +# shellcheck disable=SC1091 +. "$BASE_DIR/tools/dep_updaters/utils.sh" + +compare_dependency_version "nixpkgs-unstable" "$NEW_VERSION" "$CURRENT_VERSION" + +CURRENT_TARBALL_HASH=$(grep 'sha256 =' "$NIXPKGS_PIN_FILE" | awk -F'"' '{ print $2 }') +NEW_TARBALL_HASH=$(nix-prefetch-url --unpack "$NIXPKGS_REPO/archive/$NEW_UPSTREAM_SHA1.tar.gz") + +TMP_FILE=$(mktemp) +sed "s/$CURRENT_VERSION_SHA1/$NEW_UPSTREAM_SHA1/;s/$CURRENT_TARBALL_HASH/$NEW_TARBALL_HASH/" "$NIXPKGS_PIN_FILE" > "$TMP_FILE" +mv "$TMP_FILE" "$NIXPKGS_PIN_FILE" + +cat -<